@framers/agentos-ext-account-sources 0.4.0 → 0.6.0

package/dist/sync/SourceQuota.js

@@ -0,0 +1,336 @@
+/**
+ * @fileoverview Persistent weighted request quotas on PostgreSQL through `@framers/sql-storage-adapter`: three
+ * released core spend meters (a project's rolling minute, a mailbox's rolling minute and a project's UTC day),
+ * request slots that every worker shares and that expire after a process dies, and durable provider retry times.
+ * @module account-sources/sync/SourceQuota
+ */
+import { randomUUID } from 'node:crypto';
+import { SqlSpendMeter, spendMeterDdl } from '@framers/agentos/safety/runtime';
+import { SourceSyncFault } from './state.js';
+class QuotaDeferred extends Error {
+    reason;
+    retryAt;
+    constructor(reason, retryAt) {
+        super('source_quota_deferred');
+        this.reason = reason;
+        this.retryAt = retryAt;
+    }
+}
+class QuotaDuplicate extends Error {
+    constructor() { super('source_quota_duplicate'); }
+}
+class QuotaLate extends Error {
+    retryAt;
+    constructor(retryAt) {
+        super('source_quota_late');
+        this.retryAt = retryAt;
+    }
+}
+/** Milliseconds after the reservation's clock reading within which an attempt must start its call. */
+const sendWindowMs = 1000;
+/** Rows of each table one `compact` call deletes at most. */
+const compactLimit = 1000;
+const integer = (n) => Number.isSafeInteger(n) && n >= 0;
+const key = (s) => typeof s === 'string' && s.length > 0 && s.length <= 512;
+const dayEnd = (now) => (Math.floor(now / 86_400_000) + 1) * 86_400_000;
+function tables(prefix = 'agentos_source_quota') {
+    if (!/^[a-z][a-z0-9_]{0,27}$/.test(prefix))
+        throw new Error('invalid_source_quota_prefix');
+    return { projects: `"${prefix}_projects"`, mailboxes: `"${prefix}_mailboxes"`,
+        attempts: `"${prefix}_attempts"`, pm: `${prefix}_pm`, mm: `${prefix}_mm`, pd: `${prefix}_pd` };
+}
+/**
+ * The quota tables' DDL under a prefix of a lower-case letter and at most 27 lower-case letters, digits or underscores
+ * (`agentos_source_quota` unless given), as seven scripts for the host's migration to run with `exec`: the project,
+ * mailbox and attempt tables, the attempts' index, then a core spend meter's tables and indexes for each budget under
+ * `<prefix>_pm`, `<prefix>_mm` and `<prefix>_pd`. Throws `invalid_source_quota_prefix` for another prefix.
+ */
+export function sourceQuotaSchema(prefix = 'agentos_source_quota') {
+    const t = tables(prefix);
+    return [
+        `CREATE TABLE IF NOT EXISTS ${t.projects} (project_key TEXT PRIMARY KEY,policy TEXT NOT NULL,retry_at BIGINT NOT NULL DEFAULT 0)`,
+        `CREATE TABLE IF NOT EXISTS ${t.mailboxes} (project_key TEXT NOT NULL,mailbox_key TEXT NOT NULL,
+      retry_at BIGINT NOT NULL DEFAULT 0,PRIMARY KEY(project_key,mailbox_key))`,
+        `CREATE TABLE IF NOT EXISTS ${t.attempts} (attempt_id TEXT PRIMARY KEY,project_key TEXT NOT NULL,
+      mailbox_key TEXT NOT NULL,token TEXT NOT NULL,send_by BIGINT NOT NULL,expires_at BIGINT NOT NULL,
+      started BOOLEAN NOT NULL DEFAULT FALSE,finished BOOLEAN NOT NULL DEFAULT FALSE)`,
+        `CREATE INDEX IF NOT EXISTS "${prefix}_active" ON ${t.attempts}(project_key,finished,expires_at)`,
+        spendMeterDdl(t.pm), spendMeterDdl(t.mm), spendMeterDdl(t.pd),
+    ];
+}
+function checkedPolicy(policy) {
+    const weights = { list: policy.weights.list, read: policy.weights.read,
+        changes: policy.weights.changes, profile: policy.weights.profile, labels: policy.weights.labels };
+    const copy = { projectMinute: policy.projectMinute, mailboxMinute: policy.mailboxMinute,
+        projectDay: policy.projectDay, projectConcurrency: policy.projectConcurrency,
+        mailboxConcurrency: policy.mailboxConcurrency, timeoutMs: policy.timeoutMs,
+        slotGraceMs: policy.slotGraceMs, weights: Object.freeze(weights) };
+    const numbers = [copy.projectMinute, copy.mailboxMinute, copy.projectDay, copy.projectConcurrency,
+        copy.mailboxConcurrency, copy.timeoutMs, copy.slotGraceMs, ...Object.values(weights)];
+    if (!numbers.every((n) => integer(n) && n > 0) || copy.timeoutMs > 60_000
+        || copy.slotGraceMs < copy.timeoutMs || copy.slotGraceMs > 120_000
+        || copy.mailboxConcurrency > copy.projectConcurrency)
+        throw new Error('invalid_source_quota_policy');
+    return Object.freeze(copy);
+}
+/** Combines released meters; this class never reimplements their accounting SQL. */
+export class SourceQuota {
+    db;
+    options;
+    t;
+    policy;
+    fingerprint;
+    /**
+     * Takes a PostgreSQL adapter with transactions and runs no DDL. Throws `source_quota_requires_postgres` for another
+     * adapter, `invalid_source_quota_prefix` for a bad prefix and `invalid_source_quota_policy` for a bad policy.
+     */
+    constructor(db, options) {
+        this.db = db;
+        this.options = options;
+        if (db.kind !== 'postgres' || !db.capabilities.has('transactions'))
+            throw new Error('source_quota_requires_postgres');
+        this.t = tables(options.tablePrefix);
+        this.policy = checkedPolicy(options.policy);
+        this.fingerprint = JSON.stringify(this.policy);
+    }
+    async now(tx) {
+        const n = this.options.clock ? await this.options.clock(tx)
+            : Number((await tx.get('SELECT floor(extract(epoch from clock_timestamp()) * 1000)::bigint AS now'))?.now);
+        if (!integer(n) || n > 8_639_999_913_600_000)
+            throw new Error('invalid_source_quota_clock');
+        return n;
+    }
+    async project(tx, projectKey) {
+        if (!key(projectKey))
+            throw new Error('invalid_source_quota_key');
+        await tx.run(`INSERT INTO ${this.t.projects}(project_key,policy) VALUES($1,$2) ON CONFLICT DO NOTHING`, [projectKey, this.fingerprint]);
+        const row = await tx.get(`SELECT policy,retry_at FROM ${this.t.projects} WHERE project_key=$1 FOR UPDATE`, [projectKey]);
+        if (!row || row.policy !== this.fingerprint)
+            throw new Error('source_quota_policy_mismatch');
+        return row;
+    }
+    meter(tx, prefix, allowance, daily) {
+        return new SqlSpendMeter({ db: tx, tablePrefix: prefix, allowanceFor: () => allowance,
+            ...(daily ? { periodOf: (now) => String(Math.floor(now / 86_400_000)) } : { windowMs: 60_000 }),
+            ensureSchema: false, requireShared: true, leaseMs: 120_000, maxAttemptsPerOperation: 1,
+            unknownAfterLease: 'consume', retry: { attempts: 1, baseDelayMs: 0, maxDelayMs: 0, deadlineMs: 1000 } });
+    }
+    async reserve(request) {
+        if (![request.projectKey, request.mailboxKey, request.attemptId].every(key)
+            || !Object.hasOwn(this.policy.weights, request.method))
+            throw new SourceSyncFault('invalid_response');
+        return this.db.transaction(async (tx) => {
+            const project = await this.project(tx, request.projectKey);
+            const now = await this.now(tx);
+            if (await tx.get(`SELECT attempt_id FROM ${this.t.attempts} WHERE attempt_id=$1`, [request.attemptId]))
+                throw new QuotaDuplicate();
+            const mailbox = await tx.get(`SELECT retry_at FROM ${this.t.mailboxes}
+        WHERE project_key=$1 AND mailbox_key=$2`, [request.projectKey, request.mailboxKey]);
+            const retryAt = Math.max(Number(project.retry_at), Number(mailbox?.retry_at ?? 0));
+            if (retryAt > now)
+                throw new QuotaDeferred('provider_wait', retryAt);
+            const slots = await tx.all(`SELECT mailbox_key,expires_at FROM ${this.t.attempts} WHERE project_key=$1 AND finished=FALSE AND expires_at>$2`, [request.projectKey, now]);
+            const local = slots.filter((s) => s.mailbox_key === request.mailboxKey);
+            if (local.length >= this.policy.mailboxConcurrency) {
+                throw new QuotaDeferred('mailbox_concurrency', Math.min(...local.map((s) => Number(s.expires_at))));
+            }
+            if (slots.length >= this.policy.projectConcurrency) {
+                throw new QuotaDeferred('project_concurrency', Math.min(...slots.map((s) => Number(s.expires_at))));
+            }
+            const projectId = JSON.stringify([request.projectKey]);
+            const mailboxId = JSON.stringify([request.projectKey, request.mailboxKey]);
+            const meters = [
+                { meter: this.meter(tx, this.t.pm, this.policy.projectMinute, false), accountId: projectId,
+                    reason: 'project_minute', retryAt: now + 60_001 },
+                { meter: this.meter(tx, this.t.mm, this.policy.mailboxMinute, false), accountId: mailboxId,
+                    reason: 'mailbox_minute', retryAt: now + 60_001 },
+                { meter: this.meter(tx, this.t.pd, this.policy.projectDay, true), accountId: projectId,
+                    reason: 'project_day', retryAt: dayEnd(now) },
+            ];
+            for (const entry of meters) {
+                const reserved = await entry.meter.reserve({ accountId: entry.accountId,
+                    operationId: request.attemptId, units: this.policy.weights[request.method], now });
+                if (reserved.status !== 'reserved') {
+                    if (reserved.reason !== 'allowance_exhausted')
+                        throw new QuotaDuplicate();
+                    // Propagate past the outer transaction: nested adapter calls have no savepoint.
+                    throw new QuotaDeferred(entry.reason, entry.retryAt);
+                }
+                const settled = await entry.meter.settle({ operationId: request.attemptId, outcome: 'consumed', now });
+                if (settled.status !== 'settled' || settled.state !== 'consumed')
+                    throw new Error('source_quota_settle_failed');
+            }
+            const permit = { token: randomUUID(), sendBy: now + sendWindowMs,
+                expiresAt: now + sendWindowMs + this.policy.timeoutMs + this.policy.slotGraceMs, timeoutMs: this.policy.timeoutMs };
+            await tx.run(`INSERT INTO ${this.t.attempts}(attempt_id,project_key,mailbox_key,token,send_by,expires_at)
+        VALUES($1,$2,$3,$4,$5,$6)`, [request.attemptId, request.projectKey, request.mailboxKey,
+                permit.token, permit.sendBy, permit.expiresAt]);
+            return permit;
+        });
+    }
+    async begin(request, permit) {
+        await this.db.transaction(async (tx) => {
+            const project = await this.project(tx, request.projectKey);
+            const now = await this.now(tx);
+            const mailbox = await tx.get(`SELECT retry_at FROM ${this.t.mailboxes}
+        WHERE project_key=$1 AND mailbox_key=$2`, [request.projectKey, request.mailboxKey]);
+            const wait = Math.max(Number(project.retry_at), Number(mailbox?.retry_at ?? 0));
+            if (wait > now)
+                throw new QuotaDeferred('provider_wait', wait);
+            const updated = await tx.run(`UPDATE ${this.t.attempts} SET started=TRUE
+        WHERE attempt_id=$1 AND project_key=$2 AND mailbox_key=$3 AND token=$4
+        AND started=FALSE AND finished=FALSE AND send_by>=$5 AND expires_at>$5`, [request.attemptId, request.projectKey, request.mailboxKey, permit.token, now]);
+            if (updated.changes === 1)
+                return;
+            // This run's own attempt, still unstarted, missed only the time conditions: its send window closed before the start.
+            const unstarted = await tx.get(`SELECT attempt_id FROM ${this.t.attempts}
+        WHERE attempt_id=$1 AND project_key=$2 AND mailbox_key=$3 AND token=$4 AND started=FALSE AND finished=FALSE`, [request.attemptId, request.projectKey, request.mailboxKey, permit.token]);
+            throw unstarted ? new QuotaLate(now + sendWindowMs) : new QuotaDuplicate();
+        });
+    }
+    async release(request, permit) {
+        await this.db.transaction(async (tx) => {
+            await this.project(tx, request.projectKey);
+            await tx.run(`UPDATE ${this.t.attempts} SET finished=TRUE WHERE attempt_id=$1 AND token=$2
+        AND project_key=$3 AND mailbox_key=$4`, [request.attemptId, permit.token, request.projectKey, request.mailboxKey]);
+        });
+    }
+    /**
+     * Consume weighted capacity and call the transport once. No inline HTTP retries: a retry, after a deferral or a
+     * `late` answer among others, is a new `run` with a new attempt id.
+     */
+    async run(request, send, parent) {
+        if (parent?.aborted)
+            throw new SourceSyncFault('cancelled');
+        let permit;
+        try {
+            permit = await this.reserve(request);
+        }
+        catch (error) {
+            if (error instanceof QuotaDeferred)
+                return { kind: 'deferred', reason: error.reason, retryAt: error.retryAt };
+            if (error instanceof QuotaDuplicate)
+                return { kind: 'duplicate' };
+            if (error instanceof SourceSyncFault)
+                throw error;
+            throw new SourceSyncFault('storage_unavailable');
+        }
+        const controller = new AbortController();
+        const abort = () => controller.abort();
+        parent?.addEventListener('abort', abort, { once: true });
+        let timer;
+        let began = false;
+        try {
+            try {
+                await this.begin(request, permit);
+                began = true;
+            }
+            catch (error) {
+                if (error instanceof QuotaDeferred)
+                    return { kind: 'deferred', reason: error.reason, retryAt: error.retryAt };
+                if (error instanceof QuotaLate)
+                    return { kind: 'late', retryAt: error.retryAt };
+                if (error instanceof QuotaDuplicate)
+                    return { kind: 'duplicate' };
+                throw new SourceSyncFault('storage_unavailable');
+            }
+            if (parent?.aborted)
+                throw new SourceSyncFault('cancelled');
+            timer = setTimeout(abort, permit.timeoutMs);
+            // The callback must consume or cancel its body before resolving and honor abort.
+            const value = await send(controller.signal);
+            if (controller.signal.aborted)
+                throw new SourceSyncFault(parent?.aborted ? 'cancelled' : 'network');
+            return { kind: 'completed', value };
+        }
+        finally {
+            if (timer !== undefined)
+                clearTimeout(timer);
+            parent?.removeEventListener('abort', abort);
+            // Request units remain consumed even if begin, transport, parsing or cleanup fails.
+            try {
+                await this.release(request, permit);
+            }
+            catch {
+                if (began)
+                    throw new SourceSyncFault('storage_unavailable');
+            }
+        }
+    }
+    /** A longer provider wait survives restarts, generation changes and shorter updates. */
+    async defer(scope, retryAt) {
+        if (!key(scope.projectKey) || (scope.mailboxKey !== undefined && !key(scope.mailboxKey))
+            || !integer(retryAt) || retryAt > 8_639_999_913_600_000)
+            throw new SourceSyncFault('invalid_response');
+        try {
+            await this.db.transaction(async (tx) => {
+                await this.project(tx, scope.projectKey);
+                if (scope.mailboxKey === undefined) {
+                    await tx.run(`UPDATE ${this.t.projects} SET retry_at=GREATEST(retry_at,$2) WHERE project_key=$1`, [scope.projectKey, retryAt]);
+                }
+                else {
+                    await tx.run(`INSERT INTO ${this.t.mailboxes}(project_key,mailbox_key,retry_at) VALUES($1,$2,$3)
+            ON CONFLICT(project_key,mailbox_key) DO UPDATE SET retry_at=GREATEST(${this.t.mailboxes}.retry_at,excluded.retry_at)`, [scope.projectKey, scope.mailboxKey, retryAt]);
+                }
+            });
+        }
+        catch {
+            throw new SourceSyncFault('storage_unavailable');
+        }
+    }
+    /**
+     * Remove expired attempt evidence after 24 hours and safely old meter records, at most 1,000 rows of each table a
+     * call. Daily totals remain. Answers what it removed; while `more` is true, a further call may remove more.
+     */
+    async compact() {
+        try {
+            return await this.db.transaction(async (tx) => {
+                const now = await this.now(tx);
+                const before = Math.max(0, now - 86_400_000);
+                const attempts = (await tx.run(`DELETE FROM ${this.t.attempts} WHERE attempt_id IN (SELECT attempt_id FROM ${this.t.attempts} WHERE expires_at < $1 ORDER BY expires_at LIMIT ${compactLimit}) AND expires_at < $1`, [before])).changes;
+                const mailboxWaits = (await tx.run(`DELETE FROM ${this.t.mailboxes} WHERE (project_key,mailbox_key) IN (SELECT project_key,mailbox_key FROM ${this.t.mailboxes} WHERE retry_at <= $1 LIMIT ${compactLimit}) AND retry_at <= $1`, [now])).changes;
+                const pages = [attempts, mailboxWaits];
+                let reservations = 0;
+                let windows = 0;
+                for (const meter of [this.meter(tx, this.t.pm, this.policy.projectMinute, false),
+                    this.meter(tx, this.t.mm, this.policy.mailboxMinute, false), this.meter(tx, this.t.pd, this.policy.projectDay, true)]) {
+                    // A meter answers its settled reservations and its window rows deleted, each its own page of the limit.
+                    const purged = await meter.purge({ before, limit: compactLimit });
+                    reservations += purged.reservations;
+                    windows += purged.periods;
+                    pages.push(purged.reservations, purged.periods);
+                }
+                return { attempts, mailboxWaits, reservations, windows, more: pages.some((n) => n >= compactLimit) };
+            });
+        }
+        catch {
+            throw new SourceSyncFault('storage_unavailable');
+        }
+    }
+}
+/** Retry-After is never clamped down to the local exponential-backoff ceiling. */
+export function sourceRetryAt(now, attempt, retryAfter, random = Math.random()) {
+    if (!integer(now) || !Number.isInteger(attempt) || attempt < 1 || attempt > 5
+        || !Number.isFinite(random) || random < 0 || random > 1)
+        throw new SourceSyncFault('invalid_response');
+    const base = Math.min(60_000, 1000 * 2 ** (attempt - 1));
+    let at = now + Math.min(60_000, Math.floor(base * (1 + random)));
+    if (retryAfter !== null) {
+        const value = retryAfter.trim();
+        if (/^\d+$/.test(value)) {
+            const seconds = Number(value);
+            if (!integer(seconds) || !integer(now + seconds * 1000))
+                throw new SourceSyncFault('invalid_response');
+            at = Math.max(at, now + seconds * 1000);
+        }
+        else {
+            const parsed = Date.parse(value);
+            if (integer(parsed) && new Date(parsed).toUTCString() === value)
+                at = Math.max(at, parsed);
+        }
+    }
+    if (!integer(at) || at > 8_639_999_913_600_000)
+        throw new SourceSyncFault('invalid_response');
+    return at;
+}
+//# sourceMappingURL=SourceQuota.js.map

package/dist/sync/SourceQuota.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"SourceQuota.js","sourceRoot":"","sources":["../../src/sync/SourceQuota.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAgF7C,MAAM,aAAc,SAAQ,KAAK;IACV;IAAoC;IAAzD,YAAqB,MAAyB,EAAW,OAAe;QAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAAtF,WAAM,GAAN,MAAM,CAAmB;QAAW,YAAO,GAAP,OAAO,CAAQ;IAAoC,CAAC;CAC9G;AACD,MAAM,cAAe,SAAQ,KAAK;IAAG,gBAAgB,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC,CAAC;CAAE;AACzF,MAAM,SAAU,SAAQ,KAAK;IAAwB;IAArB,YAAqB,OAAe;QAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;QAA9C,YAAO,GAAP,OAAO,CAAQ;IAAgC,CAAC;CAAE;AACvG,sGAAsG;AACtG,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,6DAA6D;AAC7D,MAAM,YAAY,GAAG,IAAI,CAAC;AAC1B,MAAM,OAAO,GAAG,CAAC,CAAS,EAAW,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1E,MAAM,GAAG,GAAG,CAAC,CAAS,EAAW,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,GAAG,CAAC;AAC7F,MAAM,MAAM,GAAG,CAAC,GAAW,EAAU,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;AACxF,SAAS,MAAM,CAAC,MAAM,GAAG,sBAAsB;IAC7C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IAC3F,OAAO,EAAE,QAAQ,EAAE,IAAI,MAAM,YAAY,EAAE,SAAS,EAAE,IAAI,MAAM,aAAa;QAC3E,QAAQ,EAAE,IAAI,MAAM,YAAY,EAAE,EAAE,EAAE,GAAG,MAAM,KAAK,EAAE,EAAE,EAAE,GAAG,MAAM,KAAK,EAAE,EAAE,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;AACnG,CAAC;AACD;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAAM,GAAG,sBAAsB;IAC/D,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACzB,OAAO;QACL,8BAA8B,CAAC,CAAC,QAAQ,yFAAyF;QACjI,8BAA8B,CAAC,CAAC,SAAS;+EACkC;QAC3E,8BAA8B,CAAC,CAAC,QAAQ;;sFAE0C;QAClF,+BAA+B,MAAM,eAAe,CAAC,CAAC,QAAQ,mCAAmC;QACjG,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC;AACJ,CAAC;AACD,SAAS,aAAa,CAAC,MAAyB;IAC9C,MAAM,OAAO,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI;QACpE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IACpG,MAAM,IAAI,GAAG,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa;QACrF,UAAU,EAAE,MAAM,CAAC,UAAU,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;QAC5E,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS;QAC1E,WAAW,EAAE,MAAM,CAAC,WAAW,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;IACrE,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,kBAAkB;QAC/F,IAAI,CAAC,kBAAkB,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,WAAW,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;IACxF,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,MAAM;WACpE,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,WAAW,GAAG,OAAO;WAC/D,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,kBAAkB;QAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;IACvG,OAAO,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,oFAAoF;AACpF,MAAM,OAAO,WAAW;IAQO;IAAqC;IAPjD,CAAC,CAA4B;IAC7B,MAAM,CAAoB;IAC1B,WAAW,CAAS;IACrC;;;OAGG;IACH,YAA6B,EAAkB,EAAmB,OAA2B;QAAhE,OAAE,GAAF,EAAE,CAAgB;QAAmB,YAAO,GAAP,OAAO,CAAoB;QAC3F,IAAI,EAAE,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;QACtH,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClF,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IACO,KAAK,CAAC,GAAG,CAAC,EAAkB;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACzD,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CACpB,2EAA2E,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QACxF,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,qBAAqB;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QAC5F,OAAO,CAAC,CAAC;IACX,CAAC;IACO,KAAK,CAAC,OAAO,CAAC,EAAkB,EAAE,UAAkB;QAC1D,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAClE,MAAM,EAAE,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,QAAQ,2DAA2D,EACpG,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAClC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,GAAG,CAAa,+BAA+B,IAAI,CAAC,CAAC,CAAC,QAAQ,kCAAkC,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC;QACrI,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC7F,OAAO,GAAG,CAAC;IACb,CAAC;IACO,KAAK,CAAC,EAAkB,EAAE,MAAc,EAAE,SAAiB,EAAE,KAAc;QACjF,OAAO,IAAI,aAAa,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,SAAS;YACnF,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;YACvG,YAAY,EAAE,KAAK,EAAE,aAAa,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,uBAAuB,EAAE,CAAC;YACtF,iBAAiB,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;IAC7G,CAAC;IACO,KAAK,CAAC,OAAO,CAAC,OAA2B;QAC/C,IAAI,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC;eACtE,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;QACxG,OAAO,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACtC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YAAC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC3F,IAAI,MAAM,EAAE,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,CAAC,CAAC,QAAQ,sBAAsB,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBAAE,MAAM,IAAI,cAAc,EAAE,CAAC;YACnI,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,GAAG,CAAgC,wBAAwB,IAAI,CAAC,CAAC,CAAC,SAAS;gDAC1D,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YACtF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC;YACnF,IAAI,OAAO,GAAG,GAAG;gBAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE,OAAO,CAAC,CAAC;YACrE,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,GAAG,CACxB,sCAAsC,IAAI,CAAC,CAAC,CAAC,QAAQ,4DAA4D,EACjH,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;YAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,UAAU,CAAC,CAAC;YACxE,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBACnD,MAAM,IAAI,aAAa,CAAC,qBAAqB,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACtG,CAAC;YACD,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,CAAC;gBACnD,MAAM,IAAI,aAAa,CAAC,qBAAqB,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YACtG,CAAC;YACD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YACvD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YAC3E,MAAM,MAAM,GAAG;gBACb,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,SAAS,EAAE,SAAS;oBACxF,MAAM,EAAE,gBAAyB,EAAE,OAAO,EAAE,GAAG,GAAG,MAAM,EAAE;gBAC5D,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,SAAS,EAAE,SAAS;oBACxF,MAAM,EAAE,gBAAyB,EAAE,OAAO,EAAE,GAAG,GAAG,MAAM,EAAE;gBAC5D,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,EAAE,SAAS,EAAE,SAAS;oBACpF,MAAM,EAAE,aAAsB,EAAE,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;aACzD,CAAC;YACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS;oBACrE,WAAW,EAAE,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;gBACrF,IAAI,QAAQ,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;oBACnC,IAAI,QAAQ,CAAC,MAAM,KAAK,qBAAqB;wBAAE,MAAM,IAAI,cAAc,EAAE,CAAC;oBAC1E,gFAAgF;oBAChF,MAAM,IAAI,aAAa,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;gBACvD,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC;gBACvG,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,UAAU;oBAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAClH,CAAC;YACD,MAAM,MAAM,GAAW,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,MAAM,EAAE,GAAG,GAAG,YAAY;gBACtE,SAAS,EAAE,GAAG,GAAG,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACtH,MAAM,EAAE,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,QAAQ;kCACf,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU;gBACpF,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YACpD,OAAO,MAAM,CAAC;QAChB,CAAC,CAAC,CAAC;IACL,CAAC;IACO,KAAK,CAAC,KAAK,CAAC,OAA2B,EAAE,MAAc;QAC7D,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACrC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YAAC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC3F,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,GAAG,CAAgC,wBAAwB,IAAI,CAAC,CAAC,CAAC,SAAS;gDAC1D,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;YACtF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,OAAO,EAAE,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC;YAChF,IAAI,IAAI,GAAG,GAAG;gBAAE,MAAM,IAAI,aAAa,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;YAC/D,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,QAAQ;;+EAEmB,EACvE,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAClF,IAAI,OAAO,CAAC,OAAO,KAAK,CAAC;gBAAE,OAAO;YAClC,qHAAqH;YACrH,MAAM,SAAS,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,CAAC,CAAC,QAAQ;oHACsC,EAC5G,CAAC,OAAO,CAAC,SAAS,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7E,MAAM,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,GAAG,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,cAAc,EAAE,CAAC;QAC7E,CAAC,CAAC,CAAC;IACL,CAAC;IACO,KAAK,CAAC,OAAO,CAAC,OAA2B,EAAE,MAAc;QAC/D,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;YACrC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;YAC3C,MAAM,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,QAAQ;8CACE,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;QACvH,CAAC,CAAC,CAAC;IACL,CAAC;IACD;;;OAGG;IACH,KAAK,CAAC,GAAG,CAAI,OAA2B,EAAE,IAAyC,EACjF,MAAoB;QACpB,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,IAAI,eAAe,CAAC,WAAW,CAAC,CAAC;QAC5D,IAAI,MAAc,CAAC;QACnB,IAAI,CAAC;YAAC,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;QAC7C,OAAO,KAAK,EAAE,CAAC;YACb,IAAI,KAAK,YAAY,aAAa;gBAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;YAC9G,IAAI,KAAK,YAAY,cAAc;gBAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;YAClE,IAAI,KAAK,YAAY,eAAe;gBAAE,MAAM,KAAK,CAAC;YAClD,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACvC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,IAAI,KAAgD,CAAC;QACrD,IAAI,KAAK,GAAG,KAAK,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAAC,KAAK,GAAG,IAAI,CAAC;YAAC,CAAC;YACxD,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,aAAa;oBAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC9G,IAAI,KAAK,YAAY,SAAS;oBAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;gBAChF,IAAI,KAAK,YAAY,cAAc;oBAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;gBAClE,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;YACnD,CAAC;YACD,IAAI,MAAM,EAAE,OAAO;gBAAE,MAAM,IAAI,eAAe,CAAC,WAAW,CAAC,CAAC;YAC5D,KAAK,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5C,iFAAiF;YACjF,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;gBAAE,MAAM,IAAI,eAAe,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACpG,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;QACtC,CAAC;gBAAS,CAAC;YACT,IAAI,KAAK,KAAK,SAAS;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC7C,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YAC5C,oFAAoF;YACpF,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YAAC,CAAC;YAC5C,MAAM,CAAC;gBAAC,IAAI,KAAK;oBAAE,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;YAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,wFAAwF;IACxF,KAAK,CAAC,KAAK,CAAC,KAAoE,EAAE,OAAe;QAC/F,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;eACnF,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,qBAAqB;YAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;QACzG,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBACrC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;gBACzC,IAAI,KAAK,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;oBACnC,MAAM,EAAE,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,QAAQ,0DAA0D,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;gBACjI,CAAC;qBAAM,CAAC;oBACN,MAAM,EAAE,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,SAAS;mFAC6B,IAAI,CAAC,CAAC,CAAC,SAAS,8BAA8B,EACrH,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;gBACnD,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YAAC,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;QAAC,CAAC;IAC/D,CAAC;IACD;;;OAGG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAAC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,GAAG,UAAU,CAAC,CAAC;gBAC7E,MAAM,QAAQ,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,QAAQ,gDAAgD,IAAI,CAAC,CAAC,CAAC,QAAQ,oDAAoD,YAAY,uBAAuB,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;gBACxO,MAAM,YAAY,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,SAAS,4EAA4E,IAAI,CAAC,CAAC,CAAC,SAAS,+BAA+B,YAAY,sBAAsB,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;gBACjP,MAAM,KAAK,GAAG,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;gBAAC,IAAI,YAAY,GAAG,CAAC,CAAC;gBAAC,IAAI,OAAO,GAAG,CAAC,CAAC;gBAC9E,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC;oBAC9E,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC;oBACxH,wGAAwG;oBACxG,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,CAAC;oBAClE,YAAY,IAAI,MAAM,CAAC,YAAY,CAAC;oBAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC;oBAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;gBAClH,CAAC;gBACD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,YAAY,CAAC,EAAE,CAAC;YACvG,CAAC,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YAAC,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;QAAC,CAAC;IAC/D,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,OAAe,EAAE,UAAyB,EAAE,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE;IAC3G,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC;WACxE,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;IACzG,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;IACzD,IAAI,EAAE,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IACjE,IAAI,UAAU,KAAK,IAAI,EAAE,CAAC;QACxB,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;YAC9B,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,OAAO,GAAG,IAAI,CAAC;gBAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;YACvG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,GAAG,OAAO,GAAG,IAAI,CAAC,CAAC;QAC1C,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YACjC,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,KAAK,KAAK;gBAAE,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,GAAG,qBAAqB;QAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;IAC9F,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/sync/SqlAccountSourceSyncStore.d.ts

@@ -6,7 +6,8 @@
  */
 import type { StorageAdapter } from '@framers/sql-storage-adapter';
 import { SqlSourceDerivationStore, type SourceDependency, type SourceDependencyState, type SourceDerivation, type SourceOwnerScope, type SourceInvalidation, type SourceFenceTicket } from '@framers/agentos/cognition/library';
-import type { SourceSyncScope, SourceSyncState, SourceSyncPolicy, SourceSyncLease, SourceSyncTrigger, SourceSyncCheckpoint, SourceSyncCycleInput, SourceSyncPage, SourceSyncWork, SourceSyncDocument, SourceSyncMessageSpan, SourceSyncReadResult, SourceSyncWorkDisposition, SourceSyncTransition } from './types.js';
+import type { SourceSyncScope, SourceSyncState, SourceSyncPolicy, SourceSyncLease, SourceSyncTrigger, SourceSyncCheckpoint, SourceSyncCycleInput, SourceSyncPage, SourceSyncWork, SourceSyncDocument, SourceSyncMessageSpan, SourceSyncReadResult, SourceSyncWorkDisposition, SourceSyncTransition, SourceSyncRetryReason } from './types.js';
+import type { SourceSyncOperationAttempt } from './coordinatorState.js';
 /**
  * The provenance kept for one retained source: its dependency, message spans, activity time, character count and
  * content hash, never its text.
@@ -189,8 +190,47 @@ export declare class SqlAccountSourceSyncStore<Prepared = unknown> {
      */
     page(h: SourceSyncHandle, expected: SourceSyncCheckpoint, page: SourceSyncPage): Promise<SourceSyncCheckpoint | null>;
     private putWork;
-    /** Null id takes the first due item. Increment before external work; a crash leaves it pending. */
+    /**
+     * Null id takes the first due item. Increment before external work; a crash leaves it pending.
+     *
+     * An item is due when it is pending or its retry time has come; without an id the first due item by source id is
+     * taken. A pending item that already has five attempts and no recorded result, as when a process died during each
+     * read, is not taken: it waits `pollMs` as a `storage_unavailable` retry, and the attempt after a wait at five
+     * attempts starts a new count.
+     *
+     * @returns The item with its attempt counted, or null when none is due or the handle is stale.
+     */
     takeWork(h: SourceSyncHandle, id?: string | null): Promise<SourceSyncWork | null>;
+    /**
+     * Counts an attempt at one provider operation of the open sync, its anchor read or a page, before the request is
+     * sent, under the connection's row lock. `operationKey` names the operation: a SHA-256 digest in lower-case hex
+     * that changes with the generation, cycle, revision and phase, so a new key starts a new count. A recorded wait
+     * that has not passed is answered as a deferral. Five attempts with no recorded wait, as when a process died during
+     * each, answer a `storage_unavailable` deferral `pollMs` away, and the attempt after a wait at five attempts starts
+     * a new count.
+     *
+     * @returns The attempt to make, from 1 to 5, or the deferral; null for a stale handle.
+     * @throws {SourceSyncFault} `invalid_response` for a key that is not 64 lower-case hex digits.
+     */
+    reserveOperation(h: SourceSyncHandle, operationKey: string): Promise<SourceSyncOperationAttempt | null>;
+    /**
+     * Records the wait after a retriable failure of the attempt `reserveOperation` answered, when that operation and
+     * attempt are still the ones reserved.
+     *
+     * @returns Whether the wait was recorded: false for a stale handle or when another reservation replaced it.
+     * @throws {SourceSyncFault} `invalid_response` for a malformed key, an attempt outside 1 to 5, a `retryAt` that is
+     *   not after the store's clock or a reason that is not retriable.
+     */
+    deferOperation(h: SourceSyncHandle, operationKey: string, attempt: number, reason: SourceSyncRetryReason, retryAt: number): Promise<boolean>;
+    /**
+     * The earliest retry wait among the open cycle's work items: the cycle cannot commit its cursor before it.
+     *
+     * @returns The wait's reason and time, or null when no item waits or the handle is stale.
+     */
+    pendingRetry(h: SourceSyncHandle): Promise<{
+        reason: SourceSyncRetryReason;
+        retryAt: number;
+    } | null>;
     private currentWork;
     private normalized;
     private documentRows;

package/dist/sync/SqlAccountSourceSyncStore.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SqlAccountSourceSyncStore.d.ts","sourceRoot":"","sources":["../../src/sync/SqlAccountSourceSyncStore.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EAAE,KAAK,qBAAqB,EAAE,KAAK,gBAAgB,EACxE,KAAK,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EACvE,MAAM,oCAAoC,CAAC;AAC5C,OAAO,KAAK,EACV,eAAe,EAAE,eAAe,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EACtF,oBAAoB,EAAE,oBAAoB,EAAE,cAAc,EAAE,cAAc,EAC1E,kBAAkB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,yBAAyB,EAC1F,oBAAoB,EACrB,MAAM,YAAY,CAAC;AAOpB;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IACtC,QAAQ,CAAC,QAAQ,EAAE,SAAS,qBAAqB,EAAE,CAAC;IACpD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AACD,0GAA0G;AAC1G,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,0BAA0B,EAAE,MAAM,CAAC;IAC5C,QAAQ,CAAC,+BAA+B,EAAE,MAAM,CAAC;IACjD,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;CAC9C;AACD;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB,CAAC,QAAQ;IACjD;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtE,mKAAmK;IACnK,SAAS,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACpF,4GAA4G;IAC5G,KAAK,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,gBAAgB,EAC5E,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,gFAAgF;IAChF,aAAa,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7F,2FAA2F;IAC3F,KAAK,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChG,yEAAyE;IACzE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjG,+GAA+G;IAC/G,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvF;AACD,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB,CAAC,QAAQ;IAC9C,qHAAqH;IACrH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,4HAA4H;IAC5H,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;IAC9B,iCAAiC;IACjC,QAAQ,CAAC,QAAQ,EAAE,yBAAyB,CAAC,QAAQ,CAAC,CAAC;CACxD;AACD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;CACjC;AACD;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,oBAAoB,EAAE;IAC5D,IAAI,EAAE,iBAAiB,GAAG,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,aAAa,GACxE,gBAAgB,GAAG,YAAY,GAAG,YAAY,CAAC;CACpD,CAAC,CAAC;AACH;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO,CAAC;IACpE,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AACD;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AACD,gGAAgG;AAChG,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,yBAAyB,CAAC;CACjD;AAeD;;;;;;;;GAQG;AACH,qBAAa,yBAAyB,CAAC,QAAQ,GAAG,OAAO;IAc3C,OAAO,CAAC,QAAQ,CAAC,EAAE;IAAkB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAbzE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAsC;IACxD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,wBAAwB,CAAC;IACzC;;;;;OAKG;gBAC0B,EAAE,EAAE,cAAc,EAAmB,OAAO,EAAE,sBAAsB,CAAC,QAAQ,CAAC;IAM3G,OAAO,CAAC,GAAG;IAGX,OAAO,CAAC,OAAO;YAUD,GAAG;YAKH,IAAI;YAQJ,UAAU;YAIV,QAAQ;IAQtB;;;;;;OAMG;IACG,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAY7E,wFAAwF;IAClF,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAM1E,+EAA+E;IACzE,MAAM,CAAC,KAAK,GAAE,eAAe,GAAG,IAAW,EAAE,KAAK,SAAM,GAAG,OAAO,CAAC,SAAS,eAAe,EAAE,CAAC;IACpG;;;;;OAKG;IACG,GAAG,CAAC,KAAK,SAAM,GAAG,OAAO,CAAC,SAAS,eAAe,EAAE,CAAC;IAO3D;;;;;;OAMG;IACG,KAAK,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAkBnF,OAAO;YAUP,UAAU;IAMxB;;;;OAIG;IACG,KAAK,CAAC,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAMlE;;;;;;;OAOG;IACG,SAAS,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,CAAC,oBAAoB,EAAE,MAAM,GAAG,YAAY,CAAC,GAAG;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,EACrJ,qBAAqB,UAAQ,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;IAYtE;;;;;;OAMG;IACG,IAAI,CAAC,CAAC,EAAE,gBAAgB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;YAqB7G,OAAO;IAKrB,mGAAmG;IAC7F,QAAQ,CAAC,CAAC,EAAE,gBAAgB,EAAE,EAAE,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;YAe/E,WAAW;IAUzB,OAAO,CAAC,UAAU;YA4BJ,YAAY;IAK1B,0FAA0F;IACpF,SAAS,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,wBAAwB,EAAE,CAAC;IAGrF,oEAAoE;IAC9D,UAAU,CAAC,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAItE,OAAO,CAAC,MAAM;IAWd;;;;;;;;;;;;OAYG;IACG,WAAW,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,oBAAoB,EACvF,QAAQ,EAAE,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC;YA+DrD,OAAO;IAWrB;;;;OAIG;IACG,QAAQ,CAAC,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAYpE;;;;;OAKG;IACG,MAAM,CAAC,CAAC,EAAE,gBAAgB,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAgBlG;;;;;OAKG;IACG,OAAO,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,GAAE,OAAO,CAAC,oBAAoB,EAAE;QAAE,IAAI,EAAE,eAAe,GAAG,aAAa,CAAA;KAAE,CAAC,GAAG,IAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IAQnJ;;;;OAIG;IACG,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,qBAAqB,EAAE,CAAC;IAe9G,+HAA+H;IACzH,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAO5G;;;;;;;;;;OAUG;IACG,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAkBhH,MAAM;IASpB,+FAA+F;IACzF,cAAc,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAyC3D;;;;;OAKG;IACG,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IActE;;;OAGG;IACG,eAAe,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D;;;;;;;;OAQG;IACG,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,SAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAuBnE"}
+{"version":3,"file":"SqlAccountSourceSyncStore.d.ts","sourceRoot":"","sources":["../../src/sync/SqlAccountSourceSyncStore.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EAAE,KAAK,qBAAqB,EAAE,KAAK,gBAAgB,EACxE,KAAK,gBAAgB,EAAE,KAAK,kBAAkB,EAAE,KAAK,iBAAiB,EACvE,MAAM,oCAAoC,CAAC;AAC5C,OAAO,KAAK,EACV,eAAe,EAAE,eAAe,EAAE,gBAAgB,EAAE,eAAe,EAAE,iBAAiB,EACtF,oBAAoB,EAAE,oBAAoB,EAAE,cAAc,EAAE,cAAc,EAC1E,kBAAkB,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,yBAAyB,EAC1F,oBAAoB,EAA+C,qBAAqB,EACzF,MAAM,YAAY,CAAC;AAMpB,OAAO,KAAK,EAAE,0BAA0B,EAA4B,MAAM,uBAAuB,CAAC;AAElG;;;GAGG;AACH,MAAM,WAAW,wBAAwB;IACvC,QAAQ,CAAC,UAAU,EAAE,gBAAgB,CAAC;IACtC,QAAQ,CAAC,QAAQ,EAAE,SAAS,qBAAqB,EAAE,CAAC;IACpD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AACD,0GAA0G;AAC1G,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,0BAA0B,EAAE,MAAM,CAAC;IAC5C,QAAQ,CAAC,+BAA+B,EAAE,MAAM,CAAC;IACjD,QAAQ,CAAC,2BAA2B,EAAE,MAAM,CAAC;CAC9C;AACD;;;;;GAKG;AACH,MAAM,WAAW,yBAAyB,CAAC,QAAQ;IACjD;;;OAGG;IACH,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACtE,mKAAmK;IACnK,SAAS,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACpF,4GAA4G;IAC5G,KAAK,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,gBAAgB,EAC5E,QAAQ,EAAE,kBAAkB,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnE,gFAAgF;IAChF,aAAa,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7F,2FAA2F;IAC3F,KAAK,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChG,yEAAyE;IACzE,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjG,+GAA+G;IAC/G,MAAM,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACvF;AACD,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB,CAAC,QAAQ;IAC9C,qHAAqH;IACrH,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,4HAA4H;IAC5H,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,+EAA+E;IAC/E,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;IAC9B,iCAAiC;IACjC,QAAQ,CAAC,QAAQ,EAAE,yBAAyB,CAAC,QAAQ,CAAC,CAAC;CACxD;AACD;;;GAGG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;CACjC;AACD;;;GAGG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,CAAC,oBAAoB,EAAE;IAC5D,IAAI,EAAE,iBAAiB,GAAG,aAAa,GAAG,OAAO,GAAG,QAAQ,GAAG,aAAa,GACxE,gBAAgB,GAAG,YAAY,GAAG,YAAY,CAAC;CACpD,CAAC,CAAC;AACH;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,QAAQ,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO,CAAC;IACpE,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC;AACD;;;GAGG;AACH,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,KAAK,EAAE,eAAe,CAAC;IAChC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,UAAU,EAAE,oBAAoB,GAAG,IAAI,CAAC;IACjD,QAAQ,CAAC,OAAO,EAAE,iBAAiB,GAAG,IAAI,CAAC;IAC3C,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;CACzC;AACD,gGAAgG;AAChG,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,MAAM,EAAE,gBAAgB,CAAC;IAClC,QAAQ,CAAC,WAAW,EAAE,yBAAyB,CAAC;CACjD;AAeD;;;;;;;;GAQG;AACH,qBAAa,yBAAyB,CAAC,QAAQ,GAAG,OAAO;IAc3C,OAAO,CAAC,QAAQ,CAAC,EAAE;IAAkB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAbzE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAsC;IACxD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IACrC;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,wBAAwB,CAAC;IACzC;;;;;OAKG;gBAC0B,EAAE,EAAE,cAAc,EAAmB,OAAO,EAAE,sBAAsB,CAAC,QAAQ,CAAC;IAM3G,OAAO,CAAC,GAAG;IAGX,OAAO,CAAC,OAAO;YAUD,GAAG;YAKH,IAAI;YAQJ,UAAU;YAIV,QAAQ;IAQtB;;;;;;OAMG;IACG,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAY7E,wFAAwF;IAClF,QAAQ,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;IAM1E,+EAA+E;IACzE,MAAM,CAAC,KAAK,GAAE,eAAe,GAAG,IAAW,EAAE,KAAK,SAAM,GAAG,OAAO,CAAC,SAAS,eAAe,EAAE,CAAC;IACpG;;;;;OAKG;IACG,GAAG,CAAC,KAAK,SAAM,GAAG,OAAO,CAAC,SAAS,eAAe,EAAE,CAAC;IAO3D;;;;;;OAMG;IACG,KAAK,CAAC,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAkBnF,OAAO;YAUP,UAAU;IAMxB;;;;OAIG;IACG,KAAK,CAAC,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC;IAMlE;;;;;;;OAOG;IACG,SAAS,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,IAAI,CAAC,oBAAoB,EAAE,MAAM,GAAG,YAAY,CAAC,GAAG;QAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,EACrJ,qBAAqB,UAAQ,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;IAYtE;;;;;;OAMG;IACG,IAAI,CAAC,CAAC,EAAE,gBAAgB,EAAE,QAAQ,EAAE,oBAAoB,EAAE,IAAI,EAAE,cAAc,GAAG,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC;YAqB7G,OAAO;IAKrB;;;;;;;;;OASG;IACG,QAAQ,CAAC,CAAC,EAAE,gBAAgB,EAAE,EAAE,GAAE,MAAM,GAAG,IAAW,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAwB7F;;;;;;;;;;OAUG;IACG,gBAAgB,CAAC,CAAC,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,0BAA0B,GAAG,IAAI,CAAC;IA8B7G;;;;;;;OAOG;IACG,cAAc,CAAC,CAAC,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAC7E,MAAM,EAAE,qBAAqB,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAgBnE;;;;OAIG;IACG,YAAY,CAAC,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC;QAAE,MAAM,EAAE,qBAAqB,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;YAW7F,WAAW;IAUzB,OAAO,CAAC,UAAU;YA4BJ,YAAY;IAK1B,0FAA0F;IACpF,SAAS,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,SAAS,wBAAwB,EAAE,CAAC;IAGrF,oEAAoE;IAC9D,UAAU,CAAC,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAItE,OAAO,CAAC,MAAM;IAWd;;;;;;;;;;;;OAYG;IACG,WAAW,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,EAAE,oBAAoB,EACvF,QAAQ,EAAE,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC,qBAAqB,GAAG,IAAI,CAAC;YA+DrD,OAAO;IAWrB;;;;OAIG;IACG,QAAQ,CAAC,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAYpE;;;;;OAKG;IACG,MAAM,CAAC,CAAC,EAAE,gBAAgB,EAAE,QAAQ,EAAE,oBAAoB,GAAG,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAgBlG;;;;;OAKG;IACG,OAAO,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,GAAE,OAAO,CAAC,oBAAoB,EAAE;QAAE,IAAI,EAAE,eAAe,GAAG,aAAa,CAAA;KAAE,CAAC,GAAG,IAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IAQnJ;;;;OAIG;IACG,WAAW,CAAC,EAAE,EAAE,cAAc,EAAE,UAAU,EAAE,gBAAgB,GAAG,OAAO,CAAC,SAAS,qBAAqB,EAAE,CAAC;IAe9G,+HAA+H;IACzH,YAAY,CAAC,EAAE,EAAE,cAAc,EAAE,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAO5G;;;;;;;;;;OAUG;IACG,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,EAAE,iBAAiB,EAAE,MAAM,CAAC,EAAE,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,GAAG,IAAI,CAAC;YAkBhH,MAAM;IASpB,+FAA+F;IACzF,cAAc,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAyC3D;;;;;OAKG;IACG,OAAO,CAAC,KAAK,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IActE;;;OAGG;IACG,eAAe,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC;IAa5D;;;;;;;;OAQG;IACG,MAAM,CAAC,KAAK,EAAE,eAAe,EAAE,KAAK,SAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAuBnE"}

package/dist/sync/SqlAccountSourceSyncStore.js

@@ -278,23 +278,120 @@ export class SqlAccountSourceSyncStore {
       VALUES($1,$2,$3,$4,$5::jsonb) ON CONFLICT(tenant_id,owner_id,connection_id,source_id)
       DO UPDATE SET work=excluded.work`, [...params(scope), work.sourceId, JSON.stringify(work)]);
     }
-    /** Null id takes the first due item. Increment before external work; a crash leaves it pending. */
+    /**
+     * Null id takes the first due item. Increment before external work; a crash leaves it pending.
+     *
+     * An item is due when it is pending or its retry time has come; without an id the first due item by source id is
+     * taken. A pending item that already has five attempts and no recorded result, as when a process died during each
+     * read, is not taken: it waits `pollMs` as a `storage_unavailable` retry, and the attempt after a wait at five
+     * attempts starts a new count.
+     *
+     * @returns The item with its attempt counted, or null when none is due or the handle is stale.
+     */
     async takeWork(h, id = null) {
         return this.withHandle(h, async (tx, row) => {
             const rows = await tx.all(`SELECT work FROM ${this.t.work}
         WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3
         AND ($4::text IS NULL OR source_id=$4) ORDER BY source_id`, [...params(h.lease.scope), id]);
-            const work = rows.map((r) => r.work).find((w) => w.cycleId === row.checkpoint?.cycleId
-                && w.generation === h.lease.generation && (w.disposition.kind === 'pending'
-                || (w.disposition.kind === 'retry' && w.disposition.retryAt <= this.now())));
-            if (!work)
-                return null;
-            const attempts = work.disposition.kind === 'retry' && work.attempts >= 5 ? 1 : work.attempts + 1;
-            if (!validNumber(attempts))
-                throw new SourceSyncFault('invalid_response');
-            const next = { ...work, attempts, disposition: { kind: 'pending' } };
-            await this.putWork(tx, h.lease.scope, next);
-            return next;
+            const now = this.now();
+            for (const { work } of rows) {
+                if (work.cycleId !== row.checkpoint?.cycleId || work.generation !== h.lease.generation)
+                    continue;
+                if (work.disposition.kind !== 'pending' && work.disposition.kind !== 'retry')
+                    continue;
+                if (work.disposition.kind === 'retry' && work.disposition.retryAt > now)
+                    continue;
+                if (work.disposition.kind === 'pending' && work.attempts >= 5) {
+                    await this.putWork(tx, h.lease.scope, { ...work, disposition: {
+                            kind: 'retry', reason: 'storage_unavailable', retryAt: now + row.policy.pollMs,
+                        } });
+                    continue;
+                }
+                const attempts = work.disposition.kind === 'retry' && work.attempts >= 5 ? 1 : work.attempts + 1;
+                if (!validNumber(attempts))
+                    throw new SourceSyncFault('invalid_response');
+                const next = { ...work, attempts, disposition: { kind: 'pending' } };
+                await this.putWork(tx, h.lease.scope, next);
+                return next;
+            }
+            return null;
+        });
+    }
+    /**
+     * Counts an attempt at one provider operation of the open sync, its anchor read or a page, before the request is
+     * sent, under the connection's row lock. `operationKey` names the operation: a SHA-256 digest in lower-case hex
+     * that changes with the generation, cycle, revision and phase, so a new key starts a new count. A recorded wait
+     * that has not passed is answered as a deferral. Five attempts with no recorded wait, as when a process died during
+     * each, answer a `storage_unavailable` deferral `pollMs` away, and the attempt after a wait at five attempts starts
+     * a new count.
+     *
+     * @returns The attempt to make, from 1 to 5, or the deferral; null for a stale handle.
+     * @throws {SourceSyncFault} `invalid_response` for a key that is not 64 lower-case hex digits.
+     */
+    async reserveOperation(h, operationKey) {
+        if (!/^[a-f0-9]{64}$/.test(operationKey))
+            throw new SourceSyncFault('invalid_response');
+        return this.withHandle(h, async (tx, row) => {
+            const found = await tx.get(`SELECT operation FROM ${this.t.connections} WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3`, params(h.lease.scope));
+            const previous = found?.operation?.key === operationKey ? found.operation : null;
+            const now = this.now();
+            if (previous && previous.retryAt !== null && previous.retryAt > now) {
+                return { kind: 'deferred', reason: previous.reason, retryAt: previous.retryAt };
+            }
+            let attempts = previous?.attempts ?? 0;
+            if (attempts >= 5 && previous?.retryAt === null) {
+                const retryAt = now + row.policy.pollMs;
+                const pending = { key: operationKey, attempts,
+                    retryAt, reason: 'storage_unavailable' };
+                await tx.run(`UPDATE ${this.t.connections} SET operation=$4::jsonb
+          WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3`, [...params(h.lease.scope), JSON.stringify(pending)]);
+                return { kind: 'deferred', reason: pending.reason, retryAt };
+            }
+            if (attempts >= 5)
+                attempts = 0;
+            const next = { key: operationKey, attempts: attempts + 1,
+                retryAt: null, reason: 'storage_unavailable' };
+            await tx.run(`UPDATE ${this.t.connections} SET operation=$4::jsonb
+        WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3`, [...params(h.lease.scope), JSON.stringify(next)]);
+            return { kind: 'attempt', attempt: next.attempts };
+        });
+    }
+    /**
+     * Records the wait after a retriable failure of the attempt `reserveOperation` answered, when that operation and
+     * attempt are still the ones reserved.
+     *
+     * @returns Whether the wait was recorded: false for a stale handle or when another reservation replaced it.
+     * @throws {SourceSyncFault} `invalid_response` for a malformed key, an attempt outside 1 to 5, a `retryAt` that is
+     *   not after the store's clock or a reason that is not retriable.
+     */
+    async deferOperation(h, operationKey, attempt, reason, retryAt) {
+        if (!/^[a-f0-9]{64}$/.test(operationKey) || !Number.isInteger(attempt) || attempt < 1 || attempt > 5
+            || !validNumber(retryAt) || retryAt <= this.now()
+            || !['network', 'rate_limited', 'provider_unavailable', 'storage_unavailable'].includes(reason)) {
+            throw new SourceSyncFault('invalid_response');
+        }
+        const result = await this.withHandle(h, async (tx) => {
+            const updated = await tx.run(`UPDATE ${this.t.connections}
+        SET operation=jsonb_build_object('key',$4::text,'attempts',$5::int,'retryAt',$6::bigint,'reason',$7::text)
+        WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3
+        AND operation->>'key'=$4 AND (operation->>'attempts')::int=$5`, [...params(h.lease.scope), operationKey, attempt, retryAt, reason]);
+            return updated.changes === 1;
+        });
+        return result === true;
+    }
+    /**
+     * The earliest retry wait among the open cycle's work items: the cycle cannot commit its cursor before it.
+     *
+     * @returns The wait's reason and time, or null when no item waits or the handle is stale.
+     */
+    async pendingRetry(h) {
+        return this.withHandle(h, async (tx, row) => {
+            const found = await tx.get(`SELECT work FROM ${this.t.work}
+        WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3
+        AND work->>'cycleId'=$4 AND work->'disposition'->>'kind'='retry'
+        ORDER BY (work->'disposition'->>'retryAt')::bigint,source_id LIMIT 1`, [...params(h.lease.scope), row.checkpoint?.cycleId ?? '']);
+            const disposition = found?.work.disposition;
+            return disposition?.kind === 'retry' ? { reason: disposition.reason, retryAt: disposition.retryAt } : null;
         });
     }
     async currentWork(tx, scope, cp, work) {