@framers/agentos-ext-account-sources 0.5.0 → 0.6.1

package/dist/sync/AccountSourceSync.js

@@ -0,0 +1,409 @@
+/**
+ * @fileoverview The resumable synchronization coordinator for account sources: it runs one connection's sync under
+ * the SQL store's lease, from its durable checkpoint, through a supplied provider, with durable attempt counts and
+ * retry times, and runs expiry and recovery apart from provider polling.
+ * @module account-sources/sync/AccountSourceSync
+ */
+import { createHash, randomUUID } from 'node:crypto';
+import { SourceSyncFault } from './state.js';
+import { sourceRetryAt } from './SourceQuota.js';
+class SyncStale extends Error {
+    constructor() { super('source_sync_stale'); }
+}
+class SyncYield extends Error {
+    constructor() { super('source_sync_yield'); }
+}
+class SyncDeferred extends Error {
+    reason;
+    retryAt;
+    constructor(reason, retryAt) {
+        super('source_sync_deferred');
+        this.reason = reason;
+        this.retryAt = retryAt;
+    }
+}
+const retryReasons = ['network', 'rate_limited', 'provider_unavailable', 'storage_unavailable'];
+const retriable = (fault) => retryReasons.includes(fault.code);
+const digest = (parts) => createHash('sha256').update(JSON.stringify(parts)).digest('hex');
+/**
+ * One lease at a time; all durable state and authorization remain in the SQL store.
+ *
+ * `run` performs one connection's sync from its durable checkpoint and yields within its budgets; any coordinator on
+ * the same tables resumes it. `maintain` runs purge recovery and expiry for every connection, apart from provider
+ * polling. The coordinator holds no state between calls.
+ *
+ * @typeParam Prepared - The host's in-memory value for one document, passed to the store's `write` binding.
+ */
+export class AccountSourceSync {
+    options;
+    store;
+    clock;
+    runBudget;
+    operationTimeout;
+    /**
+     * @param options - The store, provider factory, preparation, lifecycle binding and optional settings.
+     * @throws {Error} `invalid_source_coordinator_policy` unless `runBudgetMs` is an integer from 1 to 90,000 and
+     *   `operationTimeoutMs` one from 1 to 60,000.
+     */
+    constructor(options) {
+        this.options = options;
+        this.store = options.store;
+        this.clock = options.clock ?? Date.now;
+        this.runBudget = options.runBudgetMs ?? 90_000;
+        this.operationTimeout = options.operationTimeoutMs ?? 30_000;
+        if (![this.runBudget, this.operationTimeout].every((n) => Number.isSafeInteger(n) && n >= 1)
+            || this.runBudget > 90_000 || this.operationTimeout > 60_000)
+            throw new Error('invalid_source_coordinator_policy');
+    }
+    now() {
+        const now = this.clock();
+        if (!Number.isSafeInteger(now) || now < 0)
+            throw new SourceSyncFault('invalid_response');
+        return now;
+    }
+    emit(state, n, started) {
+        const event = { kind: 'source_sync', state, n,
+            ...(started === undefined ? {} : { ms: Math.max(0, this.now() - started) }) };
+        try {
+            this.options.events?.(event);
+        }
+        catch { /* A diagnostic sink cannot alter a content transaction. */ }
+    }
+    async stored(work) {
+        try {
+            return await work();
+        }
+        catch (error) {
+            if (error instanceof SourceSyncFault)
+                throw error;
+            throw new SourceSyncFault('storage_unavailable');
+        }
+    }
+    at(fault, attempt, pollMs) {
+        const now = this.now();
+        return Math.max(sourceRetryAt(now, Math.min(5, Math.max(1, attempt)), null, this.options.random?.() ?? Math.random()), fault.retryAt ?? 0, attempt >= 5 ? now + pollMs : now + 1);
+    }
+    /**
+     * Runs one connection's sync under a claimed lease. It recovers the host's purges and the store's controls, claims
+     * the lease, opens a cycle when none is open (`bootstrap` without a cursor, `reconcile` once `reconcileMs` has
+     * passed since the last non-incremental cycle or when none is recorded, as after a change to the exclusions,
+     * otherwise `incremental`), drains the queued reads before asking for the next page, and commits the cursor once
+     * every item is terminal. It yields with the checkpoint kept after the policy's changed-item or history-page
+     * budget, the run budget or an aborted signal, and never waits out a retry.
+     *
+     * @param scope - The connection.
+     * @param trigger - A scheduled poll or a person's request.
+     * @param signal - Aborting it ends the run with `yielded` and aborts the call in flight.
+     * @returns How the run ended; `unavailable` claims no recorded retry.
+     */
+    async run(scope, trigger, signal) {
+        if (signal?.aborted)
+            return { kind: 'yielded' };
+        const started = this.now();
+        let handle = null;
+        let snapshot = null;
+        let checkpoint = null;
+        let changed = 0;
+        let historyPages = 0;
+        let recovered = false;
+        const checkBudget = () => {
+            if (signal?.aborted || this.now() - started >= this.runBudget)
+                throw new SyncYield();
+        };
+        const needHandle = () => { if (!handle)
+            throw new SyncStale(); return handle; };
+        const renew = async () => {
+            checkBudget();
+            const current = needHandle();
+            handle = await this.stored(() => this.store.renew(current));
+            if (!handle)
+                throw new SyncStale();
+        };
+        const context = (cycleId, cutoff, child) => ({
+            scope, generation: needHandle().lease.generation, cycleId, cutoff, policy: snapshot.policy, signal: child,
+        });
+        const external = async (work, failure) => {
+            await renew();
+            const timeout = Math.min(this.operationTimeout, this.runBudget - (this.now() - started), needHandle().lease.expiresAt - this.now() - 1000);
+            if (timeout <= 0)
+                throw new SyncYield();
+            const controller = new AbortController();
+            const abort = () => controller.abort();
+            signal?.addEventListener('abort', abort, { once: true });
+            if (signal?.aborted)
+                abort();
+            const timer = setTimeout(abort, timeout);
+            try {
+                if (controller.signal.aborted)
+                    throw new SyncYield();
+                const value = await work(controller.signal);
+                if (signal?.aborted)
+                    throw new SyncYield();
+                if (controller.signal.aborted)
+                    throw new SourceSyncFault(failure === 'storage_unavailable' ? failure : 'network');
+                return value;
+            }
+            catch (error) {
+                if (signal?.aborted)
+                    throw new SyncYield();
+                if (controller.signal.aborted)
+                    throw new SourceSyncFault(failure === 'storage_unavailable' ? failure : 'network');
+                if (error instanceof SyncYield || error instanceof SourceSyncFault)
+                    throw error;
+                throw new SourceSyncFault(failure);
+            }
+            finally {
+                clearTimeout(timer);
+                signal?.removeEventListener('abort', abort);
+            }
+        };
+        const operation = async (key, work) => {
+            await renew();
+            const current = needHandle();
+            const reserved = await this.stored(() => this.store.reserveOperation(current, key));
+            if (!reserved)
+                throw new SyncStale();
+            if (reserved.kind === 'deferred')
+                throw new SyncDeferred(reserved.reason, reserved.retryAt);
+            try {
+                return await external(work, 'invalid_response');
+            }
+            catch (error) {
+                if (!(error instanceof SourceSyncFault) || !retriable(error))
+                    throw error;
+                const reason = error.code;
+                const retryAt = this.at(error, reserved.attempt, snapshot.policy.pollMs);
+                if (!await this.stored(() => this.store.deferOperation(needHandle(), key, reserved.attempt, reason, retryAt))) {
+                    throw new SyncStale();
+                }
+                throw new SyncDeferred(error.code, retryAt);
+            }
+        };
+        const deferRead = async (work, fault) => {
+            const retryAt = this.at(fault, work.attempts, snapshot.policy.pollMs);
+            const result = await this.stored(() => this.store.applyResult(needHandle(), work, { kind: 'retry', id: work.sourceId, reason: fault.code, retryAt }, null));
+            if (!result)
+                throw new SyncStale();
+            handle = result.handle;
+            throw new SyncDeferred(fault.code, retryAt);
+        };
+        try {
+            await this.stored(() => this.options.lifecycle.recover(scope));
+            await this.stored(() => this.store.recoverControl(scope));
+            handle = await this.stored(() => this.store.claim(scope, trigger));
+            if (!handle)
+                return { kind: 'idle' };
+            snapshot = await this.stored(() => this.store.snapshot(scope));
+            if (!snapshot || snapshot.state.generation !== handle.lease.generation)
+                throw new SyncStale();
+            checkpoint = snapshot.checkpoint;
+            this.emit('started', 0);
+            if (!await this.stored(() => this.store.progress(needHandle())))
+                throw new SyncStale();
+            let source;
+            try {
+                source = await this.options.provider(needHandle);
+            }
+            catch (error) {
+                if (error instanceof SourceSyncFault)
+                    throw error;
+                throw new SourceSyncFault('storage_unavailable');
+            }
+            // An anchor read shares one attempt count with the provider call that leads to it: a recovery's with the history
+            // page that answered `history_expired`, a reconcile's with its look at the committed cursor's history. The one
+            // operation slot then never alternates between two keys, and while a call keeps failing the count reaches five
+            // attempts and the `pollMs` wait.
+            const open = async (mode, replace = false, sharedKey) => {
+                const cutoff = this.now();
+                const cycleId = randomUUID();
+                const committed = snapshot.state.cursor;
+                const key = sharedKey ?? digest(['begin', handle.lease.generation, mode, committed,
+                    snapshot.state.lastSyncAt, checkpoint?.cycleId ?? null, checkpoint?.revision ?? null]);
+                const anchor = mode === 'incremental' ? committed : await operation(key, (child) => source.begin(context(cycleId, cutoff, child)));
+                // A reconcile reads its anchor before the scan but replays history from the committed cursor: history ids
+                // only increase, so that replay also covers the changes between the cursor and the anchor, which the bounded
+                // scan may not list. It asks for that history first: when the provider answers that it has expired, nothing
+                // before the anchor can be replayed, so the cycle replays from the anchor, with one scan and one read pass,
+                // instead of finding the expiry after them and repeating both in a recovery.
+                let fromCursor = anchor;
+                if (mode === 'reconcile' && committed !== null) {
+                    const cursor = committed;
+                    try {
+                        await operation(key, (child) => source.historyPage(context(cycleId, cutoff, child), cursor, null));
+                        fromCursor = cursor;
+                    }
+                    catch (error) {
+                        if (!(error instanceof SourceSyncFault) || error.code !== 'history_expired')
+                            throw error;
+                    }
+                }
+                const next = await this.stored(() => this.store.openCycle(needHandle(), { mode, fromCursor, cutoff, cycleId }, replace));
+                if (!next)
+                    throw new SyncStale();
+                return next;
+            };
+            if (!checkpoint) {
+                const mode = snapshot.state.cursor === null ? 'bootstrap'
+                    : snapshot.lastReconcileAt === null || this.now() - snapshot.lastReconcileAt >= snapshot.policy.reconcileMs
+                        ? 'reconcile' : 'incremental';
+                checkpoint = await open(mode);
+            }
+            for (;;) {
+                checkBudget();
+                await renew();
+                if (changed >= snapshot.policy.maxChangedPerLease)
+                    throw new SyncYield();
+                const work = await this.stored(() => this.store.takeWork(needHandle()));
+                if (work) {
+                    changed++;
+                    try {
+                        let result;
+                        if (await this.stored(() => this.store.isExcluded(scope, work.sourceId))) {
+                            result = { kind: 'skipped', id: work.sourceId, reason: 'excluded' };
+                        }
+                        else {
+                            const cp = checkpoint;
+                            result = await external((child) => source.readSource(context(cp.cycleId, cp.cutoff, child), work.sourceId), 'invalid_response');
+                        }
+                        if (result.kind === 'retry') {
+                            if (result.id !== work.sourceId)
+                                throw new SourceSyncFault('invalid_response');
+                            throw new SourceSyncFault(result.reason, result.retryAt);
+                        }
+                        let prepared = null;
+                        if (result.kind === 'document') {
+                            const cp = checkpoint;
+                            const document = result.document;
+                            prepared = await external((child) => this.options.prepare(context(cp.cycleId, cp.cutoff, child), document), 'storage_unavailable');
+                            if (prepared === null)
+                                throw new SourceSyncFault('invalid_response');
+                        }
+                        const applied = await this.stored(() => this.store.applyResult(needHandle(), work, result, prepared));
+                        if (!applied)
+                            throw new SyncStale();
+                        handle = applied.handle;
+                    }
+                    catch (error) {
+                        if (error instanceof SourceSyncFault && retriable(error))
+                            await deferRead(work, error);
+                        throw error;
+                    }
+                    const progress = await this.stored(() => this.store.progress(needHandle()));
+                    if (!progress)
+                        throw new SyncStale();
+                    this.emit('progress', progress.progress.committed);
+                    continue;
+                }
+                const pending = await this.stored(() => this.store.pendingRetry(needHandle()));
+                if (pending)
+                    throw new SyncDeferred(pending.reason, Math.max(this.now() + 1, pending.retryAt));
+                if (checkpoint.phase === 'apply') {
+                    const cp = checkpoint;
+                    const state = await this.stored(() => this.store.finish(needHandle(), cp));
+                    if (!state)
+                        throw new SyncStale();
+                    handle = null;
+                    this.emit('succeeded', state.progress.committed, started);
+                    return { kind: 'completed', examined: state.progress.examined,
+                        committed: state.progress.committed, bounded: state.progress.bounded };
+                }
+                if (checkpoint.phase === 'history' && historyPages >= snapshot.policy.maxHistoryPagesPerLease)
+                    throw new SyncYield();
+                const cp = checkpoint;
+                const pageKey = digest(['page', cp.generation, cp.cycleId, cp.revision, cp.phase]);
+                try {
+                    if (cp.phase === 'history')
+                        historyPages++;
+                    const page = await operation(pageKey, (child) => cp.phase === 'scan'
+                        ? source.scanPage(context(cp.cycleId, cp.cutoff, child), cp.nextPageToken)
+                        : source.historyPage(context(cp.cycleId, cp.cutoff, child), cp.fromCursor, cp.nextPageToken));
+                    checkpoint = await this.stored(() => this.store.page(needHandle(), cp, page));
+                    if (!checkpoint)
+                        throw new SyncStale();
+                }
+                catch (error) {
+                    if (error instanceof SourceSyncFault && error.code === 'history_expired' && cp.phase === 'history') {
+                        if (recovered)
+                            throw new SyncDeferred('provider_unavailable', this.now() + snapshot.policy.pollMs);
+                        // The recovery's anchor read continues this page's attempt count (see `open`).
+                        recovered = true;
+                        checkpoint = await open('recovery', true, pageKey);
+                        continue;
+                    }
+                    throw error;
+                }
+            }
+        }
+        catch (error) {
+            if (error instanceof SyncStale)
+                return { kind: 'stale' };
+            if (!handle)
+                return { kind: 'unavailable' };
+            try {
+                if (error instanceof SyncYield || (error instanceof SourceSyncFault && error.code === 'cancelled')) {
+                    if (!await this.stored(() => this.store.release(needHandle())))
+                        return { kind: 'stale' };
+                    this.emit('yielded', changed, started);
+                    return { kind: 'yielded' };
+                }
+                if (error instanceof SourceSyncFault && (error.code === 'grant_rejected' || error.code === 'insufficient_scope')) {
+                    const reason = error.code;
+                    if (!await this.stored(() => this.options.lifecycle.loseGrant(needHandle(), reason)))
+                        return { kind: 'stale' };
+                    this.emit('reconnect', 0, started);
+                    return { kind: 'reconnect' };
+                }
+                if (error instanceof SourceSyncFault && (error.code === 'domain_policy' || error.code === 'invalid_response')) {
+                    const reason = error.code;
+                    if (!await this.stored(() => this.store.release(needHandle(), { type: 'sync_failed', reason })))
+                        return { kind: 'stale' };
+                    return { kind: 'failed', reason: error.code };
+                }
+                const deferred = error instanceof SyncDeferred ? error : {
+                    reason: 'storage_unavailable',
+                    retryAt: Math.max(this.now() + (snapshot?.policy.pollMs ?? 900_000), error instanceof SourceSyncFault ? error.retryAt ?? 0 : 0),
+                };
+                const retryAt = Math.max(this.now() + 1, deferred.retryAt);
+                if (!await this.stored(() => this.store.release(needHandle(), { type: 'sync_deferred', reason: deferred.reason, retryAt })))
+                    return { kind: 'stale' };
+                this.emit('retry', changed, started);
+                return { kind: 'deferred', reason: deferred.reason, retryAt };
+            }
+            catch {
+                return { kind: 'unavailable' };
+            }
+        }
+    }
+    /**
+     * A bounded scheduler batch. The host persists next between ticks to avoid starving high keys.
+     *
+     * Visits up to `limit` connections after `after` in key order, paused, blocked and disconnected ones included, and
+     * for each runs the lifecycle's recovery, the store's control recovery and the expiry of up to 100 sources. A
+     * failure is counted and the next connection still runs. No provider is called.
+     *
+     * @param after - The `next` of the previous batch, or null to start from the first connection.
+     * @param limit - At most this many connections, an integer from 1 to 1,000.
+     * @throws {SourceSyncFault} `storage_unavailable` when the connections cannot be listed, a `limit` outside that
+     *   range included.
+     */
+    async maintain(after = null, limit = 100) {
+        const scopes = await this.stored(() => this.store.scopes(after, limit));
+        let expired = 0;
+        let failed = 0;
+        for (const scope of scopes) {
+            try {
+                await this.stored(() => this.options.lifecycle.recover(scope));
+                await this.stored(() => this.store.recoverControl(scope));
+                expired += await this.stored(() => this.store.expire(scope, 100));
+            }
+            catch {
+                failed++;
+            }
+        }
+        if (expired > 0)
+            this.emit('purged', expired);
+        return { scanned: scopes.length, expired, failed,
+            next: scopes.length === limit ? scopes[scopes.length - 1] : null };
+    }
+}
+//# sourceMappingURL=AccountSourceSync.js.map

package/dist/sync/AccountSourceSync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"AccountSourceSync.js","sourceRoot":"","sources":["../../src/sync/AccountSourceSync.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AASrD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAmEjD,MAAM,SAAU,SAAQ,KAAK;IAAG,gBAAgB,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAAE;AAC/E,MAAM,SAAU,SAAQ,KAAK;IAAG,gBAAgB,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;CAAE;AAC/E,MAAM,YAAa,SAAQ,KAAK;IACT;IAAwC;IAA7D,YAAqB,MAA6B,EAAW,OAAe;QAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;QAAzF,WAAM,GAAN,MAAM,CAAuB;QAAW,YAAO,GAAP,OAAO,CAAQ;IAAmC,CAAC;CACjH;AACD,MAAM,YAAY,GAAqC,CAAC,SAAS,EAAE,cAAc,EAAE,sBAAsB,EAAE,qBAAqB,CAAC,CAAC;AAClI,MAAM,SAAS,GAAG,CAAC,KAAsB,EAAuE,EAAE,CAChH,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,IAA6B,CAAC,CAAC;AAC7D,MAAM,MAAM,GAAG,CAAC,KAA0C,EAAU,EAAE,CACpE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAEnE;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IAUC;IATZ,KAAK,CAAsC;IAC3C,KAAK,CAAe;IACpB,SAAS,CAAS;IAClB,gBAAgB,CAAS;IAC1C;;;;OAIG;IACH,YAA6B,OAA2C;QAA3C,YAAO,GAAP,OAAO,CAAoC;QACtE,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAAC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,WAAW,IAAI,MAAM,CAAC;QAC/C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,IAAI,MAAM,CAAC;QAC7D,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;eACvF,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,IAAI,CAAC,gBAAgB,GAAG,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;IACvH,CAAC;IACO,GAAG;QACT,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC;YAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;QACzF,OAAO,GAAG,CAAC;IACb,CAAC;IACO,IAAI,CAAC,KAA+B,EAAE,CAAS,EAAE,OAAgB;QACvE,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;YAC5D,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;QAChF,IAAI,CAAC;YAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,2DAA2D,CAAC,CAAC;IAC7G,CAAC;IACO,KAAK,CAAC,MAAM,CAAI,IAAsB;QAC5C,IAAI,CAAC;YAAC,OAAO,MAAM,IAAI,EAAE,CAAC;QAAC,CAAC;QAC5B,OAAO,KAAK,EAAE,CAAC;YAAC,IAAI,KAAK,YAAY,eAAe;gBAAE,MAAM,KAAK,CAAC;YAAC,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;QAAC,CAAC;IACxH,CAAC;IACO,EAAE,CAAC,KAAsB,EAAE,OAAe,EAAE,MAAc;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,EAAE,IAAI,EACxE,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,CAAC,OAAO,IAAI,CAAC,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IAC1G,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,GAAG,CAAC,KAAsB,EAAE,OAA0B,EAAE,MAAoB;QAChF,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,IAAI,MAAM,GAA4B,IAAI,CAAC;QAC3C,IAAI,QAAQ,GAA8B,IAAI,CAAC;QAC/C,IAAI,UAAU,GAAgC,IAAI,CAAC;QACnD,IAAI,OAAO,GAAG,CAAC,CAAC;QAAC,IAAI,YAAY,GAAG,CAAC,CAAC;QAAC,IAAI,SAAS,GAAG,KAAK,CAAC;QAC7D,MAAM,WAAW,GAAG,GAAG,EAAE;YACvB,IAAI,MAAM,EAAE,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,IAAI,IAAI,CAAC,SAAS;gBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;QACvF,CAAC,CAAC;QACF,MAAM,UAAU,GAAG,GAAqB,EAAE,GAAG,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,SAAS,EAAE,CAAC,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;QAClG,MAAM,KAAK,GAAG,KAAK,IAAI,EAAE;YACvB,WAAW,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YAC5D,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;QACrC,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,CAAC,OAAe,EAAE,MAAc,EAAE,KAAkB,EAAqB,EAAE,CAAC,CAAC;YAC3F,KAAK,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,QAAS,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK;SAC3G,CAAC,CAAC;QACH,MAAM,QAAQ,GAAG,KAAK,EAAK,IAAwC,EAAE,OAAmD,EAAc,EAAE;YACtI,MAAM,KAAK,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,EACrF,UAAU,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;YACpD,IAAI,OAAO,IAAI,CAAC;gBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;YACzC,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;YACvC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,IAAI,MAAM,EAAE,OAAO;gBAAE,KAAK,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACzC,IAAI,CAAC;gBACH,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;gBAC5C,IAAI,MAAM,EAAE,OAAO;oBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAM,IAAI,eAAe,CAAC,OAAO,KAAK,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBAClH,OAAO,KAAK,CAAC;YACf,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,MAAM,EAAE,OAAO;oBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBAC3C,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAM,IAAI,eAAe,CAAC,OAAO,KAAK,qBAAqB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBAClH,IAAI,KAAK,YAAY,SAAS,IAAI,KAAK,YAAY,eAAe;oBAAE,MAAM,KAAK,CAAC;gBAChF,MAAM,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;YACrC,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,KAAK,CAAC,CAAC;gBAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACnE,CAAC;QACH,CAAC,CAAC;QACF,MAAM,SAAS,GAAG,KAAK,EAAK,GAAW,EAAE,IAAwC,EAAc,EAAE;YAC/F,MAAM,KAAK,EAAE,CAAC;YAAC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;YAC5C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;YACpF,IAAI,CAAC,QAAQ;gBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YACrC,IAAI,QAAQ,CAAC,IAAI,KAAK,UAAU;gBAAE,MAAM,IAAI,YAAY,CAAC,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;YAC5F,IAAI,CAAC;gBAAC,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,kBAAkB,CAAC,CAAC;YAAC,CAAC;YACxD,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;oBAAE,MAAM,KAAK,CAAC;gBAC1E,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;gBAAC,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC,OAAO,EAAE,QAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACrG,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,EAAE,CAAC;oBAC9G,MAAM,IAAI,SAAS,EAAE,CAAC;gBACxB,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC,CAAC;QACF,MAAM,SAAS,GAAG,KAAK,EAAE,IAAoB,EAAE,KAAiE,EAAkB,EAAE;YAClI,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,QAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACvE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,IAAI,EAC9E,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;YAC5E,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC3D,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC,CAAC;QACF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/D,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1D,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC;YACnE,IAAI,CAAC,MAAM;gBAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YACrC,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/D,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,KAAK,CAAC,UAAU,KAAK,MAAM,CAAC,KAAK,CAAC,UAAU;gBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YAC9F,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YACxB,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;gBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YACvF,IAAI,MAAiC,CAAC;YACtC,IAAI,CAAC;gBAAC,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAAC,CAAC;YACzD,OAAO,KAAK,EAAE,CAAC;gBAAC,IAAI,KAAK,YAAY,eAAe;oBAAE,MAAM,KAAK,CAAC;gBAAC,MAAM,IAAI,eAAe,CAAC,qBAAqB,CAAC,CAAC;YAAC,CAAC;YACtH,iHAAiH;YACjH,+GAA+G;YAC/G,+GAA+G;YAC/G,kCAAkC;YAClC,MAAM,IAAI,GAAG,KAAK,EAAE,IAAoB,EAAE,OAAO,GAAG,KAAK,EAAE,SAAkB,EAAiC,EAAE;gBAC9G,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAAC,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;gBAAC,MAAM,SAAS,GAAG,QAAS,CAAC,KAAK,CAAC,MAAM,CAAC;gBAClG,MAAM,GAAG,GAAG,SAAS,IAAI,MAAM,CAAC,CAAC,OAAO,EAAE,MAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,EAAE,SAAS;oBACjF,QAAS,CAAC,KAAK,CAAC,UAAU,EAAE,UAAU,EAAE,OAAO,IAAI,IAAI,EAAE,UAAU,EAAE,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC;gBAC1F,MAAM,MAAM,GAAG,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,SAAU,CAAC,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,EACtE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBAC5D,0GAA0G;gBAC1G,6GAA6G;gBAC7G,4GAA4G;gBAC5G,4GAA4G;gBAC5G,6EAA6E;gBAC7E,IAAI,UAAU,GAAG,MAAM,CAAC;gBACxB,IAAI,IAAI,KAAK,WAAW,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;oBAC/C,MAAM,MAAM,GAAW,SAAS,CAAC;oBACjC,IAAI,CAAC;wBACH,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;wBACnG,UAAU,GAAG,MAAM,CAAC;oBACtB,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB;4BAAE,MAAM,KAAK,CAAC;oBAC3F,CAAC;gBACH,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC;gBACzH,IAAI,CAAC,IAAI;oBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBAAC,OAAO,IAAI,CAAC;YAChD,CAAC,CAAC;YACF,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,IAAI,GAAmB,QAAQ,CAAC,KAAK,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW;oBACvE,CAAC,CAAC,QAAQ,CAAC,eAAe,KAAK,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,eAAe,IAAI,QAAQ,CAAC,MAAM,CAAC,WAAW;wBACzG,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,aAAa,CAAC;gBAClC,UAAU,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;YACD,SAAS,CAAC;gBACR,WAAW,EAAE,CAAC;gBACd,MAAM,KAAK,EAAE,CAAC;gBACd,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,CAAC,kBAAkB;oBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBACzE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBACxE,IAAI,IAAI,EAAE,CAAC;oBACT,OAAO,EAAE,CAAC;oBACV,IAAI,CAAC;wBACH,IAAI,MAA4B,CAAC;wBACjC,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;4BACzE,MAAM,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;wBACtE,CAAC;6BAAM,CAAC;4BACN,MAAM,EAAE,GAAyB,UAAU,CAAC;4BAC5C,MAAM,GAAG,MAAM,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE,kBAAkB,CAAC,CAAC;wBAClI,CAAC;wBACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;4BAC5B,IAAI,MAAM,CAAC,EAAE,KAAK,IAAI,CAAC,QAAQ;gCAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;4BAC/E,MAAM,IAAI,eAAe,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;wBAC3D,CAAC;wBACD,IAAI,QAAQ,GAAoB,IAAI,CAAC;wBACrC,IAAI,MAAM,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;4BAC/B,MAAM,EAAE,GAAyB,UAAU,CAAC;4BAAC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;4BAC9E,QAAQ,GAAG,MAAM,QAAQ,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,QAAQ,CAAC,EAAE,qBAAqB,CAAC,CAAC;4BACnI,IAAI,QAAQ,KAAK,IAAI;gCAAE,MAAM,IAAI,eAAe,CAAC,kBAAkB,CAAC,CAAC;wBACvE,CAAC;wBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;wBACtG,IAAI,CAAC,OAAO;4BAAE,MAAM,IAAI,SAAS,EAAE,CAAC;wBAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;oBAC/D,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,KAAK,YAAY,eAAe,IAAI,SAAS,CAAC,KAAK,CAAC;4BAAE,MAAM,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;wBACvF,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;oBAC5E,IAAI,CAAC,QAAQ;wBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;oBACrC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;oBACnD,SAAS;gBACX,CAAC;gBACD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;gBAC/E,IAAI,OAAO;oBAAE,MAAM,IAAI,YAAY,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC/F,IAAI,UAAU,CAAC,KAAK,KAAK,OAAO,EAAE,CAAC;oBACjC,MAAM,EAAE,GAAyB,UAAU,CAAC;oBAC5C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;oBAC3E,IAAI,CAAC,KAAK;wBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;oBAClC,MAAM,GAAG,IAAI,CAAC;oBACd,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;oBAC1D,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;wBAC3D,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC;gBAC3E,CAAC;gBACD,IAAI,UAAU,CAAC,KAAK,KAAK,SAAS,IAAI,YAAY,IAAI,QAAQ,CAAC,MAAM,CAAC,uBAAuB;oBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBACrH,MAAM,EAAE,GAAyB,UAAU,CAAC;gBAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;gBACnF,IAAI,CAAC;oBACH,IAAI,EAAE,CAAC,KAAK,KAAK,SAAS;wBAAE,YAAY,EAAE,CAAC;oBAC3C,MAAM,IAAI,GAAG,MAAM,SAAS,CAAiB,OAAO,EAClD,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,KAAK,KAAK,MAAM;wBAC5B,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC;wBAC1E,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;oBAClG,UAAU,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC,CAAC;oBAC9E,IAAI,CAAC,UAAU;wBAAE,MAAM,IAAI,SAAS,EAAE,CAAC;gBACzC,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,IAAI,KAAK,iBAAiB,IAAI,EAAE,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;wBACnG,IAAI,SAAS;4BAAE,MAAM,IAAI,YAAY,CAAC,sBAAsB,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;wBACnG,+EAA+E;wBAC/E,SAAS,GAAG,IAAI,CAAC;wBAAC,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;wBAAC,SAAS;oBACjF,CAAC;oBACD,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,SAAS;gBAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;YACzD,IAAI,CAAC,MAAM;gBAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;YAC5C,IAAI,CAAC;gBACH,IAAI,KAAK,YAAY,SAAS,IAAI,CAAC,KAAK,YAAY,eAAe,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC,EAAE,CAAC;oBACnG,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;wBAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;oBACzF,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;oBAAC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;gBACrE,CAAC;gBACD,IAAI,KAAK,YAAY,eAAe,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,gBAAgB,IAAI,KAAK,CAAC,IAAI,KAAK,oBAAoB,CAAC,EAAE,CAAC;oBACjH,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;oBAAC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,MAAM,CAAC,CAAC;wBAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;oBAC1I,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;oBAAC,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;gBACnE,CAAC;gBACD,IAAI,KAAK,YAAY,eAAe,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,eAAe,IAAI,KAAK,CAAC,IAAI,KAAK,kBAAkB,CAAC,EAAE,CAAC;oBAC9G,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC;oBAAC,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,CAAC,CAAC;wBAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;oBACrJ,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChD,CAAC;gBACD,MAAM,QAAQ,GAAG,KAAK,YAAY,YAAY,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;oBACvD,MAAM,EAAE,qBAA8B;oBACtC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,EACjE,KAAK,YAAY,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;iBAC7D,CAAC;gBACF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,EAC1D,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;oBAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;gBAC1F,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBACrC,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAChE,CAAC;YAAC,MAAM,CAAC;gBAAC,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;YAAC,CAAC;QAC7C,CAAC;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,QAAQ,CAAC,QAAgC,IAAI,EAAE,KAAK,GAAG,GAAG;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;QACxE,IAAI,OAAO,GAAG,CAAC,CAAC;QAAC,IAAI,MAAM,GAAG,CAAC,CAAC;QAChC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC/D,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC1D,OAAO,IAAI,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YACpE,CAAC;YAAC,MAAM,CAAC;gBAAC,MAAM,EAAE,CAAC;YAAC,CAAC;QACvB,CAAC;QACD,IAAI,OAAO,GAAG,CAAC;YAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM;YAC9C,IAAI,EAAE,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IACxE,CAAC;CACF"}

package/dist/sync/SourceQuota.d.ts

@@ -45,7 +45,10 @@ export interface SourceQuotaRequest {
 export type SourceQuotaReason = 'project_minute' | 'mailbox_minute' | 'project_day' | 'project_concurrency' | 'mailbox_concurrency' | 'provider_wait';
 /**
  * What `SourceQuota.run` answers: the transport's value; a deferral with the earliest time, in epoch milliseconds, to
- * try again, nothing sent; or `duplicate` for an attempt id seen before, which never sends.
+ * try again, nothing sent; `late` when the call could not start within a second of its reservation, after a long wait
+ * for the project's lock or a stalled process: nothing was sent and the units stay spent, and the attempt is retryable
+ * by a new `run` with a new attempt id from `retryAt`, in epoch milliseconds, a second after the late start was found;
+ * or `duplicate` for an attempt id seen before, which never sends.
  */
 export type SourceQuotaResult<T> = {
     readonly kind: 'completed';
@@ -54,9 +57,28 @@ export type SourceQuotaResult<T> = {
     readonly kind: 'deferred';
     readonly reason: SourceQuotaReason;
     readonly retryAt: number;
+} | {
+    readonly kind: 'late';
+    readonly retryAt: number;
 } | {
     readonly kind: 'duplicate';
 };
+/**
+ * What one `SourceQuota.compact` call deleted. Each table gives up at most 1,000 rows a call: `more` is true when one
+ * gave that many, so a further call may find more to delete, and false when every table gave fewer.
+ */
+export interface SourceQuotaCompaction {
+    /** Attempt records deleted a day after their slots expired. */
+    readonly attempts: number;
+    /** Mailbox provider waits deleted once passed. */
+    readonly mailboxWaits: number;
+    /** Reservations settled more than a day ago, deleted from the three budgets' meters together. */
+    readonly reservations: number;
+    /** The two minute budgets' account rows idle for a day, deleted together. */
+    readonly windows: number;
+    /** Whether a table gave a full page of 1,000 rows; a host calls `compact` again while it is true. */
+    readonly more: boolean;
+}
 /** The quota's table prefix, policy and clock. */
 export interface SourceQuotaOptions {
     /** Prefix of the quota tables, the one given to `sourceQuotaSchema`; `agentos_source_quota` unless given. */
@@ -91,15 +113,21 @@ export declare class SourceQuota {
     private reserve;
     private begin;
     private release;
-    /** Consume weighted capacity and call the transport once. No inline HTTP retries. */
+    /**
+     * Consume weighted capacity and call the transport once. No inline HTTP retries: a retry, after a deferral or a
+     * `late` answer among others, is a new `run` with a new attempt id.
+     */
     run<T>(request: SourceQuotaRequest, send: (signal: AbortSignal) => Promise<T>, parent?: AbortSignal): Promise<SourceQuotaResult<T>>;
     /** A longer provider wait survives restarts, generation changes and shorter updates. */
     defer(scope: {
         readonly projectKey: string;
         readonly mailboxKey?: string;
     }, retryAt: number): Promise<void>;
-    /** Remove expired attempt evidence after 24 hours and safely old meter records. Daily totals remain. */
-    compact(): Promise<void>;
+    /**
+     * Remove expired attempt evidence after 24 hours and safely old meter records, at most 1,000 rows of each table a
+     * call. Daily totals remain. Answers what it removed; while `more` is true, a further call may remove more.
+     */
+    compact(): Promise<SourceQuotaCompaction>;
 }
 /** Retry-After is never clamped down to the local exponential-backoff ceiling. */
 export declare function sourceRetryAt(now: number, attempt: number, retryAfter: string | null, random?: number): number;

package/dist/sync/SourceQuota.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"SourceQuota.d.ts","sourceRoot":"","sources":["../../src/sync/SourceQuota.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAInE,wFAAwF;AACxF,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,CAAC;AACnF;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,4EAA4E;IAC5E,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,8EAA8E;IAC9E,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,2DAA2D;IAC3D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,iEAAiE;IACjE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,mEAAmE;IACnE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,sEAAsE;IACtE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,wGAAwG;IACxG,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,wCAAwC;IACxC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC;CAC/D;AACD,sGAAsG;AACtG,MAAM,WAAW,kBAAkB;IACjC,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,yDAAyD;IACzD,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AACD,+GAA+G;AAC/G,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,aAAa,GAC/E,qBAAqB,GAAG,qBAAqB,GAAG,eAAe,CAAC;AACpE;;;GAGG;AACH,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAC3B;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC3F;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,CAAC;AACnC,kDAAkD;AAClD,MAAM,WAAW,kBAAkB;IACjC,6GAA6G;IAC7G,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,wDAAwD;IACxD,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,mFAAmF;IACnF,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CAC1D;AAeD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,SAAyB,GAAG,SAAS,MAAM,EAAE,CAYpF;AAgBD,oFAAoF;AACpF,qBAAa,WAAW;IAQV,OAAO,CAAC,QAAQ,CAAC,EAAE;IAAkB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAPzE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAA4B;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC;;;OAGG;gBAC0B,EAAE,EAAE,cAAc,EAAmB,OAAO,EAAE,kBAAkB;YAK/E,GAAG;YAOH,OAAO;IAQrB,OAAO,CAAC,KAAK;YAMC,OAAO;YAiDP,KAAK;YAcL,OAAO;IAOrB,qFAAqF;IAC/E,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EACjF,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAoCtD,wFAAwF;IAClF,KAAK,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBjH,wGAAwG;IAClG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;CAa/B;AAED,kFAAkF;AAClF,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,SAAgB,GAAG,MAAM,CAkBrH"}
+{"version":3,"file":"SourceQuota.d.ts","sourceRoot":"","sources":["../../src/sync/SourceQuota.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAInE,wFAAwF;AACxF,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,SAAS,GAAG,QAAQ,CAAC;AACnF;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,4EAA4E;IAC5E,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,8EAA8E;IAC9E,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,2DAA2D;IAC3D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,iEAAiE;IACjE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,mEAAmE;IACnE,QAAQ,CAAC,kBAAkB,EAAE,MAAM,CAAC;IACpC,sEAAsE;IACtE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,wGAAwG;IACxG,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,wCAAwC;IACxC,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,iBAAiB,EAAE,MAAM,CAAC,CAAC,CAAC;CAC/D;AACD,sGAAsG;AACtG,MAAM,WAAW,kBAAkB;IACjC,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,uDAAuD;IACvD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,yDAAyD;IACzD,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,mEAAmE;IACnE,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AACD,+GAA+G;AAC/G,MAAM,MAAM,iBAAiB,GAAG,gBAAgB,GAAG,gBAAgB,GAAG,aAAa,GAC/E,qBAAqB,GAAG,qBAAqB,GAAG,eAAe,CAAC;AACpE;;;;;;GAMG;AACH,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAC3B;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC3F;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,CAAC;AACnC;;;GAGG;AACH,MAAM,WAAW,qBAAqB;IACpC,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,kDAAkD;IAClD,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,iGAAiG;IACjG,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,6EAA6E;IAC7E,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,qGAAqG;IACrG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB;AACD,kDAAkD;AAClD,MAAM,WAAW,kBAAkB;IACjC,6GAA6G;IAC7G,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,wDAAwD;IACxD,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;IACnC,mFAAmF;IACnF,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,cAAc,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CAC1D;AAoBD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,SAAyB,GAAG,SAAS,MAAM,EAAE,CAYpF;AAgBD,oFAAoF;AACpF,qBAAa,WAAW;IAQV,OAAO,CAAC,QAAQ,CAAC,EAAE;IAAkB,OAAO,CAAC,QAAQ,CAAC,OAAO;IAPzE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAA4B;IAC9C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC;;;OAGG;gBAC0B,EAAE,EAAE,cAAc,EAAmB,OAAO,EAAE,kBAAkB;YAK/E,GAAG;YAOH,OAAO;IAQrB,OAAO,CAAC,KAAK;YAMC,OAAO;YAiDP,KAAK;YAmBL,OAAO;IAOrB;;;OAGG;IACG,GAAG,CAAC,CAAC,EAAE,OAAO,EAAE,kBAAkB,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EACjF,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;IAqCtD,wFAAwF;IAClF,KAAK,CAAC,KAAK,EAAE;QAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBjH;;;OAGG;IACG,OAAO,IAAI,OAAO,CAAC,qBAAqB,CAAC;CAiBhD;AAED,kFAAkF;AAClF,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,IAAI,EAAE,MAAM,SAAgB,GAAG,MAAM,CAkBrH"}

package/dist/sync/SourceQuota.js

@@ -19,6 +19,17 @@ class QuotaDeferred extends Error {
 class QuotaDuplicate extends Error {
     constructor() { super('source_quota_duplicate'); }
 }
+class QuotaLate extends Error {
+    retryAt;
+    constructor(retryAt) {
+        super('source_quota_late');
+        this.retryAt = retryAt;
+    }
+}
+/** Milliseconds after the reservation's clock reading within which an attempt must start its call. */
+const sendWindowMs = 1000;
+/** Rows of each table one `compact` call deletes at most. */
+const compactLimit = 1000;
 const integer = (n) => Number.isSafeInteger(n) && n >= 0;
 const key = (s) => typeof s === 'string' && s.length > 0 && s.length <= 512;
 const dayEnd = (now) => (Math.floor(now / 86_400_000) + 1) * 86_400_000;
@@ -149,8 +160,8 @@ export class SourceQuota {
                 if (settled.status !== 'settled' || settled.state !== 'consumed')
                     throw new Error('source_quota_settle_failed');
             }
-            const permit = { token: randomUUID(), sendBy: now + 1000,
-                expiresAt: now + 1000 + this.policy.timeoutMs + this.policy.slotGraceMs, timeoutMs: this.policy.timeoutMs };
+            const permit = { token: randomUUID(), sendBy: now + sendWindowMs,
+                expiresAt: now + sendWindowMs + this.policy.timeoutMs + this.policy.slotGraceMs, timeoutMs: this.policy.timeoutMs };
             await tx.run(`INSERT INTO ${this.t.attempts}(attempt_id,project_key,mailbox_key,token,send_by,expires_at)
         VALUES($1,$2,$3,$4,$5,$6)`, [request.attemptId, request.projectKey, request.mailboxKey,
                 permit.token, permit.sendBy, permit.expiresAt]);
@@ -169,8 +180,12 @@ export class SourceQuota {
             const updated = await tx.run(`UPDATE ${this.t.attempts} SET started=TRUE
         WHERE attempt_id=$1 AND project_key=$2 AND mailbox_key=$3 AND token=$4
         AND started=FALSE AND finished=FALSE AND send_by>=$5 AND expires_at>$5`, [request.attemptId, request.projectKey, request.mailboxKey, permit.token, now]);
-            if (updated.changes !== 1)
-                throw new QuotaDuplicate();
+            if (updated.changes === 1)
+                return;
+            // This run's own attempt, still unstarted, missed only the time conditions: its send window closed before the start.
+            const unstarted = await tx.get(`SELECT attempt_id FROM ${this.t.attempts}
+        WHERE attempt_id=$1 AND project_key=$2 AND mailbox_key=$3 AND token=$4 AND started=FALSE AND finished=FALSE`, [request.attemptId, request.projectKey, request.mailboxKey, permit.token]);
+            throw unstarted ? new QuotaLate(now + sendWindowMs) : new QuotaDuplicate();
         });
     }
     async release(request, permit) {
@@ -180,7 +195,10 @@ export class SourceQuota {
         AND project_key=$3 AND mailbox_key=$4`, [request.attemptId, permit.token, request.projectKey, request.mailboxKey]);
         });
     }
-    /** Consume weighted capacity and call the transport once. No inline HTTP retries. */
+    /**
+     * Consume weighted capacity and call the transport once. No inline HTTP retries: a retry, after a deferral or a
+     * `late` answer among others, is a new `run` with a new attempt id.
+     */
     async run(request, send, parent) {
         if (parent?.aborted)
             throw new SourceSyncFault('cancelled');
@@ -210,6 +228,8 @@ export class SourceQuota {
             catch (error) {
                 if (error instanceof QuotaDeferred)
                     return { kind: 'deferred', reason: error.reason, retryAt: error.retryAt };
+                if (error instanceof QuotaLate)
+                    return { kind: 'late', retryAt: error.retryAt };
                 if (error instanceof QuotaDuplicate)
                     return { kind: 'duplicate' };
                 throw new SourceSyncFault('storage_unavailable');
@@ -258,17 +278,29 @@ export class SourceQuota {
             throw new SourceSyncFault('storage_unavailable');
         }
     }
-    /** Remove expired attempt evidence after 24 hours and safely old meter records. Daily totals remain. */
+    /**
+     * Remove expired attempt evidence after 24 hours and safely old meter records, at most 1,000 rows of each table a
+     * call. Daily totals remain. Answers what it removed; while `more` is true, a further call may remove more.
+     */
     async compact() {
         try {
-            await this.db.transaction(async (tx) => {
+            return await this.db.transaction(async (tx) => {
                 const now = await this.now(tx);
-                await tx.run(`DELETE FROM ${this.t.attempts} WHERE attempt_id IN (SELECT attempt_id FROM ${this.t.attempts} WHERE expires_at < $1 ORDER BY expires_at LIMIT 1000) AND expires_at < $1`, [Math.max(0, now - 86_400_000)]);
-                await tx.run(`DELETE FROM ${this.t.mailboxes} WHERE (project_key,mailbox_key) IN (SELECT project_key,mailbox_key FROM ${this.t.mailboxes} WHERE retry_at <= $1 LIMIT 1000) AND retry_at <= $1`, [now]);
+                const before = Math.max(0, now - 86_400_000);
+                const attempts = (await tx.run(`DELETE FROM ${this.t.attempts} WHERE attempt_id IN (SELECT attempt_id FROM ${this.t.attempts} WHERE expires_at < $1 ORDER BY expires_at LIMIT ${compactLimit}) AND expires_at < $1`, [before])).changes;
+                const mailboxWaits = (await tx.run(`DELETE FROM ${this.t.mailboxes} WHERE (project_key,mailbox_key) IN (SELECT project_key,mailbox_key FROM ${this.t.mailboxes} WHERE retry_at <= $1 LIMIT ${compactLimit}) AND retry_at <= $1`, [now])).changes;
+                const pages = [attempts, mailboxWaits];
+                let reservations = 0;
+                let windows = 0;
                 for (const meter of [this.meter(tx, this.t.pm, this.policy.projectMinute, false),
                     this.meter(tx, this.t.mm, this.policy.mailboxMinute, false), this.meter(tx, this.t.pd, this.policy.projectDay, true)]) {
-                    await meter.purge({ before: Math.max(0, now - 86_400_000), limit: 1000 });
+                    // A meter answers its settled reservations and its window rows deleted, each its own page of the limit.
+                    const purged = await meter.purge({ before, limit: compactLimit });
+                    reservations += purged.reservations;
+                    windows += purged.periods;
+                    pages.push(purged.reservations, purged.periods);
                 }
+                return { attempts, mailboxWaits, reservations, windows, more: pages.some((n) => n >= compactLimit) };
             });
         }
         catch {