@framers/agentos-ext-account-sources 0.10.0 → 0.11.1

package/dist/sync/SqlAccountSourceSyncStore.js

@@ -142,6 +142,21 @@ const addedByMessage = (stored, read) => {
  */
 const sameCopy = (stored, read) => sameText(stored, read) && stored.dependency.version === read.dependency.version
     && stored.contentHash === read.contentHash;
+/**
+ * The host's allowance, checked: its three amounts nonnegative safe integers, and its period's end one too when given.
+ *
+ * @throws {Error} `invalid_source_allowance` for any other value.
+ */
+const checkedAllowance = (allowance) => {
+    if (![allowance.maximumConnectionDocuments, allowance.maximumConnectionHeldCharacters,
+        allowance.additionalIndexedCharacters].every(validNumber)
+        || (allowance.indexingPeriodEndsAt !== undefined && !validNumber(allowance.indexingPeriodEndsAt))) {
+        throw new Error('invalid_source_allowance');
+    }
+    return allowance;
+};
+/** The end of the indexing period the allowance names, or null when it names none or the period has ended by `now`. */
+const periodOf = (allowance, now) => allowance.indexingPeriodEndsAt !== undefined && allowance.indexingPeriodEndsAt > now ? allowance.indexingPeriodEndsAt : null;
 /**
  * Durable synchronization state for account sources on PostgreSQL. Every change to an owner's content runs under
  * that owner's fence (`fence`), and a change made with a handle checks its lease, generation and host liveness in
@@ -150,8 +165,11 @@ const sameCopy = (stored, read) => sameText(stored, read) && stored.dependency.v
  * before purging it, so a purge that fails leaves nothing derived from it deliverable. A read that repeats the text of
  * a copy the store holds is never charged again, one that changes it is charged, by the hash the store keeps of each
  * message's text, its new and changed messages and the text beside them, and with the host's `refresh` binding a read
- * of that very copy keeps it in place. Provider reads, embedding and model calls stay outside the store. The
- * constructor runs no DDL: the host's migration runs `sourceSyncSchema` and core's `sourceDerivationSchema` first.
+ * of that very copy keeps it in place. A host whose allowance names the end of its indexing period has each source's
+ * indexed text weighed for the rest of the period too, past its copy's expiry, and a source the allowance could not
+ * take settled with no read until it can, for a reconcile interval at a time. Provider reads, embedding and model calls
+ * stay outside the store. The constructor runs no DDL: the host's migration runs `sourceSyncSchema` and core's
+ * `sourceDerivationSchema` first.
  *
  * @typeParam Prepared - The host's in-memory value for one document, passed through to its `write` binding.
  */
@@ -186,6 +204,88 @@ export class SqlAccountSourceSyncStore {
             throw new Error('invalid_source_sync_clock');
         return n;
     }
+    /** Whether the charges table was found in place; a table found stays found, one missing is looked for again. */
+    chargesTable = false;
+    /**
+     * Whether the charges table of `sourceSyncSchema` is in place, which a database migrated by an earlier release's
+     * statements lacks: without it the store records no charge and no deferral.
+     */
+    async chargesHeld(tx) {
+        if (!this.chargesTable) {
+            const found = await tx.get(`SELECT to_regclass($1::text) IS NOT NULL AS present`, [this.t.charges]);
+            this.chargesTable = found?.present === true;
+        }
+        return this.chargesTable;
+    }
+    /** The source's charge record in the period ending at `period`, or null when it has none there. */
+    async chargeRecord(tx, scope, sourceId, period) {
+        return tx.get(`SELECT period_ends_at,charged,deferred FROM ${this.t.charges}
+      WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4 AND period_ends_at=$5`, [...params(scope), sourceId, period]);
+    }
+    /** Writes the source's charge record for the period ending at `period`, in place of any earlier one. */
+    async recordCharge(tx, scope, sourceId, period, charged, deferred) {
+        await tx.run(`INSERT INTO ${this.t.charges}(tenant_id,owner_id,connection_id,source_id,period_ends_at,charged,deferred)
+      VALUES($1,$2,$3,$4,$5,$6::jsonb,$7::jsonb) ON CONFLICT(tenant_id,owner_id,connection_id,source_id)
+      DO UPDATE SET period_ends_at=excluded.period_ends_at,charged=excluded.charged,deferred=excluded.deferred`, [...params(scope), sourceId, period, charged === null ? null : JSON.stringify(charged),
+            deferred === null ? null : JSON.stringify(deferred)]);
+    }
+    /** Removes the source's charge record, when the table is in place. */
+    async forgetCharge(tx, scope, sourceId) {
+        if (!await this.chargesHeld(tx))
+            return;
+        await tx.run(`DELETE FROM ${this.t.charges} WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3
+      AND source_id=$4`, [...params(scope), sourceId]);
+    }
+    /** Whether the source has a charge record, in any period, when the table is in place. */
+    async recorded(scope, sourceId) {
+        if (!await this.chargesHeld(this.db))
+            return false;
+        return !!await this.db.get(`SELECT 1 FROM ${this.t.charges} WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3
+      AND source_id=$4`, [...params(scope), sourceId]);
+    }
+    /**
+     * Removes up to `limit` charge records whose period has ended by the store's clock, the earliest first: every
+     * owner's, or `owner`'s alone, or only those of `connectionId` of that owner. Answers how many went.
+     */
+    async forgetEnded(owner, connectionId, limit) {
+        if (!await this.chargesHeld(this.db))
+            return 0;
+        const ended = `period_ends_at<=$1 AND ($2::text IS NULL OR (tenant_id=$2 AND owner_id=$3))
+      AND ($4::text IS NULL OR connection_id=$4)`;
+        // The period's end is checked on the row itself as well, so a record a read moved to a later period since the
+        // look stays.
+        const removed = await this.db.run(`DELETE FROM ${this.t.charges} WHERE ${ended}
+      AND (tenant_id,owner_id,connection_id,source_id) IN (SELECT tenant_id,owner_id,connection_id,source_id
+        FROM ${this.t.charges} WHERE ${ended} ORDER BY period_ends_at,tenant_id,owner_id,connection_id,source_id LIMIT $5)`, [this.now(), owner?.tenantId ?? null, owner?.ownerId ?? null, connectionId, limit]);
+        return removed.changes;
+    }
+    /**
+     * The listed sources `page` settles as `indexing_limit` skips with no read: each holds a deferral of the host's
+     * current indexing period at the connection's generation, recorded less than the policy's `reconcileMs` ago, the store
+     * holds no copy of it within its expiry, and the room the allowance leaves is no larger than the room it was skipped
+     * with. The allowance is read only when a listed source holds such a deferral. Past the reconcile interval the source
+     * is read again, so a change at the source, a deletion or an edit that makes it fit, reaches the store, as a
+     * reconcile reads every listed source again.
+     */
+    async deferredOf(tx, scope, policy, generation, ids) {
+        if (ids.length === 0 || !await this.chargesHeld(tx))
+            return new Set();
+        const now = this.now();
+        const marks = await tx.all(`SELECT c.source_id,c.period_ends_at,c.deferred FROM ${this.t.charges} c
+        WHERE c.tenant_id=$1 AND c.owner_id=$2 AND c.connection_id=$3 AND c.source_id=ANY($4::text[])
+        AND c.deferred IS NOT NULL AND c.period_ends_at>$5
+        AND NOT EXISTS(SELECT 1 FROM ${this.t.documents} d WHERE d.tenant_id=c.tenant_id AND d.owner_id=c.owner_id
+          AND d.connection_id=c.connection_id AND d.source_id=c.source_id AND d.expires_at>$5)`, [...params(scope), [...ids], now]);
+        const current = marks.filter((mark) => mark.deferred.generation === generation && validNumber(mark.deferred.limit)
+            && validNumber(mark.deferred.at) && now < mark.deferred.at + policy.reconcileMs);
+        if (current.length === 0)
+            return new Set();
+        const allowance = checkedAllowance(await this.options.bindings.allowance(tx, scope));
+        const period = periodOf(allowance, now);
+        const room = Math.min(policy.maxIndexedCharacters, allowance.additionalIndexedCharacters);
+        return new Set(current.filter((mark) => period !== null && Number(mark.period_ends_at) === period
+            && room <= mark.deferred.limit).map((mark) => mark.source_id));
+    }
     fenceOn(db) {
         return new SqlSourceDerivationStore(db, {
             tablePrefix: this.options.derivationPrefix, clock: this.clock,
@@ -386,8 +486,17 @@ export class SqlAccountSourceSyncStore {
      * counts they are weighed against, so a read given up before is given up again at its next failure; a read the
      * provider answered starts a count of its own.
      *
+     * A source an `indexing_limit` skip deferred in the host's current indexing period is queued settled, as the same
+     * skip, with no read, no purge and no deletion record, while the connection keeps the generation of the skip, less
+     * than the policy's `reconcileMs` has passed since it, the store holds no copy of the source and the allowance leaves
+     * no more room than the skip had: the host's `allowance` binding is read for that, only when a listed source holds
+     * such a deferral. The skip counts among the cycle's settled and skipped items, and not as a provider's answer. Once
+     * the period turns, the room grows, the generation moves or the reconcile interval has passed, the source is queued
+     * pending and read again, so a change at the source reaches the store within about one reconcile interval.
+     *
      * @throws {SourceSyncFault} `invalid_response` for an id longer than 512 characters, and the page rules'
      *   `invalid_source_sync_page` for a page the checkpoint does not admit.
+     * @throws {Error} `invalid_source_allowance` when the allowance read for a deferral is not valid.
      */
     async page(h, expected, page) {
         return this.withHandle(h, async (tx, row) => {
@@ -420,19 +529,25 @@ export class SqlAccountSourceSyncStore {
                         earlier.set(work.sourceId, work);
                 }
             }
+            // The sources the allowance deferred, settled here as their skip, so the run reads none of them.
+            const deferred = await this.deferredOf(tx, s, row.policy, cp.generation, ids);
             for (const id of ids) {
                 const before = earlier.get(id);
                 if (tally && before && isTerminalSourceWork(before.disposition)) {
                     tally = { ...tally, settled: tally.settled - 1,
                         skipped: tally.skipped - (before.disposition.kind === 'skipped' ? 1 : 0) };
                 }
+                const settled = deferred.has(id);
                 const failing = before !== undefined && !answers(before.disposition) ? before : undefined;
                 const work = { cycleId: cp.cycleId, generation: cp.generation, sourceId: id,
-                    discoveredRevision: next.checkpoint.revision, attempts: 0, disposition: { kind: 'pending' },
+                    discoveredRevision: next.checkpoint.revision, attempts: 0,
+                    disposition: settled ? { kind: 'skipped', reason: 'indexing_limit' } : { kind: 'pending' },
                     ...(failing?.failures !== undefined ? { failures: failing.failures } : {}),
                     ...(failing?.answeredAtFailure !== undefined ? { answeredAtFailure: failing.answeredAtFailure } : {}),
                     ...(failing?.preparedAtFailure !== undefined ? { preparedAtFailure: failing.preparedAtFailure } : {}) };
                 await this.putWork(tx, s, work);
+                if (tally && settled)
+                    tally = { ...tally, settled: tally.settled + 1, skipped: tally.skipped + 1 };
             }
             // Counts that would fall below zero were not the store's: count them again from the rows just written.
             if (!validTally(tally))
@@ -726,6 +841,16 @@ export class SqlAccountSourceSyncStore {
      * does not begin with the copy's text, so with the allowance spent it removes the thread's copy, and the thread stays
      * out of the store until a read of it fits the allowance whole.
      *
+     * An allowance that names the end of the host's indexing period (`indexingPeriodEndsAt`), with the charges table in
+     * place, has the read weighed against the newest text the store wrote or refreshed for the source in that period as
+     * well, by the same rules, and charged the smaller amount: a source read again after its copy expired and was
+     * removed is charged nothing when its text is unchanged and only what it adds otherwise, as a reply's separator and
+     * text. Each written or refreshed read becomes the period's text for the source. An `indexing_limit` skip that keeps
+     * no copy records a deferral with the room the allowance left it and the time, which `page` honours for a reconcile
+     * interval, and the record then keeps no provenance: nothing in it confirms the text of a source the store holds no
+     * copy of. A `gone` result and a skip the provider answers remove the source's record, before the application and
+     * with the withdrawal below, so a host purge that throws cannot keep it; the store's own skips keep it.
+     *
      * A `gone` result and a skip the provider answers withdraw the old copy before that, in a transaction of their own
      * (`withdraw`), and so does an `expired` skip, before its application runs again: the copy's expiry moves to the
      * store's clock, so from then on `readCurrent` answers it expired and the fence neither commits nor delivers a result
@@ -741,9 +866,10 @@ export class SqlAccountSourceSyncStore {
      *   stale handle or work item.
      * @throws {SourceSyncFault} `invalid_response` for a result that does not match its work item or is malformed, a skip
      *   that carries `attempt_limit` among them.
-     * @throws {Error} `invalid_source_allowance` when the host's allowance is not made of nonnegative safe integers, and
-     *   `invalid_source_sync_charge` should the characters a read adds come out below zero or fractional, which the
-     *   ordered spans of a valid read and an intact copy rule out; nothing is charged or written.
+     * @throws {Error} `invalid_source_allowance` when the host's allowance is not made of nonnegative safe integers, its
+     *   period's end included when given, and `invalid_source_sync_charge` should the characters a read adds come out
+     *   below zero or fractional, which the ordered spans of a valid read and an intact copy rule out; nothing is charged
+     *   or written.
      */
     async applyResult(h, work, result, prepared) {
         // The purge below runs in the transaction that records the result, so a binding that throws rolls the copy's
@@ -751,9 +877,11 @@ export class SqlAccountSourceSyncStore {
         // connection's copies, so a copy not held at this read cannot appear before the purge. A result that carries
         // `attempt_limit`, the store's own reason, withdraws nothing and is refused below.
         // The read result's type leaves `attempt_limit` out; a provider that breaks the type at run time still withdraws
-        // nothing, so the comparison is made on the reason as the store's wider type.
+        // nothing, so the comparison is made on the reason as the store's wider type. The source's charge record goes in
+        // the withdrawal's transaction too, which runs for it even with no copy held, so the purge's rollback keeps none.
         if ((result.kind === 'gone' || (result.kind === 'skipped' && result.reason !== 'attempt_limit'))
-            && await this.holds(h.lease.scope, work.sourceId) && await this.withdraw(h, work, result) === null)
+            && (await this.holds(h.lease.scope, work.sourceId) || await this.recorded(h.lease.scope, work.sourceId))
+            && await this.withdraw(h, work, result, true) === null)
             return null;
         // `withdrawn` once the copy of a read the weighing below finds expired has been withdrawn (`WithdrawFirst`).
         const apply = (withdrawn) => this.withHandle(h, async (tx, row) => {
@@ -814,34 +942,44 @@ export class SqlAccountSourceSyncStore {
             let kept = false;
             let charge = 0;
             let shortened = null;
+            // With the host's indexing period named and the charges table in place: the period's end (`period`), the
+            // source's record in it (`record`), and the deferral an indexing skip that keeps no copy records (`deferral`).
+            let period = null;
+            let record = null;
+            let deferral = null;
             const excluded = await tx.get(`SELECT 1 FROM ${this.t.exclusions}
         WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(s), work.sourceId]);
             if (result.kind === 'document' && excluded)
                 disposition = { kind: 'skipped', reason: 'excluded' };
             else if (result.kind === 'document') {
                 const read = this.normalized(h, work, result.document, row.policy);
-                const allowance = await this.options.bindings.allowance(tx, s);
-                if (!Object.values(allowance).every(validNumber))
-                    throw new Error('invalid_source_allowance');
+                const allowance = checkedAllowance(await this.options.bindings.allowance(tx, s));
                 const maximumDocuments = allowance.maximumConnectionDocuments;
                 const maximumCharacters = allowance.maximumConnectionHeldCharacters;
                 const now = this.now();
+                period = periodOf(allowance, now);
+                if (period !== null && !await this.chargesHeld(tx))
+                    period = null;
+                record = period === null ? null : await this.chargeRecord(tx, s, work.sourceId, period);
                 const selected = this.choose([...all.filter((d) => d !== old), read], row.policy, maximumDocuments, maximumCharacters, now);
-                // A copy still within its expiry and window was charged when it was admitted: a read that repeats its text
-                // adds nothing to index, and one that also repeats its version and content hash is that copy. Any other read
-                // adds the messages the copy does not hold unchanged, with the text beside them, or, against a copy whose
-                // messages keep no text hash, what follows the copy's text when the read begins with it. A copy past its
-                // expiry or window is no longer held, whether or not `expire` has removed it yet.
+                // A copy within its expiry and window was charged when it was admitted, and the newest text the store
+                // wrote or refreshed for the source in the host's current period was charged by then: a read that repeats
+                // either text adds nothing to index, and one that also repeats the copy's version and content hash is that
+                // copy. Any other read adds the messages the copy or the period's text does not hold unchanged, with the text
+                // beside them, or, against a copy whose messages keep no text hash, what follows the copy's text when the read
+                // begins with it; the smaller amount is the one charged. A copy past its expiry or window is not held,
+                // whether or not `expire` has removed it yet, while the period's text is weighed until the period ends.
                 const holding = old !== undefined && retainable(old, row.policy, now) ? old : undefined;
-                const repeats = holding !== undefined && sameText(holding, read);
                 const text = result.document.read.content.text;
-                const indexed = repeats ? 0 : holding === undefined ? read.characters
-                    : addedByMessage(holding, read)
-                        ?? (beginsWith(holding, read, text) ? read.characters - holding.characters : read.characters);
+                const weigh = (base) => !base ? read.characters
+                    : sameText(base, read) ? 0
+                        : addedByMessage(base, read) ?? (beginsWith(base, read, text) ? read.characters - base.characters : read.characters);
+                const indexed = Math.min(weigh(holding), weigh(record?.charged));
                 // Each amount is the read's characters less parts of its text that do not overlap, so it is never negative;
                 // a negative or fractional one would mean a malformed copy, and is refused rather than charged.
                 if (!validNumber(indexed))
                     throw new Error('invalid_source_sync_charge');
+                const room = Math.min(row.policy.maxIndexedCharacters, allowance.additionalIndexedCharacters);
                 // A read whose cache lifetime has ended by the time it is weighed is an ordinary expired result, never a
                 // malformed one: a document admitted with seconds of its lifetime left and prepared for longer comes back past
                 // it, and refusing it as `invalid_response` would end the run with `sync_failed`, which clears the schedule.
@@ -858,7 +996,7 @@ export class SqlAccountSourceSyncStore {
                         ? 'document_limit' : 'held_limit';
                     disposition = { kind: 'skipped', reason };
                 }
-                else if (indexed > Math.min(row.policy.maxIndexedCharacters, allowance.additionalIndexedCharacters)) {
+                else if (indexed > room) {
                     disposition = { kind: 'skipped', reason: 'indexing_limit' };
                     // The copy stays where the read begins with its text and the retained limits still admit it as it is, so an
                     // edit of a message it holds, a reply that cuts its oldest message at the cap, or a lowered ceiling,
@@ -874,6 +1012,12 @@ export class SqlAccountSourceSyncStore {
                         const expiresAt = Math.min(holding.dependency.expiresAt, read.dependency.expiresAt, ...messages.map((m) => m.windowExpiresAt));
                         shortened = { ...holding, messages, dependency: { ...holding.dependency, expiresAt } };
                     }
+                    else if (period !== null) {
+                        // With no copy left, nothing a later read of the source decides needs it soon: `page` settles the source
+                        // as this skip with no read until the period turns, the room grows, the generation moves or a reconcile
+                        // interval has passed. A kept copy is read again at each listing, so a change at the source reaches it.
+                        deferral = { generation: h.lease.generation, limit: room, at: now };
+                    }
                 }
                 else {
                     refreshed = holding !== undefined && sameCopy(holding, read)
@@ -922,6 +1066,18 @@ export class SqlAccountSourceSyncStore {
           VALUES($1,$2,$3,$4,$5::jsonb,$6)`, [...params(s), work.sourceId,
                     JSON.stringify(normalized), normalized.dependency.expiresAt]);
             }
+            // The period's record takes the text this application wrote or refreshed, which the index holds for the source
+            // in the period, or the deferral beside the text the period already charged for the source, if it charged one: a
+            // read once the room grows is weighed against that text, so the period never charges it twice. A source the period
+            // never indexed keeps the deferral alone, with no message ids or hashes. A source the provider answers gone or
+            // skipped keeps no record, which the withdrawal before this transaction removed: the provider has said the store
+            // may not hold it. The store's own skips keep it, since the source may fit again in the period; an exclusion
+            // removed it in `exclude`.
+            if (period !== null && normalized !== null)
+                await this.recordCharge(tx, s, work.sourceId, period, normalized, null);
+            else if (period !== null && deferral !== null) {
+                await this.recordCharge(tx, s, work.sourceId, period, record?.charged ?? null, deferral);
+            }
             // A slow binding cannot extend permission or finish after its lease expired.
             if (normalized && normalized.dependency.expiresAt <= this.now())
                 throw new StaleSync();
@@ -942,7 +1098,7 @@ export class SqlAccountSourceSyncStore {
         catch (error) {
             if (!(error instanceof WithdrawFirst))
                 throw error;
-            if (await this.withdraw(h, work, { kind: 'skipped', id: work.sourceId, reason: 'expired' }) === null)
+            if (await this.withdraw(h, work, { kind: 'skipped', id: work.sourceId, reason: 'expired' }, false) === null)
                 return null;
             return apply(true);
         }
@@ -957,13 +1113,15 @@ export class SqlAccountSourceSyncStore {
      * before the one that removes it: the copy's expiry, and its dependency's, move to the store's clock. `readCurrent`
      * then answers it expired, so the fence neither commits nor delivers a result derived from it, and `expire` and
      * `sweepExpired` remove it should the removal fail. The handle, the work item and the result are checked as
-     * `applyResult` checks them, and a copy already past its expiry is left as it is.
+     * `applyResult` checks them, and a copy already past its expiry is left as it is. With `forget`, for a provider's
+     * answer, the source's charge record goes in the same transaction, so its message ids and hashes are gone whatever
+     * the removal after it does; the store's own `expired` skip keeps it.
      *
      * @returns True once the copy is withdrawn or none is held within its expiry; null for a stale handle or work item.
      * @throws {SourceSyncFault} `invalid_response` for a result that does not match its work item or a skip reason the
      *   store does not know, with nothing withdrawn.
      */
-    async withdraw(h, work, result) {
+    async withdraw(h, work, result, forget) {
         return this.withHandle(h, async (tx, row) => {
             const s = h.lease.scope;
             await this.currentWork(tx, s, row.checkpoint, work);
@@ -980,6 +1138,8 @@ export class SqlAccountSourceSyncStore {
                 await tx.run(`UPDATE ${this.t.documents} SET document=$5::jsonb,expires_at=$6
           WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(s), work.sourceId, JSON.stringify(withdrawn), now]);
             }
+            if (forget)
+                await this.forgetCharge(tx, s, work.sourceId);
             return true;
         });
     }
@@ -1250,13 +1410,22 @@ export class SqlAccountSourceSyncStore {
         }
         return states;
     }
-    /** Removes the host's content and this store's provenance in the affected scope; the fence calls it inside an invalidation. */
+    /**
+     * Removes the host's content and this store's provenance in the affected scope; the fence calls it inside an
+     * invalidation. A connection's or an owner's purge removes their charge records too. One source's purge, which its
+     * expiry and every replacement of its copy run, leaves the source's record: the record outlives the copy until its
+     * period ends, and `exclude` and a provider's `gone` or skip remove it themselves.
+     */
     async purgeSources(tx, scope, affected) {
         await this.options.bindings.purge(tx, scope, affected);
         const args = [scope.tenantId, scope.ownerId, affected.kind === 'owner' ? null : affected.connectionId,
             affected.kind === 'source' ? affected.sourceId : null];
         await tx.run(`DELETE FROM ${this.t.documents} WHERE tenant_id=$1 AND owner_id=$2
       AND ($3::text IS NULL OR connection_id=$3) AND ($4::text IS NULL OR source_id=$4)`, args);
+        if ((affected.kind === 'connection' || affected.kind === 'owner') && await this.chargesHeld(tx)) {
+            await tx.run(`DELETE FROM ${this.t.charges} WHERE tenant_id=$1 AND owner_id=$2
+        AND ($3::text IS NULL OR connection_id=$3)`, args.slice(0, 3));
+        }
     }
     /**
      * Applies a host control: completes any earlier barrier, then blocks the owner, moves the connection, records a
@@ -1406,9 +1575,7 @@ export class SqlAccountSourceSyncStore {
                 await remove(barrier.sourceId);
             else if (barrier.action === 'prune') {
                 const docs = await this.documentRows(tx, scope);
-                const allowance = await this.options.bindings.allowance(tx, scope);
-                if (!Object.values(allowance).every(validNumber))
-                    throw new Error('invalid_source_allowance');
+                const allowance = checkedAllowance(await this.options.bindings.allowance(tx, scope));
                 const kept = this.choose(docs, row.policy, allowance.maximumConnectionDocuments, allowance.maximumConnectionHeldCharacters, this.now());
                 for (const doc of docs)
                     if (!kept.includes(doc))
@@ -1442,8 +1609,9 @@ export class SqlAccountSourceSyncStore {
     }
     /**
      * Removes one source and every result derived from it, and keeps a content-free exclusion so later syncs skip it.
-     * The connection moves to a new generation and its open cycle is dropped. Its committed cursor is kept, and the next
-     * cycle is a reconcile: a fresh bounded scan, then history replayed from that cursor.
+     * The source's charge record goes in the same transaction as the exclusion. The connection moves to a new generation
+     * and its open cycle is dropped. Its committed cursor is kept, and the next cycle is a reconcile: a fresh bounded
+     * scan, then history replayed from that cursor.
      *
      * @throws {SourceSyncFault} `invalid_response` for an empty id or one longer than 512 characters.
      */
@@ -1460,6 +1628,7 @@ export class SqlAccountSourceSyncStore {
             this.exclusionsChanged(row);
             await this.clearCycle(tx, scope, row);
             await tx.run(`INSERT INTO ${this.t.exclusions} VALUES($1,$2,$3,$4) ON CONFLICT DO NOTHING`, [...params(scope), sourceId]);
+            await this.forgetCharge(tx, scope, sourceId);
             row.barrier = { token: randomUUID(), action: 'exclude', sourceId };
             await this.rebind(tx, scope, row.state.generation);
             await this.save(tx, scope, row);
@@ -1492,13 +1661,15 @@ export class SqlAccountSourceSyncStore {
      * Removes retained sources whose expiry has passed, with every result derived from them, also while the connection
      * is paused or the owner's fence is inactive, then expires the owner's derived results. An inactive owner, such as a
      * suspended account or one a control's barrier holds, stays inactive; reads refuse an expired source regardless.
-     * Each source goes in a transaction of its own, so a later failure leaves the removals before it in place.
+     * Each source goes in a transaction of its own, so a later failure leaves the removals before it in place. Last, it
+     * removes up to `limit` of the connection's charge records whose indexing period has ended by the store's clock.
      *
-     * @param limit - At most this many sources, an integer from 1 to 1,000.
+     * @param limit - At most this many sources, and this many ended charge records, an integer from 1 to 1,000.
      * @returns How many sources were removed.
-     * @throws {Error} `invalid_sync_batch` for another limit, and the failure itself when a removal or the results'
-     *   expiry fails: once a source was removed, a `SourceSyncExpiryError` with the failure's message, the failure as its
-     *   `cause` and the removals made before it in `partial`, counted in `removed` and named in `sourceIds`.
+     * @throws {Error} `invalid_sync_batch` for another limit, and the failure itself when a removal, the results' expiry
+     *   or the removal of ended charge records fails: once a source was removed, a `SourceSyncExpiryError` with the
+     *   failure's message, the failure as its `cause` and the removals made before it in `partial`, counted in `removed`
+     *   and named in `sourceIds`.
      */
     async expire(scope, limit = 100) {
         if (!Number.isInteger(limit) || limit < 1 || limit > 1000)
@@ -1512,6 +1683,7 @@ export class SqlAccountSourceSyncStore {
                 if (await this.expireSource(scope, row.source_id))
                     removed.push(row.source_id);
             await this.fence.expireResults(scope, limit);
+            await this.forgetEnded(scope, scope.connectionId, limit);
         }
         catch (error) {
             // The removals before the failure have committed, so their count and their sources go to the caller with it.
@@ -1547,11 +1719,14 @@ export class SqlAccountSourceSyncStore {
      * indexing skip that keeps a copy may have shortened. An owner whose removal fails, as when its host purge keeps
      * failing, is counted and left out of the call's later reads, so a call with up to `limit` such owners never holds
      * back another owner's copies. An owner whose derived results then fail to expire is counted too, once; the call
-     * still answers every source it removed, and the other owners' results still expire.
+     * still answers every source it removed, and the other owners' results still expire. Last, it removes up to `limit`
+     * charge records whose indexing period has ended by the store's clock, every owner's or `owner`'s, the earliest end
+     * first; a failure there counts once, after every copy the call could remove has gone.
      *
-     * @param limit - At most this many sources, an integer from 1 to 1,000.
+     * @param limit - At most this many sources, and this many ended charge records, an integer from 1 to 1,000.
      * @param owner - The owner whose sources alone are swept, or null for every owner.
-     * @returns The sources removed, and the owners whose removal or whose derived results' expiry failed.
+     * @returns The sources removed, and the owners whose removal or whose derived results' expiry failed, with one more
+     *   when the removal of ended charge records failed.
      * @throws {Error} `invalid_sync_batch` for another limit.
      */
     async sweepExpired(limit = 100, owner = null) {
@@ -1605,12 +1780,19 @@ export class SqlAccountSourceSyncStore {
                     failed++;
             }
         }
+        // A record of an ended period is weighed against nothing, so it goes once the copies have.
+        try {
+            await this.forgetEnded(owner, null, limit);
+        }
+        catch {
+            failed++;
+        }
         return { removed, failed };
     }
     /**
      * Removes a stopped owner's rows from the store in one transaction: every connection of the owner, with its work
-     * rows, retained-source provenance and exclusions, once each connection is disconnected with no barrier, lease, work
-     * or retained source. `remove` runs first in the same transaction, after the owner fence's row lock and the
+     * rows, retained-source provenance, exclusions and charge records, once each connection is disconnected with no
+     * barrier, lease, work or retained source. `remove` runs first in the same transaction, after the owner fence's row lock and the
      * connections' row locks, for a lifecycle's own rows; `SourceRevocations.removeOwner` passes its own. The core
      * fence's owner row and its deletion records stay: they hold no content, keep the owner's epoch ahead of any old
      * handle, and a restore replays the deletion from them.
@@ -1636,7 +1818,7 @@ export class SqlAccountSourceSyncStore {
                 }
                 if (!await remove(tx))
                     throw new StaleSync();
-                // The work rows, provenance and exclusions go with their connection (ON DELETE CASCADE).
+                // The work rows, provenance, exclusions and charge records go with their connection (ON DELETE CASCADE).
                 await tx.run(`DELETE FROM ${this.t.connections} WHERE tenant_id=$1 AND owner_id=$2`, where);
             });
             return true;