@framers/agentos-ext-account-sources 0.7.2 → 0.8.0

package/dist/sync/SqlAccountSourceSyncStore.js

@@ -8,6 +8,19 @@ import { createHash, randomUUID } from 'node:crypto';
 import { SqlSourceDerivationStore, createSourceDerivation, } from '@framers/agentos/cognition/library';
 import { createSourceSyncState, transitionSourceSync, sourceSyncDue, openSourceSyncCycle, advanceSourceSyncPage, committableSourceCursor, assertSourceSyncPolicy, SourceSyncFault, } from './state.js';
 import { sourceSyncTables } from './schema.js';
+/**
+ * Thrown by `expire` when it fails after removing at least one source. Each removal committed in a transaction of its
+ * own, so `partial` counts them in `removed`, with the failure itself in `failed`; `cause` holds the failure, whose
+ * message this error carries.
+ */
+export class SourceSyncExpiryError extends Error {
+    partial;
+    constructor(removed, cause) {
+        super(cause instanceof Error ? cause.message : String(cause), { cause });
+        this.name = 'SourceSyncExpiryError';
+        this.partial = { removed, failed: 1 };
+    }
+}
 class StaleSync extends Error {
     constructor() { super('stale_source_sync'); }
 }
@@ -17,12 +30,37 @@ const validId = (s) => typeof s === 'string' && s.length > 0 && s.length <= 512;
 const skipReasons = ["not_permitted", "no_store", "revalidation_required", "invalid_cache_header", "invalid_bounds", "expired", "excluded", "out_of_window", "excluded_label", "no_eligible_content", "response_too_large", "structure_limit", "unsupported_charset", "invalid_encoding", "document_limit", "held_limit", "indexing_limit"];
 const newest = (a, b) => b.activityAt - a.activityAt || (a.dependency.sourceId < b.dependency.sourceId ? -1
     : a.dependency.sourceId > b.dependency.sourceId ? 1 : 0);
+/** Whether a retained copy may still be held: before its expiry, with every message inside the policy's window. */
+const retainable = (doc, policy, now) => doc.dependency.expiresAt > now && !doc.messages.some((m) => m.receivedAt + policy.windowMs <= now);
+/** Whether two spans place the same message at the same offsets of the text, with the same receive time and cut. */
+const sameSpan = (a, b) => a.messageId === b.messageId && a.start === b.start && a.end === b.end
+    && a.receivedAt === b.receivedAt && a.truncated === b.truncated;
+/** SHA-256 of a sanitized text in lower-case hex. */
+const textHashOf = (text) => createHash('sha256').update(text).digest('hex');
+/**
+ * Whether a read repeats the retained copy's text: the same text hash, length, activity time and message spans, so it
+ * adds nothing to index. For a copy an earlier release wrote, which has no text hash, the version and content hash
+ * stand in for it.
+ */
+const sameText = (stored, read) => (stored.textHash !== undefined ? stored.textHash === read.textHash
+    : stored.dependency.version === read.dependency.version && stored.contentHash === read.contentHash)
+    && stored.characters === read.characters && stored.activityAt === read.activityAt
+    && stored.messages.length === read.messages.length
+    && stored.messages.every((message, index) => sameSpan(message, read.messages[index]));
+/**
+ * Whether a read holds exactly the retained copy: its text (`sameText`) at the same version and content hash. Only the
+ * expiry and the message windows, which each read decides afresh, may differ.
+ */
+const sameCopy = (stored, read) => sameText(stored, read) && stored.dependency.version === read.dependency.version
+    && stored.contentHash === read.contentHash;
 /**
  * Durable synchronization state for account sources on PostgreSQL. Every change to an owner's content runs under
  * that owner's fence (`fence`), and a change made with a handle checks its lease, generation and host liveness in
- * the same transaction. Applying a source purges its old copy and every derived result that depends on it before the
- * replacement is written. Provider reads, embedding and model calls stay outside the store. The constructor runs no
- * DDL: the host's migration runs `sourceSyncSchema` and core's `sourceDerivationSchema` first.
+ * the same transaction. Applying a changed source purges its old copy and every derived result that depends on it
+ * before the replacement is written. A read that repeats the text of a copy the store holds is never charged again,
+ * and with the host's `refresh` binding a read of that very copy keeps it in place. Provider reads, embedding and model
+ * calls stay outside the store. The constructor runs no DDL: the host's migration runs `sourceSyncSchema` and core's
+ * `sourceDerivationSchema` first.
  *
  * @typeParam Prepared - The host's in-memory value for one document, passed through to its `write` binding.
  */
@@ -437,7 +475,7 @@ export class SqlAccountSourceSyncStore {
             generation: h.lease.generation, messageIds: [...ids], version: version, expiresAt: doc.cache.expiresAt };
         createSourceDerivation([dependency]);
         return { dependency, messages: doc.messages.map((m) => ({ ...m })),
-            activityAt: doc.activityAt, characters: text.length, contentHash: doc.contentHash };
+            activityAt: doc.activityAt, characters: text.length, contentHash: doc.contentHash, textHash: textHashOf(text) };
     }
     async documentRows(tx, scope) {
         const rows = await tx.all(`SELECT document FROM ${this.t.documents}
@@ -457,7 +495,7 @@ export class SqlAccountSourceSyncStore {
         const selected = [];
         let held = 0;
         for (const doc of [...rows].sort(newest)) {
-            if (doc.dependency.expiresAt <= now || doc.messages.some((m) => m.receivedAt + policy.windowMs <= now))
+            if (!retainable(doc, policy, now))
                 continue;
             if (selected.length >= Math.min(policy.maxRetainedDocuments, maximumDocuments))
                 break;
@@ -470,14 +508,23 @@ export class SqlAccountSourceSyncStore {
     }
     /**
      * Applies one read result for a work item taken in this cycle, under a live lease and the owner fence. A retry
-     * keeps the work unresolved. Any other result first purges the source's old copy and every result derived from it;
-     * a document then replaces it in the same transaction (host write, indexing charge and provenance) when it is not
-     * excluded and fits the retained and indexing limits, evicting this connection's older sources the limits no
-     * longer admit. A document that is excluded or does not fit is recorded as a skip with its reason.
+     * keeps the work unresolved. A document is weighed first: excluded, outside the retained limits or, unless it
+     * repeats the text of a copy the store still holds, over the indexing allowance, it is recorded as a skip with its
+     * reason. A document that fits replaces the source's old copy in the same transaction (host write, indexing charge
+     * and provenance), after the old copy and every result derived from it are purged, evicting this connection's older
+     * sources the limits no longer admit. A read that repeats the text and message spans of a copy the store still holds,
+     * within its expiry and window, adds nothing to index and is never charged. When it also repeats the copy's version
+     * and content hash, the host's `refresh` binding keeps the copy, with every result derived from it, and gives it the
+     * read's expiry and message windows; otherwise the copy is written again from a fresh preparation. A skip and every
+     * other result purge the old copy, except an `indexing_limit` skip that finds the copy still held, still admitted as
+     * it is by the retained limits and the read beginning with its text, every message of it at the same span: that
+     * copy, admitted and charged under an earlier read, stays with the shorter of its own and the read's expiry.
      *
-     * @param prepared - The host's prepared value for a document result; null for any other result.
-     * @returns The disposition and the handle later calls use, whose ticket reflects the purge; null for a stale
-     *   handle or work item.
+     * @param prepared - The host's prepared value for a document result; null for any other result, and for a document
+     *   weighed before its preparation.
+     * @returns The disposition and the handle later calls use, whose ticket reflects any purge; `pending` with the same
+     *   handle, and nothing recorded, for a document without its prepared value that the store has to write; null for a
+     *   stale handle or work item.
      * @throws {SourceSyncFault} `invalid_response` for a result that does not match its work item or is malformed.
      * @throws {Error} `invalid_source_allowance` when the host's allowance is not made of nonnegative safe integers.
      */
@@ -505,46 +552,92 @@ export class SqlAccountSourceSyncStore {
                 : result.kind === 'skipped' ? { kind: 'skipped', reason: result.reason }
                     : { kind: 'pending' };
             let victims = [];
+            // The old copy stays when the read is that copy and the host refreshes it in place (`refreshed`), or when an
+            // indexing skip keeps it (`kept`, with `shortened` its provenance); a read that repeats its text is written again
+            // without a charge (`charge`).
+            let refreshed = false;
+            let kept = false;
+            let charge = true;
+            let shortened = null;
             const excluded = await tx.get(`SELECT 1 FROM ${this.t.exclusions}
         WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(s), work.sourceId]);
             if (result.kind === 'document' && excluded)
                 disposition = { kind: 'skipped', reason: 'excluded' };
             else if (result.kind === 'document') {
-                normalized = this.normalized(h, work, result.document, row.policy);
+                const read = this.normalized(h, work, result.document, row.policy);
                 const allowance = await this.options.bindings.allowance(tx, s);
                 if (!Object.values(allowance).every(validNumber))
                     throw new Error('invalid_source_allowance');
                 const maximumDocuments = allowance.maximumConnectionDocuments;
                 const maximumCharacters = allowance.maximumConnectionHeldCharacters;
-                const selected = this.choose([...all.filter((d) => d !== old), normalized], row.policy, maximumDocuments, maximumCharacters, this.now());
-                if (!selected.includes(normalized)) {
+                const now = this.now();
+                const selected = this.choose([...all.filter((d) => d !== old), read], row.policy, maximumDocuments, maximumCharacters, now);
+                // A copy still within its expiry and window was charged when it was admitted: a read that repeats its text
+                // adds nothing to index, and one that also repeats its version and content hash is that copy. A copy past its
+                // expiry or window is no longer held, whether or not `expire` has removed it yet.
+                const holding = old !== undefined && retainable(old, row.policy, now) ? old : undefined;
+                const repeats = holding !== undefined && sameText(holding, read);
+                if (!selected.includes(read)) {
                     const reason = selected.length >= Math.min(row.policy.maxRetainedDocuments, maximumDocuments)
                         ? 'document_limit' : 'held_limit';
                     disposition = { kind: 'skipped', reason };
-                    normalized = null;
                 }
-                else if (normalized.characters > Math.min(row.policy.maxIndexedCharacters, allowance.additionalIndexedCharacters)) {
+                else if (!repeats && read.characters > Math.min(row.policy.maxIndexedCharacters, allowance.additionalIndexedCharacters)) {
                     disposition = { kind: 'skipped', reason: 'indexing_limit' };
-                    normalized = null;
+                    // The copy stays where the read begins with its text and the retained limits still admit it as it is, so an
+                    // edit of a message it holds, or a lowered ceiling, removes it.
+                    kept = holding !== undefined && holding.dependency.generation === read.dependency.generation
+                        && holding.textHash !== undefined
+                        && holding.textHash === textHashOf(result.document.read.content.text.slice(0, holding.characters))
+                        && holding.messages.every((message) => read.messages.some((span) => sameSpan(message, span)))
+                        && this.choose(all, row.policy, maximumDocuments, maximumCharacters, now).includes(holding);
+                    if (kept && holding !== undefined) {
+                        // The kept copy takes the shorter of its own and the read's expiry and message windows, never a longer one.
+                        const windows = new Map(read.messages.map((m) => [m.messageId, m.windowExpiresAt]));
+                        const messages = holding.messages.map((m) => ({ ...m,
+                            windowExpiresAt: Math.min(m.windowExpiresAt, windows.get(m.messageId) ?? m.windowExpiresAt) }));
+                        const expiresAt = Math.min(holding.dependency.expiresAt, read.dependency.expiresAt, ...messages.map((m) => m.windowExpiresAt));
+                        shortened = { ...holding, messages, dependency: { ...holding.dependency, expiresAt } };
+                    }
                 }
                 else {
-                    if (prepared === null)
-                        throw new SourceSyncFault('invalid_response');
+                    refreshed = holding !== undefined && sameCopy(holding, read)
+                        && holding.dependency.generation === read.dependency.generation
+                        && typeof this.options.bindings.refresh === 'function';
+                    if (!refreshed && prepared === null) {
+                        const pending = { kind: 'pending' };
+                        return { handle: h, disposition: pending };
+                    }
+                    normalized = read;
+                    charge = !repeats;
                     victims = all.filter((d) => d !== old && !selected.includes(d)).map((d) => d.dependency.sourceId);
-                    disposition = { kind: 'applied', version: normalized.dependency.version };
+                    disposition = { kind: 'applied', version: read.dependency.version };
                 }
             }
             let ticket = h.ticket;
             const localFence = this.fenceOn(tx);
-            for (const sourceId of [...new Set([...victims, work.sourceId])].sort()) {
+            const purged = refreshed || kept ? victims : [...victims, work.sourceId];
+            for (const sourceId of [...new Set(purged)].sort()) {
                 const invalidated = await localFence.invalidate(s, { kind: 'source', connectionId: s.connectionId, sourceId }, async () => { }, { expected: ticket });
                 if (invalidated.kind !== 'applied')
                     throw new StaleSync();
                 ticket = invalidated.ticket;
             }
-            if (normalized && result.kind === 'document' && prepared !== null) {
+            if (shortened) {
+                await tx.run(`UPDATE ${this.t.documents} SET document=$5::jsonb,expires_at=$6
+          WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(s), work.sourceId,
+                    JSON.stringify(shortened), shortened.dependency.expiresAt]);
+            }
+            if (normalized && result.kind === 'document' && refreshed) {
+                await this.options.bindings.refresh?.(tx, s, normalized.dependency, result.document);
+                await tx.run(`UPDATE ${this.t.documents} SET document=$5::jsonb,expires_at=$6
+          WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(s), work.sourceId,
+                    JSON.stringify(normalized), normalized.dependency.expiresAt]);
+            }
+            else if (normalized && result.kind === 'document' && prepared !== null) {
                 await this.options.bindings.write(tx, s, normalized.dependency, result.document, prepared);
-                await this.options.bindings.chargeIndexed(tx, s, normalized.characters);
+                if (charge)
+                    await this.options.bindings.chargeIndexed(tx, s, normalized.characters);
                 await tx.run(`INSERT INTO ${this.t.documents}(tenant_id,owner_id,connection_id,source_id,document,expires_at)
           VALUES($1,$2,$3,$4,$5::jsonb,$6)`, [...params(s), work.sourceId,
                     JSON.stringify(normalized), normalized.dependency.expiresAt]);
@@ -940,12 +1033,15 @@ export class SqlAccountSourceSyncStore {
     }
     /**
      * Removes retained sources whose expiry has passed, with every result derived from them, also while the connection
-     * is paused, then expires the owner's derived results. Sources are removed only while the owner's fence is active;
-     * reads refuse an expired source regardless.
+     * is paused or the owner's fence is inactive, then expires the owner's derived results. An inactive owner, such as a
+     * suspended account or one a control's barrier holds, stays inactive; reads refuse an expired source regardless.
+     * Each source goes in a transaction of its own, so a later failure leaves the removals before it in place.
      *
      * @param limit - At most this many sources, an integer from 1 to 1,000.
      * @returns How many sources were removed.
-     * @throws {Error} `invalid_sync_batch` for another limit.
+     * @throws {Error} `invalid_sync_batch` for another limit, and the failure itself when a removal or the results'
+     *   expiry fails: once a source was removed, a `SourceSyncExpiryError` with the failure's message, the failure as its
+     *   `cause` and the removals made before it in `partial`.
      */
     async expire(scope, limit = 100) {
         if (!Number.isInteger(limit) || limit < 1 || limit > 1000)
@@ -954,29 +1050,145 @@ export class SqlAccountSourceSyncStore {
       WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND expires_at <= $4
       ORDER BY expires_at,source_id LIMIT $5`, [...params(scope), this.now(), limit]);
         let removed = 0;
-        for (const row of rows) {
-            const ticket = await this.fence.capture(scope);
-            if (!ticket)
+        try {
+            for (const row of rows)
+                if (await this.expireSource(scope, row.source_id))
+                    removed++;
+            await this.fence.expireResults(scope, limit);
+        }
+        catch (error) {
+            // The removals before the failure have committed, so their count goes to the caller with it.
+            throw removed > 0 ? new SourceSyncExpiryError(removed, error) : error;
+        }
+        return removed;
+    }
+    /**
+     * Removes one retained source whose expiry has passed, with every result derived from it, under its owner's row
+     * lock, whether or not the owner's fence is active: the fence's invalidation advances the owner's epoch, so no handle
+     * from before it applies anything, and leaves an inactive owner inactive.
+     *
+     * @returns Whether it was removed: false when it is gone or its expiry has not passed, as after a later read moved it.
+     */
+    async expireSource(scope, sourceId) {
+        return this.db.transaction(async (tx) => {
+            const fence = this.fenceOn(tx);
+            // The owner fence's row lock first, as every store transaction takes it; an answer of false changes nothing.
+            await fence.activateOwner(scope, async () => false);
+            const current = await tx.get(`SELECT expires_at FROM ${this.t.documents}
+        WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(scope), sourceId]);
+            if (!current || Number(current.expires_at) > this.now())
+                return false;
+            await fence.invalidate(scope, { kind: 'source', connectionId: scope.connectionId, sourceId }, async () => { });
+            return true;
+        });
+    }
+    /**
+     * Removes retained sources whose expiry has passed from every connection, or from `owner`'s alone, the earliest
+     * expiry first, each with every result derived from it as `expire` does for one connection, however far a
+     * maintenance walk is from them and whether or not their owner's fence is active; then expires the derived results
+     * of each owner it removed a source of. The store's own expiry decides, which a policy change's prune and an
+     * indexing skip that keeps a copy may have shortened. An owner whose removal fails, as when its host purge keeps
+     * failing, is counted and left out of the call's later reads, so a call with up to `limit` such owners never holds
+     * back another owner's copies. An owner whose derived results then fail to expire is counted too, once; the call
+     * still answers every source it removed, and the other owners' results still expire.
+     *
+     * @param limit - At most this many sources, an integer from 1 to 1,000.
+     * @param owner - The owner whose sources alone are swept, or null for every owner.
+     * @returns The sources removed, and the owners whose removal or whose derived results' expiry failed.
+     * @throws {Error} `invalid_sync_batch` for another limit.
+     */
+    async sweepExpired(limit = 100, owner = null) {
+        if (!Number.isInteger(limit) || limit < 1 || limit > 1000)
+            throw new Error('invalid_sync_batch');
+        let removed = 0;
+        let failed = 0;
+        const swept = new Map();
+        // The owners whose removal failed, which each later read leaves out.
+        const passed = new Set();
+        const passedTenants = [];
+        const passedOwners = [];
+        // Each read after the first follows a newly failed owner, and the call ends once more than `limit` owners have
+        // failed, so at most `limit` + 2 reads run.
+        for (;;) {
+            const rows = await this.db.all(`SELECT d.tenant_id,d.owner_id,d.connection_id,d.source_id FROM ${this.t.documents} d
+          WHERE d.expires_at <= $1 AND ($2::text IS NULL OR (d.tenant_id=$2 AND d.owner_id=$3))
+          AND NOT EXISTS(SELECT 1 FROM unnest($5::text[],$6::text[]) AS p(tenant_id,owner_id)
+            WHERE p.tenant_id=d.tenant_id AND p.owner_id=d.owner_id)
+          ORDER BY d.expires_at,d.tenant_id,d.owner_id,d.connection_id,d.source_id LIMIT $4`, [this.now(), owner?.tenantId ?? null, owner?.ownerId ?? null, limit - removed, passedTenants, passedOwners]);
+            let passedNow = false;
+            for (const row of rows) {
+                const ownerKey = JSON.stringify([row.tenant_id, row.owner_id]);
+                if (passed.has(ownerKey))
+                    continue;
+                try {
+                    if (await this.expireSource({ tenantId: row.tenant_id, ownerId: row.owner_id, connectionId: row.connection_id }, row.source_id)) {
+                        removed++;
+                        swept.set(ownerKey, { tenantId: row.tenant_id, ownerId: row.owner_id });
+                    }
+                }
+                catch {
+                    failed++;
+                    passed.add(ownerKey);
+                    passedTenants.push(row.tenant_id);
+                    passedOwners.push(row.owner_id);
+                    passedNow = true;
+                }
+            }
+            if (!passedNow || removed >= limit || passed.size > limit)
                 break;
+        }
+        // An owner whose derived results cannot be expired counts once, as a failed removal does; the copies already
+        // removed stay counted, and every other owner's results still expire.
+        for (const [ownerKey, sweptOwner] of swept) {
             try {
-                const result = await this.fence.apply(ticket, async (tx) => {
-                    const current = await tx.get(`SELECT expires_at FROM ${this.t.documents}
-            WHERE tenant_id=$1 AND owner_id=$2 AND connection_id=$3 AND source_id=$4`, [...params(scope), row.source_id]);
-                    if (!current || Number(current.expires_at) > this.now())
-                        return false;
-                    await this.fenceOn(tx).invalidate(scope, { kind: 'source', connectionId: scope.connectionId, sourceId: row.source_id }, async () => { }, { expected: ticket });
-                    return true;
-                });
-                if (result.kind === 'applied' && result.value)
-                    removed++;
+                await this.fence.expireResults(sweptOwner, limit);
             }
-            catch (error) {
-                if (!(error instanceof StaleSync))
-                    throw error;
+            catch {
+                if (!passed.has(ownerKey))
+                    failed++;
             }
         }
-        await this.fence.expireResults(scope, limit);
-        return removed;
+        return { removed, failed };
+    }
+    /**
+     * Removes a stopped owner's rows from the store in one transaction: every connection of the owner, with its work
+     * rows, retained-source provenance and exclusions, once each connection is disconnected with no barrier, lease, work
+     * or retained source. `remove` runs first in the same transaction, after the owner fence's row lock and the
+     * connections' row locks, for a lifecycle's own rows; `SourceRevocations.removeOwner` passes its own. The core
+     * fence's owner row and its deletion records stay: they hold no content, keep the owner's epoch ahead of any old
+     * handle, and a restore replays the deletion from them.
+     *
+     * @param remove - The lifecycle's removal in the same transaction; answering false rolls the whole call back.
+     * @returns Whether the owner's rows were removed: false, changing nothing, while a connection of the owner has not
+     *   stopped or when `remove` answers false.
+     */
+    async removeOwner(owner, remove = async () => true) {
+        const where = [owner.tenantId, owner.ownerId];
+        try {
+            await this.db.transaction(async (tx) => {
+                // The owner fence's row lock first, as every store transaction takes it; an answer of false changes nothing.
+                await this.fenceOn(tx).activateOwner(owner, async () => false);
+                const rows = await tx.all(`SELECT * FROM ${this.t.connections}
+          WHERE tenant_id=$1 AND owner_id=$2 ORDER BY connection_id FOR UPDATE`, where);
+                if (rows.some((row) => row.barrier !== null || row.lease_token !== null
+                    || row.state.status !== 'disconnected'))
+                    throw new StaleSync();
+                if (await tx.get(`SELECT 1 FROM ${this.t.documents} WHERE tenant_id=$1 AND owner_id=$2 LIMIT 1`, where)
+                    || await tx.get(`SELECT 1 FROM ${this.t.work} WHERE tenant_id=$1 AND owner_id=$2 LIMIT 1`, where)) {
+                    throw new StaleSync();
+                }
+                if (!await remove(tx))
+                    throw new StaleSync();
+                // The work rows, provenance and exclusions go with their connection (ON DELETE CASCADE).
+                await tx.run(`DELETE FROM ${this.t.connections} WHERE tenant_id=$1 AND owner_id=$2`, where);
+            });
+            return true;
+        }
+        catch (error) {
+            if (error instanceof StaleSync)
+                return false;
+            throw error;
+        }
     }
 }
 //# sourceMappingURL=SqlAccountSourceSyncStore.js.map