@fgv/ts-agent-memory 5.1.0-46 → 5.1.0-48

package/lib/packlets/store/fileTreeMemoryStore.js

@@ -9,6 +9,7 @@ const ts_utils_1 = require("@fgv/ts-utils");
 const ts_json_base_1 = require("@fgv/ts-json-base");
 const types_1 = require("../types");
 const converters_1 = require("../converters");
+const vectorMaintenance_1 = require("./vectorMaintenance");
 const index_1 = require("../index");
 const scopeEncoding_1 = require("./scopeEncoding");
 /** The on-disk extension for a memory record file. */
@@ -29,11 +30,13 @@ const MEMORY_FILE_EXTENSION = '.md';
  */
 class FileTreeMemoryStore {
     constructor(params) {
+        var _a;
         this._root = params.root;
         this._registry = params.registry;
         this._writePolicies = params.writePolicies;
         this._codecs = params.codecs;
         this._rankProjectors = params.rankProjectors;
+        this._embedKinds = (_a = params.embedKinds) !== null && _a !== void 0 ? _a : undefined;
         this._defaultCodec = params.defaultCodec;
         this._defaultPolicy = params.defaultPolicy;
         this._scopeEncoding = params.scopeEncoding;
@@ -42,10 +45,14 @@ class FileTreeMemoryStore {
         this._hasher = new ts_utils_1.Hash.Crc32Normalizer();
         this._observers = params.observers;
         this._logger = params.logger;
-        this._vectorIndex = params.vectorIndex;
-        this._embed = params.embed;
-        this._fragmentIndex = params.fragmentIndex;
-        this._fragmentEmbedder = params.fragmentEmbedder;
+        this._vectors = new vectorMaintenance_1.VectorMaintenance({
+            vectorIndex: params.vectorIndex,
+            embed: params.embed,
+            fragmentIndex: params.fragmentIndex,
+            fragmentEmbedder: params.fragmentEmbedder,
+            warn: (message) => this._warnSwallowed(message),
+            embedsKind: (kind) => this.embedsKind(kind)
+        });
         this._skippedRecords = [];
         this._seq = 0;
         this._observationSeq = 0;
@@ -78,6 +85,7 @@ class FileTreeMemoryStore {
                 writePolicies: (_a = params.writePolicies) !== null && _a !== void 0 ? _a : new Map(),
                 codecs: (_b = params.codecs) !== null && _b !== void 0 ? _b : new Map(),
                 rankProjectors: (_c = params.rankProjectors) !== null && _c !== void 0 ? _c : new Map(),
+                embedKinds: params.embedKinds,
                 defaultCodec: params.defaultCodec,
                 defaultPolicy,
                 scopeEncoding: (_d = params.scopeEncoding) !== null && _d !== void 0 ? _d : scopeEncoding_1.defaultMemoryScopeEncoding,
@@ -171,7 +179,14 @@ class FileTreeMemoryStore {
     }
     /** {@inheritDoc IMemoryStore.asRecordSource} */
     asRecordSource() {
-        return { list: () => this.listScoped() };
+        return {
+            // Filtered to the kinds that participate in the record vector index. This
+            // source exists to drive `IVectorIndex` rebuilds, so a kind excluded from the
+            // index has no business being re-embedded on open — which is where the cost
+            // is worst, since a rebuild embeds the whole vault serially. With no
+            // `embedKinds` declaration every kind passes and this is the identity filter.
+            list: async () => (await this.listScoped()).onSuccess((scoped) => (0, ts_utils_1.succeed)(scoped.filter((s) => this.embedsKind(s.record.envelope.kind))))
+        };
     }
     /**
      * Collapse temporal records to the single version valid at `asOf` per entity;
@@ -213,7 +228,8 @@ class FileTreeMemoryStore {
             outcome: result.isSuccess() ? 'success' : 'failure',
             id: result.isSuccess() ? result.value.record.envelope.id : record.envelope.id,
             provenance: record.envelope.provenance,
-            error: result.isFailure() ? result.message : undefined
+            error: result.isFailure() ? result.message : undefined,
+            embed: result.isSuccess() ? result.value.embed : undefined
         });
         // Fire one `'delete'` observation per record evicted by cap-cull. The
         // evicted records are always in the same `(scope, kind)` cohort as the
@@ -269,7 +285,8 @@ class FileTreeMemoryStore {
             kind,
             outcome: details.outcome,
             error: details.error,
-            provenance: details.provenance
+            provenance: details.provenance,
+            embed: details.embed
         };
         const awaited = [];
         for (const observer of this._observers) {
@@ -351,9 +368,11 @@ class FileTreeMemoryStore {
      * vector index, the on-disk file, and the derived index never interleave.
      */
     async _writeResolved(record, body, scope, idStem, hash) {
-        var _a;
         const policy = this._policyFor(record.envelope.kind);
-        const dedupScope = (_a = policy.dedupScope) !== null && _a !== void 0 ? _a : types_1.DEFAULT_DEDUP_SCOPE;
+        // Read through the public accessor so the write path and every external
+        // caller (the ingest orchestrator's layer-1) resolve dedup granularity from
+        // ONE place — there is no second declaration site.
+        const dedupScope = this.dedupScopeFor(record.envelope.kind);
         // Content-hash dedup runs BEFORE policy. Its granularity is the kind's
         // `dedupScope`:
         //   - 'content': an identical { kind, body, links } triple ANYWHERE in the
@@ -420,89 +439,27 @@ class FileTreeMemoryStore {
         }
         return this._buildRecord(record, body, existing, policy, hash)
             .onSuccess((built) => (0, ts_utils_1.succeed)(this._stampRank(built)))
-            .thenOnSuccess((built) => this._embedOnWrite(built, scope))
-            .thenOnSuccess((built) => this._embedFragmentsOnWrite(built, scope))
-            .onSuccess((embeddedBuilt) => this._persist(embeddedBuilt, scope, idStem))
-            .thenOnSuccess(async (persisted) => {
+            .thenOnSuccess((built) => this._vectors.embedOnWrite(built, scope))
+            .thenOnSuccess(async (embedded) => (await this._vectors.embedFragmentsOnWrite(embedded.record, scope)).onSuccess((withFragments) => (0, ts_utils_1.succeed)(Object.assign(Object.assign({}, embedded), { record: withFragments }))))
+            .onSuccess((embedded) => this._persist(embedded.record, scope, idStem).onSuccess((persisted) => (0, ts_utils_1.succeed)({ persisted, stale: embedded.stale, embed: embedded.embed })))
+            .thenOnSuccess(async ({ persisted, stale, embed }) => {
             // Everything after the authoritative `_persist` commit is best-effort and
             // never turns a committed write into a `Failure`:
+            //   - a stale vector superseded by a decline is pruned here rather than at
+            //     the point of the decline, because a `_persist` that fails leaves the
+            //     PREVIOUS content on disk — content that vector is still an accurate
+            //     embedding of. Pruning early would delete a correct vector on behalf
+            //     of a write that never landed;
             //   - a cull-oldest eviction that fails is logged (the per-(scope,kind)
             //     cap may be transiently exceeded; the next write's admission restores
             //     it), and only the successfully-evicted ids flow onward;
             //   - vector pruning of the evicted cohort is likewise best-effort.
+            await this._vectors.pruneStaleVector(stale);
             const evicted = this._applyEvictions(decision, scope);
-            await this._removeEvictedVectors(evicted, scope);
-            return (0, ts_utils_1.succeed)({ record: persisted, evicted });
+            await this._vectors.removeEvictedVectors(evicted, scope);
+            return (0, ts_utils_1.succeed)({ record: persisted, evicted, embed });
         });
     }
-    /**
-     * Best-effort embed-on-write. When a vector index AND an embedder are wired,
-     * embeds the built record, `add`s the vector (replace semantics handle a same-id
-     * re-embed — no explicit remove), and stamps the returned `embeddingRef`. A
-     * failure (returned `fail` OR a thrown/rejected hook) is logged and the
-     * unembedded record is returned unchanged — the put still persists, and the
-     * derived index is reconciled by a later `rebuild`. A pass-through no-op when
-     * unwired (byte-identical record).
-     *
-     * Always succeeds (`Result` is the chain's shape, never a vector-induced
-     * failure).
-     */
-    async _embedOnWrite(built, scope) {
-        if (this._vectorIndex === undefined || this._embed === undefined) {
-            return (0, ts_utils_1.succeed)(built);
-        }
-        const vectorIndex = this._vectorIndex;
-        const embed = this._embed;
-        const target = { scope, id: built.envelope.id };
-        const embedded = await this._tryVectorOp(() => embed(built), `embedding '${built.envelope.id}'`);
-        if (embedded.isFailure()) {
-            return (0, ts_utils_1.succeed)(built);
-        }
-        const added = await this._tryVectorOp(() => vectorIndex.add(target, embedded.value), `vector add for '${built.envelope.id}'`);
-        if (added.isFailure()) {
-            return (0, ts_utils_1.succeed)(built);
-        }
-        return (0, ts_utils_1.succeed)({ envelope: Object.assign(Object.assign({}, built.envelope), { embeddingRef: added.value }), body: built.body });
-    }
-    /**
-     * Best-effort fragment-embed-on-write. When a fragment index AND a fragment
-     * embedder are wired, chunks + embeds the built record and replaces its
-     * fragments in the index (`addFragments` is whole-record-replace, so a re-authored
-     * document never leaves stale fragments behind — no explicit remove needed). A
-     * failure (returned `fail` OR a thrown/rejected hook) is logged and the record is
-     * returned unchanged — the put still persists, and the fragment index is a derived
-     * view a later `rebuild` reconciles. Unlike {@link FileTreeMemoryStore._embedOnWrite}
-     * it stamps nothing on the record (fragments have no per-record `embeddingRef`
-     * analog). A pass-through no-op when unwired (byte-identical record).
-     */
-    async _embedFragmentsOnWrite(built, scope) {
-        if (this._fragmentIndex === undefined || this._fragmentEmbedder === undefined) {
-            return (0, ts_utils_1.succeed)(built);
-        }
-        const fragmentIndex = this._fragmentIndex;
-        const fragmentEmbedder = this._fragmentEmbedder;
-        const target = { scope, id: built.envelope.id };
-        const embedded = await this._tryVectorOp(() => fragmentEmbedder(built), `fragment embedding '${built.envelope.id}'`);
-        if (embedded.isFailure()) {
-            return (0, ts_utils_1.succeed)(built);
-        }
-        await this._tryVectorOp(() => fragmentIndex.addFragments(target, embedded.value), `fragment add for '${built.envelope.id}'`);
-        return (0, ts_utils_1.succeed)(built);
-    }
-    /**
-     * Best-effort fragment removal. A no-op unless the full fragment lifecycle is
-     * wired (both an index AND an embedder), so an unwired store does no fragment
-     * work and behaves byte-identically. Failures are logged, never surfaced — a
-     * committed delete/eviction must not fail because a derived fragment index could
-     * not be pruned.
-     */
-    async _removeFragmentsBestEffort(target) {
-        if (this._fragmentIndex === undefined || this._fragmentEmbedder === undefined) {
-            return;
-        }
-        const fragmentIndex = this._fragmentIndex;
-        await this._tryVectorOp(() => fragmentIndex.remove(target), `fragment removal for '${target.id}'`);
-    }
     /**
      * Evict the records named by a `cull-oldest` decision, best-effort. Runs only
      * after the authoritative `_persist`, so a failed eviction is logged (never
@@ -526,49 +483,6 @@ class FileTreeMemoryStore {
         }
         return evicted;
     }
-    /**
-     * Best-effort vector removal for each evicted record (never fails the put).
-     * Every evicted record is in the same `scope` as the incoming write (the
-     * cull-oldest cohort is the incoming record's `(scope, kind)` cohort), so that
-     * scope qualifies each removal target.
-     */
-    async _removeEvictedVectors(evicted, scope) {
-        for (const id of evicted) {
-            await this._removeVectorBestEffort({ scope, id });
-            await this._removeFragmentsBestEffort({ scope, id });
-        }
-    }
-    /**
-     * Run a consumer-supplied vector hook, normalizing a thrown/rejected hook into a
-     * `Failure` and logging any failure at `warn`. Best-effort: the caller proceeds
-     * regardless, since the index is rebuildable.
-     */
-    async _tryVectorOp(op, label) {
-        let result;
-        try {
-            result = await op();
-        }
-        catch (err) {
-            result = (0, ts_utils_1.fail)(`${label} threw: ${String(err)}`);
-        }
-        if (result.isFailure()) {
-            this._warnSwallowed(`memory: ${label} failed (best-effort; derived index left for rebuild): ${result.message}`);
-        }
-        return result;
-    }
-    /**
-     * Best-effort vector removal. A no-op unless the full vector lifecycle is wired
-     * (both an index AND an embedder), so an unwired store does no vector work and
-     * behaves byte-identically. Failures are logged, never surfaced — a committed
-     * delete/eviction must not fail because a derived index could not be pruned.
-     */
-    async _removeVectorBestEffort(target) {
-        if (this._vectorIndex === undefined || this._embed === undefined) {
-            return;
-        }
-        const vectorIndex = this._vectorIndex;
-        await this._tryVectorOp(() => vectorIndex.remove(target), `vector removal for '${target.id}'`);
-    }
     /**
      * Build the record to persist. On a first write the incoming envelope is the
      * base (final content equals the incoming content, so the dedup `hash` is
@@ -638,8 +552,7 @@ class FileTreeMemoryStore {
             return this._deleteFile(scope, idStem)
                 .onSuccess(() => this._index.patch('delete', { scope, record: existing }))
                 .thenOnSuccess(async () => {
-                await this._removeVectorBestEffort({ scope, id: existing.envelope.id });
-                await this._removeFragmentsBestEffort({ scope, id: existing.envelope.id });
+                await this._vectors.removeAll({ scope, id: existing.envelope.id });
                 return (0, ts_utils_1.succeed)(existing.envelope.id);
             });
         });
@@ -677,7 +590,6 @@ class FileTreeMemoryStore {
      * version's `valid_at` lower-bounds its interval.
      */
     async _putVersioned(record, body, codec, scope) {
-        var _a;
         const envelope = record.envelope;
         const entityId = envelope.entityId;
         const kind = envelope.kind;
@@ -689,7 +601,8 @@ class FileTreeMemoryStore {
         const priorCurrents = versions.filter(types_1.isVersionCurrent);
         const current = (0, types_1.selectCurrentVersion)(versions);
         const policy = this._policyFor(kind);
-        const dedupScope = (_a = policy.dedupScope) !== null && _a !== void 0 ? _a : types_1.DEFAULT_DEDUP_SCOPE;
+        // Same single-owner read as the flat path (see `_writeResolved`).
+        const dedupScope = this.dedupScopeFor(kind);
         return this._contentHash(kind, body, envelope.links).thenOnSuccess((hash) => {
             // Entity-scoped dedup: a re-put is a no-op only when the CURRENT content AND
             // its mutable metadata are unchanged (does not spawn a redundant version).
@@ -725,11 +638,15 @@ class FileTreeMemoryStore {
                     .withErrorFormat((msg) => `memory put '${entityId}': ${msg}`)
                     .thenOnSuccess((versionStem) => this._buildVersionedRecord(record, body, current, policy, hash, versionStem, validAt, now, seq)
                     .onSuccess((built) => (0, ts_utils_1.succeed)(this._stampRank(built)))
-                    .thenOnSuccess((built) => this._embedOnWrite(built, scope))
-                    .thenOnSuccess((built) => this._embedFragmentsOnWrite(built, scope))
-                    .onSuccess((embeddedBuilt) => this._persist(embeddedBuilt, scope, versionStem))
-                    .onSuccess((persisted) => this._invalidateCurrents(scope, priorCurrents, validAt, now).onSuccess(() => (0, ts_utils_1.succeed)(persisted)))
-                    .onSuccess((persisted) => (0, ts_utils_1.succeed)({ record: persisted, evicted: [] })));
+                    .thenOnSuccess((built) => this._vectors.embedOnWrite(built, scope))
+                    .thenOnSuccess(async (embedded) => (await this._vectors.embedFragmentsOnWrite(embedded.record, scope)).onSuccess((withFragments) => (0, ts_utils_1.succeed)(Object.assign(Object.assign({}, embedded), { record: withFragments }))))
+                    .onSuccess((embedded) => this._persist(embedded.record, scope, versionStem).onSuccess((persisted) => (0, ts_utils_1.succeed)({ persisted, stale: embedded.stale, embed: embedded.embed })))
+                    .onSuccess(({ persisted, stale, embed }) => this._invalidateCurrents(scope, priorCurrents, validAt, now).onSuccess(() => (0, ts_utils_1.succeed)({ persisted, stale, embed })))
+                    // Post-commit and best-effort, exactly as on the flat path.
+                    .thenOnSuccess(async ({ persisted, stale, embed }) => {
+                    await this._vectors.pruneStaleVector(stale);
+                    return (0, ts_utils_1.succeed)({ record: persisted, evicted: [], embed });
+                }));
             });
         });
     }
@@ -937,6 +854,17 @@ class FileTreeMemoryStore {
         var _a;
         return (_a = this._writePolicies.get(kind)) !== null && _a !== void 0 ? _a : this._defaultPolicy;
     }
+    /** {@inheritDoc IMemoryStore.dedupScopeFor} */
+    dedupScopeFor(kind) {
+        var _a;
+        return (_a = this._policyFor(kind).dedupScope) !== null && _a !== void 0 ? _a : types_1.DEFAULT_DEDUP_SCOPE;
+    }
+    /** {@inheritDoc IMemoryStore.embedsKind} */
+    embedsKind(kind) {
+        // No declaration means every kind participates — the behavior that predates
+        // this param, so an existing consumer is unaffected.
+        return this._embedKinds === undefined || this._embedKinds.has(kind);
+    }
     /**
      * Read and validate the record at `<scope>/<idStem>.md`, returning `undefined`
      * when the scope directory or file is absent. Verifies the on-disk id ↔