@fgv/ts-agent-memory 5.1.0-47 → 5.1.0-49

package/dist/packlets/store/fileTreeMemoryStore.js

@@ -5,7 +5,8 @@
 import { Hash, Logging, fail, mapResults, mapSuccess, succeed } from '@fgv/ts-utils';
 import { FileTree } from '@fgv/ts-json-base';
 import { DEFAULT_DEDUP_SCOPE, KnowledgeLwwPolicy, isTemporalIdentityCodec, isTemporalRecord, isVersionCurrent, selectCurrentVersion, selectVersionAsOf } from '../types';
-import { parseMemoryFile, serializeMemoryFile } from '../converters';
+import { parseMemoryFile, serializeMemoryFile, splitFrontmatter } from '../converters';
+import { VectorMaintenance } from './vectorMaintenance';
 import { MemoryIndex } from '../index';
 import { defaultMemoryScopeEncoding } from './scopeEncoding';
 /** The on-disk extension for a memory record file. */
@@ -26,11 +27,13 @@ const MEMORY_FILE_EXTENSION = '.md';
  */
 export class FileTreeMemoryStore {
     constructor(params) {
+        var _a;
         this._root = params.root;
         this._registry = params.registry;
         this._writePolicies = params.writePolicies;
         this._codecs = params.codecs;
         this._rankProjectors = params.rankProjectors;
+        this._embedKinds = (_a = params.embedKinds) !== null && _a !== void 0 ? _a : undefined;
         this._defaultCodec = params.defaultCodec;
         this._defaultPolicy = params.defaultPolicy;
         this._scopeEncoding = params.scopeEncoding;
@@ -39,10 +42,14 @@ export class FileTreeMemoryStore {
         this._hasher = new Hash.Crc32Normalizer();
         this._observers = params.observers;
         this._logger = params.logger;
-        this._vectorIndex = params.vectorIndex;
-        this._embed = params.embed;
-        this._fragmentIndex = params.fragmentIndex;
-        this._fragmentEmbedder = params.fragmentEmbedder;
+        this._vectors = new VectorMaintenance({
+            vectorIndex: params.vectorIndex,
+            embed: params.embed,
+            fragmentIndex: params.fragmentIndex,
+            fragmentEmbedder: params.fragmentEmbedder,
+            warn: (message) => this._warnSwallowed(message),
+            embedsKind: (kind) => this.embedsKind(kind)
+        });
         this._skippedRecords = [];
         this._seq = 0;
         this._observationSeq = 0;
@@ -75,6 +82,7 @@ export class FileTreeMemoryStore {
                 writePolicies: (_a = params.writePolicies) !== null && _a !== void 0 ? _a : new Map(),
                 codecs: (_b = params.codecs) !== null && _b !== void 0 ? _b : new Map(),
                 rankProjectors: (_c = params.rankProjectors) !== null && _c !== void 0 ? _c : new Map(),
+                embedKinds: params.embedKinds,
                 defaultCodec: params.defaultCodec,
                 defaultPolicy,
                 scopeEncoding: (_d = params.scopeEncoding) !== null && _d !== void 0 ? _d : defaultMemoryScopeEncoding,
@@ -168,7 +176,14 @@ export class FileTreeMemoryStore {
     }
     /** {@inheritDoc IMemoryStore.asRecordSource} */
     asRecordSource() {
-        return { list: () => this.listScoped() };
+        return {
+            // Filtered to the kinds that participate in the record vector index. This
+            // source exists to drive `IVectorIndex` rebuilds, so a kind excluded from the
+            // index has no business being re-embedded on open — which is where the cost
+            // is worst, since a rebuild embeds the whole vault serially. With no
+            // `embedKinds` declaration every kind passes and this is the identity filter.
+            list: async () => (await this.listScoped()).onSuccess((scoped) => succeed(scoped.filter((s) => this.embedsKind(s.record.envelope.kind))))
+        };
     }
     /**
      * Collapse temporal records to the single version valid at `asOf` per entity;
@@ -203,6 +218,10 @@ export class FileTreeMemoryStore {
         }
         return result;
     }
+    /** {@inheritDoc IMemoryStore.reconcileRank} */
+    async reconcileRank(kind) {
+        return this._enqueue(() => this._reconcileRankLocked(kind));
+    }
     /** {@inheritDoc IMemoryStore.put} */
     async put(record) {
         const result = await this._enqueue(() => this._putLocked(record));
@@ -210,7 +229,8 @@ export class FileTreeMemoryStore {
             outcome: result.isSuccess() ? 'success' : 'failure',
             id: result.isSuccess() ? result.value.record.envelope.id : record.envelope.id,
             provenance: record.envelope.provenance,
-            error: result.isFailure() ? result.message : undefined
+            error: result.isFailure() ? result.message : undefined,
+            embed: result.isSuccess() ? result.value.embed : undefined
         });
         // Fire one `'delete'` observation per record evicted by cap-cull. The
         // evicted records are always in the same `(scope, kind)` cohort as the
@@ -266,7 +286,8 @@ export class FileTreeMemoryStore {
             kind,
             outcome: details.outcome,
             error: details.error,
-            provenance: details.provenance
+            provenance: details.provenance,
+            embed: details.embed
         };
         const awaited = [];
         for (const observer of this._observers) {
@@ -419,89 +440,27 @@ export class FileTreeMemoryStore {
         }
         return this._buildRecord(record, body, existing, policy, hash)
             .onSuccess((built) => succeed(this._stampRank(built)))
-            .thenOnSuccess((built) => this._embedOnWrite(built, scope))
-            .thenOnSuccess((built) => this._embedFragmentsOnWrite(built, scope))
-            .onSuccess((embeddedBuilt) => this._persist(embeddedBuilt, scope, idStem))
-            .thenOnSuccess(async (persisted) => {
+            .thenOnSuccess((built) => this._vectors.embedOnWrite(built, scope))
+            .thenOnSuccess(async (embedded) => (await this._vectors.embedFragmentsOnWrite(embedded.record, scope)).onSuccess((withFragments) => succeed(Object.assign(Object.assign({}, embedded), { record: withFragments }))))
+            .onSuccess((embedded) => this._persist(embedded.record, scope, idStem).onSuccess((persisted) => succeed({ persisted, stale: embedded.stale, embed: embedded.embed })))
+            .thenOnSuccess(async ({ persisted, stale, embed }) => {
             // Everything after the authoritative `_persist` commit is best-effort and
             // never turns a committed write into a `Failure`:
+            //   - a stale vector superseded by a decline is pruned here rather than at
+            //     the point of the decline, because a `_persist` that fails leaves the
+            //     PREVIOUS content on disk — content that vector is still an accurate
+            //     embedding of. Pruning early would delete a correct vector on behalf
+            //     of a write that never landed;
             //   - a cull-oldest eviction that fails is logged (the per-(scope,kind)
             //     cap may be transiently exceeded; the next write's admission restores
             //     it), and only the successfully-evicted ids flow onward;
             //   - vector pruning of the evicted cohort is likewise best-effort.
+            await this._vectors.pruneStaleVector(stale);
             const evicted = this._applyEvictions(decision, scope);
-            await this._removeEvictedVectors(evicted, scope);
-            return succeed({ record: persisted, evicted });
+            await this._vectors.removeEvictedVectors(evicted, scope);
+            return succeed({ record: persisted, evicted, embed });
         });
     }
-    /**
-     * Best-effort embed-on-write. When a vector index AND an embedder are wired,
-     * embeds the built record, `add`s the vector (replace semantics handle a same-id
-     * re-embed — no explicit remove), and stamps the returned `embeddingRef`. A
-     * failure (returned `fail` OR a thrown/rejected hook) is logged and the
-     * unembedded record is returned unchanged — the put still persists, and the
-     * derived index is reconciled by a later `rebuild`. A pass-through no-op when
-     * unwired (byte-identical record).
-     *
-     * Always succeeds (`Result` is the chain's shape, never a vector-induced
-     * failure).
-     */
-    async _embedOnWrite(built, scope) {
-        if (this._vectorIndex === undefined || this._embed === undefined) {
-            return succeed(built);
-        }
-        const vectorIndex = this._vectorIndex;
-        const embed = this._embed;
-        const target = { scope, id: built.envelope.id };
-        const embedded = await this._tryVectorOp(() => embed(built), `embedding '${built.envelope.id}'`);
-        if (embedded.isFailure()) {
-            return succeed(built);
-        }
-        const added = await this._tryVectorOp(() => vectorIndex.add(target, embedded.value), `vector add for '${built.envelope.id}'`);
-        if (added.isFailure()) {
-            return succeed(built);
-        }
-        return succeed({ envelope: Object.assign(Object.assign({}, built.envelope), { embeddingRef: added.value }), body: built.body });
-    }
-    /**
-     * Best-effort fragment-embed-on-write. When a fragment index AND a fragment
-     * embedder are wired, chunks + embeds the built record and replaces its
-     * fragments in the index (`addFragments` is whole-record-replace, so a re-authored
-     * document never leaves stale fragments behind — no explicit remove needed). A
-     * failure (returned `fail` OR a thrown/rejected hook) is logged and the record is
-     * returned unchanged — the put still persists, and the fragment index is a derived
-     * view a later `rebuild` reconciles. Unlike {@link FileTreeMemoryStore._embedOnWrite}
-     * it stamps nothing on the record (fragments have no per-record `embeddingRef`
-     * analog). A pass-through no-op when unwired (byte-identical record).
-     */
-    async _embedFragmentsOnWrite(built, scope) {
-        if (this._fragmentIndex === undefined || this._fragmentEmbedder === undefined) {
-            return succeed(built);
-        }
-        const fragmentIndex = this._fragmentIndex;
-        const fragmentEmbedder = this._fragmentEmbedder;
-        const target = { scope, id: built.envelope.id };
-        const embedded = await this._tryVectorOp(() => fragmentEmbedder(built), `fragment embedding '${built.envelope.id}'`);
-        if (embedded.isFailure()) {
-            return succeed(built);
-        }
-        await this._tryVectorOp(() => fragmentIndex.addFragments(target, embedded.value), `fragment add for '${built.envelope.id}'`);
-        return succeed(built);
-    }
-    /**
-     * Best-effort fragment removal. A no-op unless the full fragment lifecycle is
-     * wired (both an index AND an embedder), so an unwired store does no fragment
-     * work and behaves byte-identically. Failures are logged, never surfaced — a
-     * committed delete/eviction must not fail because a derived fragment index could
-     * not be pruned.
-     */
-    async _removeFragmentsBestEffort(target) {
-        if (this._fragmentIndex === undefined || this._fragmentEmbedder === undefined) {
-            return;
-        }
-        const fragmentIndex = this._fragmentIndex;
-        await this._tryVectorOp(() => fragmentIndex.remove(target), `fragment removal for '${target.id}'`);
-    }
     /**
      * Evict the records named by a `cull-oldest` decision, best-effort. Runs only
      * after the authoritative `_persist`, so a failed eviction is logged (never
@@ -525,49 +484,6 @@ export class FileTreeMemoryStore {
         }
         return evicted;
     }
-    /**
-     * Best-effort vector removal for each evicted record (never fails the put).
-     * Every evicted record is in the same `scope` as the incoming write (the
-     * cull-oldest cohort is the incoming record's `(scope, kind)` cohort), so that
-     * scope qualifies each removal target.
-     */
-    async _removeEvictedVectors(evicted, scope) {
-        for (const id of evicted) {
-            await this._removeVectorBestEffort({ scope, id });
-            await this._removeFragmentsBestEffort({ scope, id });
-        }
-    }
-    /**
-     * Run a consumer-supplied vector hook, normalizing a thrown/rejected hook into a
-     * `Failure` and logging any failure at `warn`. Best-effort: the caller proceeds
-     * regardless, since the index is rebuildable.
-     */
-    async _tryVectorOp(op, label) {
-        let result;
-        try {
-            result = await op();
-        }
-        catch (err) {
-            result = fail(`${label} threw: ${String(err)}`);
-        }
-        if (result.isFailure()) {
-            this._warnSwallowed(`memory: ${label} failed (best-effort; derived index left for rebuild): ${result.message}`);
-        }
-        return result;
-    }
-    /**
-     * Best-effort vector removal. A no-op unless the full vector lifecycle is wired
-     * (both an index AND an embedder), so an unwired store does no vector work and
-     * behaves byte-identically. Failures are logged, never surfaced — a committed
-     * delete/eviction must not fail because a derived index could not be pruned.
-     */
-    async _removeVectorBestEffort(target) {
-        if (this._vectorIndex === undefined || this._embed === undefined) {
-            return;
-        }
-        const vectorIndex = this._vectorIndex;
-        await this._tryVectorOp(() => vectorIndex.remove(target), `vector removal for '${target.id}'`);
-    }
     /**
      * Build the record to persist. On a first write the incoming envelope is the
      * base (final content equals the incoming content, so the dedup `hash` is
@@ -637,8 +553,7 @@ export class FileTreeMemoryStore {
             return this._deleteFile(scope, idStem)
                 .onSuccess(() => this._index.patch('delete', { scope, record: existing }))
                 .thenOnSuccess(async () => {
-                await this._removeVectorBestEffort({ scope, id: existing.envelope.id });
-                await this._removeFragmentsBestEffort({ scope, id: existing.envelope.id });
+                await this._vectors.removeAll({ scope, id: existing.envelope.id });
                 return succeed(existing.envelope.id);
             });
         });
@@ -724,11 +639,15 @@ export class FileTreeMemoryStore {
                     .withErrorFormat((msg) => `memory put '${entityId}': ${msg}`)
                     .thenOnSuccess((versionStem) => this._buildVersionedRecord(record, body, current, policy, hash, versionStem, validAt, now, seq)
                     .onSuccess((built) => succeed(this._stampRank(built)))
-                    .thenOnSuccess((built) => this._embedOnWrite(built, scope))
-                    .thenOnSuccess((built) => this._embedFragmentsOnWrite(built, scope))
-                    .onSuccess((embeddedBuilt) => this._persist(embeddedBuilt, scope, versionStem))
-                    .onSuccess((persisted) => this._invalidateCurrents(scope, priorCurrents, validAt, now).onSuccess(() => succeed(persisted)))
-                    .onSuccess((persisted) => succeed({ record: persisted, evicted: [] })));
+                    .thenOnSuccess((built) => this._vectors.embedOnWrite(built, scope))
+                    .thenOnSuccess(async (embedded) => (await this._vectors.embedFragmentsOnWrite(embedded.record, scope)).onSuccess((withFragments) => succeed(Object.assign(Object.assign({}, embedded), { record: withFragments }))))
+                    .onSuccess((embedded) => this._persist(embedded.record, scope, versionStem).onSuccess((persisted) => succeed({ persisted, stale: embedded.stale, embed: embedded.embed })))
+                    .onSuccess(({ persisted, stale, embed }) => this._invalidateCurrents(scope, priorCurrents, validAt, now).onSuccess(() => succeed({ persisted, stale, embed })))
+                    // Post-commit and best-effort, exactly as on the flat path.
+                    .thenOnSuccess(async ({ persisted, stale, embed }) => {
+                    await this._vectors.pruneStaleVector(stale);
+                    return succeed({ record: persisted, evicted: [], embed });
+                }));
             });
         });
     }
@@ -893,6 +812,101 @@ export class FileTreeMemoryStore {
     _contentHash(kind, body, links) {
         return this._hasher.computeHash({ kind, body, links });
     }
+    /**
+     * The locked body of {@link FileTreeMemoryStore.reconcileRank}.
+     *
+     * @remarks
+     * Re-reads each record's file rather than trusting the in-memory index, for
+     * two reasons: the index holds converted bodies on some paths and raw ones on
+     * others, and re-serializing a *converted* body could change the bytes on disk
+     * — a reconcile of an ordering field has no business rewriting content.
+     * `splitFrontmatter` hands back the body text unconverted, so the round trip
+     * carries the authored characters through untouched and only the envelope
+     * moves. The parsed record is additionally put through `_verifyLoaded`, the
+     * same id-vs-filename and scope-derived-entityId check the load paths apply —
+     * without it, reconcile would be the one path that accepts and rewrites a file
+     * the store would refuse to load.
+     *
+     * "Untouched" is not quite "byte-identical", and the exception is line
+     * endings: `splitFrontmatter` strips a trailing `\r` per line and
+     * `joinFrontmatter` writes `\n` delimiters, so a CRLF-authored file comes back
+     * LF-normalized. That is **the store's behavior on every write path, not
+     * something reconcile introduces** — an ordinary `put` normalizes the same way
+     * — so reconcile does not rewrite content that a subsequent write would have
+     * left alone. The property being claimed here is the narrower and load-bearing
+     * one: no body is round-tripped through its registered Converter.
+     *
+     * The projector is fed an `IMemoryRecord<string>` carrying that raw body,
+     * which is exactly the shape {@link FileTreeMemoryStore._stampRank} hands it
+     * on the write path — so a projector cannot see one thing on a write and
+     * another on a reconcile. `_stampRank` itself is reused verbatim, which also
+     * inherits its throw semantics (logged at `warn`, `rank` cleared).
+     */
+    async _reconcileRankLocked(kind) {
+        if (!this._rankProjectors.has(kind)) {
+            return fail(`memory reconcileRank '${kind}': no rank projector is registered for this kind`);
+        }
+        const targets = this._index
+            .entries()
+            .filter((entry) => entry.record.envelope.kind === kind);
+        let restamped = 0;
+        for (const target of targets) {
+            const applied = this._restampOne(target.scope, target.record.envelope.id);
+            if (applied.isFailure()) {
+                return fail(`memory reconcileRank '${kind}': ${applied.message}`);
+            }
+            if (applied.value) {
+                restamped++;
+            }
+        }
+        return succeed(restamped);
+    }
+    /**
+     * Re-apply the rank projector to one record on disk. Returns whether `rank`
+     * actually changed — an unchanged rank writes nothing, so a reconcile over an
+     * already-consistent store touches no files.
+     */
+    _restampOne(scope, id) {
+        return this._resolveScopeDir(scope).onSuccess((scopeDir) => {
+            /* c8 ignore next 3 - defensive: the scope dir exists for any indexed record */
+            if (scopeDir === undefined) {
+                return fail(`'${id}': scope '${scope}' not found`);
+            }
+            return scopeDir.getChildren().onSuccess((children) => {
+                const targetName = `${id}${MEMORY_FILE_EXTENSION}`;
+                const file = children.find((c) => c.type === 'file' && c.name === targetName);
+                /* c8 ignore next 3 - defensive: the file exists for any indexed record */
+                if (file === undefined) {
+                    return fail(`'${id}': file not found`);
+                }
+                return file.getRawContents().onSuccess((raw) => 
+                // `parseMemoryFile` validates the body through the registered Converter,
+                // so a corrupt record is refused rather than silently rewritten, and
+                // `_verifyLoaded` re-applies the same id-vs-filename and
+                // scope-derived-entityId checks the two load paths apply. Reconcile is a
+                // read-then-write, and a file can change on disk after the index was
+                // built, so skipping them would make this the one path that accepts —
+                // and rewrites — a record the store would otherwise refuse to load.
+                // `splitFrontmatter` then supplies the body text for the write.
+                parseMemoryFile(raw, this._registry)
+                    .withErrorFormat((msg) => `'${id}': ${msg}`)
+                    .onSuccess((parsedRecord) => this._verifyLoaded(scope, file, parsedRecord))
+                    .onSuccess((parsed) => splitFrontmatter(raw)
+                    .withErrorFormat((msg) => `'${id}': ${msg}`)
+                    .onSuccess((parts) => {
+                    const before = parsed.envelope.rank;
+                    const stamped = this._stampRank({
+                        envelope: parsed.envelope,
+                        body: parts.body
+                    });
+                    if (stamped.envelope.rank === before) {
+                        return succeed(false);
+                    }
+                    return this._persist(stamped, scope, id).onSuccess(() => succeed(true));
+                })));
+            });
+        });
+    }
     /**
      * Stamp the store-computed {@link IMemoryEnvelope.rank} onto a fully-built,
      * fully-stamped record by running the kind's registered {@link RankProjector}.
@@ -941,6 +955,12 @@ export class FileTreeMemoryStore {
         var _a;
         return (_a = this._policyFor(kind).dedupScope) !== null && _a !== void 0 ? _a : DEFAULT_DEDUP_SCOPE;
     }
+    /** {@inheritDoc IMemoryStore.embedsKind} */
+    embedsKind(kind) {
+        // No declaration means every kind participates — the behavior that predates
+        // this param, so an existing consumer is unaffected.
+        return this._embedKinds === undefined || this._embedKinds.has(kind);
+    }
     /**
      * Read and validate the record at `<scope>/<idStem>.md`, returning `undefined`
      * when the scope directory or file is absent. Verifies the on-disk id ↔