@fgv/ts-agent-memory 5.1.0-48 → 5.1.0-50

package/dist/packlets/store/fileTreeMemoryStore.js

@@ -5,10 +5,14 @@
 import { Hash, Logging, fail, mapResults, mapSuccess, succeed } from '@fgv/ts-utils';
 import { FileTree } from '@fgv/ts-json-base';
 import { DEFAULT_DEDUP_SCOPE, KnowledgeLwwPolicy, isTemporalIdentityCodec, isTemporalRecord, isVersionCurrent, selectCurrentVersion, selectVersionAsOf } from '../types';
-import { parseMemoryFile, serializeMemoryFile } from '../converters';
+import { parseMemoryFile, serializeMemoryFile, splitFrontmatter } from '../converters';
 import { VectorMaintenance } from './vectorMaintenance';
+import { computeCoverage } from './storeCoverage';
+import { reconcileVectors } from './storeReconcile';
 import { MemoryIndex } from '../index';
 import { defaultMemoryScopeEncoding } from './scopeEncoding';
+import { isWholeVaultScan } from './listSelection';
+import { vectorRecordSource } from './vectorRecordSource';
 /** The on-disk extension for a memory record file. */
 const MEMORY_FILE_EXTENSION = '.md';
 /**
@@ -124,7 +128,7 @@ export class FileTreeMemoryStore {
                 // version whose `invalid_at` is null/absent) from the entity subtree,
                 // read off the derived index. `asOf` resolution is via `list({ asOf })`
                 // and the temporal retrievers.
-                return succeed(this._readVersionedCurrent(addr.scope));
+                return this._readVersionedCurrent(addr.scope);
             }
             return this._readRecord(addr.scope, addr.idStem);
         }));
@@ -140,50 +144,130 @@ export class FileTreeMemoryStore {
         return this._readRecord(scope, id);
     }
     /** {@inheritDoc IMemoryStore.list} */
-    async list(filter) {
-        const matches = this._index
-            .entries()
-            .filter((entry) => {
-            if ((filter === null || filter === void 0 ? void 0 : filter.scope) !== undefined && entry.scope !== filter.scope) {
+    async list(selection) {
+        // Deliberately NOT annotated `: boolean`. An explicit annotation discards the
+        // type predicate, and TypeScript's aliased-condition narrowing is what lets
+        // the ternary below see `selection` as a filter on the false branch. With the
+        // annotation this line compiles and the next one does not.
+        const scan = isWholeVaultScan(selection);
+        const filter = scan ? {} : selection;
+        if (!scan && filter.scope === undefined && filter.kind === undefined && filter.tag === undefined) {
+            // `asOf` alone does not narrow — it collapses versions, it does not exclude
+            // entities — so a selection carrying only `asOf` lands here too.
+            return fail('memory list: a selection must narrow by scope, kind or tag; ' +
+                'pass scanEveryRecord() to read every record in the vault deliberately');
+        }
+        // Select over envelopes, project temporally, and materialize LAST — so a
+        // narrowed list reads only the files it is going to return.
+        const selected = this._index.entries().filter((entry) => {
+            if (filter.scope !== undefined && entry.scope !== filter.scope) {
                 return false;
             }
-            if ((filter === null || filter === void 0 ? void 0 : filter.kind) !== undefined && entry.record.envelope.kind !== filter.kind) {
+            if (filter.kind !== undefined && entry.envelope.kind !== filter.kind) {
                 return false;
             }
-            if ((filter === null || filter === void 0 ? void 0 : filter.tag) !== undefined && !entry.record.envelope.tags.includes(filter.tag)) {
+            if (filter.tag !== undefined && !entry.envelope.tags.includes(filter.tag)) {
                 return false;
             }
             return true;
-        })
-            .map((entry) => entry.record);
-        if ((filter === null || filter === void 0 ? void 0 : filter.asOf) === undefined) {
-            // No temporal projection requested: byte-identical to the pre-temporal
-            // behavior (the flat-path guarantee — every version is returned).
-            return succeed(matches);
-        }
-        return succeed(FileTreeMemoryStore._projectAsOf(matches, filter.asOf));
+        });
+        const projected = selection.asOf === undefined
+            ? // No temporal projection requested: the flat-path guarantee — every
+                // version is returned.
+                selected
+            : FileTreeMemoryStore._projectAsOf(selected, selection.asOf);
+        return this._materialize(projected);
+    }
+    /** {@inheritDoc IMemoryStore.coverage} */
+    coverage() {
+        return Promise.resolve(computeCoverage({
+            entries: this._index.entries(),
+            hasRankProjector: (kind) => this._rankProjectors.has(kind),
+            anyRankProjector: this._rankProjectors.size > 0,
+            embedsKind: (kind) => this.embedsKind(kind),
+            vectorIndex: this._vectors.vectorIndex,
+            fragmentIndex: this._vectors.fragmentIndex
+        }));
+    }
+    /** {@inheritDoc IMemoryStore.listEntries} */
+    async listEntries() {
+        return succeed(this._index.entries());
     }
     /** {@inheritDoc IMemoryStore.listScoped} */
     async listScoped() {
-        // The derived index already carries each record's scope
-        // ({@link IIndexedMemoryRecord.scope}), so the scoped projection is a direct
-        // map — the record's `(scope, id)` is exactly the address the vector index
-        // keys on. No filter/temporal projection: the seam re-embeds the whole vault.
-        return succeed(this._index.entries().map((entry) => ({
-            target: { scope: entry.scope, id: entry.record.envelope.id },
-            record: entry.record
-        })));
+        // The derived index already carries each record's scope, so the scoped
+        // projection keys straight off it — `(scope, id)` is exactly the address the
+        // vector index uses. Bodies ARE materialized here: this seam feeds an
+        // embedder, which is the one consumer that genuinely needs every body. No
+        // filter/temporal projection — it re-embeds the whole vault by contract.
+        //
+        // **Complete or failed — never quietly short.** This call is the sole feed
+        // for `asRecordSource()`, and therefore for `IVectorIndex.rebuild`'s coverage
+        // report. A record dropped here would land in none of `records` / `excluded`
+        // / `indexed` / `declined` / `skipped`, so a caller computing coverage would
+        // undercount *in the direction of looking healthier* — which is precisely the
+        // failure `vectorRecordSource`'s own tally exists to prevent, reintroduced one
+        // layer down. A loud failure is recoverable; a silent undercount is not.
+        //
+        // Two consequences worth stating rather than discovering. Before the index
+        // was projected this method read nothing and could not fail, because the
+        // index held the records themselves; it now costs a full-vault read and is
+        // fallible. And it is NOT write-locked, so a concurrent `put` whose cap-cull
+        // physically evicts a record between the snapshot and the read will fail it.
+        // That window is real and is tracked in `docs/FUTURE.md` — the answer is to
+        // make the loss *countable*, not to swallow it.
+        return mapResults(this._index.entries().map((entry) => {
+            const target = { scope: entry.scope, id: entry.envelope.id };
+            return this._resolveRequired(entry).onSuccess((record) => succeed({ target, record }));
+        }));
+    }
+    /** {@inheritDoc IMemoryRecordResolver.resolveRecord} */
+    resolveRecord(scope, id) {
+        return this._readRecord(scope, id);
+    }
+    /**
+     * Materialize a selected set of entries into records, dropping any that have
+     * vanished since selection.
+     *
+     * @remarks
+     * A miss is not a failure. Selection reads the in-memory index and
+     * materialization reads storage, so a record deleted in between is a legitimate
+     * race and yields a shorter list rather than an error. A read that FAILS is a
+     * real fault and propagates.
+     */
+    _materialize(entries) {
+        return mapResults(entries.map((entry) => this._readRecord(entry.scope, entry.envelope.id))).onSuccess((records) => succeed(records.filter((r) => r !== undefined)));
+    }
+    /**
+     * Materialize one entry, treating "gone" as a fault rather than a miss.
+     *
+     * @remarks
+     * For paths where a vanished record really does mean the index and the vault
+     * disagree, rather than that something legitimately removed it in between.
+     *
+     * Three of the four callers hold the write lock, so nothing can have removed
+     * the record since the entry was read. `get()`'s versioned path does not, and
+     * is safe only because temporal kinds never physically delete a version — they
+     * invalidate in place, and cap-cull does not apply to them. **If eviction is
+     * ever added to the temporal path, that caller must change**, or it
+     * reintroduces the race this method exists to detect.
+     *
+     * `listScoped` also does not hold the lock, and uses this deliberately anyway:
+     * it feeds a coverage report, so a silent drop there is worse than a loud
+     * failure. See its comment, and `docs/FUTURE.md` for the eviction window.
+     *
+     * The drop-tolerant counterpart is {@link FileTreeMemoryStore._materialize},
+     * for readers where a record that vanished between selection and
+     * materialization is a miss rather than a fault.
+     */
+    _resolveRequired(entry) {
+        return this._readRecord(entry.scope, entry.envelope.id).onSuccess((record) => record === undefined
+            ? fail(`memory: index entry '${entry.scope}/${entry.envelope.id}' has no record in the vault`)
+            : succeed(record));
     }
     /** {@inheritDoc IMemoryStore.asRecordSource} */
     asRecordSource() {
-        return {
-            // Filtered to the kinds that participate in the record vector index. This
-            // source exists to drive `IVectorIndex` rebuilds, so a kind excluded from the
-            // index has no business being re-embedded on open — which is where the cost
-            // is worst, since a rebuild embeds the whole vault serially. With no
-            // `embedKinds` declaration every kind passes and this is the identity filter.
-            list: async () => (await this.listScoped()).onSuccess((scoped) => succeed(scoped.filter((s) => this.embedsKind(s.record.envelope.kind))))
-        };
+        return vectorRecordSource(this);
     }
     /**
      * Collapse temporal records to the single version valid at `asOf` per entity;
@@ -192,21 +276,21 @@ export class FileTreeMemoryStore {
      * filtering is deferred — OQ-9). An entity with no version valid at `asOf`
      * contributes nothing.
      */
-    static _projectAsOf(records, asOf) {
+    static _projectAsOf(entries, asOf) {
         const passthrough = [];
         const groups = new Map();
-        for (const record of records) {
-            if (!isTemporalRecord(record)) {
-                passthrough.push(record);
+        for (const entry of entries) {
+            if (!isTemporalRecord(entry)) {
+                passthrough.push(entry);
                 continue;
             }
-            const key = `${record.envelope.kind}\0${record.envelope.entityId}`;
+            const key = `${entry.envelope.kind}\0${entry.envelope.entityId}`;
             const existing = groups.get(key);
             if (existing === undefined) {
-                groups.set(key, [record]);
+                groups.set(key, [entry]);
             }
             else {
-                existing.push(record);
+                existing.push(entry);
             }
         }
         const result = [...passthrough];
@@ -218,6 +302,10 @@ export class FileTreeMemoryStore {
         }
         return result;
     }
+    /** {@inheritDoc IMemoryStore.reconcile} */
+    async reconcile(kind, artifact) {
+        return this._enqueue(() => this._reconcileLocked(kind, artifact));
+    }
     /** {@inheritDoc IMemoryStore.put} */
     async put(record) {
         const result = await this._enqueue(() => this._putLocked(record));
@@ -387,8 +475,11 @@ export class FileTreeMemoryStore {
             // mutableFields) actually applies. Content-dedup must never shadow LWW for
             // the same entity.
             const duplicate = this._findByContentHash(scope, hash, record.envelope.id);
-            if (duplicate !== undefined) {
-                return succeed({ record: duplicate, evicted: [] });
+            if (duplicate.isFailure()) {
+                return fail(duplicate.message);
+            }
+            if (duplicate.value !== undefined) {
+                return succeed({ record: duplicate.value, evicted: [] });
             }
         }
         return this._readRecord(scope, idStem).thenOnSuccess((existing) => {
@@ -409,9 +500,8 @@ export class FileTreeMemoryStore {
             // (a replace, not a grow). The same-id `existing` record is threaded
             // separately into `_buildRecord` for the merge-patch. Knowledge LWW ignores
             // this argument, so its behavior is unchanged by the wider cohort.
-            const cohort = this._admissionCohort(scope, record.envelope.kind, idStem);
-            return policy
-                .admit(record, cohort)
+            return this._admissionCohort(scope, record.envelope.kind, idStem)
+                .onSuccess((cohort) => policy.admit(record, cohort))
                 .thenOnSuccess((decision) => this._admitWrite(record, body, scope, idStem, hash, policy, existing, decision));
         });
     }
@@ -561,7 +651,16 @@ export class FileTreeMemoryStore {
      * written, or fully invalidated / soft-deleted).
      */
     _readVersionedCurrent(scope) {
-        return selectCurrentVersion(this._versionsForEntity(scope));
+        // Select over ENVELOPES, then materialize the one winner — which is what
+        // `IMemoryStore.get`'s docstring promises and what `selectCurrentVersion`
+        // being generic over `IEnvelopeCarrier` exists for. Materializing every
+        // version first (as this did) cost N file reads and N body validations to
+        // return one record, and made that docstring false.
+        const current = selectCurrentVersion(this._index.entries().filter((entry) => entry.scope === scope));
+        if (current === undefined) {
+            return succeed(undefined);
+        }
+        return this._resolveRequired(current);
     }
     /**
      * Every persisted version of the entity whose subtree is `scope`. All version
@@ -569,10 +668,13 @@ export class FileTreeMemoryStore {
      * entityId), so a scope filter over the index isolates one entity's versions.
      */
     _versionsForEntity(scope) {
-        return this._index
+        // Selected on the envelope (scope), materialized after — so a versioned write
+        // reads only that entity's versions, never the vault. Bounded by the entity's
+        // version count.
+        return mapResults(this._index
             .entries()
             .filter((entry) => entry.scope === scope)
-            .map((entry) => entry.record);
+            .map((entry) => this._resolveRequired(entry)));
     }
     /**
      * Versioned write (invalidate-don't-delete). Builds the new version's content
@@ -594,7 +696,11 @@ export class FileTreeMemoryStore {
         // still-current version at snapshot time — normally one, but two-or-more if a
         // prior invalidation partially failed; invalidating all of them lets the write
         // self-heal a stuck state (P2-7).
-        const versions = this._versionsForEntity(scope);
+        const snapshot = this._versionsForEntity(scope);
+        if (snapshot.isFailure()) {
+            return fail(snapshot.message);
+        }
+        const versions = snapshot.value;
         const priorCurrents = versions.filter(isVersionCurrent);
         const current = selectCurrentVersion(versions);
         const policy = this._policyFor(kind);
@@ -720,7 +826,11 @@ export class FileTreeMemoryStore {
      * builds on it), so a hard delete would defeat the purpose.
      */
     async _deleteVersioned(entityId, scope) {
-        const versions = this._versionsForEntity(scope);
+        const snapshot = this._versionsForEntity(scope);
+        if (snapshot.isFailure()) {
+            return fail(snapshot.message);
+        }
+        const versions = snapshot.value;
         const currents = versions.filter(isVersionCurrent);
         const current = selectCurrentVersion(versions);
         if (current === undefined) {
@@ -767,10 +877,12 @@ export class FileTreeMemoryStore {
      * across first-writes and updates.
      */
     _admissionCohort(scope, kind, idStem) {
-        return this._index
+        // Envelope-only selection; the cohort is per-(scope, kind) and bounded by the
+        // cull cap, so materializing it is small by construction.
+        return mapResults(this._index
             .entries()
-            .filter((entry) => entry.scope === scope && entry.record.envelope.kind === kind && entry.record.envelope.id !== idStem)
-            .map((entry) => entry.record);
+            .filter((entry) => entry.scope === scope && entry.envelope.kind === kind && entry.envelope.id !== idStem)
+            .map((entry) => this._resolveRequired(entry)));
     }
     /**
      * Find a record in `scope` whose `contentHash` equals `hash`, if any,
@@ -778,12 +890,13 @@ export class FileTreeMemoryStore {
      * dedup skip the same-id record so it does not shadow the LWW update path.
      */
     _findByContentHash(scope, hash, excludeId) {
+        // The clearest win of the projection: the hash lives on the envelope, so this
+        // scans envelopes and reads exactly ONE file — the match — where it used to
+        // hold every body in the scope to look at one field.
         const match = this._index
             .entries()
-            .find((entry) => entry.scope === scope &&
-            entry.record.envelope.contentHash === hash &&
-            entry.record.envelope.id !== excludeId);
-        return match === null || match === void 0 ? void 0 : match.record;
+            .find((entry) => entry.scope === scope && entry.envelope.contentHash === hash && entry.envelope.id !== excludeId);
+        return match === undefined ? succeed(undefined) : this._resolveRequired(match);
     }
     /**
      * True when `incoming`'s caller-authored mutable metadata (`tags` / `provenance`)
@@ -808,6 +921,128 @@ export class FileTreeMemoryStore {
     _contentHash(kind, body, links) {
         return this._hasher.computeHash({ kind, body, links });
     }
+    /**
+     * The rank branch of {@link FileTreeMemoryStore.reconcile}, under the write lock.
+     *
+     * @remarks
+     * Re-reads each record's file rather than trusting the in-memory index, for
+     * two reasons: the index holds converted bodies on some paths and raw ones on
+     * others, and re-serializing a *converted* body could change the bytes on disk
+     * — a reconcile of an ordering field has no business rewriting content.
+     * `splitFrontmatter` hands back the body text unconverted, so the round trip
+     * carries the authored characters through untouched and only the envelope
+     * moves. The parsed record is additionally put through `_verifyLoaded`, the
+     * same id-vs-filename and scope-derived-entityId check the load paths apply —
+     * without it, reconcile would be the one path that accepts and rewrites a file
+     * the store would refuse to load.
+     *
+     * "Untouched" is not quite "byte-identical", and the exception is line
+     * endings: `splitFrontmatter` strips a trailing `\r` per line and
+     * `joinFrontmatter` writes `\n` delimiters, so a CRLF-authored file comes back
+     * LF-normalized. That is **the store's behavior on every write path, not
+     * something reconcile introduces** — an ordinary `put` normalizes the same way
+     * — so reconcile does not rewrite content that a subsequent write would have
+     * left alone. The property being claimed here is the narrower and load-bearing
+     * one: no body is round-tripped through its registered Converter.
+     *
+     * The projector is fed an `IMemoryRecord<string>` carrying that raw body,
+     * which is exactly the shape {@link FileTreeMemoryStore._stampRank} hands it
+     * on the write path — so a projector cannot see one thing on a write and
+     * another on a reconcile. `_stampRank` itself is reused verbatim, which also
+     * inherits its throw semantics (logged at `warn`, `rank` cleared).
+     */
+    async _reconcileLocked(kind, artifact) {
+        // Envelope-only selection: the walk needs `(scope, id)` and the kind, and
+        // each branch materializes only the records it decides to repair.
+        const targets = this._index
+            .entries()
+            .filter((entry) => entry.envelope.kind === kind);
+        if (artifact === 'rank') {
+            return this._reconcileRankLocked(kind, targets);
+        }
+        return reconcileVectors({
+            kind,
+            artifact,
+            targets,
+            maintenance: this._vectors,
+            embedsKind: (k) => this.embedsKind(k),
+            resolve: (scope, id) => this._readRecord(scope, id),
+            stampRef: (scope, id, ref) => this._rewriteEnvelope(scope, id, (r) => r.envelope.embeddingRef === ref
+                ? undefined
+                : { envelope: Object.assign(Object.assign({}, r.envelope), { embeddingRef: ref }), body: r.body })
+        });
+    }
+    /** The rank branch of {@link FileTreeMemoryStore._reconcileLocked}. */
+    async _reconcileRankLocked(kind, targets) {
+        if (!this._rankProjectors.has(kind)) {
+            return fail(`memory reconcile '${kind}' rank: no rank projector is registered for this kind`);
+        }
+        // No materialization at all: `_rewriteEnvelope` re-reads each record itself.
+        let repaired = 0;
+        for (const target of targets) {
+            const applied = this._rewriteEnvelope(target.scope, target.envelope.id, (r) => {
+                const stamped = this._stampRank(r);
+                return stamped.envelope.rank === r.envelope.rank ? undefined : stamped;
+            });
+            if (applied.isFailure()) {
+                return fail(`memory reconcile '${kind}' rank: ${applied.message}`);
+            }
+            if (applied.value) {
+                repaired++;
+            }
+        }
+        return succeed({ artifact: 'rank', kind, examined: targets.length, repaired, failed: [] });
+    }
+    /**
+     * Re-apply the rank projector to one record on disk. Returns whether `rank`
+     * actually changed — an unchanged rank writes nothing, so a reconcile over an
+     * already-consistent store touches no files.
+     */
+    _rewriteEnvelope(scope, id, mutate) {
+        return this._resolveScopeDir(scope).onSuccess((scopeDir) => {
+            /* c8 ignore next 3 - defensive: the scope dir exists for any indexed record */
+            if (scopeDir === undefined) {
+                return fail(`'${id}': scope '${scope}' not found`);
+            }
+            return scopeDir.getChildren().onSuccess((children) => {
+                const targetName = `${id}${MEMORY_FILE_EXTENSION}`;
+                const file = children.find((c) => c.type === 'file' && c.name === targetName);
+                /* c8 ignore next 3 - defensive: the file exists for any indexed record */
+                if (file === undefined) {
+                    return fail(`'${id}': file not found`);
+                }
+                return file.getRawContents().onSuccess((raw) => 
+                // `parseMemoryFile` validates the body through the registered Converter,
+                // so a corrupt record is refused rather than silently rewritten, and
+                // `_verifyLoaded` re-applies the same id-vs-filename and
+                // scope-derived-entityId checks the two load paths apply. Reconcile is a
+                // read-then-write, and a file can change on disk after the index was
+                // built, so skipping them would make this the one path that accepts —
+                // and rewrites — a record the store would otherwise refuse to load.
+                // `splitFrontmatter` then supplies the body text for the write.
+                parseMemoryFile(raw, this._registry)
+                    .withErrorFormat((msg) => `'${id}': ${msg}`)
+                    .onSuccess((parsedRecord) => this._verifyLoaded(scope, file, parsedRecord))
+                    .onSuccess((parsed) => splitFrontmatter(raw)
+                    .withErrorFormat((msg) => `'${id}': ${msg}`)
+                    .onSuccess((parts) => {
+                    // The mutator says "nothing to change" with `undefined` rather
+                    // than by returning an equal record: persisting unconditionally
+                    // would bump `updated` on every record of the kind, trading a
+                    // wrong value for a wrong timestamp, and a deep comparison here
+                    // would have to know which fields each caller touches.
+                    const stamped = mutate({
+                        envelope: parsed.envelope,
+                        body: parts.body
+                    });
+                    if (stamped === undefined) {
+                        return succeed(false);
+                    }
+                    return this._persist(stamped, scope, id).onSuccess(() => succeed(true));
+                })));
+            });
+        });
+    }
     /**
      * Stamp the store-computed {@link IMemoryEnvelope.rank} onto a fully-built,
      * fully-stamped record by running the kind's registered {@link RankProjector}.
@@ -996,8 +1231,8 @@ export class FileTreeMemoryStore {
     _initialIndex(onRecordError) {
         return this._collectEntries(this._root, [], onRecordError).onSuccess((entries) => this._index.rebuild(entries).onSuccess(() => {
             for (const entry of entries) {
-                if (entry.record.envelope.seq > this._seq) {
-                    this._seq = entry.record.envelope.seq;
+                if (entry.envelope.seq > this._seq) {
+                    this._seq = entry.envelope.seq;
                 }
             }
             return succeed(true);
@@ -1038,11 +1273,17 @@ export class FileTreeMemoryStore {
      * failure passes through untouched so the historical error is byte-identical.
      */
     _loadRecordFile(scope, child, onRecordError) {
-        return child
+        return (child
             .getRawContents()
             .onSuccess((raw) => parseMemoryFile(raw, this._registry))
             .onSuccess((parsedRecord) => this._verifyLoaded(scope, child, parsedRecord))
-            .onSuccess((verified) => succeed([{ scope, record: verified }]))
+            // Parse → validate → PROJECT → discard, per file. The body is read and
+            // fully validated (which is what gives `onRecordError` its meaning), then
+            // dropped here rather than carried into the index. Peak body residency
+            // across the whole open is therefore ONE record, not N — which is the
+            // resident-memory moment this whole surface exists to fix, and it is why
+            // the projection had to reach `rebuild` and not just the read methods.
+            .onSuccess((verified) => succeed([{ scope, envelope: verified.envelope }]))
             .onFailure((message) => {
             if (onRecordError === 'skip') {
                 const path = `${scope}/${child.name}`;
@@ -1051,7 +1292,7 @@ export class FileTreeMemoryStore {
                 this._warnSwallowed(error);
             }
             return fail(message);
-        });
+        }));
     }
 }
 /**