@crewhaus/memory-store 0.2.4 → 0.3.1

package/dist/index.d.ts

@@ -1,26 +1,110 @@
 import { CrewhausError } from "@crewhaus/errors";
 export declare const DEFAULT_ROOT_DIR = ".crewhaus/memories";
+/** Schema version stamped on every new write. Absent = v1 (read lazily). */
+export declare const MEMORY_SCHEMA_VERSION = 2;
+/** Where a memory came from, and which tool runs prove it (toolUseIds). */
+export type MemoryProvenance = {
+    readonly sessionId?: string;
+    /** toolUseIds from the source session's event log — proof the fact is
+     *  grounded in tool results, not just assistant narration. */
+    readonly evidence?: readonly string[];
+};
 export type MemoryEntry = {
     readonly id: string;
     readonly text: string;
     readonly tags: readonly string[];
     readonly createdAt: string;
+    /** 2 on new writes; absent = v1 entry (read untouched). */
+    readonly schemaVersion?: number;
+    /** Epoch ms after which the entry is expired (filtered from recall). */
+    readonly expiresAt?: number;
+    /** Entry id that supersedes this one (folded from tombstones at read). */
+    readonly supersededBy?: string;
+    readonly provenance?: MemoryProvenance;
+};
+/** Lifecycle status of an entry, materialized at read time. */
+export type MemoryEntryStatus = "live" | "superseded" | "expired";
+export type MemoryListItem = {
+    readonly entry: MemoryEntry;
+    readonly status: MemoryEntryStatus;
 };
 export type MemoryRecallResult = {
     readonly entry: MemoryEntry;
+    /** BM25 score (no embedder) or fused RRF score (hybrid recall). */
     readonly score: number;
 };
+/**
+ * Minimal structural interface for the hybrid-recall embedder. The
+ * `@crewhaus/embedder` package's `Embedder` satisfies it (use
+ * `createEmbedder({ model: "mock/deterministic" })` for offline tests).
+ * Declared structurally here so memory-store keeps zero runtime deps.
+ */
+export interface MemoryEmbedder {
+    embed(texts: ReadonlyArray<string>): Promise<number[][]>;
+}
 export interface MemoryStoreOptions {
     readonly rootDir?: string;
     readonly specName: string;
     readonly now?: () => Date;
+    /**
+     * Optional embedder enabling hybrid BM25 + embedding recall (RRF).
+     * Absent → BM25-only ranking, byte-identical to the pre-v2 behavior.
+     */
+    readonly embedder?: MemoryEmbedder;
 }
+export type RememberOptions = {
+    /** Time-to-live in ms; the entry expires at `now + ttlMs`. */
+    readonly ttlMs?: number;
+    readonly provenance?: MemoryProvenance;
+};
+export type ForgetOptions = {
+    /** Recorded on the tombstone line for later audit. */
+    readonly reason?: string;
+};
+export type SweepResult = {
+    /** Entries newly tombstoned as expired by this sweep. */
+    readonly swept: number;
+    /** Live entries remaining after the sweep. */
+    readonly live: number;
+};
+export type CompactResult = {
+    /** Lines kept (live entries + preserved unknown lines). */
+    readonly kept: number;
+    /** Lines dropped (dead entries, tombstones, unparseable lines). */
+    readonly dropped: number;
+};
 export interface MemoryStore {
     /** Append a new memory. Returns the assigned entry. */
-    remember(text: string, tags?: readonly string[]): Promise<MemoryEntry>;
-    /** Top-k matches for the query, ranked by BM25-style score. */
+    remember(text: string, tags?: readonly string[], opts?: RememberOptions): Promise<MemoryEntry>;
+    /** Top-k matches for the query. Superseded/expired entries are filtered.
+     *  BM25-ranked without an embedder; hybrid RRF-ranked with one. */
     recall(query: string, k?: number): Promise<readonly MemoryRecallResult[]>;
-    /** Diagnostic: how many entries are stored. */
+    /**
+     * Explicit forgetting. `idOrQuery` that looks like an entry id
+     * (`mem_<16hex>`) tombstones exactly that entry (no text fallback — a
+     * missing id forgets nothing); anything else is a query and tombstones
+     * every live entry with a positive BM25 match (the same match set
+     * `recall(query, Infinity)` would return). The file stays append-only:
+     * a supersede tombstone line is appended per forgotten entry. Returns
+     * the entries that were forgotten.
+     */
+    forget(idOrQuery: string, opts?: ForgetOptions): Promise<readonly MemoryEntry[]>;
+    /**
+     * TTL sweep: appends an `expired` tombstone for every live entry whose
+     * `expiresAt` has passed. Deterministic and idempotent — re-running at
+     * the same time appends nothing new.
+     */
+    sweep(nowMs?: number): Promise<SweepResult>;
+    /**
+     * Growth-bounding rewrite (TODO #53 F7): drops tombstoned/expired
+     * entries and tombstone lines, preserving live entries (v1 lines
+     * untouched) and parseable-but-unknown lines verbatim. Atomic via
+     * tmp + rename.
+     */
+    compact(): Promise<CompactResult>;
+    /** Every entry in file order with its materialized lifecycle status. */
+    list(): Promise<readonly MemoryListItem[]>;
+    /** Diagnostic: how many LIVE entries are stored. */
     size(): Promise<number>;
     /** Diagnostic: where on disk this store writes. */
     path(): string;
@@ -29,6 +113,15 @@ export declare class MemoryStoreError extends CrewhausError {
     readonly name = "MemoryStoreError";
     constructor(message: string, cause?: unknown);
 }
+/** An append-only tombstone line folding a lifecycle change over an entry. */
+export type MemoryTombstone = {
+    readonly tombstone: "superseded" | "expired";
+    readonly target: string;
+    readonly at: string;
+    readonly schemaVersion: number;
+    readonly supersededBy?: string;
+    readonly reason?: string;
+};
 /**
  * Construct a memory store for a given spec. The store is lazy — the
  * underlying file is created on the first `remember()` call.
@@ -59,6 +152,9 @@ export declare const DEFAULT_AUTO_RECALL_K = 5;
 export type CapturableTurn = {
     readonly input: string;
     readonly output: string;
+    /** toolUseIds whose tool_result succeeded during this turn — the proof
+     *  substrate for provenance.evidence on captured facts. */
+    readonly toolUseIds?: readonly string[];
 };
 /**
  * Decide, from a memory config and a completed-turn count, whether an
@@ -84,8 +180,63 @@ export type SessionEvent = {
  * so the auto-capture codegen and CLI both consume one extractor without
  * importing the CLI's feedback module. Synthetic (runtime-injected) user
  * messages and tool-result echoes are not turns.
+ *
+ * v2: each turn also carries the toolUseIds of `tool_result` events that
+ * succeeded (`isError !== true`) between its user message and the next —
+ * the proof links auto-capture stamps into `provenance.evidence`.
  */
 export declare function turnsFromEvents(events: readonly SessionEvent[]): CapturableTurn[];
+/**
+ * Per-child event-read cap for the sub-agent capture walk. A researcher
+ * fan-out can leave arbitrarily large child session JSONLs behind; the walk
+ * reads AT MOST this many events per child (readers get the cap so they can
+ * stop early; the walker slices defensively regardless). 2000 events is
+ * roughly a few hundred turns — far beyond what a single-turn sub-agent
+ * produces, small enough that a 50-child fan-out stays bounded.
+ */
+export declare const MAX_CAPTURE_EVENTS_PER_CHILD = 2000;
+/**
+ * Read one child session's parsed events. `maxEvents` is the per-child cap
+ * (see {@link MAX_CAPTURE_EVENTS_PER_CHILD}) — implementations SHOULD stop
+ * reading past it (e.g. slice the parsed JSONL lines); the walker slices
+ * defensively either way. Return `undefined` (or throw) for an unreadable /
+ * missing child log — the walker skips that child.
+ */
+export type ChildSessionEventsReader = (childSessionId: string, maxEvents: number) => readonly SessionEvent[] | undefined | Promise<readonly SessionEvent[] | undefined>;
+/** One child session's capturable turns, keyed for provenance/tagging. */
+export type ChildCapturableTurns = {
+    /** The child's own sessionId — stamped into `provenance.sessionId`. */
+    readonly sessionId: string;
+    /** The sub-agent definition name — becomes the `subagent:<name>` tag. */
+    readonly name: string;
+    readonly turns: CapturableTurn[];
+    /** True when the reader supplied more events than the cap (input was cut). */
+    readonly truncated: boolean;
+};
+export type TurnsWithChildren = {
+    /** The parent session's turns — identical to `turnsFromEvents(events)`. */
+    readonly turns: CapturableTurn[];
+    readonly children: ChildCapturableTurns[];
+};
+/**
+ * `turnsFromEvents` extended with the sub-agent walk: follows the parent
+ * log's `sub_agent_start` brackets (payload `{name, childSessionId}` — the
+ * spawner appends one per spawn, before the matching `sub_agent_end`) and
+ * lazily reads each referenced child session's events through the injected
+ * reader, capped per child. Children are deduped by `childSessionId`;
+ * unreadable child logs are skipped. This is THE shared helper both memory
+ * wiring paths (target-cli codegen and the apps/cli interpreter) consume so
+ * researcher sub-agent findings stop being structurally invisible to
+ * auto-capture.
+ */
+export declare function turnsFromEventsWithChildren(events: readonly SessionEvent[], readChildEvents: ChildSessionEventsReader, opts?: {
+    maxEventsPerChild?: number;
+}): Promise<TurnsWithChildren>;
+/** A durable fact plus the toolUseIds that ground it (may be empty). */
+export type DurableFact = {
+    readonly text: string;
+    readonly evidence: readonly string[];
+};
 /**
  * Extract durable, self-contained facts worth remembering from a session's
  * turns. Deterministic (no model call) so it runs offline and in tests: it
@@ -99,9 +250,46 @@ export declare function summarizeDurableFacts(turns: readonly CapturableTurn[],
     maxFacts?: number;
     maxLen?: number;
 }): string[];
+/**
+ * `summarizeDurableFacts` carrying each fact's proof links: the toolUseIds
+ * of the source turn's successful tool results (design §2.4 proof-linked
+ * capture). Same extraction/dedupe rules; the string-returning wrapper above
+ * stays for pre-v2 callers.
+ */
+export declare function summarizeDurableFactsWithEvidence(turns: readonly CapturableTurn[], opts?: {
+    maxFacts?: number;
+    maxLen?: number;
+}): DurableFact[];
+export type CaptureFactsOptions = {
+    /** Stamped into each written entry's `provenance.sessionId`. */
+    readonly sessionId?: string;
+    /** TTL applied to each written entry. */
+    readonly ttlMs?: number;
+};
 /**
  * Idempotently persist facts into a store, skipping any whose text (case- and
  * whitespace-insensitively) already matches an existing entry. Returns the
  * entries actually written. Re-running the same auto-capture never duplicates.
+ *
+ * Facts may be plain strings or `DurableFact`s; the latter carry their proof
+ * toolUseIds into `provenance.evidence`. When `opts.sessionId` is given (the
+ * auto-capture path) it is stamped into `provenance.sessionId`.
+ */
+export declare function captureFacts(store: MemoryStore, facts: ReadonlyArray<string | DurableFact>, tags?: readonly string[], opts?: CaptureFactsOptions): Promise<MemoryEntry[]>;
+export type CaptureChildFactsOptions = {
+    /** TTL applied to each written entry. */
+    readonly ttlMs?: number;
+    /** Per-child fact cap forwarded to `summarizeDurableFactsWithEvidence`. */
+    readonly maxFactsPerChild?: number;
+};
+/**
+ * v0.3.0 §7.1 — persist the durable facts of walked child sessions (from
+ * {@link turnsFromEventsWithChildren}). Each child's facts land with
+ * `provenance.sessionId` set to the CHILD's sessionId (so proof/evidence
+ * resolution walks the right log) and a `subagent:<name>` tag appended to
+ * `baseTags` (so researcher findings are grep-able by sub-agent). Same
+ * idempotent dedupe as {@link captureFacts}. Returns everything written.
  */
-export declare function captureFacts(store: MemoryStore, facts: readonly string[], tags?: readonly string[]): Promise<MemoryEntry[]>;
+export declare function captureChildFacts(store: MemoryStore, children: ReadonlyArray<ChildCapturableTurns>, baseTags?: readonly string[], opts?: CaptureChildFactsOptions): Promise<MemoryEntry[]>;
+export declare function isMemoryEntry(value: unknown): value is MemoryEntry;
+export declare function isMemoryTombstone(value: unknown): value is MemoryTombstone;

package/dist/index.js

@@ -14,16 +14,57 @@
  * Why simple BM25 instead of embeddings: zero deps, deterministic for
  * tests, and the working set is small. When a user grows into a much
  * larger memory bank, swap the search backend behind the `MemoryStore`
- * interface — `recall()` is the only consumer-visible signature.
+ * interface — `recall()` is the only consumer-visible signature. As a
+ * middle step, `createMemoryStore({ embedder })` upgrades recall to a
+ * hybrid BM25 + embedding-similarity ranking (see "Hybrid recall").
  *
  * File path: `<rootDir>/<specName>.jsonl` where rootDir defaults to
  * `.crewhaus/memories/`. One file per spec keeps memories scoped.
+ *
+ * ## Schema v2 (0.3.0 memory release, design §3.4)
+ *
+ * New writes stamp `schemaVersion: 2` and may carry three additive
+ * fields: `expiresAt` (epoch ms — explicit forgetting via TTL),
+ * `supersededBy` (entry id — supersede, never delete), and
+ * `provenance {sessionId?, evidence?: toolUseId[]}` (where a fact came
+ * from and which tool runs prove it). Lines without `schemaVersion`
+ * are v1 entries and are read untouched — old readers skip unknown
+ * fields, new readers accept v1 lines as-is, so mixed files work in
+ * both directions.
+ *
+ * ## Explicit forgetting (append-only tombstones)
+ *
+ * The file stays append-only: `forget()` and `sweep()` never rewrite
+ * lines — they append tombstone lines (`{tombstone: "superseded" |
+ * "expired", target: <id>, at, …}`) that the reader folds over the
+ * entries. `recall()`/`size()` see only live entries. `compact()` is
+ * the growth-bounding primitive (closes TODO #53 F7): it rewrites the
+ * file atomically (tmp + rename) dropping tombstoned/expired entries
+ * and the tombstone lines themselves; parseable lines it does not
+ * recognise (future line kinds) are preserved verbatim.
+ *
+ * ## Hybrid recall (design §3.4)
+ *
+ * With no `embedder` option, recall is byte-identical to the v1
+ * BM25-only ranking (the regression-guarded default). With an
+ * `embedder` (structurally compatible with `@crewhaus/embedder`'s
+ * `Embedder`; use `mock/…` for offline tests), recall becomes a
+ * reciprocal-rank fusion (RRF, k=60) of the BM25 ranking and a
+ * cosine-similarity ranking over embeddings of `text + tags`.
+ * Tool-grounded facts — entries whose `provenance.evidence` is
+ * non-empty — receive a documented rank boost in the fused score: one
+ * extra reciprocal-rank vote at rank 1 (`1/(60+1)`), as if a third
+ * ranker had put every proof-backed fact first. This is the design's
+ * write-path-governance rule: facts proven by tool runs outrank
+ * pure-text claims of equal textual relevance.
  */
-import { existsSync, mkdirSync } from "node:fs";
+import { existsSync, mkdirSync, renameSync, writeFileSync } from "node:fs";
 import { appendFile, readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { CrewhausError } from "@crewhaus/errors";
 export const DEFAULT_ROOT_DIR = ".crewhaus/memories";
+/** Schema version stamped on every new write. Absent = v1 (read lazily). */
+export const MEMORY_SCHEMA_VERSION = 2;
 export class MemoryStoreError extends CrewhausError {
     name = "MemoryStoreError";
     constructor(message, cause) {
@@ -32,6 +73,15 @@ export class MemoryStoreError extends CrewhausError {
 }
 const DEFAULT_K = 5;
 const ID_PREFIX = "mem_";
+const ENTRY_ID_RE = /^mem_[0-9a-f]{16}$/;
+/** Reciprocal-rank-fusion constant (the standard k=60). */
+const RRF_K = 60;
+/**
+ * Documented rank boost for tool-grounded facts in hybrid recall: one
+ * extra reciprocal-rank vote at rank 1. Applied ONLY on the fused
+ * (embedder-present) path so BM25-only ranking stays byte-identical.
+ */
+const PROOF_BOOST = 1 / (RRF_K + 1);
 /**
  * Construct a memory store for a given spec. The store is lazy — the
  * underlying file is created on the first `remember()` call.
@@ -45,13 +95,18 @@ export function createMemoryStore(opts) {
     }
     const rootDir = opts.rootDir ?? DEFAULT_ROOT_DIR;
     const now = opts.now ?? (() => new Date());
+    const embedder = opts.embedder;
     const filePath = join(rootDir, `${opts.specName}.jsonl`);
+    // Embedding cache — entries are immutable once written, so id-keyed
+    // vectors never go stale; the cache only saves re-embedding across
+    // recall() calls within one store instance.
+    const embeddingCache = new Map();
     async function ensureRootDir() {
         if (!existsSync(rootDir)) {
             mkdirSync(rootDir, { recursive: true });
         }
     }
-    async function loadAll() {
+    async function readLines() {
         if (!existsSync(filePath))
             return [];
         let raw;
@@ -61,14 +116,18 @@ export function createMemoryStore(opts) {
         catch {
             return [];
         }
+        return raw.split("\n").filter((l) => l.trim() !== "");
+    }
+    async function loadAll() {
         const entries = [];
-        for (const line of raw.split("\n")) {
-            if (!line.trim())
-                continue;
+        const tombstones = [];
+        for (const line of await readLines()) {
             try {
                 const parsed = JSON.parse(line);
                 if (isMemoryEntry(parsed))
                     entries.push(parsed);
+                else if (isMemoryTombstone(parsed))
+                    tombstones.push(parsed);
             }
             catch {
                 // Malformed line — skip. Append-only writes mean partial-write
@@ -76,7 +135,36 @@ export function createMemoryStore(opts) {
                 // shouldn't block recall on the others.
             }
         }
-        return entries;
+        return { entries, tombstones };
+    }
+    /** Fold tombstones + TTL over the raw entries into status-carrying items. */
+    function materialize(loaded, nowMs) {
+        const superseded = new Map();
+        const expired = new Set();
+        for (const t of loaded.tombstones) {
+            if (t.tombstone === "superseded")
+                superseded.set(t.target, t);
+            else
+                expired.add(t.target);
+        }
+        return loaded.entries.map((entry) => {
+            const sup = superseded.get(entry.id);
+            if (sup !== undefined || entry.supersededBy !== undefined) {
+                const supersededBy = entry.supersededBy ?? sup?.supersededBy;
+                return {
+                    entry: supersededBy !== undefined ? { ...entry, supersededBy } : entry,
+                    status: "superseded",
+                };
+            }
+            if (expired.has(entry.id) || (entry.expiresAt !== undefined && nowMs >= entry.expiresAt)) {
+                return { entry, status: "expired" };
+            }
+            return { entry, status: "live" };
+        });
+    }
+    async function loadLive() {
+        const items = materialize(await loadAll(), now().getTime());
+        return items.filter((i) => i.status === "live").map((i) => i.entry);
     }
     function mkId() {
         // 8-byte random hex; collision probability with < 10k entries per
@@ -88,21 +176,157 @@ export function createMemoryStore(opts) {
         }
         return `${ID_PREFIX}${hex}`;
     }
+    async function appendTombstones(targets, kind, reason) {
+        if (targets.length === 0)
+            return;
+        await ensureRootDir();
+        const at = now().toISOString();
+        const lines = targets
+            .map((e) => {
+            const t = {
+                tombstone: kind,
+                target: e.id,
+                at,
+                schemaVersion: MEMORY_SCHEMA_VERSION,
+                ...(reason !== undefined ? { reason } : {}),
+            };
+            return JSON.stringify(t);
+        })
+            .join("\n");
+        await appendFile(filePath, `${lines}\n`, { mode: 0o600 });
+    }
+    /**
+     * BM25-style scoring over the given entries — the exact pre-v2 math:
+     *   - tf  = term frequency in this document
+     *   - idf = log((N - df + 0.5) / (df + 0.5))
+     *   - score = sum over terms of idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (dl / avgdl)))
+     * Returns only positive-scoring entries, sorted descending (stable).
+     */
+    function bm25Rank(entries, query) {
+        const queryTerms = tokenize(query);
+        if (queryTerms.length === 0 || entries.length === 0)
+            return [];
+        const k1 = 1.5;
+        const b = 0.75;
+        const docs = entries.map((entry) => ({
+            entry,
+            terms: tokenize(`${entry.text} ${entry.tags.join(" ")}`),
+        }));
+        const N = docs.length;
+        const avgdl = docs.reduce((sum, d) => sum + d.terms.length, 0) / Math.max(1, N);
+        // Document frequency for each query term.
+        const df = new Map();
+        for (const t of new Set(queryTerms)) {
+            df.set(t, docs.filter((d) => d.terms.includes(t)).length);
+        }
+        const results = [];
+        for (const d of docs) {
+            let score = 0;
+            for (const t of queryTerms) {
+                const tf = d.terms.filter((x) => x === t).length;
+                if (tf === 0)
+                    continue;
+                const dfi = df.get(t) ?? 0;
+                const idf = Math.log((N - dfi + 0.5) / (dfi + 0.5) + 1);
+                const dl = d.terms.length;
+                const norm = tf * (k1 + 1);
+                const denom = tf + k1 * (1 - b + (b * dl) / Math.max(1, avgdl));
+                score += idf * (norm / denom);
+            }
+            if (score > 0)
+                results.push({ entry: d.entry, score });
+        }
+        results.sort((a, b2) => b2.score - a.score);
+        return results;
+    }
+    /** Embed `text + tags` for the given entries, id-cached across calls. */
+    async function embeddingsFor(entries) {
+        if (embedder === undefined)
+            return new Map();
+        const missing = entries.filter((e) => !embeddingCache.has(e.id));
+        if (missing.length > 0) {
+            const vectors = await embedder.embed(missing.map((e) => `${e.text} ${e.tags.join(" ")}`));
+            missing.forEach((e, i) => {
+                const v = vectors[i];
+                if (v !== undefined)
+                    embeddingCache.set(e.id, v);
+            });
+        }
+        const out = new Map();
+        for (const e of entries) {
+            const v = embeddingCache.get(e.id);
+            if (v !== undefined)
+                out.set(e.id, v);
+        }
+        return out;
+    }
+    /**
+     * Hybrid recall: reciprocal-rank fusion (k=60) of the BM25 ranking and
+     * a cosine-similarity ranking, plus the documented `PROOF_BOOST` for
+     * entries with non-empty `provenance.evidence`. Candidates are the
+     * union of both rankers' positive matches.
+     */
+    async function hybridRank(entries, query, emb) {
+        const bmRanked = bm25Rank(entries, query);
+        const [queryVec] = await emb.embed([query]);
+        const entryVecs = await embeddingsFor(entries);
+        const simRanked = [];
+        if (queryVec !== undefined) {
+            for (const entry of entries) {
+                const v = entryVecs.get(entry.id);
+                if (v === undefined)
+                    continue;
+                const sim = cosineSimilarity(queryVec, v);
+                if (sim > 0)
+                    simRanked.push({ entry, sim });
+            }
+            simRanked.sort((a, b) => b.sim - a.sim);
+        }
+        const fused = new Map();
+        const vote = (entry, rank) => {
+            const prev = fused.get(entry.id);
+            const inc = 1 / (RRF_K + rank);
+            if (prev === undefined)
+                fused.set(entry.id, { entry, score: inc });
+            else
+                prev.score += inc;
+        };
+        bmRanked.forEach((r, i) => vote(r.entry, i + 1));
+        simRanked.forEach((r, i) => vote(r.entry, i + 1));
+        for (const f of fused.values()) {
+            if (f.entry.provenance?.evidence !== undefined && f.entry.provenance.evidence.length > 0) {
+                f.score += PROOF_BOOST;
+            }
+        }
+        return [...fused.values()]
+            .sort((a, b) => b.score - a.score)
+            .map((f) => ({ entry: f.entry, score: f.score }));
+    }
     return {
-        async remember(text, tags = []) {
+        async remember(text, tags = [], rememberOpts = {}) {
             if (typeof text !== "string" || text.length === 0) {
                 throw new MemoryStoreError("remember(): text must be a non-empty string");
             }
+            if (rememberOpts.ttlMs !== undefined && rememberOpts.ttlMs <= 0) {
+                throw new MemoryStoreError("remember(): ttlMs must be a positive number of milliseconds");
+            }
+            const createdAt = now();
             const entry = {
                 id: mkId(),
                 text,
                 tags: [...tags],
-                createdAt: now().toISOString(),
+                createdAt: createdAt.toISOString(),
+                schemaVersion: MEMORY_SCHEMA_VERSION,
+                ...(rememberOpts.ttlMs !== undefined
+                    ? { expiresAt: createdAt.getTime() + rememberOpts.ttlMs }
+                    : {}),
+                ...(rememberOpts.provenance !== undefined ? { provenance: rememberOpts.provenance } : {}),
             };
             await ensureRootDir();
-            // TODO(#53 F7): this append-only store grows unbounded — add a size/age
-            // cap or prune-on-write (e.g. keep the newest N or evict by recall score)
-            // so long-lived harnesses don't accumulate an ever-growing memory file.
+            // #53 F7 (unbounded growth) is addressed by the explicit-forgetting
+            // primitives: `sweep()` tombstones expired entries, `forget()`
+            // supersedes stale ones, and `compact()` rewrites the file dropping
+            // dead lines (`crewhaus memory sweep --compact` runs both).
             await appendFile(filePath, `${JSON.stringify(entry)}\n`, { mode: 0o600 });
             return entry;
         },
@@ -110,58 +334,115 @@ export function createMemoryStore(opts) {
             if (typeof query !== "string" || query.length === 0) {
                 throw new MemoryStoreError("recall(): query must be a non-empty string");
             }
-            const all = await loadAll();
-            if (all.length === 0)
+            const live = await loadLive();
+            if (live.length === 0)
                 return [];
-            const queryTerms = tokenize(query);
-            if (queryTerms.length === 0)
+            if (tokenize(query).length === 0)
                 return [];
-            // BM25-style scoring:
-            //   - tf  = term frequency in this document
-            //   - idf = log((N - df + 0.5) / (df + 0.5))
-            //   - score = sum over terms of idf * (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (dl / avgdl)))
-            const k1 = 1.5;
-            const b = 0.75;
-            const docs = all.map((entry) => ({
-                entry,
-                terms: tokenize(`${entry.text} ${entry.tags.join(" ")}`),
-            }));
-            const N = docs.length;
-            const avgdl = docs.reduce((sum, d) => sum + d.terms.length, 0) / Math.max(1, N);
-            // Document frequency for each query term.
-            const df = new Map();
-            for (const t of new Set(queryTerms)) {
-                df.set(t, docs.filter((d) => d.terms.includes(t)).length);
+            const ranked = embedder === undefined ? bm25Rank(live, query) : await hybridRank(live, query, embedder);
+            return ranked.slice(0, Math.max(0, k));
+        },
+        async forget(idOrQuery, forgetOpts = {}) {
+            if (typeof idOrQuery !== "string" || idOrQuery.length === 0) {
+                throw new MemoryStoreError("forget(): idOrQuery must be a non-empty string");
+            }
+            const live = await loadLive();
+            let targets;
+            if (ENTRY_ID_RE.test(idOrQuery)) {
+                // Id-shaped input NEVER falls back to text matching — forgetting
+                // a missing id must forget nothing.
+                targets = live.filter((e) => e.id === idOrQuery);
+            }
+            else {
+                targets = bm25Rank(live, idOrQuery).map((r) => r.entry);
             }
-            const results = [];
-            for (const d of docs) {
-                let score = 0;
-                for (const t of queryTerms) {
-                    const tf = d.terms.filter((x) => x === t).length;
-                    if (tf === 0)
-                        continue;
-                    const dfi = df.get(t) ?? 0;
-                    const idf = Math.log((N - dfi + 0.5) / (dfi + 0.5) + 1);
-                    const dl = d.terms.length;
-                    const norm = tf * (k1 + 1);
-                    const denom = tf + k1 * (1 - b + (b * dl) / Math.max(1, avgdl));
-                    score += idf * (norm / denom);
+            await appendTombstones(targets, "superseded", forgetOpts.reason);
+            return targets;
+        },
+        async sweep(nowMs) {
+            const at = nowMs ?? now().getTime();
+            const loaded = await loadAll();
+            const items = materialize(loaded, at);
+            // Only past-expiry entries that carry NO tombstone yet get one, so
+            // re-running at the same time appends nothing (idempotent).
+            const tombstoned = new Set(loaded.tombstones.map((t) => t.target));
+            const fresh = items
+                .filter((i) => i.status === "expired" &&
+                i.entry.expiresAt !== undefined &&
+                at >= i.entry.expiresAt &&
+                !tombstoned.has(i.entry.id))
+                .map((i) => i.entry);
+            await appendTombstones(fresh, "expired");
+            const live = items.filter((i) => i.status === "live").length;
+            return { swept: fresh.length, live };
+        },
+        async compact() {
+            const lines = await readLines();
+            if (lines.length === 0)
+                return { kept: 0, dropped: 0 };
+            const loaded = await loadAll();
+            const items = materialize(loaded, now().getTime());
+            const liveIds = new Set(items.filter((i) => i.status === "live").map((i) => i.entry.id));
+            const kept = [];
+            let dropped = 0;
+            for (const line of lines) {
+                let parsed;
+                try {
+                    parsed = JSON.parse(line);
+                }
+                catch {
+                    dropped += 1; // unparseable partial-write junk
+                    continue;
                 }
-                if (score > 0)
-                    results.push({ entry: d.entry, score });
+                if (isMemoryEntry(parsed)) {
+                    if (liveIds.has(parsed.id))
+                        kept.push(line);
+                    else
+                        dropped += 1;
+                    continue;
+                }
+                if (isMemoryTombstone(parsed)) {
+                    dropped += 1; // its target is gone — the tombstone has no referent
+                    continue;
+                }
+                // Parseable but unknown line kind (a future writer's) — preserve
+                // verbatim so compact never destroys forward-compatible data.
+                kept.push(line);
             }
-            results.sort((a, b) => b.score - a.score);
-            return results.slice(0, Math.max(0, k));
+            const tmpPath = `${filePath}.tmp`;
+            writeFileSync(tmpPath, kept.length > 0 ? `${kept.join("\n")}\n` : "", { mode: 0o600 });
+            renameSync(tmpPath, filePath);
+            embeddingCache.clear();
+            return { kept: kept.length, dropped };
+        },
+        async list() {
+            return materialize(await loadAll(), now().getTime());
         },
         async size() {
-            const all = await loadAll();
-            return all.length;
+            const live = await loadLive();
+            return live.length;
         },
         path() {
             return filePath;
         },
     };
 }
+function cosineSimilarity(a, b) {
+    const n = Math.min(a.length, b.length);
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < n; i++) {
+        const x = a[i] ?? 0;
+        const y = b[i] ?? 0;
+        dot += x * y;
+        na += x * x;
+        nb += y * y;
+    }
+    if (na === 0 || nb === 0)
+        return 0;
+    return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
 export const DEFAULT_AUTO_CAPTURE_THRESHOLD = 1;
 export const DEFAULT_AUTO_RECALL_K = 5;
 /**
@@ -188,6 +469,10 @@ export function deriveMemoryDecision(config, completedTurns) {
  * so the auto-capture codegen and CLI both consume one extractor without
  * importing the CLI's feedback module. Synthetic (runtime-injected) user
  * messages and tool-result echoes are not turns.
+ *
+ * v2: each turn also carries the toolUseIds of `tool_result` events that
+ * succeeded (`isError !== true`) between its user message and the next —
+ * the proof links auto-capture stamps into `provenance.evidence`.
  */
 export function turnsFromEvents(events) {
     const turns = [];
@@ -198,6 +483,7 @@ export function turnsFromEvents(events) {
         turns.push({
             input: current.input,
             output: current.texts.length > 0 ? current.texts[current.texts.length - 1] : "",
+            ...(current.toolUseIds.length > 0 ? { toolUseIds: current.toolUseIds } : {}),
         });
     };
     for (const ev of events) {
@@ -205,7 +491,7 @@ export function turnsFromEvents(events) {
             const text = userEventText(ev.payload);
             if (text !== undefined) {
                 flush();
-                current = { input: text, texts: [] };
+                current = { input: text, texts: [], toolUseIds: [] };
             }
         }
         else if (ev.kind === "assistant_message" && current !== undefined) {
@@ -213,6 +499,11 @@ export function turnsFromEvents(events) {
             if (t !== "")
                 current.texts.push(t);
         }
+        else if (ev.kind === "tool_result" && current !== undefined) {
+            const id = toolResultUseId(ev.payload);
+            if (id !== undefined)
+                current.toolUseIds.push(id);
+        }
     }
     flush();
     return turns;
@@ -250,6 +541,73 @@ function assistantEventText(payload) {
         .map((b) => b.text)
         .join("\n");
 }
+/** The toolUseId of a successful `tool_result` event payload, if any. */
+function toolResultUseId(payload) {
+    if (payload === null || typeof payload !== "object")
+        return undefined;
+    const p = payload;
+    if (typeof p.toolUseId !== "string" || p.toolUseId === "")
+        return undefined;
+    // Errored tool runs are not evidence — the proof ladder rejects them.
+    if (p.isError === true)
+        return undefined;
+    return p.toolUseId;
+}
+// -------- v0.3.0 §7.1: capture walks sub-agent brackets into child logs --------
+/**
+ * Per-child event-read cap for the sub-agent capture walk. A researcher
+ * fan-out can leave arbitrarily large child session JSONLs behind; the walk
+ * reads AT MOST this many events per child (readers get the cap so they can
+ * stop early; the walker slices defensively regardless). 2000 events is
+ * roughly a few hundred turns — far beyond what a single-turn sub-agent
+ * produces, small enough that a 50-child fan-out stays bounded.
+ */
+export const MAX_CAPTURE_EVENTS_PER_CHILD = 2000;
+/**
+ * `turnsFromEvents` extended with the sub-agent walk: follows the parent
+ * log's `sub_agent_start` brackets (payload `{name, childSessionId}` — the
+ * spawner appends one per spawn, before the matching `sub_agent_end`) and
+ * lazily reads each referenced child session's events through the injected
+ * reader, capped per child. Children are deduped by `childSessionId`;
+ * unreadable child logs are skipped. This is THE shared helper both memory
+ * wiring paths (target-cli codegen and the apps/cli interpreter) consume so
+ * researcher sub-agent findings stop being structurally invisible to
+ * auto-capture.
+ */
+export async function turnsFromEventsWithChildren(events, readChildEvents, opts = {}) {
+    const cap = Math.max(1, opts.maxEventsPerChild ?? MAX_CAPTURE_EVENTS_PER_CHILD);
+    const children = [];
+    const seen = new Set();
+    for (const ev of events) {
+        if (ev.kind !== "sub_agent_start")
+            continue;
+        const p = ev.payload;
+        const childSessionId = typeof p?.childSessionId === "string" ? p.childSessionId : undefined;
+        if (childSessionId === undefined || childSessionId === "" || seen.has(childSessionId)) {
+            continue;
+        }
+        seen.add(childSessionId);
+        const name = typeof p?.name === "string" && p.name !== "" ? p.name : "unknown";
+        let childEvents;
+        try {
+            childEvents = await readChildEvents(childSessionId, cap);
+        }
+        catch {
+            childEvents = undefined;
+        }
+        if (childEvents === undefined)
+            continue;
+        const truncated = childEvents.length > cap;
+        const capped = truncated ? childEvents.slice(0, cap) : childEvents;
+        children.push({
+            sessionId: childSessionId,
+            name,
+            turns: turnsFromEvents(capped),
+            truncated,
+        });
+    }
+    return { turns: turnsFromEvents(events), children };
+}
 /**
  * Extract durable, self-contained facts worth remembering from a session's
  * turns. Deterministic (no model call) so it runs offline and in tests: it
@@ -260,6 +618,15 @@ function assistantEventText(payload) {
  * injected.
  */
 export function summarizeDurableFacts(turns, opts = {}) {
+    return summarizeDurableFactsWithEvidence(turns, opts).map((f) => f.text);
+}
+/**
+ * `summarizeDurableFacts` carrying each fact's proof links: the toolUseIds
+ * of the source turn's successful tool results (design §2.4 proof-linked
+ * capture). Same extraction/dedupe rules; the string-returning wrapper above
+ * stays for pre-v2 callers.
+ */
+export function summarizeDurableFactsWithEvidence(turns, opts = {}) {
     const maxFacts = opts.maxFacts ?? 8;
     const maxLen = opts.maxLen ?? 240;
     const seen = new Set();
@@ -282,7 +649,7 @@ export function summarizeDurableFacts(turns, opts = {}) {
         if (seen.has(key))
             continue;
         seen.add(key);
-        facts.push(fact);
+        facts.push({ text: fact, evidence: [...(t.toolUseIds ?? [])] });
         if (facts.length >= maxFacts)
             break;
     }
@@ -292,8 +659,12 @@ export function summarizeDurableFacts(turns, opts = {}) {
  * Idempotently persist facts into a store, skipping any whose text (case- and
  * whitespace-insensitively) already matches an existing entry. Returns the
  * entries actually written. Re-running the same auto-capture never duplicates.
+ *
+ * Facts may be plain strings or `DurableFact`s; the latter carry their proof
+ * toolUseIds into `provenance.evidence`. When `opts.sessionId` is given (the
+ * auto-capture path) it is stamped into `provenance.sessionId`.
  */
-export async function captureFacts(store, facts, tags = ["auto-capture"]) {
+export async function captureFacts(store, facts, tags = ["auto-capture"], opts = {}) {
     const written = [];
     if (facts.length === 0)
         return written;
@@ -301,30 +672,101 @@ export async function captureFacts(store, facts, tags = ["auto-capture"]) {
     // re-run is a no-op. recall() needs a query; we normalize existing text by
     // recalling each fact and checking for an exact normalized match.
     const norm = (s) => s.toLowerCase().replace(/\s+/g, " ").trim();
+    const asFact = (f) => typeof f === "string" ? { text: f, evidence: [] } : f;
     const existing = new Set();
     for (const fact of facts) {
-        for (const r of await store.recall(fact, 20))
+        for (const r of await store.recall(asFact(fact).text, 20))
             existing.add(norm(r.entry.text));
     }
     const writtenNorms = new Set();
-    for (const fact of facts) {
-        const n = norm(fact);
+    for (const raw of facts) {
+        const fact = asFact(raw);
+        const n = norm(fact.text);
         if (existing.has(n) || writtenNorms.has(n))
             continue;
         writtenNorms.add(n);
-        written.push(await store.remember(fact, tags));
+        const provenance = opts.sessionId !== undefined || fact.evidence.length > 0
+            ? {
+                ...(opts.sessionId !== undefined ? { sessionId: opts.sessionId } : {}),
+                ...(fact.evidence.length > 0 ? { evidence: fact.evidence } : {}),
+            }
+            : undefined;
+        written.push(await store.remember(fact.text, tags, {
+            ...(provenance !== undefined ? { provenance } : {}),
+            ...(opts.ttlMs !== undefined ? { ttlMs: opts.ttlMs } : {}),
+        }));
     }
     return written;
 }
-function isMemoryEntry(value) {
+/**
+ * v0.3.0 §7.1 — persist the durable facts of walked child sessions (from
+ * {@link turnsFromEventsWithChildren}). Each child's facts land with
+ * `provenance.sessionId` set to the CHILD's sessionId (so proof/evidence
+ * resolution walks the right log) and a `subagent:<name>` tag appended to
+ * `baseTags` (so researcher findings are grep-able by sub-agent). Same
+ * idempotent dedupe as {@link captureFacts}. Returns everything written.
+ */
+export async function captureChildFacts(store, children, baseTags = ["auto-capture"], opts = {}) {
+    const written = [];
+    for (const child of children) {
+        const facts = summarizeDurableFactsWithEvidence(child.turns, opts.maxFactsPerChild !== undefined ? { maxFacts: opts.maxFactsPerChild } : {});
+        if (facts.length === 0)
+            continue;
+        written.push(...(await captureFacts(store, facts, [...baseTags, `subagent:${child.name}`], {
+            sessionId: child.sessionId,
+            ...(opts.ttlMs !== undefined ? { ttlMs: opts.ttlMs } : {}),
+        })));
+    }
+    return written;
+}
+export function isMemoryEntry(value) {
     if (typeof value !== "object" || value === null)
         return false;
     const v = value;
-    return (typeof v["id"] === "string" &&
+    const baseOk = typeof v["id"] === "string" &&
         typeof v["text"] === "string" &&
         Array.isArray(v["tags"]) &&
         v["tags"].every((t) => typeof t === "string") &&
-        typeof v["createdAt"] === "string");
+        typeof v["createdAt"] === "string";
+    if (!baseOk)
+        return false;
+    // Tombstone lines carry `tombstone` + `target` and never the entry base
+    // fields, so they can't reach here — but guard anyway.
+    if (v["tombstone"] !== undefined)
+        return false;
+    // v2 optional fields must be well-typed WHEN PRESENT; a line with a
+    // mangled optional field is treated as malformed (skipped), never
+    // half-read.
+    if (v["schemaVersion"] !== undefined && typeof v["schemaVersion"] !== "number")
+        return false;
+    if (v["expiresAt"] !== undefined && typeof v["expiresAt"] !== "number")
+        return false;
+    if (v["supersededBy"] !== undefined && typeof v["supersededBy"] !== "string")
+        return false;
+    if (v["provenance"] !== undefined && !isProvenance(v["provenance"]))
+        return false;
+    return true;
+}
+function isProvenance(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const v = value;
+    if (v["sessionId"] !== undefined && typeof v["sessionId"] !== "string")
+        return false;
+    if (v["evidence"] !== undefined &&
+        !(Array.isArray(v["evidence"]) &&
+            v["evidence"].every((e) => typeof e === "string"))) {
+        return false;
+    }
+    return true;
+}
+export function isMemoryTombstone(value) {
+    if (typeof value !== "object" || value === null)
+        return false;
+    const v = value;
+    return ((v["tombstone"] === "superseded" || v["tombstone"] === "expired") &&
+        typeof v["target"] === "string" &&
+        typeof v["at"] === "string");
 }
 function tokenize(text) {
     return text

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@crewhaus/memory-store",
-  "version": "0.2.4",
+  "version": "0.3.1",
   "type": "module",
   "description": "M4.2 — persistent cross-session memory store. File-backed JSONL with simple BM25-style text search. Per-spec scoped.",
   "main": "dist/index.js",
@@ -15,7 +15,10 @@
     "test": "bun test src"
   },
   "dependencies": {
-    "@crewhaus/errors": "0.2.4"
+    "@crewhaus/errors": "0.3.1"
+  },
+  "devDependencies": {
+    "@crewhaus/embedder": "0.3.1"
   },
   "license": "Apache-2.0",
   "author": {