@seanhogg/builderforce-memory 2026.9.34 → 2026.10.1

package/src/memory/SemanticRecall.ts

@@ -0,0 +1,204 @@
+/**
+ * memory/SemanticRecall.ts – The query side of {@link MemoryStore}: semantic
+ * (SSM-embedding cosine, Jaccard fallback) and hybrid (RRF + MMR) recall over
+ * an already-loaded fact list, plus the derived caches that make it cheap:
+ *
+ *   - an LRU content→embedding cache (EVM-8);
+ *   - a persistent dense vector store keyed by `key@seq` and the HNSW index
+ *     built from it, reused until the fact set changes (EVM-1 / EVM-1b).
+ *
+ * Every cache is model-specific and is dropped when the runtime identity
+ * changes, or when the owning store calls {@link SemanticRecall.invalidate}
+ * (clear, cross-context write).
+ */
+
+import { tokenize, jaccardSimilarity } from '../similarity/index.js';
+import { hybridRetrieve, denseSearch, HnswIndex, type RetrievalCandidate, type HybridRetrieveOptions } from '../retrieval/index.js';
+import type { MemoryEntry, RankedRecall, ScoredMemory, SSMRuntimeRef } from './types.js';
+
+export interface SemanticRecallOptions {
+    /** Fact count at/above which dense search uses the HNSW index. */
+    annThreshold: number;
+    /** Max content→embedding vectors held in the LRU cache. */
+    embedCacheMax: number;
+    /** Hits scoring below this are dropped (0 = keep everything). */
+    minRelevance: number;
+}
+
+interface DenseView {
+    byKey: Map<string, MemoryEntry>;
+    items: Array<{ id: string; vector: Float32Array }>;
+    index: HnswIndex | null;
+}
+
+export class SemanticRecall {
+    private readonly _embedCache = new Map<string, Float32Array>();
+    private _embedRuntime: unknown = null;
+    private readonly _denseVectors = new Map<string, Float32Array>();
+    private _denseIndex: HnswIndex | null = null;
+    /** Signature of the fact set the current index was built from. */
+    private _denseSig = '';
+    private _denseRuntime: unknown = null;
+
+    constructor(private readonly _opts: SemanticRecallOptions) {}
+
+    /** Drops every derived cache (the fact set changed underneath us). */
+    invalidate(): void {
+        this._embedCache.clear();
+        this.resetDense();
+    }
+
+    /** Drops the dense vectors + index but keeps content embeddings. */
+    resetDense(): void {
+        this._denseVectors.clear();
+        this._denseIndex = null;
+        this._denseSig = '';
+    }
+
+    /**
+     * Top-K entries of `all` by real 0..1 relevance: SSM-embedding cosine when
+     * `runtime.embed` works, Jaccard word overlap otherwise.
+     */
+    async similar(all: MemoryEntry[], query: string, topK: number, runtime?: SSMRuntimeRef): Promise<ScoredMemory[]> {
+        if (all.length === 0) return [];
+        if (runtime != null && typeof runtime.embed === 'function') {
+            const queryVec = await this._embed(runtime, query);
+            if (queryVec) {
+                // EVM-1b: sync the persistent vector store + index (embeds only the
+                // delta), then query it — no per-call re-embed of every fact.
+                const dense = await this._syncDense(runtime, all);
+                if (dense) {
+                    const hits = dense.index
+                        ? dense.index.search(queryVec, topK)
+                        : denseSearch(queryVec, dense.items, topK, this._opts.annThreshold);
+                    return this._floor(hits
+                        .map((h) => { const entry = dense.byKey.get(h.id); return entry ? { entry, score: h.score } : null; })
+                        .filter((e): e is ScoredMemory => !!e));
+                }
+            }
+            // Any embedding failure falls through to the Jaccard path below.
+        }
+        const queryTokens = new Set(tokenize(query));
+        const scored = all.map((entry) => ({ entry, score: jaccardSimilarity(queryTokens, new Set(tokenize(entry.content))) }));
+        scored.sort((a, b) => b.score - a.score);
+        return this._floor(scored).slice(0, topK);
+    }
+
+    /**
+     * Hybrid recall: dense + BM25 fused by the shared `hybridRetrieve` primitive,
+     * with an honest `method` — `'embedding'` only when the query AND at least
+     * one candidate really embedded.
+     */
+    async ranked(
+        all: MemoryEntry[],
+        query: string,
+        topK: number,
+        runtime?: SSMRuntimeRef,
+        opts?: HybridRetrieveOptions,
+    ): Promise<RankedRecall> {
+        if (all.length === 0) return { hits: [], method: 'lexical' };
+        const canEmbed = runtime != null && typeof runtime.embed === 'function';
+        const queryVec = canEmbed ? (await this._embed(runtime, query)) ?? undefined : undefined;
+        const candidates: RetrievalCandidate[] = [];
+        for (const entry of all) {
+            const vector = queryVec ? (await this._embed(runtime, entry.content)) ?? undefined : undefined;
+            candidates.push({ id: entry.key, text: entry.content, vector });
+        }
+        const dense = !!queryVec && candidates.some((c) => !!c.vector);
+        const hits = hybridRetrieve({ text: query, vector: queryVec }, candidates, { topK, annThreshold: this._opts.annThreshold, ...opts });
+        const byKey = new Map(all.map((e) => [e.key, e]));
+        return {
+            hits: this._floor(hits
+                .map((h) => { const entry = byKey.get(h.id); return entry ? { entry, score: h.score } : null; })
+                .filter((h): h is ScoredMemory => !!h)),
+            method: dense ? 'embedding' : 'lexical',
+        };
+    }
+
+    private _floor(hits: ScoredMemory[]): ScoredMemory[] {
+        const min = this._opts.minRelevance;
+        return min > 0 ? hits.filter((h) => h.score >= min) : hits;
+    }
+
+    /**
+     * Cached embedding for `text`, computed via `runtime.embed()` on a miss.
+     * `null` (never throws) when embedding is unavailable, so callers fall back
+     * to lexical similarity.
+     */
+    private async _embed(runtime: SSMRuntimeRef, text: string): Promise<Float32Array | null> {
+        if (runtime !== this._embedRuntime) {
+            this._embedCache.clear();
+            this._embedRuntime = runtime;
+        }
+        const cached = this._embedCache.get(text);
+        if (cached) {
+            // LRU touch: re-insert so this entry becomes most-recently-used. (EVM-8)
+            this._embedCache.delete(text);
+            this._embedCache.set(text, cached);
+            return cached;
+        }
+        let vec: unknown;
+        try {
+            vec = await runtime.embed(text);
+        } catch {
+            // Embedding unavailable (no GPU, destroyed runtime, etc.) — the
+            // documented lexical fallback, reported to callers as `method`.
+            return null;
+        }
+        if (!(vec instanceof Float32Array) || vec.length === 0) return null;
+        if (this._embedCache.size >= this._opts.embedCacheMax) {
+            const lru = this._embedCache.keys().next().value;
+            if (lru !== undefined) this._embedCache.delete(lru);
+        }
+        this._embedCache.set(text, vec);
+        return vec;
+    }
+
+    /**
+     * Synchronise the dense vector store + HNSW index to `all`: embeds ONLY
+     * new/changed facts (keyed by `key@seq`), prunes removed ones, and rebuilds
+     * the index only when the fact set changed. `null` when any embedding is
+     * unavailable (caller falls back to lexical).
+     */
+    private async _syncDense(runtime: SSMRuntimeRef, all: MemoryEntry[]): Promise<DenseView | null> {
+        if (runtime !== this._denseRuntime) {
+            this.resetDense();
+            this._denseRuntime = runtime;
+        }
+        const vkeyOf = (e: MemoryEntry): string => `${e.key}@${e.seq ?? 0}`;
+        const currentVkeys = new Set<string>();
+        let sig = `${all.length}`;
+        for (const e of all) { const k = vkeyOf(e); currentVkeys.add(k); sig += '|' + k; }
+
+        const byKey = new Map<string, MemoryEntry>();
+        const items: Array<{ id: string; vector: Float32Array }> = [];
+        for (const entry of all) {
+            const vk = vkeyOf(entry);
+            let vec = this._denseVectors.get(vk);
+            if (!vec) {
+                const v = await this._embed(runtime, entry.content);
+                if (!v) return null;
+                vec = v;
+                this._denseVectors.set(vk, vec);
+            }
+            items.push({ id: entry.key, vector: vec });
+            byKey.set(entry.key, entry);
+        }
+        if (this._denseVectors.size > currentVkeys.size) {
+            for (const k of this._denseVectors.keys()) {
+                if (!currentVkeys.has(k)) this._denseVectors.delete(k);
+            }
+        }
+        if (all.length >= this._opts.annThreshold) {
+            if (sig !== this._denseSig || this._denseIndex === null) {
+                const index = new HnswIndex();
+                for (const it of items) index.add(it.id, it.vector);
+                this._denseIndex = index;
+            }
+        } else {
+            this._denseIndex = null;
+        }
+        this._denseSig = sig;
+        return { byKey, items, index: this._denseIndex };
+    }
+}

package/src/memory/records.ts

@@ -0,0 +1,79 @@
+/**
+ * memory/records.ts – Mapping between the public {@link MemoryEntry} shape and
+ * the durable {@link StoredFactRecord}, plus the TTL and ordering rules every
+ * MemoryStore read shares.
+ */
+
+import type { RememberRecordInput } from '../storage/ScopedFactRepository.js';
+import type { MemoryScope, StoredFactRecord } from '../storage/types.js';
+import type { MemoryEntry, RememberOptions } from './types.js';
+
+/** True once `ttlMs` has elapsed since the entry's (source) timestamp. */
+export function isExpired(entry: { timestamp: number; ttlMs?: number }): boolean {
+    if (entry.ttlMs == null) return false;
+    return Date.now() > entry.timestamp + entry.ttlMs;
+}
+
+/**
+ * Newest first; same-millisecond ties break by the durable write sequence so
+ * ordering is deterministic (and survives restarts).
+ */
+export function newestFirst(a: StoredFactRecord, b: StoredFactRecord): number {
+    return b.timestamp - a.timestamp || b.seq - a.seq;
+}
+
+/** Non-expired records, newest first, as public entries. */
+export function liveEntries(records: StoredFactRecord[]): MemoryEntry[] {
+    return records.filter((r) => !isExpired(r)).sort(newestFirst).map(toEntry);
+}
+
+export function toEntry(record: StoredFactRecord): MemoryEntry {
+    return {
+        key        : record.key,
+        content    : record.content,
+        timestamp  : record.timestamp,
+        seq        : record.seq,
+        ttlMs      : record.ttlMs,
+        type       : record.type,
+        tags       : record.tags,
+        importance : record.importance,
+        revision   : record.revision,
+        opId       : record.opId,
+        scope      : { ...record.scope },
+        provenance : record.provenance,
+        pendingSync: record.pendingSync,
+    };
+}
+
+/** A `remember` call's options as a repository write (store default TTL applied). */
+export function rememberInput(content: string, opts: RememberOptions | undefined, defaultTtlMs: number | undefined): RememberRecordInput {
+    return {
+        content,
+        timestamp : opts?.timestamp,
+        ttlMs     : opts?.ttlMs ?? defaultTtlMs,
+        type      : opts?.type,
+        tags      : opts?.tags,
+        importance: opts?.importance,
+        provenance: opts?.provenance,
+    };
+}
+
+/** An exported entry as an import record for `scope` (id/revision reassigned by the repository). */
+export function toImportRecord(e: MemoryEntry, scope: MemoryScope): StoredFactRecord {
+    return {
+        id         : '',
+        key        : e.key,
+        scope,
+        content    : e.content,
+        timestamp  : e.timestamp,
+        seq        : e.seq ?? 0,
+        revision   : 0,
+        opId       : e.opId ?? '',
+        ttlMs      : e.ttlMs,
+        type       : e.type,
+        tags       : e.tags,
+        importance : e.importance,
+        provenance : e.provenance,
+        pendingSync: false,
+    };
+}

package/src/memory/ttlSweeper.ts

@@ -0,0 +1,29 @@
+/**
+ * memory/ttlSweeper.ts – The background timer behind
+ * `MemoryStore.startTtlSweeper`: TTL is otherwise evaluated lazily on read, so
+ * an idle store would keep dead entries forever. (EVM-7)
+ */
+
+export class TtlSweeper {
+    private _timer: ReturnType<typeof setInterval> | null = null;
+
+    /** Runs `sweep` every `intervalMs`. Idempotent while running. */
+    start(intervalMs: number, sweep: () => Promise<unknown>): void {
+        if (this._timer) return;
+        const timer = setInterval(() => { void sweep(); }, intervalMs);
+        // Don't keep a Node process alive just for the sweeper.
+        const maybeUnref = timer as unknown as { unref?: () => void };
+        /* istanbul ignore else -- unref exists in Node; absent in the browser */
+        if (typeof maybeUnref.unref === 'function') {
+            maybeUnref.unref();
+        }
+        this._timer = timer;
+    }
+
+    stop(): void {
+        if (this._timer) {
+            clearInterval(this._timer);
+            this._timer = null;
+        }
+    }
+}

package/src/memory/types.ts

@@ -0,0 +1,149 @@
+/**
+ * memory/types.ts – The public shapes of {@link MemoryStore}: entries, recall
+ * results, write options and store options.
+ */
+
+import type { FactProvenance, FactStoragePort, MemoryScope } from '../storage/types.js';
+
+export type FactType = 'text' | 'json' | 'number' | 'boolean';
+
+export interface MemoryEntry {
+    key        : string;
+    content    : string;
+    timestamp  : number;
+    /**
+     * Monotonic write sequence, used only to break `timestamp` ties so that
+     * ordering stays deterministic when several entries are written within the
+     * same millisecond (Date.now() resolution). Higher = written later.
+     * Optional for backward compatibility with entries persisted/imported
+     * before this field existed (those sort as seq 0).
+     */
+    seq?       : number;
+    /** Optional time-to-live in milliseconds. */
+    ttlMs?     : number;
+    /** Semantic type of the stored value. Default: 'text'. */
+    type?      : FactType;
+    /** Tags for grouping and filtering. */
+    tags?      : string[];
+    /** Importance weight in the range 0–1. Default: 0.5. */
+    importance?: number;
+    /**
+     * Durable revision of this record (the store generation that wrote it).
+     * Present on every record written through the repository; absent only on
+     * entries fabricated by callers for import.
+     */
+    revision?  : number;
+    /** Durable identity of the operation that wrote this record. */
+    opId?      : string;
+    /** Ownership partition this entry was recalled from. */
+    scope?     : MemoryScope;
+    /** Where the fact came from; preserved across export/import (M06). */
+    provenance?: FactProvenance;
+    /** True while a sync acknowledgement for this record is outstanding. */
+    pendingSync?: boolean;
+}
+
+/**
+ * Which ranker produced a recall ordering. Matches the vocabulary the gateway's
+ * Evermind recall/validate surfaces already report, so a host can render one chip
+ * ("Semantic recall" / "Lexical recall (fallback)") regardless of which layer
+ * answered.
+ */
+export type RecallMethod = 'embedding' | 'lexical';
+
+/** A recalled entry with its relevance score (higher = closer). */
+export interface ScoredMemory {
+    entry: MemoryEntry;
+    score: number;
+}
+
+/** A ranked recall: the hits plus the ranker that ordered them. */
+export interface RankedRecall {
+    hits: ScoredMemory[];
+    method: RecallMethod;
+}
+
+export interface RememberOptions {
+    /** Override the store-level defaultTtlMs for this entry. */
+    ttlMs?     : number;
+    type?      : FactType;
+    tags?      : string[];
+    importance?: number;
+    /** Provenance to record with the fact (surface, importer, source time). */
+    provenance?: FactProvenance;
+    /** Explicit source timestamp (import/replay). Default: now. */
+    timestamp? : number;
+}
+
+export interface MemoryStoreOptions {
+    /** IndexedDB database name. Default: 'ssmjs'. Ignored when `storage` is supplied. */
+    dbName?       : string;
+    /** Key used for weight storage within the 'weights' object store. Default: 'ssmjs-weights'. */
+    weightsKey?   : string;
+    /**
+     * IDBFactory to use instead of the global `indexedDB` (browser path only).
+     * fake-indexeddb is supported for tests, but it is IN-MEMORY: on Node.js,
+     * pass `storage` from `NodeFileStorageAdapter.open(...)` for durable facts.
+     */
+    idbFactory?   : IDBFactory;
+    /**
+     * Durable storage port backing this store (M01). When omitted, an
+     * IndexedDB adapter over `dbName` is used.
+     */
+    storage?      : FactStoragePort;
+    /**
+     * Explicit ownership partition for every fact in this store (M03).
+     * Default: the unbound scope — personal, never-connected memory.
+     */
+    scope?        : MemoryScope;
+    /**
+     * Enable the sync journal, tombstones and pendingSync tracking (M04/M06).
+     * Unsynced facts are then never evicted by `maxEntries`. Default: false
+     * (legacy retention semantics — nothing to sync, nothing pending).
+     */
+    sync?         : boolean;
+    /** Identity of this context for operation ids and cross-context notes. */
+    contextId?    : string;
+    /**
+     * Default TTL applied to new entries when no per-entry ttlMs is provided.
+     * Entries with an expired TTL are filtered from recallAll() and related methods.
+     */
+    defaultTtlMs? : number;
+    /**
+     * Fact count at/above which semantic recall switches from an exact O(N)
+     * cosine scan to an O(log N) HNSW ANN index. Default 256 — small stores stay
+     * exact (simpler + faster), large stores stay fast. (EVM-1)
+     */
+    annThreshold? : number;
+    /**
+     * Hard cap on stored facts. When set, each write evicts the lowest-value
+     * SYNCED entries (expired first, then lowest importance, then oldest) so the
+     * store stays bounded. Facts with unsynced operations are never evicted
+     * (M06). Unset = unbounded (legacy). (EVM-7)
+     */
+    maxEntries?   : number;
+    /**
+     * Max content→embedding vectors held in the LRU recall cache. Default 2000.
+     * (EVM-8 — bounded LRU instead of clear-on-overflow.)
+     */
+    embedCacheMax?: number;
+    /**
+     * Minimum relevance (0..1) a hit must reach to be recalled by
+     * `recallSimilarScored` / `recallRanked` (and their wrappers). Below it, a
+     * weak match returns nothing rather than noise. Default 0 (off): recall
+     * returns its best-ranked hits exactly as before this option existed. Opt in
+     * with e.g. 0.05 to drop zero-overlap matches (a query sharing no token with
+     * any fact scores 0 on the lexical path).
+     */
+    minRelevance? : number;
+}
+
+// Minimal interface to avoid importing SSMRuntime (circular dep)
+export interface SaveLoadRuntime {
+    save(opts?: { storage: 'indexedDB'; key: string }): Promise<void>;
+    load(opts?: { key: string }): Promise<boolean>;
+}
+
+// Forward-declared to avoid circular dependency at import time
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type SSMRuntimeRef = any;