memtrails 0.0.0-stage → 0.2.0

package/dist/core/store.d.ts

@@ -0,0 +1,198 @@
+export type MemoryKind = 'decision' | 'procedure' | 'snippet' | 'template' | 'pitfall' | 'fact' | 'architecture' | 'preference' | 'reference' | 'episode' | 'note' | 'observation'
+/** What memory knows about one subject, gathered from its items (profiles.ts) — written again from them, never merged. */
+ | 'profile';
+export type MemoryScope = 'project' | 'global' | 'person';
+/** `merged`: a text another item absorbed — kept whole, out of recall, so a merge never loses what either side said. */
+export type MemoryStatus = 'active' | 'superseded' | 'archived' | 'merged';
+/** A request as captured: short-term memory. */
+export interface StmEntry {
+    uid: string;
+    fingerprint: string;
+    engine: string;
+    session: string;
+    turn: number;
+    cwd: string | null;
+    project: string;
+    title: string;
+    ts: number;
+    synthetic: boolean;
+    typed: 'human' | 'unknown';
+    request: string;
+    reply: string;
+    said: string;
+    steps: string[];
+    files: string[];
+    /** Read by the consolidation worker. */
+    done: boolean;
+    /** The record it is (records.ts): its own id, where it came from, who wrote it, where to open it, and your metadata. */
+    record?: {
+        id: string;
+        source?: string;
+        author?: string;
+        url?: string;
+        metadata?: Record<string, string | number | boolean | null>;
+    };
+}
+/** What long-term memory keeps. */
+export interface MemoryItem {
+    uid: string;
+    kind: MemoryKind;
+    scope: MemoryScope;
+    project: string;
+    engine: string;
+    session: string | null;
+    cwd: string | null;
+    /** Where it came from, for a person: the conversation's title, the note's name. */
+    where: string;
+    /** Last learned or refreshed. */
+    ts: number;
+    firstTs: number;
+    /** The key: a content title and a ~150-word summary (what is embedded). */
+    title: string;
+    summary: string;
+    body: string;
+    files: string[];
+    appliesWhen: string;
+    /** verified / stated (consolidation keeps only these), curated (a saved note), observed (an episode). */
+    confidence: 'verified' | 'stated' | 'curated' | 'observed';
+    /** 1–5, from consolidation. */
+    importance: number;
+    status: MemoryStatus;
+    supersededBy?: string;
+    /** When it was superseded — kept beside when it was learned (`ts`), so what was true when stays answerable. */
+    supersededAt?: number;
+    /** For an episode: how its request ended (success, partial, fail, uncertain). */
+    outcome?: 'success' | 'partial' | 'fail' | 'uncertain';
+    /** For a `merged` record: the item that absorbed it. */
+    mergedInto?: string;
+    /** The `merged` records this item absorbed: the texts it was made from, whole, in the log. */
+    absorbed?: string[];
+    /** The requests it was learned from (episode uids), newest last, at most 50. */
+    provenance: string[];
+    /** How many times consolidation learned it again (a duplicate merged into it). */
+    seen: number;
+    fingerprint: string;
+    /** Requests the person would type when it is needed — searched beside the key (absent before round 2: the worker
+     *  fills them, see Consolidator.fillCues). */
+    cues?: string[];
+    /** Where it came from (a guard that keeps only trusted sources): the person's own words, or a result the agent confirmed —
+     *  with the words that show it. Absent on a desktop, where every source is the person's own machine. */
+    origin?: MemoryOrigin;
+    /** Items about the same subject that say different things (consolidation and sleep link them; memory_explore walks
+     *  them), newest last. */
+    related?: string[];
+    /** What it is about — files, services, commands, named things — as the conversation named them: the concept graph's
+     *  links (memory_explore walks them; absent before the graph: the worker's backlog fills them). */
+    concepts?: string[];
+    /** For an observation: when what it tells happened or applies — an absolute date as precise as the conversation made
+     *  it (YYYY-MM-DD, YYYY-MM or YYYY), resolved from the request's own date; absent when the conversation gave none.
+     *  `ts` stays when it was said. */
+    eventAt?: string;
+    /** For an observation of something that lasted (a state, a habit, a job, a stay): when it began and when it ended,
+     *  as precise as the conversation made them; absent when it gave none. */
+    startAt?: string;
+    endAt?: string;
+    /** For an observation: done (it happened), plan (intended, not yet done), ongoing, or state (how things are). */
+    phase?: 'done' | 'plan' | 'ongoing' | 'state';
+    /** For an observation: who or what it is about, named as the conversation names them ("the person", "Melanie"). */
+    about?: string;
+}
+/** What a kept thing was learned from, and the exact words that show it (a guard checks they are there). */
+export interface MemoryOrigin {
+    by: 'user' | 'result';
+    evidence: string;
+}
+export declare function fingerprint(...parts: Array<string | readonly string[] | null | undefined>): string;
+/** An append-only log of records by uid: a later record for the same uid replaces the earlier one. What consolidation,
+ *  sleep and capture read and write, wherever the records are kept. */
+export interface MemoryLog<T extends {
+    uid: string;
+}, M> {
+    size(): number;
+    meta(uid: string): M | undefined;
+    entries(): Iterable<[string, M]>;
+    load(): Promise<void>;
+    get(uid: string): T | null;
+    put(recs: T[]): void;
+    /** Forget these records: gone from memory, and from the file when it is next compacted. */
+    remove(uids: string[]): void;
+    /** Every live record, oldest first. */
+    all(): Iterable<T>;
+    compact(): void;
+    /** Resolves once every `put` is kept; rejects with the first write that failed. */
+    flush(): Promise<void>;
+}
+/** An append-only JSONL log, read back by byte offset. `meta` is what stays in memory for each record. */
+export declare class JsonlLog<T extends {
+    uid: string;
+}, M> implements MemoryLog<T, M> {
+    private readonly path;
+    private readonly project;
+    private readonly live;
+    private lines;
+    private end;
+    constructor(path: string, project: (rec: T) => M);
+    size(): number;
+    meta(uid: string): M | undefined;
+    entries(): Generator<[string, M]>;
+    load(): Promise<void>;
+    get(uid: string): T | null;
+    put(recs: T[]): void;
+    remove(uids: string[]): void;
+    /** Every live record, in file order. */
+    all(): Generator<T>;
+    /** A file is written when `put` returns. */
+    flush(): Promise<void>;
+    /** Rewrite the log without replaced lines once it is twice what is live. */
+    compact(): void;
+}
+export type StmMeta = {
+    fp: string;
+    done: boolean;
+    ts: number;
+    session: string;
+    project: string;
+    synthetic: boolean;
+};
+export type LtmMeta = {
+    fp: string;
+    status: MemoryStatus;
+    kind: MemoryKind;
+    scope: MemoryScope;
+    project: string;
+    ts: number;
+    importance: number;
+    cued: boolean;
+    mapped: boolean;
+};
+export declare const stmMetaOf: (r: StmEntry) => StmMeta;
+export declare const ltmMetaOf: (r: MemoryItem) => LtmMeta;
+export declare const stmLog: (path: string) => JsonlLog<StmEntry, StmMeta>;
+export declare const ltmLog: (path: string) => JsonlLog<MemoryItem, LtmMeta>;
+export type StmStore = MemoryLog<StmEntry, StmMeta>;
+export type LtmStore = MemoryLog<MemoryItem, LtmMeta>;
+/** How often each long-term item was shown and opened — what the sleep pass reads as "use". */
+export interface UsageLog {
+    of(uid: string): {
+        shown: number;
+        opened: number;
+        lastUsed: number;
+    };
+    note(uids: string[], what: 'shown' | 'opened', now?: number): void;
+    flush(): void | Promise<void>;
+}
+/** How often each long-term item was shown with a turn and opened by an agent. Saved at most once a minute. */
+export declare class UsageBook implements UsageLog {
+    private readonly path;
+    private data;
+    private dirty;
+    private timer;
+    constructor(path: string);
+    of(uid: string): {
+        shown: number;
+        opened: number;
+        lastUsed: number;
+    };
+    note(uids: string[], what: 'shown' | 'opened', now?: number): void;
+    flush(): void;
+}

package/dist/core/store.js

@@ -0,0 +1,190 @@
+/**
+ * What a memory holds, on disk, as two append-only logs in its home:
+ *
+ *   - `records.jsonl` — SHORT-TERM memory: every record as it was added (records.ts), kept for good; also the queue the
+ *     consolidation worker reads (`done`);
+ *   - `knowledge.jsonl` — LONG-TERM memory: what consolidation kept (consolidate.ts) — typed items, dated observations,
+ *     kept episodes, profiles — with their keys, status (active / superseded / archived) and provenance.
+ *
+ * The logs are the truth; Qdrant is derived from them (rebuilt without asking a model, after a Qdrant upgrade or a
+ * lost index). In memory: each record's place in its file and a small projection — never the text. A later line for
+ * the same uid replaces the earlier one; a log is rewritten without replaced lines once it is twice what is live.
+ *
+ * `usage.json`: how often each long-term item was shown and opened (the sleep pass's "use", MemRL-style).
+ * `recall.jsonl`: what each search showed, for what, and what was opened after (recall-log.ts).
+ *
+ * The logs are behind `MemoryLog`: these files, or any store you implement (a database table per user). Reads are from what `load()` brought in; `put` writes behind and `flush()`
+ * waits for it (a file write is done when `put` returns).
+ */
+import { createHash } from 'node:crypto';
+import { appendFileSync, closeSync, createReadStream, existsSync, mkdirSync, openSync, readFileSync, readSync, renameSync, statSync, writeFileSync } from 'node:fs';
+import { dirname } from 'node:path';
+import { createInterface } from 'node:readline';
+export function fingerprint(...parts) {
+    const h = createHash('sha1');
+    for (const p of parts)
+        h.update(typeof p === 'string' || p == null ? (p ?? '') : p.join('\u0001')).update('\u0000');
+    return h.digest('hex').slice(0, 16);
+}
+/** An append-only JSONL log, read back by byte offset. `meta` is what stays in memory for each record. */
+export class JsonlLog {
+    path;
+    project;
+    live = new Map();
+    lines = 0;
+    end = 0;
+    constructor(path, project) {
+        this.path = path;
+        this.project = project;
+    }
+    size() { return this.live.size; }
+    meta(uid) { return this.live.get(uid)?.meta; }
+    *entries() { for (const [uid, s] of this.live)
+        yield [uid, s.meta]; }
+    async load() {
+        this.live.clear();
+        this.lines = 0;
+        this.end = 0;
+        if (!existsSync(this.path))
+            return;
+        let at = 0;
+        for await (const line of createInterface({ input: createReadStream(this.path, { encoding: 'utf8' }), crlfDelay: Infinity })) {
+            const len = Buffer.byteLength(line, 'utf8') + 1;
+            if (line) {
+                try {
+                    const rec = JSON.parse(line);
+                    if (rec.removed)
+                        this.live.delete(rec.uid);
+                    else
+                        this.live.set(rec.uid, { at, len, meta: this.project(rec) });
+                    this.lines++;
+                }
+                catch { /* a torn last line */ }
+            }
+            at += len;
+        }
+        this.end = at;
+    }
+    get(uid) {
+        const s = this.live.get(uid);
+        if (!s)
+            return null;
+        const fd = openSync(this.path, 'r');
+        try {
+            const buf = Buffer.alloc(s.len);
+            readSync(fd, buf, 0, s.len, s.at);
+            return JSON.parse(buf.toString('utf8').trimEnd());
+        }
+        catch {
+            return null;
+        }
+        finally {
+            closeSync(fd);
+        }
+    }
+    put(recs) {
+        if (!recs.length)
+            return;
+        mkdirSync(dirname(this.path), { recursive: true });
+        if (!this.end && existsSync(this.path))
+            this.end = statSync(this.path).size;
+        const chunks = [];
+        let at = this.end;
+        for (const rec of recs) {
+            const b = Buffer.from(`${JSON.stringify(rec)}\n`, 'utf8');
+            this.live.set(rec.uid, { at, len: b.length, meta: this.project(rec) });
+            chunks.push(b);
+            at += b.length;
+        }
+        appendFileSync(this.path, Buffer.concat(chunks), { mode: 0o600 });
+        this.end = at;
+        this.lines += recs.length;
+    }
+    remove(uids) {
+        const gone = uids.filter((u) => this.live.has(u));
+        if (!gone.length)
+            return;
+        mkdirSync(dirname(this.path), { recursive: true });
+        const b = Buffer.from(gone.map((uid) => `${JSON.stringify({ uid, removed: true })}\n`).join(''), 'utf8');
+        appendFileSync(this.path, b, { mode: 0o600 });
+        if (!this.end)
+            this.end = statSync(this.path).size - b.length;
+        this.end += b.length;
+        this.lines += gone.length;
+        for (const uid of gone)
+            this.live.delete(uid);
+    }
+    /** Every live record, in file order. */
+    *all() {
+        const order = [...this.live.entries()].sort((a, b) => a[1].at - b[1].at);
+        for (const [uid] of order) {
+            const r = this.get(uid);
+            if (r)
+                yield r;
+        }
+    }
+    /** A file is written when `put` returns. */
+    async flush() { }
+    /** Rewrite the log without replaced lines once it is twice what is live. */
+    compact() {
+        if (this.lines < 2 * this.live.size + 1000)
+            return;
+        const recs = [...this.all()];
+        const tmp = `${this.path}.compact`;
+        writeFileSync(tmp, recs.map((r) => `${JSON.stringify(r)}\n`).join(''), { mode: 0o600 });
+        renameSync(tmp, this.path);
+        // Offsets again, from the file just written.
+        this.live.clear();
+        let at = 0;
+        for (const r of recs) {
+            const len = Buffer.byteLength(`${JSON.stringify(r)}\n`, 'utf8');
+            this.live.set(r.uid, { at, len, meta: this.project(r) });
+            at += len;
+        }
+        this.end = at;
+        this.lines = recs.length;
+    }
+}
+export const stmMetaOf = (r) => ({ fp: r.fingerprint, done: r.done, ts: r.ts, session: r.session, project: r.project, synthetic: r.synthetic });
+export const ltmMetaOf = (r) => ({ fp: r.fingerprint, status: r.status, kind: r.kind, scope: r.scope, project: r.project, ts: r.ts, importance: r.importance, cued: !!r.cues?.length, mapped: !!r.concepts?.length });
+export const stmLog = (path) => new JsonlLog(path, stmMetaOf);
+export const ltmLog = (path) => new JsonlLog(path, ltmMetaOf);
+/** How often each long-term item was shown with a turn and opened by an agent. Saved at most once a minute. */
+export class UsageBook {
+    path;
+    data = {};
+    dirty = false;
+    timer = null;
+    constructor(path) {
+        this.path = path;
+        try {
+            this.data = JSON.parse(readFileSync(path, 'utf8'));
+        }
+        catch {
+            this.data = {};
+        }
+    }
+    of(uid) {
+        return this.data[uid] ?? { shown: 0, opened: 0, lastUsed: 0 };
+    }
+    note(uids, what, now = Date.now()) {
+        for (const uid of uids) {
+            const u = (this.data[uid] ??= { shown: 0, opened: 0, lastUsed: 0 });
+            u[what] += 1;
+            u.lastUsed = now;
+        }
+        this.dirty = true;
+        this.timer ??= setTimeout(() => { this.timer = null; this.flush(); }, 60_000);
+        this.timer.unref?.();
+    }
+    flush() {
+        if (!this.dirty)
+            return;
+        try {
+            mkdirSync(dirname(this.path), { recursive: true });
+            writeFileSync(this.path, JSON.stringify(this.data), { mode: 0o600 });
+            this.dirty = false;
+        }
+        catch { /* read-only: kept in memory */ }
+    }
+}

package/dist/core/text.d.ts

@@ -0,0 +1,31 @@
+/**
+ * What gets indexed from an item's text, besides its content key: the keyword vector (BM25 as a Qdrant sparse vector —
+ * term frequencies here, the IDF on Qdrant's side with `modifier: idf`) and the small-to-big children.
+ *
+ * Measured on a private history of real work: the keyword channel beside the dense one put a needed
+ * item in the top 5 for +5–8 points; ~96-word children beat 192-word ones; adding raw text to the key itself hurt.
+ */
+import type { SparseVector } from './qdrant-client.js';
+/** Lower case, compatibility forms unified, accents dropped from Latin, Greek and Cyrillic letters — "bază" and
+ *  "baza" are the same word to a person typing either. Marks that make the word elsewhere (Devanagari, Thai) stay. */
+export declare function fold(text: string): string;
+/** Words for the keyword channel, in any language: folded (fold()); identifiers kept whole AND split at `_ . - /`; in
+ *  scripts without spaces (Chinese, Japanese, Thai…) the segmenter's words. */
+export declare function tokens(text: string): string[];
+/** A text's first `n` words, as written (whitespace flattened). */
+export declare function firstWords(text: string, n: number): string;
+/** 32-bit FNV-1a: a term's index in the sparse vector. */
+export declare function termId(term: string): number;
+/** A document's BM25 term weights (Qdrant applies the IDF). `avgLen`: the typical length of what is indexed. */
+export declare function docSparse(text: string, avgLen?: number): SparseVector;
+/** A query's terms, each once. */
+export declare function querySparse(texts: readonly string[]): SparseVector;
+export declare const CHILD_WORDS = 96;
+export declare const CHILD_STRIDE = 72;
+/** ~96-word windows of a body, a quarter overlapping, all of it — searched, never shown: the item is what comes back.
+ *  Words as the segmenter finds them, so a body in a script without spaces is windowed too. */
+export declare function children(text: string, words?: number, stride?: number): string[];
+export declare const cut: (s: string, n: number) => string;
+/** One spelling per concept, the graph's key: folded like a word (case, accents, compatibility forms), inner spaces
+ *  collapsed, stray punctuation off the ends — "Gateway.conf " and "gateway.conf" are one node. */
+export declare function conceptKey(name: string): string;

package/dist/core/text.js

@@ -0,0 +1,101 @@
+const K1 = 1.2;
+const B = 0.75;
+/** Scripts written without spaces between words: there the ICU word segmenter finds the words. */
+const UNSPACED = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Thai}\p{Script=Lao}\p{Script=Khmer}\p{Script=Myanmar}]/u;
+const WORDS = new Intl.Segmenter(undefined, { granularity: 'word' });
+/** Lower case, compatibility forms unified, accents dropped from Latin, Greek and Cyrillic letters — "bază" and
+ *  "baza" are the same word to a person typing either. Marks that make the word elsewhere (Devanagari, Thai) stay. */
+export function fold(text) {
+    return text.normalize('NFKD').replace(/([\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}])\p{M}+/gu, '$1').normalize('NFKC').toLowerCase();
+}
+/** Words for the keyword channel, in any language: folded (fold()); identifiers kept whole AND split at `_ . - /`; in
+ *  scripts without spaces (Chinese, Japanese, Thai…) the segmenter's words. */
+export function tokens(text) {
+    const out = [];
+    for (const raw of fold(text).split(/[^\p{L}\p{N}\p{M}_.\-/]+/u)) {
+        const w = raw.replace(/^[._\-/]+|[._\-/]+$/g, '');
+        if (!w)
+            continue;
+        if (UNSPACED.test(w)) {
+            for (const seg of WORDS.segment(w))
+                if (seg.isWordLike)
+                    out.push(seg.segment);
+            continue;
+        }
+        if (w.length < 2 || w.length > 60)
+            continue;
+        out.push(w);
+        if (/[_.\-/]/.test(w))
+            for (const p of w.split(/[_.\-/]+/))
+                if (p.length >= 2 && p !== w)
+                    out.push(p);
+    }
+    return out;
+}
+/** Where each word of a text starts and ends — the segmenter's words, so a text without spaces counts right. */
+function wordSpans(text) {
+    const at = [];
+    for (const seg of WORDS.segment(text))
+        if (seg.isWordLike)
+            at.push([seg.index, seg.index + seg.segment.length]);
+    return at;
+}
+const flat = (s) => s.replace(/\s+/g, ' ').trim();
+/** A text's first `n` words, as written (whitespace flattened). */
+export function firstWords(text, n) {
+    const at = wordSpans(text);
+    return flat(at.length > n ? text.slice(0, at[n - 1][1]) : text);
+}
+/** 32-bit FNV-1a: a term's index in the sparse vector. */
+export function termId(term) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < term.length; i++) {
+        h ^= term.charCodeAt(i);
+        h = Math.imul(h, 0x01000193) >>> 0;
+    }
+    return h >>> 0;
+}
+/** A document's BM25 term weights (Qdrant applies the IDF). `avgLen`: the typical length of what is indexed. */
+export function docSparse(text, avgLen = 220) {
+    const ts = tokens(text);
+    const tf = new Map();
+    for (const t of ts) {
+        const id = termId(t);
+        tf.set(id, (tf.get(id) ?? 0) + 1);
+    }
+    const norm = K1 * (1 - B + (B * ts.length) / avgLen);
+    const indices = [...tf.keys()].sort((a, b) => a - b);
+    return { indices, values: indices.map((i) => { const f = tf.get(i); return (f * (K1 + 1)) / (f + norm); }) };
+}
+/** A query's terms, each once. */
+export function querySparse(texts) {
+    const ids = [...new Set(texts.flatMap((t) => tokens(t)).map(termId))].sort((a, b) => a - b);
+    return { indices: ids, values: ids.map(() => 1) };
+}
+export const CHILD_WORDS = 96;
+export const CHILD_STRIDE = 72;
+/** ~96-word windows of a body, a quarter overlapping, all of it — searched, never shown: the item is what comes back.
+ *  Words as the segmenter finds them, so a body in a script without spaces is windowed too. */
+export function children(text, words = CHILD_WORDS, stride = CHILD_STRIDE) {
+    const at = wordSpans(text);
+    if (!at.length)
+        return flat(text) ? [flat(text)] : [];
+    if (at.length <= words)
+        return [flat(text)];
+    const out = [];
+    let end = 0;
+    for (let i = 0; i < at.length - Math.floor(words / 2); i += stride) {
+        end = Math.min(i + words, at.length);
+        out.push(flat(text.slice(at[i][0], at[end - 1][1])));
+    }
+    // The last words, when the stride stopped short of them: a window ending at the text's end.
+    if (end < at.length)
+        out.push(flat(text.slice(at[Math.max(0, at.length - words)][0], at[at.length - 1][1])));
+    return out;
+}
+export const cut = (s, n) => (s.length > n ? `${s.slice(0, n)}…` : s);
+/** One spelling per concept, the graph's key: folded like a word (case, accents, compatibility forms), inner spaces
+ *  collapsed, stray punctuation off the ends — "Gateway.conf " and "gateway.conf" are one node. */
+export function conceptKey(name) {
+    return fold(name).replace(/\s+/g, ' ').replace(/^[\s.,:;"'`()[\]]+|[\s.,:;"'`()[\]]+$/g, '');
+}

package/dist/core/words.d.ts

@@ -0,0 +1,32 @@
+/** Words kept of a memory (64 lost most of the gain). */
+export declare const WORD_LIMIT = 128;
+/** Unit vectors, one a word, flat: `words × dims` numbers. */
+export interface WordVectors {
+    words: number;
+    dims: number;
+    data: Float32Array;
+}
+export type WordEncoder = (texts: string[], kind: 'query' | 'document') => Promise<WordVectors[]>;
+/** A memory's words as signs, one bit a number, a word to a whole number of bytes. */
+export interface PackedWords {
+    words: number;
+    dims: number;
+    bits: Uint8Array;
+}
+export declare function packWords(v: WordVectors, limit?: number): PackedWords;
+/** The sum, over the question's words, of the closest word of the memory. */
+export declare function maxSim(query: WordVectors, memory: PackedWords): number;
+/** The packed words by text, in memory and appended to a file: key, word count, numbers a word, the bits. Derived, like
+ *  the index: losing it only means encoding again. */
+export declare class WordStore {
+    private readonly file?;
+    private readonly byKey;
+    constructor(file?: string | undefined);
+    get size(): number;
+    get(text: string): PackedWords | undefined;
+    load(): Promise<void>;
+    put(entries: Array<[string, PackedWords]>): Promise<void>;
+}
+/** A reranker for memory_search (RecallDeps.rerank) that needs no server: what is not stored yet is encoded here, once.
+ *  Null — the search keeps its own order — when the encoder fails or takes longer than it was given. */
+export declare function wordReranker(encode: WordEncoder, store: WordStore): (query: string, documents: string[], timeoutMs: number) => Promise<number[] | null>;

package/dist/core/words.js

@@ -0,0 +1,127 @@
+/**
+ * Word by word: memory_search's order with no server. A memory's one vector averages it, so one exact detail in a long
+ * memory is drowned out. The embedder has a vector per word before it averages them; a memory scores the sum, over the
+ * question's words, of its closest word (MaxSim, as ColBERT does). Over the search's best 30, fused by rank as the
+ * reranker is (reranker.ts `rerankedOrder`). No training, no word list, any language.
+ *
+ * Measured (docs/BENCHMARKS.md): all 2,011 questions, answer shown at 5 50.3 → 53.1 (+80/−24), at 10
+ * 57.1 → 58.4 (+67/−40); no refinement of the score beat this plain sum; other languages against English memories gain
+ * more (Romanian at 5 43.0 → 48.2). One bit a number keeps nearly all of it, and 128 words are enough where 64 are not.
+ *
+ * A memory's words are kept small, the question's whole: its cosine with a word stored as signs is the sum of its
+ * numbers with those signs, over √dims. Kept by the text's hash, so an unchanged text is never encoded twice.
+ */
+import { createHash } from 'node:crypto';
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { dirname } from 'node:path';
+/** Words kept of a memory (64 lost most of the gain). */
+export const WORD_LIMIT = 128;
+const KEY_BYTES = 8;
+export function packWords(v, limit = WORD_LIMIT) {
+    const words = Math.min(v.words, limit);
+    const stride = Math.ceil(v.dims / 8);
+    const bits = new Uint8Array(words * stride);
+    for (let w = 0; w < words; w++) {
+        for (let d = 0; d < v.dims; d++)
+            if (v.data[w * v.dims + d] >= 0)
+                bits[w * stride + (d >> 3)] |= 1 << (d & 7);
+    }
+    return { words, dims: v.dims, bits };
+}
+/** The sum, over the question's words, of the closest word of the memory. */
+export function maxSim(query, memory) {
+    if (!query.words || !memory.words || query.dims !== memory.dims)
+        return Number.NEGATIVE_INFINITY;
+    const { dims } = query;
+    const stride = Math.ceil(dims / 8);
+    const norm = Math.sqrt(dims);
+    let total = 0;
+    for (let q = 0; q < query.words; q++) {
+        const at = q * dims;
+        let best = Number.NEGATIVE_INFINITY;
+        for (let w = 0; w < memory.words; w++) {
+            const row = w * stride;
+            let sum = 0;
+            for (let d = 0; d < dims; d++) {
+                const x = query.data[at + d];
+                sum += memory.bits[row + (d >> 3)] & (1 << (d & 7)) ? x : -x;
+            }
+            if (sum > best)
+                best = sum;
+        }
+        total += best / norm;
+    }
+    return total;
+}
+const keyOf = (text) => createHash('sha256').update(text).digest('hex').slice(0, KEY_BYTES * 2);
+/** The packed words by text, in memory and appended to a file: key, word count, numbers a word, the bits. Derived, like
+ *  the index: losing it only means encoding again. */
+export class WordStore {
+    file;
+    byKey = new Map();
+    constructor(file) {
+        this.file = file;
+    }
+    get size() { return this.byKey.size; }
+    get(text) { return this.byKey.get(keyOf(text)); }
+    async load() {
+        if (!this.file)
+            return;
+        const b = await readFile(this.file).catch(() => null);
+        if (!b)
+            return;
+        for (let at = 0; at + KEY_BYTES + 4 <= b.length;) {
+            const key = b.subarray(at, at + KEY_BYTES).toString('hex');
+            const words = b.readUInt16LE(at + KEY_BYTES);
+            const dims = b.readUInt16LE(at + KEY_BYTES + 2);
+            const size = words * Math.ceil(dims / 8);
+            const start = at + KEY_BYTES + 4;
+            if (start + size > b.length)
+                break; // cut short by a crash: what came before is whole
+            this.byKey.set(key, { words, dims, bits: new Uint8Array(b.subarray(start, start + size)) });
+            at = start + size;
+        }
+    }
+    async put(entries) {
+        const parts = [];
+        for (const [text, p] of entries) {
+            const key = keyOf(text);
+            this.byKey.set(key, p);
+            const head = Buffer.alloc(KEY_BYTES + 4);
+            Buffer.from(key, 'hex').copy(head);
+            head.writeUInt16LE(p.words, KEY_BYTES);
+            head.writeUInt16LE(p.dims, KEY_BYTES + 2);
+            parts.push(head, Buffer.from(p.bits));
+        }
+        if (!this.file || !parts.length)
+            return;
+        await mkdir(dirname(this.file), { recursive: true });
+        await appendFile(this.file, Buffer.concat(parts));
+    }
+}
+/** A reranker for memory_search (RecallDeps.rerank) that needs no server: what is not stored yet is encoded here, once.
+ *  Null — the search keeps its own order — when the encoder fails or takes longer than it was given. */
+export function wordReranker(encode, store) {
+    return async (query, documents, timeoutMs) => {
+        let timer;
+        const late = new Promise((resolve) => { timer = setTimeout(() => resolve(null), timeoutMs); });
+        const work = (async () => {
+            const missing = [...new Set(documents.filter((t) => !store.get(t)))];
+            if (missing.length) {
+                const encoded = await encode(missing, 'document');
+                await store.put(missing.map((t, i) => [t, packWords(encoded[i])]));
+            }
+            const [q] = await encode([query], 'query');
+            return documents.map((t) => maxSim(q, store.get(t)));
+        })();
+        try {
+            return await Promise.race([work, late]);
+        }
+        catch {
+            return null;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    };
+}