minnimemory 1.0.0-beta.1

package/dist/router.d.ts

@@ -0,0 +1,141 @@
+/**
+ * The router: deterministic lexical ranking over section-level units (interface rule O4).
+ *
+ * Two decisions here, both from the 2026-09-04 interface audit
+ * (interface audit, 2026-09-04):
+ *
+ *   1. The retrieval unit is a section, not an OnDemandMemory file. A file-level hit on a real
+ *      memory file cost 1,303 to 8,825 tokens, and the largest file was the changelog, so better
+ *      recall at file granularity opened exactly the content whose non-opening produced the
+ *      saving. At section granularity the same queries cost 17 to 60 tokens where the file had
+ *      structure. Inside an episodic section each dated entry is its own unit, which turns a
+ *      changelog from dead weight into cheap, addressable history.
+ *
+ *   2. Ranking is BM25 over stemmed tokens, with the unit's heading path and its OnDemandMemory
+ *      file's list triggers weighted three times the body. The previous router scored one point per
+ *      trigger whose words all appeared exactly; on five realistic tasks it found a unit for
+ *      two. This one found all five. It stays inside O4's constraints: no embeddings, no model
+ *      call, no network, deterministic, ties broken by document order.
+ *
+ * The stemmer is deliberately crude (a suffix list, not Porter): it only has to make `deploy`,
+ * `deploys` and `deployment` one term, and it must never produce different output on different
+ * machines. Every parameter is a named constant so the interface doc can quote it.
+ */
+import type { Manifest } from "./compile.js";
+import type { MemoryKind } from "./episodic.js";
+/** BM25 term-frequency saturation. */
+export declare const BM25_K1 = 1.2;
+/**
+ * BM25 length normalisation. 0.75 is the textbook value; it made a 21-token changelog line
+ * outrank the 4,467-token section that actually held the answer on two of five audit tasks.
+ * 0.3 keeps some preference for short units without letting length alone decide.
+ */
+export declare const BM25_B = 0.3;
+/** Heading-path and trigger terms count this many times a body term. */
+export declare const HEADING_WEIGHT = 3;
+/** Default cap on tokens a recall returns; the agent widens with outline() if it needs more. */
+export declare const RECALL_MAX_TOKENS = 1500;
+/**
+ * A unit scoring under this fraction of the best hit is not returned even when the cap has
+ * room. The cap is a ceiling, not a target: on the audit corpus a query whose answer was two
+ * changelog entries (169 tokens) came back with six (1,470) because the tail fit; at 0.3 the
+ * tail still fit (it scored 34 to 40% of the best), so the floor is half the best hit.
+ */
+export declare const RECALL_MIN_SCORE_RATIO = 0.5;
+export interface Unit {
+    /** the OnDemandMemory file this unit came from */
+    onDemandFile: string;
+    /** OnDemandMemory file path, relative to the compiled directory */
+    file: string;
+    /** this unit's own heading, or the first 60 characters of a dated entry */
+    heading: string;
+    /** heading path from the OnDemandMemory file's top heading down to and including this unit */
+    path: string[];
+    kind: "section" | "entry";
+    /** 1-indexed, inclusive, within the OnDemandMemory file */
+    startLine: number;
+    endLine: number;
+    tokens: number;
+    content: string;
+}
+/**
+ * Crude suffix stemmer. Short words are left alone: stripping "s" from "bus" or "ed" from
+ * "red" does more harm than joining their plurals does good.
+ */
+export declare function stem(word: string): string;
+/** Lower-cased, stopword-free, stemmed terms of a piece of text. */
+export declare function tokenize(text: string): string[];
+/**
+ * Split one OnDemandMemory file's content into retrieval units. Every heading-delimited section
+ * is a unit; a section whose bullets are mostly dated (a changelog, a session log) is split
+ * further into one unit per entry, with any continuation lines kept with their entry. A heading
+ * with no body is a container, not a unit: its text still reaches the router through its
+ * children's heading path.
+ */
+export declare function unitsOf(onDemandFile: {
+    name: string;
+    file: string;
+}, content: string): Unit[];
+interface Doc {
+    tf: Map<string, number>;
+    len: number;
+}
+export interface SearchIndex {
+    units: Unit[];
+    docs: Doc[];
+    df: Map<string, number>;
+    avgLen: number;
+}
+/**
+ * Build the ranking index once per manifest. `triggers` are the editable per-OnDemandMemory-file
+ * trigger words from the manifest; they join every unit of that file at heading weight, so a
+ * person who adds a word to the list steers recall without touching the file text.
+ */
+export declare function buildSearchIndex(units: Unit[], triggers?: Map<string, string[]>): SearchIndex;
+export interface RankedUnit {
+    unit: Unit;
+    score: number;
+}
+/**
+ * BM25 over the index. Returns only units with a positive score, best first, ties in document
+ * order, so the same query against the same files ranks the same way on every machine.
+ */
+export declare function rankUnits(index: SearchIndex, query: string, limit?: number): RankedUnit[];
+/**
+ * The best units that fit under a token cap, in rank order, dropping the tail that scores under
+ * RECALL_MIN_SCORE_RATIO of the best hit. Always returns the top unit even when it alone
+ * exceeds the cap: an answer that is too long is better than no answer, and the caller sees the
+ * token count and can decide.
+ */
+export declare function selectUnits(ranked: RankedUnit[], maxTokens?: number): RankedUnit[];
+/**
+ * Trigger words for an OnDemandMemory file, chosen against the other OnDemandMemory files
+ * rather than by raw frequency: a term scores by how often it appears here, times how rare it
+ * is across the set, with heading terms weighted. The previous frequency ranking produced
+ * `bars, free, merged` for a trading-app memory file (audit 2026-09-04). Returns surface words,
+ * not stems, so the list stays readable; two words with one stem count once.
+ */
+export declare function selectTriggers(onDemandFiles: {
+    name: string;
+    heading: string;
+    content: string;
+}[], limit?: number): Map<string, string[]>;
+export interface RankedOnDemandFile {
+    name: string;
+    file: string;
+    kind?: MemoryKind;
+    triggers: string[];
+    tokens: number;
+    score: number;
+}
+/**
+ * Deterministic keyword-overlap ranking: one point per trigger whose words all appear as whole
+ * words in the query. Whole words, not substrings, so the trigger "app" does not match "happy"
+ * and "api" does not match "capital". Stable-sorted by score desc, then by manifest order, so the
+ * same query always ranks the same way - reproducibility matters more than ranking
+ * sophistication here. The older trigger-overlap ranker over the manifest alone, kept for
+ * bench.ts's agent-driven-route model (needs no file content, unlike rankUnits above); moved
+ * here from recall.ts (2026-09-13 audit round 2, D7c) to sit beside the rest of the ranking code.
+ */
+export declare function rankOnDemandFiles(manifest: Manifest, query: string): RankedOnDemandFile[];
+export {};

package/dist/router.js

@@ -0,0 +1,314 @@
+/**
+ * The router: deterministic lexical ranking over section-level units (interface rule O4).
+ *
+ * Two decisions here, both from the 2026-09-04 interface audit
+ * (interface audit, 2026-09-04):
+ *
+ *   1. The retrieval unit is a section, not an OnDemandMemory file. A file-level hit on a real
+ *      memory file cost 1,303 to 8,825 tokens, and the largest file was the changelog, so better
+ *      recall at file granularity opened exactly the content whose non-opening produced the
+ *      saving. At section granularity the same queries cost 17 to 60 tokens where the file had
+ *      structure. Inside an episodic section each dated entry is its own unit, which turns a
+ *      changelog from dead weight into cheap, addressable history.
+ *
+ *   2. Ranking is BM25 over stemmed tokens, with the unit's heading path and its OnDemandMemory
+ *      file's list triggers weighted three times the body. The previous router scored one point per
+ *      trigger whose words all appeared exactly; on five realistic tasks it found a unit for
+ *      two. This one found all five. It stays inside O4's constraints: no embeddings, no model
+ *      call, no network, deterministic, ties broken by document order.
+ *
+ * The stemmer is deliberately crude (a suffix list, not Porter): it only has to make `deploy`,
+ * `deploys` and `deployment` one term, and it must never produce different output on different
+ * machines. Every parameter is a named constant so the interface doc can quote it.
+ */
+import { isEpisodicBody, RE_DATED_BULLET, sections, STOPWORDS, weightedFragments } from "./text.js";
+import { estimateTokens } from "./tokenizer.js";
+/** BM25 term-frequency saturation. */
+export const BM25_K1 = 1.2;
+/**
+ * BM25 length normalisation. 0.75 is the textbook value; it made a 21-token changelog line
+ * outrank the 4,467-token section that actually held the answer on two of five audit tasks.
+ * 0.3 keeps some preference for short units without letting length alone decide.
+ */
+export const BM25_B = 0.3;
+/** Heading-path and trigger terms count this many times a body term. */
+export const HEADING_WEIGHT = 3;
+/** Default cap on tokens a recall returns; the agent widens with outline() if it needs more. */
+export const RECALL_MAX_TOKENS = 1500;
+/**
+ * A unit scoring under this fraction of the best hit is not returned even when the cap has
+ * room. The cap is a ceiling, not a target: on the audit corpus a query whose answer was two
+ * changelog entries (169 tokens) came back with six (1,470) because the tail fit; at 0.3 the
+ * tail still fit (it scored 34 to 40% of the best), so the floor is half the best hit.
+ */
+export const RECALL_MIN_SCORE_RATIO = 0.5;
+const RE_SUFFIX = /(?:ations?|ments?|tions?|ings?|ers?|ed|es|s)$/;
+/**
+ * Crude suffix stemmer. Short words are left alone: stripping "s" from "bus" or "ed" from
+ * "red" does more harm than joining their plurals does good.
+ */
+export function stem(word) {
+    if (word.length <= 4)
+        return word;
+    if (word.endsWith("ies"))
+        return `${word.slice(0, -3)}y`;
+    const stripped = word.replace(RE_SUFFIX, "");
+    return stripped.length >= 3 ? stripped : word;
+}
+/** Lower-cased, stopword-free, stemmed terms of a piece of text. */
+export function tokenize(text) {
+    const words = text.toLowerCase().match(/[a-z][a-z0-9_-]{1,}/g) ?? [];
+    const out = [];
+    for (const w of words) {
+        if (STOPWORDS.has(w))
+            continue;
+        const s = stem(w);
+        if (s.length >= 3 && !STOPWORDS.has(s))
+            out.push(s);
+    }
+    return out;
+}
+const RE_BULLET = /^\s*[-*+]\s+/;
+function entryHeading(line) {
+    const text = line.replace(RE_BULLET, "").replace(/\*\*/g, "").trim();
+    return text.length > 60 ? `${text.slice(0, 57)}...` : text;
+}
+/**
+ * Split one OnDemandMemory file's content into retrieval units. Every heading-delimited section
+ * is a unit; a section whose bullets are mostly dated (a changelog, a session log) is split
+ * further into one unit per entry, with any continuation lines kept with their entry. A heading
+ * with no body is a container, not a unit: its text still reaches the router through its
+ * children's heading path.
+ */
+export function unitsOf(onDemandFile, content) {
+    const lines = content.split(/\r?\n/);
+    const secs = sections(lines);
+    const units = [];
+    const stack = [];
+    for (const sec of secs) {
+        while (stack.length > 0 && (stack[stack.length - 1]?.level ?? 0) >= sec.level)
+            stack.pop();
+        if (sec.level > 0)
+            stack.push({ level: sec.level, heading: sec.heading });
+        const path = stack.map((s) => s.heading);
+        // sections() gives 1-indexed inclusive start and end; the heading is the first line.
+        const bodyStart = sec.level > 0 ? sec.startLine + 1 : sec.startLine;
+        const bodyLines = lines.slice(bodyStart - 1, sec.endLine);
+        if (bodyLines.every((l) => l.trim() === ""))
+            continue;
+        const episodic = isEpisodicBody(bodyLines);
+        if (!episodic) {
+            const text = lines.slice(sec.startLine - 1, sec.endLine).join("\n").trimEnd();
+            units.push({
+                onDemandFile: onDemandFile.name,
+                file: onDemandFile.file,
+                heading: sec.heading,
+                path,
+                kind: "section",
+                startLine: sec.startLine,
+                endLine: sec.endLine,
+                tokens: estimateTokens(text),
+                content: text,
+            });
+            continue;
+        }
+        // One unit per dated entry. Lines before the first entry (an intro sentence) stay with the
+        // section heading as their own unit so nothing is lost.
+        let entryStart = -1;
+        const flush = (endLine) => {
+            if (entryStart < 0)
+                return;
+            const text = lines.slice(entryStart - 1, endLine).join("\n").trimEnd();
+            const heading = entryHeading(lines[entryStart - 1] ?? "");
+            units.push({
+                onDemandFile: onDemandFile.name,
+                file: onDemandFile.file,
+                heading,
+                path: [...path, heading],
+                kind: "entry",
+                startLine: entryStart,
+                endLine,
+                tokens: estimateTokens(text),
+                content: text,
+            });
+            entryStart = -1;
+        };
+        const introEnd = bodyLines.findIndex((l) => RE_DATED_BULLET.test(l));
+        if (introEnd > 0 && bodyLines.slice(0, introEnd).some((l) => l.trim() !== "")) {
+            const text = lines.slice(sec.startLine - 1, bodyStart - 1 + introEnd).join("\n").trimEnd();
+            units.push({
+                onDemandFile: onDemandFile.name,
+                file: onDemandFile.file,
+                heading: sec.heading,
+                path,
+                kind: "section",
+                startLine: sec.startLine,
+                endLine: bodyStart - 1 + introEnd,
+                tokens: estimateTokens(text),
+                content: text,
+            });
+        }
+        for (let i = 0; i < bodyLines.length; i++) {
+            const lineNo = bodyStart + i;
+            if (RE_DATED_BULLET.test(bodyLines[i] ?? "")) {
+                flush(lineNo - 1);
+                entryStart = lineNo;
+            }
+        }
+        flush(sec.endLine);
+    }
+    return units;
+}
+/**
+ * Build the ranking index once per manifest. `triggers` are the editable per-OnDemandMemory-file
+ * trigger words from the manifest; they join every unit of that file at heading weight, so a
+ * person who adds a word to the list steers recall without touching the file text.
+ */
+export function buildSearchIndex(units, triggers = new Map()) {
+    const docs = units.map((u) => {
+        const tf = new Map();
+        const add = (terms, weight) => {
+            for (const t of terms)
+                tf.set(t, (tf.get(t) ?? 0) + weight);
+        };
+        add(tokenize(u.path.join(" ")), HEADING_WEIGHT);
+        add(tokenize((triggers.get(u.onDemandFile) ?? []).join(" ")), HEADING_WEIGHT);
+        // The body without its heading line, which is already counted at heading weight.
+        add(tokenize(u.kind === "section" ? u.content.split("\n").slice(1).join("\n") : u.content), 1);
+        let len = 0;
+        for (const n of tf.values())
+            len += n;
+        return { tf, len };
+    });
+    const df = new Map();
+    for (const d of docs)
+        for (const t of d.tf.keys())
+            df.set(t, (df.get(t) ?? 0) + 1);
+    const avgLen = docs.length === 0 ? 0 : docs.reduce((n, d) => n + d.len, 0) / docs.length;
+    return { units, docs, df, avgLen };
+}
+/**
+ * BM25 over the index. Returns only units with a positive score, best first, ties in document
+ * order, so the same query against the same files ranks the same way on every machine.
+ */
+export function rankUnits(index, query, limit = 20) {
+    const terms = [...new Set(tokenize(query))];
+    if (terms.length === 0 || index.units.length === 0)
+        return [];
+    const n = index.units.length;
+    const scored = [];
+    index.docs.forEach((d, i) => {
+        let score = 0;
+        for (const t of terms) {
+            const f = d.tf.get(t) ?? 0;
+            if (f === 0)
+                continue;
+            const dfT = index.df.get(t) ?? 0;
+            const idf = Math.log(1 + (n - dfT + 0.5) / (dfT + 0.5));
+            const norm = 1 - BM25_B + (BM25_B * d.len) / (index.avgLen || 1);
+            score += (idf * (f * (BM25_K1 + 1))) / (f + BM25_K1 * norm);
+        }
+        if (score > 0)
+            scored.push({ i, score: Math.round(score * 1000) / 1000 });
+    });
+    return scored
+        .sort((a, b) => b.score - a.score || a.i - b.i)
+        .slice(0, limit)
+        .map(({ i, score }) => ({ unit: index.units[i], score }));
+}
+/**
+ * The best units that fit under a token cap, in rank order, dropping the tail that scores under
+ * RECALL_MIN_SCORE_RATIO of the best hit. Always returns the top unit even when it alone
+ * exceeds the cap: an answer that is too long is better than no answer, and the caller sees the
+ * token count and can decide.
+ */
+export function selectUnits(ranked, maxTokens = RECALL_MAX_TOKENS) {
+    const out = [];
+    const floor = (ranked[0]?.score ?? 0) * RECALL_MIN_SCORE_RATIO;
+    let used = 0;
+    for (const r of ranked) {
+        if (out.length > 0 && (r.score < floor || used + r.unit.tokens > maxTokens))
+            break;
+        out.push(r);
+        used += r.unit.tokens;
+    }
+    return out;
+}
+/**
+ * Trigger words for an OnDemandMemory file, chosen against the other OnDemandMemory files
+ * rather than by raw frequency: a term scores by how often it appears here, times how rare it
+ * is across the set, with heading terms weighted. The previous frequency ranking produced
+ * `bars, free, merged` for a trading-app memory file (audit 2026-09-04). Returns surface words,
+ * not stems, so the list stays readable; two words with one stem count once.
+ */
+export function selectTriggers(onDemandFiles, limit = 6) {
+    const surface = new Map();
+    const perFile = onDemandFiles.map((m) => {
+        const tf = new Map();
+        const add = (text, weight) => {
+            const words = text.toLowerCase().match(/[a-z][a-z0-9_-]{2,}/g) ?? [];
+            for (const w of words) {
+                if (STOPWORDS.has(w))
+                    continue;
+                const s = stem(w);
+                if (s.length < 3)
+                    continue;
+                tf.set(s, (tf.get(s) ?? 0) + weight);
+                // Keep the shortest surface form seen for a stem, which is usually the base word.
+                const seen = surface.get(s);
+                if (!seen || w.length < seen.length)
+                    surface.set(s, w);
+            }
+        };
+        add(m.heading, HEADING_WEIGHT);
+        for (const { text, weight } of weightedFragments(m.content))
+            add(text, weight);
+        return tf;
+    });
+    const df = new Map();
+    for (const tf of perFile)
+        for (const t of tf.keys())
+            df.set(t, (df.get(t) ?? 0) + 1);
+    const n = onDemandFiles.length;
+    const out = new Map();
+    onDemandFiles.forEach((m, i) => {
+        const tf = perFile[i];
+        const ranked = [...tf.entries()]
+            .map(([t, f]) => {
+            // Plain idf: a term in every file scores zero and drops out, which is the point. With
+            // one or two files idf is degenerate, so every term counts and headings still lead.
+            const rarity = n >= 3 ? Math.log(n / (df.get(t) ?? 1)) : 1;
+            return { t, score: f * rarity };
+        })
+            .filter((x) => x.score > 0)
+            .sort((a, b) => b.score - a.score || a.t.localeCompare(b.t))
+            .slice(0, limit)
+            .map(({ t }) => surface.get(t) ?? t);
+        out.set(m.name, ranked.length > 0 ? ranked : [m.name.replace(/_/g, " ")]);
+    });
+    return out;
+}
+/**
+ * Deterministic keyword-overlap ranking: one point per trigger whose words all appear as whole
+ * words in the query. Whole words, not substrings, so the trigger "app" does not match "happy"
+ * and "api" does not match "capital". Stable-sorted by score desc, then by manifest order, so the
+ * same query always ranks the same way - reproducibility matters more than ranking
+ * sophistication here. The older trigger-overlap ranker over the manifest alone, kept for
+ * bench.ts's agent-driven-route model (needs no file content, unlike rankUnits above); moved
+ * here from recall.ts (2026-09-13 audit round 2, D7c) to sit beside the rest of the ranking code.
+ */
+export function rankOnDemandFiles(manifest, query) {
+    const words = new Set(query.toLowerCase().match(/[a-z0-9][a-z0-9_-]*/g) ?? []);
+    const matches = (trigger) => trigger
+        .toLowerCase()
+        .split(/\s+/)
+        .filter(Boolean)
+        .every((w) => words.has(w));
+    return manifest.onDemandFiles
+        .map((m, i) => {
+        const score = m.triggers.reduce((n, t) => n + (matches(t) ? 1 : 0), 0);
+        return { name: m.name, file: m.file, kind: m.kind, triggers: m.triggers, tokens: m.tokens, score, _i: i };
+    })
+        .sort((a, b) => b.score - a.score || a._i - b._i)
+        .map(({ _i, ...rest }) => rest);
+}

package/dist/rules.d.ts

@@ -0,0 +1,32 @@
+/**
+ * The MM001-MM010 rule set.
+ *
+ * Every rule is pure: it reads a Workspace and returns Findings. Nothing here touches the
+ * filesystem or the network, which is what makes the whole engine testable from fixtures.
+ */
+import { GRANULARITY_TOKENS, type Block } from "./text.js";
+import { type Finding, type MemoryFile, type Rule } from "./types.js";
+export interface DuplicateGroup {
+    occurrences: {
+        file: MemoryFile;
+        block: Block;
+    }[];
+    wastedTokens: number;
+}
+/**
+ * Blocks repeated verbatim (modulo list-marker/heading-hash/whitespace normalisation) across two
+ * or more files. Shared by MM005 and the multi-file scan (scan.ts) so both see the same picture
+ * of what a reorganize plan should merge.
+ */
+export declare function findDuplicateBlocks(files: MemoryFile[]): DuplicateGroup[];
+/**
+ * The hook is the descriptive text, with the list marker, any leading markdown link, and a
+ * separator dash stripped. That is the part a human writes and the part worth budgeting;
+ * the link is structural overhead the author cannot shorten much.
+ */
+export declare function hookText(line: string): string;
+/** MM008 for one file. Exported so `init` can refuse to copy a credential into new files. */
+export declare function secretFindings(file: MemoryFile): Finding[];
+export { GRANULARITY_TOKENS };
+export declare const RULES: Rule[];
+export declare function ruleById(id: string): Rule | undefined;