@fonlan/dsh-gitmemo 0.4.0 → 0.5.0

package/lib/mem.js

@@ -34,9 +34,20 @@ export const LOCK_FILE_NAME = ".mem.gitmemo.lock";
 export const MIGRATION_JOURNAL_NAME = ".mem.gitmemo-migration.json";
 /** Max entry content size. */
 export const MAX_CONTENT_BYTES = 1024 * 1024;
+/** Upper bound on stored entry keywords (mem_write / mem_replace). */
+export const MAX_WRITE_KEYWORDS = 12;
+/**
+ * Upper bound on query keywords for one mem_search call. Deliberately higher
+ * than the write bound: recall is a fixed-string OR over the query terms, so a
+ * wider (still human-curated) query only widens recall, while every matched
+ * term still contributes to the score and the write side keeps entries tight.
+ */
+export const MAX_SEARCH_KEYWORDS = 15;
 /** Error type thrown by every gitmemo operation. */
 export class GitMemoError extends Error {
 }
+/** Accepted `searchScoring` values, for config validation and tool schemas. */
+export const SEARCH_SCORING_MODES = ["count", "weighted"];
 const UNSAFE_CONTROL_RE = /[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/;
 const TRAILER_INJECT_RE = /^\s*GitMemo-[A-Za-z0-9-]+:/im;
 const COMMIT_RE = /^[0-9a-f]{40}$/;
@@ -256,8 +267,8 @@ export function validateWriteInput(input, legacy = false) {
         throw new GitMemoError("gitmemo: keywords must be an array");
     const rawKeywords = input.keywords.map((k) => (typeof k === "string" ? k.trim() : "")).filter((k) => k.length > 0);
     const minKeywords = legacy ? 0 : 2;
-    if (rawKeywords.length < minKeywords || rawKeywords.length > 12) {
-        throw new GitMemoError(`gitmemo: keywords must contain ${minKeywords === 0 ? "0-12" : "2-12"} entries, got ${rawKeywords.length}`);
+    if (rawKeywords.length < minKeywords || rawKeywords.length > MAX_WRITE_KEYWORDS) {
+        throw new GitMemoError(`gitmemo: keywords must contain ${minKeywords === 0 ? "0" : "2"}-${MAX_WRITE_KEYWORDS} entries, got ${rawKeywords.length}`);
     }
     const keywords = [];
     const seen = new Set();
@@ -271,7 +282,7 @@ export function validateWriteInput(input, legacy = false) {
         keywords.push(kw);
     }
     if (keywords.length < minKeywords) {
-        throw new GitMemoError(`gitmemo: keywords must contain ${minKeywords === 0 ? "0-12" : "2-12"} distinct entries after normalization, got ${keywords.length}`);
+        throw new GitMemoError(`gitmemo: keywords must contain ${minKeywords === 0 ? "0" : "2"}-${MAX_WRITE_KEYWORDS} distinct entries after normalization, got ${keywords.length}`);
     }
     let related_branches = [];
     if (input.related_branches !== undefined) {
@@ -495,6 +506,23 @@ export function recencyBonus(committerTime, nowMs = Date.now()) {
         return 0;
     return Math.round(((0.5 * (180 - ageDays)) / 150) * 100) / 100;
 }
+/**
+ * Rarity weight of a keyword matched by `df` of `n` candidates, in the
+ * BM25-style form already used across this project. A term matched by every
+ * candidate is worth almost nothing; a term matched by one candidate out of
+ * many is worth the most.
+ *
+ * Used by `searchScoring: "weighted"` so a distinctive match can outrank a
+ * pile of common ones. The weight is computed over the *recall page*, which
+ * costs no extra git call (the candidate commit messages are already in hand)
+ * and is the pool the ordering actually has to get right.
+ */
+export function keywordRarity(df, n) {
+    if (n <= 0)
+        return 0;
+    const present = Math.min(Math.max(df, 0), n);
+    return Math.log(1 + (n - present + 0.5) / (present + 0.5));
+}
 /** Commit trailer emitted only for topic pages (absence means "task"). */
 const KIND_TRAILER_TOPIC = "GitMemo-Kind: topic";
 /** ADD commit message (plan 4.1). */
@@ -1248,10 +1276,15 @@ export class GitMemo {
             }
             return value;
         };
+        const scoring = config.searchScoring ?? "weighted";
+        if (!SEARCH_SCORING_MODES.includes(scoring)) {
+            throw new GitMemoError(`gitmemo: searchScoring must be one of ${SEARCH_SCORING_MODES.join(" | ")}, got ${JSON.stringify(config.searchScoring)}`);
+        }
         this.config = {
             searchLimit: positiveInt(config.searchLimit, 20, "searchLimit"),
             gitTimeoutMs: positiveInt(config.gitTimeoutMs, 60000, "gitTimeoutMs"),
-            lockTimeoutMs: positiveInt(config.lockTimeoutMs, 30000, "lockTimeoutMs")
+            lockTimeoutMs: positiveInt(config.lockTimeoutMs, 30000, "lockTimeoutMs"),
+            searchScoring: scoring
         };
         this.memDir = join(this.root, ".mem");
     }
@@ -1299,7 +1332,7 @@ export class GitMemo {
      * messages only, active-entry filtering, field-level matching and scoring,
      * snapshot-stable pagination, in-process LRU.
      *
-     * @param keywords - 1-12 keywords (legacy callers may pass a comma-separated string).
+     * @param keywords - 1-15 keywords (legacy callers may pass a comma-separated string).
      * @param options.skip - pagination offset.
      * @param options.snapshot - pass back the snapshot from the first page.
      */
@@ -1312,7 +1345,7 @@ export class GitMemo {
                 .map((k) => k.trim())
                 .filter((k) => k.length > 0)
                 .map((k) => normalizeKeyword(k));
-            adapterWarning = "gitmemo: deprecation — mem_search.keywords as a comma-separated string; pass an array of 1-12 keywords";
+            adapterWarning = `gitmemo: deprecation — mem_search.keywords as a comma-separated string; pass an array of 1-${MAX_SEARCH_KEYWORDS} keywords`;
         }
         else {
             if (!Array.isArray(keywords))
@@ -1324,8 +1357,8 @@ export class GitMemo {
             adapterWarning = (adapterWarning ? adapterWarning + " " : "") +
                 "gitmemo: deprecation — mem_search.mode is gone; recall is always OR";
         }
-        if (normalized.length < 1 || normalized.length > 12) {
-            throw new GitMemoError("gitmemo: search requires 1-12 non-empty keywords");
+        if (normalized.length < 1 || normalized.length > MAX_SEARCH_KEYWORDS) {
+            throw new GitMemoError(`gitmemo: search requires 1-${MAX_SEARCH_KEYWORDS} non-empty keywords`);
         }
         const skip = options.skip ?? 0;
         if (!Number.isInteger(skip) || skip < 0)
@@ -1348,7 +1381,11 @@ export class GitMemo {
                 : undefined;
             return { ...resolved, blockedNote: note };
         });
-        const cacheKey = this.memDir + "\x00" + snapshot + "\x00" + normalized.join("\x00");
+        // The scoring mode is part of the key: two handles on the SAME repo with
+        // different `searchScoring` must not share a cached ranking. (Omitting it
+        // made an A/B of the two modes return identical results — the second
+        // handle simply read the first one's cached computation.)
+        const cacheKey = this.memDir + "\x00" + snapshot + "\x00" + this.config.searchScoring + "\x00" + normalized.join("\x00");
         let computation = searchCache.get(cacheKey);
         if (computation === undefined) {
             computation = await this.computeSearch(snapshot, normalized, legacy);
@@ -1383,6 +1420,15 @@ export class GitMemo {
         const diagnostics = [...(legacySnapshot?.diagnostics ?? [])];
         const seenFiles = new Set();
         const nowMs = Date.now();
+        // Two passes: collect the matching candidates first, then (in weighted
+        // mode) score each match by how rare its keyword is *within this recall
+        // page*. Saturation-free counting treats a distinctive term like
+        // "biscuit" exactly like a ubiquitous one, which is what the LongMemEval
+        // measurements pinned as the engine's ranking defect: evidence was almost
+        // always reachable (recall_all@50 = 0.998) but ranked poorly
+        // (ndcg_any@5 = 0.686). Rarity re-weights the same matches without
+        // changing which entries match at all, so recall is untouched by design.
+        const scored = [];
         for (const record of candidates) {
             if (record.added.length !== 1) {
                 if (record.added.length > 1)
@@ -1409,6 +1455,24 @@ export class GitMemo {
             }
             if (matched.length === 0)
                 continue;
+            scored.push({ record, matched });
+        }
+        let weights;
+        if (this.config.searchScoring === "weighted" && scored.length > 0) {
+            weights = new Map();
+            for (const kw of kws) {
+                let df = 0;
+                for (const entry of scored) {
+                    if (entry.matched.includes(kw))
+                        df += 1;
+                }
+                weights.set(kw, keywordRarity(df, scored.length));
+            }
+        }
+        for (const { record, matched } of scored) {
+            const base = weights === undefined
+                ? matched.length
+                : matched.reduce((sum, kw) => sum + (weights.get(kw) ?? 0), 0);
             hits.push({
                 hash: record.hash,
                 title: record.message.subject,
@@ -1416,7 +1480,7 @@ export class GitMemo {
                 summary: record.message.summary,
                 keywords: record.message.keywords,
                 kind: record.message.kind,
-                score: Math.round((matched.length + recencyBonus(record.committerTime, nowMs)) * 100) / 100,
+                score: Math.round((base + recencyBonus(record.committerTime, nowMs)) * 100) / 100,
                 matched_keywords: matched
             });
         }

package/lib/systemone.d.ts

@@ -0,0 +1,250 @@
+/**
+ * dsh-gitmemo — System-one recall gate.
+ *
+ * Optional narrowing stage for `mem_search`. After the engine produces a page
+ * of lexical candidates, a System-one model (TypeSafe Jev by default, or any
+ * endpoint speaking the same request/answer contract) is asked one typed
+ * question per candidate, in a single request. Candidates the model judges
+ * irrelevant are dropped from the page the root agent sees, which cuts the
+ * tokens injected into the conversation.
+ *
+ * Design rules (deliberate, see README):
+ *  - FAIL-OPEN: the gate may only ever REMOVE noise. Any transport, HTTP,
+ *    timeout, or parse failure keeps every candidate and reports `degraded`,
+ *    so a broken or unconfigured endpoint can never hide a memory.
+ *  - NEVER EMPTY: when the gate would drop every candidate on a page, the
+ *    `minKeep` best-ranked candidates are retained. A filter that silently
+ *    returns nothing is indistinguishable from "no memory exists", which is
+ *    the one failure mode a recall path must not have.
+ *  - AUDITABLE: per-candidate values and the keep/drop verdict are returned in
+ *    the tool output's `gated` block, so the decision can be inspected rather
+ *    than trusted.
+ *
+ * This module imports nothing from the harness and performs no I/O of its own
+ * beyond `fetch`, so it is unit-testable with a fake fetch implementation.
+ *
+ * @module dsh-gitmemo/systemone
+ */
+/** Question types this gate can drive. */
+export type SystemOneMode = "noul" | "score";
+/** One candidate memory offered to the gate. */
+export interface GateCandidate {
+    /** Commit hash — the identity used to keep/drop the candidate. */
+    hash: string;
+    /** Entry title (commit subject). */
+    title: string;
+    /** Entry summary (commit body). */
+    summary: string;
+    /** Structured keywords stored with the entry. */
+    keywords: string[];
+    /** Entry kind: `task` or `topic`. */
+    kind: string;
+}
+/** Resolved gate settings for one search. */
+export interface GateOptions {
+    /** Absolute endpoint URL accepting the System-one request contract. */
+    endpoint: string;
+    /** Model identifier sent in the request body (e.g. `jev-latest`). */
+    model: string;
+    /** Question type used to judge each candidate. */
+    mode: SystemOneMode;
+    /** noul mode: keep a candidate when its probability is >= this value (0..1). */
+    threshold: number;
+    /** score mode: keep a candidate when its score is >= this value. */
+    scoreMin: number;
+    /** Retain at least this many candidates when the model rejects the whole page. */
+    minKeep: number;
+    /** What the caller does with the values; see {@link GatePolicy}. */
+    policy?: GatePolicy;
+    /**
+     * `filter` policy only: never drop more than this fraction of the judged page,
+     * lowest-scoring first. Bounds the damage a mis-calibrated threshold can do —
+     * an offline sweep kept recall_all@5 at 0.8875 with a 20 % cap, against 0.6875
+     * uncapped. Default 0.25.
+     */
+    maxDropFraction?: number;
+    /** Never send more than this many candidates in one request. */
+    maxCandidates: number;
+    /** Truncate the task text to this many characters. */
+    maxTaskChars: number;
+    /** End-to-end request deadline in milliseconds. */
+    timeoutMs: number;
+    /** Literal API key, when one is configured in the settings section. */
+    apiKey?: string;
+    /** Late-bound credential lookup, used when no literal key is configured. */
+    resolveApiKey?: () => Promise<string | undefined>;
+    /** Injectable fetch, for tests. */
+    fetchImpl?: typeof fetch;
+}
+/**
+ * One candidate's verdict, retained for auditability.
+ *
+ * `value` is `null` when no readable answer came back for that candidate;
+ * `null` rather than `NaN` because the harness rejects non-finite numbers in
+ * session events and tool output must stay losslessly JSON-serializable.
+ */
+export interface GateVerdict {
+    hash: string;
+    /** Probability (noul) or score (score) returned for this candidate, or null. */
+    value: number | null;
+    /** Whether the candidate survived the gate. */
+    keep: boolean;
+}
+/** Token accounting reported by the endpoint, when it reports any. */
+export interface GateUsage {
+    /** Input tokens billed for the request (Jev bills input only). */
+    inputTokens?: number;
+    /** Output tokens; free on Jev, but reported. */
+    outputTokens?: number;
+}
+/** Outcome of one gate evaluation. */
+export interface GateDecision {
+    /** Hashes retained, in their original order. */
+    kept: string[];
+    /** Hashes removed, in their original order. */
+    dropped: string[];
+    /** Per-candidate verdicts, in candidate order. */
+    verdicts: GateVerdict[];
+    /** Policy actually applied; see {@link GatePolicy}. */
+    policy: GatePolicy;
+    /** True when `maxDropFraction` restored candidates a threshold would have removed. */
+    bounded: boolean;
+    /**
+     * Candidate hashes ordered by the judge's value, best first (nulls last and
+     * kept stable). This is what the `rerank` policy applies; in `filter` mode it
+     * is still reported so a caller can log or inspect the judge's ordering.
+     */
+    ordered: string[];
+    /** Question type actually used. */
+    mode: SystemOneMode;
+    /** Model identifier the request named. */
+    model: string;
+    /** True when the gate failed open (every candidate retained as a result). */
+    degraded: boolean;
+    /**
+     * Why the gate degraded — or, on a `false` {@link degraded} page, that it
+     * floored: no candidate cleared the threshold and `minKeep` was applied.
+     */
+    reason?: string;
+    /**
+     * Candidates actually sent to the endpoint, i.e. `min(maxCandidates, page
+     * size)`. Reported separately from the page size so a truncated judgement is
+     * never mistaken for a full one.
+     */
+    judged: number;
+    /**
+     * Candidates past `maxCandidates`, which were never evaluated and therefore
+     * can never be dropped. Always `0` unless the page exceeded the cap.
+     */
+    untouched: number;
+    /** Endpoint-reported token usage, so the gate's own cost is observable. */
+    usage?: GateUsage;
+}
+/** Score levels offered to the model in `score` mode (ordered low to high). */
+export declare const SCORE_LEVELS: readonly ["unrelated to the task", "same general area but not actionable for this task", "partially relevant background", "substantially relevant and worth reusing", "directly reusable conclusions for this exact task"];
+/**
+ * The judgement asked of the model, verbatim. Kept in English on purpose:
+ * Jev's primary training language is English and the vendor documents lower
+ * accuracy for other scripts, so only the (possibly CJK) candidate text and
+ * task text are non-English.
+ */
+export declare const NOUL_INSTRUCTION = "Does the `candidate` memory record contain conclusions, decisions, constraints, or reusable findings that are directly relevant to the `task`? Answer yes only if reading that memory would inform or change how the task is carried out. Answer no when it is merely on a related topic, is superseded, or concerns a different area.";
+/**
+ * Clarifies what yes and no mean for {@link NOUL_INSTRUCTION}.
+ *
+ * A noul `criteria` is an object with exactly `true` and `false` keys — a bare
+ * string is rejected by the endpoint's validation, so this shape is required.
+ */
+export declare const NOUL_CRITERIA: {
+    readonly true: "yes — the memory is directly reusable for this task";
+    readonly false: "no — unrelated, superseded, or only loosely related to this task";
+};
+/** The judgement asked of the model in `score` mode. */
+export declare const SCORE_INSTRUCTION = "How relevant is the `candidate` memory record to the `task`? Rate how reusable its conclusions, decisions, constraints, or findings are for this exact task.";
+/**
+ * What the caller does with the judge's values.
+ *
+ * - `rerank` (default): keep every judged candidate and let the values reorder
+ *   the page. Cannot lose a memory by construction.
+ * - `filter`: drop candidates below the threshold, then apply `maxDropFraction`.
+ *
+ * Measured on LongMemEval `_S` (500 questions, identical ingested memories): at
+ * threshold 0.5 the filter held 343 gold memories out of the prompt across 254
+ * instances and took LLM-as-Judge from 0.7840 to 0.3800, while the same values
+ * used as a reranker scored 0.8020 on the same context. The judge's AUC is only
+ * ~0.71-0.75, so no threshold can make a hard drop safe; the same signal is a
+ * fine ordering feature. See evals/longmemeval/README.md.
+ */
+export type GatePolicy = "rerank" | "filter";
+/** Accepted `policy` values, for config validation and tool schemas. */
+export declare const GATE_POLICIES: readonly GatePolicy[];
+/**
+ * Build the System-one request body: the task is the shared `state`, and each
+ * candidate travels inside its own question's structured `instructions`, so
+ * every candidate is judged against the same task in one request and no
+ * candidate payload is repeated across questions.
+ *
+ * @param task - the agent's current task, used as the shared state.
+ * @param candidates - candidates to judge, already capped by the caller.
+ * @param options - resolved gate settings.
+ * @returns a JSON-serializable request body.
+ */
+export declare function buildRequest(task: string, candidates: readonly GateCandidate[], options: Pick<GateOptions, "mode" | "model" | "maxTaskChars">): {
+    state: {
+        task: string;
+    };
+    model: string;
+    questions: Record<string, unknown>;
+};
+/**
+ * Extract one numeric answer per question id from a System-one response.
+ *
+ * Tolerates both documented envelope shapes (a top-level map of answers, or
+ * the same map nested under `answers`) and both `noul`/`score` field spellings,
+ * so an endpoint that is merely contract-compatible still works.
+ *
+ * @param payload - parsed response JSON.
+ * @param mode - the question type that was asked.
+ * @param count - number of questions sent (ids are `m0`..`m<count-1>`).
+ * @returns the value per question index, or undefined where absent/unreadable.
+ */
+export declare function parseAnswers(payload: unknown, mode: SystemOneMode, count: number): Array<number | undefined>;
+/**
+ * Read the endpoint's token accounting.
+ *
+ * @param payload - parsed response JSON.
+ * @returns camel-cased usage, or undefined when the endpoint reported none.
+ */
+export declare function parseUsage(payload: unknown): GateUsage | undefined;
+/**
+ * Turn per-candidate values into a keep/drop decision.
+ *
+ * Applies the configured threshold, then the `minKeep` floor that guarantees a
+ * non-empty page whenever candidates existed. Candidates with a missing or
+ * unreadable answer are KEPT (fail-open at the per-candidate level).
+ *
+ * @param candidates - candidates in page order.
+ * @param values - value per candidate index (undefined when unreadable).
+ * @param options - resolved gate settings.
+ * @returns kept/dropped hash lists and per-candidate verdicts.
+ */
+export declare function decide(candidates: readonly GateCandidate[], values: ReadonlyArray<number | undefined>, options: Pick<GateOptions, "mode" | "threshold" | "scoreMin" | "minKeep" | "policy" | "maxDropFraction">): {
+    kept: string[];
+    dropped: string[];
+    verdicts: GateVerdict[];
+    floored: boolean;
+    bounded: boolean;
+};
+/**
+ * Evaluate the gate for one search page.
+ *
+ * Never throws and never returns an empty `kept` list while candidates exist:
+ * every failure path degrades to "keep everything" with a reason.
+ *
+ * @param task - the agent's current task text (shared state).
+ * @param candidates - page candidates; only the first `maxCandidates` are judged.
+ * @param options - resolved gate settings.
+ * @param signal - caller cancellation.
+ * @returns the decision to apply to the page.
+ */
+export declare function evaluateGate(task: string, candidates: readonly GateCandidate[], options: GateOptions, signal?: AbortSignal): Promise<GateDecision>;