@remnic/core 1.1.12 → 1.1.14

package/src/memory-worth-bench.test.ts

@@ -0,0 +1,71 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import { runMemoryWorthBench } from "./memory-worth-bench.js";
+
+/**
+ * Issue #560 PR 5 — bench verifies the filter wins against a no-filter
+ * baseline on synthetic cases. If this test ever fails, do NOT ship the
+ * default flip — something changed in the scorer math.
+ */
+
+test("memory-worth bench: filter improves precision@5 on every seed case", () => {
+  const result = runMemoryWorthBench({ cases: 50, seed: 0xdeadbeef });
+  // Sanity.
+  assert.equal(result.cases, 50);
+  assert.equal(result.k, 5);
+  // The synthetic corpus is designed so the "true positives" only reach the
+  // top 5 when the filter is applied — filter-off precision should be 0,
+  // filter-on should land near the 3/5 ceiling (all three gold items
+  // promoted). Pinning the boundary both ways catches scorer regressions.
+  assert.equal(result.precisionAtK_off, 0, "baseline must leave zero gold in top 5");
+  assert.ok(
+    result.precisionAtK_on >= 0.58,
+    `filter-on precision ${result.precisionAtK_on} should be ≥ 0.58`,
+  );
+  assert.ok(
+    result.delta > 0.5,
+    `delta ${result.delta} must be a substantial lift, not a tie`,
+  );
+  assert.equal(
+    result.filterWinsOrTies,
+    true,
+    "filter-on must win or tie on every case in the seeded set",
+  );
+});
+
+test("memory-worth bench: result object shape is stable", () => {
+  // If a new field lands on MemoryWorthBenchResult, the default-flip
+  // decision should consciously re-evaluate — pin the shape here so that
+  // review catches the change.
+  const result = runMemoryWorthBench({ cases: 5 });
+  const keys = Object.keys(result).sort();
+  assert.deepEqual(keys, [
+    "cases",
+    "delta",
+    "filterWinsOrTies",
+    "k",
+    "precisionAtK_off",
+    "precisionAtK_on",
+  ]);
+});
+
+test("memory-worth bench: rejects non-positive-integer case counts", () => {
+  // Codex P2: 0 would divide by zero and return NaN; fractional inputs
+  // inflate the average. Both would produce misleading default-flip
+  // justification — fail loudly instead.
+  assert.throws(() => runMemoryWorthBench({ cases: 0 }), /positive integer/);
+  assert.throws(() => runMemoryWorthBench({ cases: -5 }), /positive integer/);
+  assert.throws(() => runMemoryWorthBench({ cases: 2.5 }), /positive integer/);
+  assert.throws(() => runMemoryWorthBench({ cases: Number.NaN }), /positive integer/);
+  assert.throws(
+    () => runMemoryWorthBench({ cases: Number.POSITIVE_INFINITY }),
+    /positive integer/,
+  );
+});
+
+test("memory-worth bench: deterministic under fixed seed", () => {
+  const a = runMemoryWorthBench({ cases: 20, seed: 42 });
+  const b = runMemoryWorthBench({ cases: 20, seed: 42 });
+  assert.deepEqual(a, b, "identical seed + cases must produce identical results");
+});

package/src/memory-worth-bench.ts

@@ -0,0 +1,265 @@
+/**
+ * Issue #560 PR 5 — Memory Worth recall filter benchmark.
+ *
+ * Self-contained precision benchmark for `applyMemoryWorthFilter`. Seeds a
+ * synthetic corpus where a small subset of memories have known-bad outcome
+ * history, then compares top-K precision with the filter off vs. on.
+ *
+ * Why a dedicated in-package file (rather than the full `@remnic/bench`
+ * harness): the filter is a pure function over candidate scores and counter
+ * data; it doesn't need QMD, the orchestrator, or the schema-tier fixtures.
+ * Running it as a plain `tsx` script keeps the signal tight — any drift in
+ * the scorer's math shows up as a precision delta here, no integration
+ * wiring required.
+ *
+ * The `runMemoryWorthBench()` export is the programmatic entry point;
+ * `runMemoryWorthBenchCli()` is what `tsx` calls when this file is executed
+ * directly. Both return (or print) a structured result so CI can gate on
+ * it if we later want to.
+ *
+ * Verdict for PR 5: run the bench once, confirm filter-on ≥ filter-off on
+ * precision@K across every seed, and only then flip the default to `true`.
+ */
+
+import {
+  applyMemoryWorthFilter,
+  type MemoryWorthCounters,
+} from "./memory-worth-filter.js";
+
+/**
+ * One synthetic query + candidate pool + relevance labels.
+ *
+ * Candidates are scored by the pretend retrieval tier (`baseScore`); the
+ * ground-truth relevance is `isRelevant`; `counters` seeds each candidate's
+ * outcome history. Some "bad" candidates have baseline scores just above
+ * the "good" candidates — the filter should be able to demote them.
+ */
+interface BenchCase {
+  id: string;
+  candidates: {
+    path: string;
+    baseScore: number;
+    isRelevant: boolean;
+    counters?: MemoryWorthCounters;
+  }[];
+  /** Top-K used for precision@K. */
+  k: number;
+}
+
+/**
+ * Deterministic pseudo-random number generator (mulberry32) so the bench
+ * produces identical results across runs, making precision changes easy to
+ * attribute to code rather than seed drift.
+ */
+function mulberry32(seed: number): () => number {
+  let t = seed >>> 0;
+  return () => {
+    t = (t + 0x6d2b79f5) >>> 0;
+    let r = t;
+    r = Math.imul(r ^ (r >>> 15), r | 1);
+    r ^= r + Math.imul(r ^ (r >>> 7), r | 61);
+    return ((r ^ (r >>> 14)) >>> 0) / 4294967296;
+  };
+}
+
+/**
+ * Generate a synthetic test case.
+ *
+ * Corpus design:
+ *   - 20 candidates per case.
+ *   - Top 5 by base score include 2 "traps" — memories that the retriever
+ *     ranks highly but whose outcome history is 10/0 failures. These should
+ *     be sunk by the filter.
+ *   - 3 lower-ranked memories are genuinely relevant with 10/0 success
+ *     history. The filter should promote them into the top 5.
+ *   - Remaining 15 are noise at the neutral prior (no counter data).
+ *
+ * `k = 5`, so the ideal precision@K is 3/5 = 0.6 (three genuinely relevant
+ * items, after filter promotion). Without the filter, precision@5 is 0/5
+ * because the top 5 by base score are the 2 traps + 3 irrelevant neutral
+ * items.
+ */
+function buildCase(caseIndex: number, rng: () => number): BenchCase {
+  const candidates: BenchCase["candidates"] = [];
+  // 2 high-ranked traps: high base score, bad outcome history, NOT relevant.
+  for (let i = 0; i < 2; i += 1) {
+    candidates.push({
+      path: `case-${caseIndex}-trap-${i}.md`,
+      baseScore: 0.95 - i * 0.02,
+      isRelevant: false,
+      counters: { mw_success: 0, mw_fail: 10 },
+    });
+  }
+  // 3 high-ranked irrelevant neutral items.
+  for (let i = 0; i < 3; i += 1) {
+    candidates.push({
+      path: `case-${caseIndex}-noise-high-${i}.md`,
+      baseScore: 0.9 - i * 0.02,
+      isRelevant: false,
+    });
+  }
+  // 3 lower-ranked TRUE POSITIVES with strong success history — the filter
+  // must float these into the top 5.
+  for (let i = 0; i < 3; i += 1) {
+    candidates.push({
+      path: `case-${caseIndex}-gold-${i}.md`,
+      baseScore: 0.7 - i * 0.05,
+      isRelevant: true,
+      counters: { mw_success: 10, mw_fail: 0 },
+    });
+  }
+  // 12 irrelevant noise candidates at random lower scores.
+  for (let i = 0; i < 12; i += 1) {
+    candidates.push({
+      path: `case-${caseIndex}-noise-low-${i}.md`,
+      baseScore: rng() * 0.5,
+      isRelevant: false,
+    });
+  }
+  // Shuffle to remove any input-order bias.
+  for (let i = candidates.length - 1; i > 0; i -= 1) {
+    const j = Math.floor(rng() * (i + 1));
+    [candidates[i], candidates[j]] = [candidates[j]!, candidates[i]!];
+  }
+  return { id: `case-${caseIndex}`, candidates, k: 5 };
+}
+
+function computePrecisionAtK(
+  orderedPaths: string[],
+  relevant: Set<string>,
+  k: number,
+): number {
+  const topK = orderedPaths.slice(0, k);
+  if (topK.length === 0) return 0;
+  let hits = 0;
+  for (const p of topK) if (relevant.has(p)) hits += 1;
+  return hits / Math.min(k, topK.length);
+}
+
+export interface MemoryWorthBenchResult {
+  cases: number;
+  k: number;
+  /** Mean precision@K with the filter disabled. */
+  precisionAtK_off: number;
+  /** Mean precision@K with the filter enabled. */
+  precisionAtK_on: number;
+  /** `on - off`; positive means filter helps, zero means tied. */
+  delta: number;
+  /** Filter-on wins at least as often as it loses, case-by-case. */
+  filterWinsOrTies: boolean;
+}
+
+/**
+ * Run the benchmark over N synthetic cases using a fixed PRNG seed. Returns
+ * aggregate precision numbers + a boolean verdict.
+ */
+export function runMemoryWorthBench(options?: {
+  cases?: number;
+  seed?: number;
+  now?: Date;
+}): MemoryWorthBenchResult {
+  const requestedCases = options?.cases ?? 50;
+  // Reject non-positive-integer case counts. Passing 0 would divide by zero
+  // and produce a NaN precision that the `filterWinsOrTies` boolean would
+  // still mark as `true` — dangerously misleading since this result is used
+  // to justify the default flip. Fractional values would inflate precision
+  // because the loop rounds up (Array.from ceil) but the average divides
+  // by the fractional input.
+  if (
+    !Number.isFinite(requestedCases) ||
+    !Number.isInteger(requestedCases) ||
+    requestedCases < 1
+  ) {
+    throw new Error(
+      `runMemoryWorthBench: cases must be a positive integer; got ${requestedCases}`,
+    );
+  }
+  const numCases = requestedCases;
+  const rng = mulberry32(options?.seed ?? 0xdeadbeef);
+  const now = options?.now ?? new Date("2026-01-01T00:00:00.000Z");
+
+  let sumOff = 0;
+  let sumOn = 0;
+  let onWinsOrTies = 0;
+
+  for (let i = 0; i < numCases; i += 1) {
+    const c = buildCase(i, rng);
+    const relevant = new Set(
+      c.candidates.filter((x) => x.isRelevant).map((x) => x.path),
+    );
+
+    // Filter OFF: sort by baseScore descending.
+    const off = [...c.candidates]
+      .sort((a, b) => b.baseScore - a.baseScore)
+      .map((x) => x.path);
+    const pOff = computePrecisionAtK(off, relevant, c.k);
+
+    // Filter ON: build counter map and apply the filter.
+    const counters = new Map<string, MemoryWorthCounters>();
+    for (const cand of c.candidates) {
+      if (cand.counters) counters.set(cand.path, cand.counters);
+    }
+    const filtered = applyMemoryWorthFilter(
+      c.candidates.map((x) => ({ path: x.path, score: x.baseScore })),
+      { counters, now },
+    );
+    const on = filtered.map((x) => x.path);
+    const pOn = computePrecisionAtK(on, relevant, c.k);
+
+    sumOff += pOff;
+    sumOn += pOn;
+    if (pOn >= pOff) onWinsOrTies += 1;
+  }
+
+  const avgOff = sumOff / numCases;
+  const avgOn = sumOn / numCases;
+  return {
+    cases: numCases,
+    k: 5,
+    precisionAtK_off: avgOff,
+    precisionAtK_on: avgOn,
+    delta: avgOn - avgOff,
+    filterWinsOrTies: onWinsOrTies === numCases,
+  };
+}
+
+/**
+ * CLI entry point — run the bench and print a structured result. Exits
+ * non-zero if the filter ever loses to the no-filter baseline (so CI can
+ * gate on this in the future).
+ */
+export function runMemoryWorthBenchCli(): void {
+  const result = runMemoryWorthBench();
+  // eslint-disable-next-line no-console
+  console.log(JSON.stringify(result, null, 2));
+  if (!result.filterWinsOrTies) {
+    // eslint-disable-next-line no-console
+    console.error("memory-worth bench: filter lost to no-filter baseline on at least one case");
+    process.exit(1);
+  }
+}
+
+// When this file is invoked directly (e.g. `tsx memory-worth-bench.ts`),
+// run the CLI. Use url.pathToFileURL to produce a normalized file:// URL
+// from `process.argv[1]` — that handles Windows drive letters,
+// URL-encoded characters (spaces, etc.), and symlinked entrypoints. A
+// naïve `file://${process.argv[1]}` comparison fails on all three and
+// would silently skip runMemoryWorthBenchCli() on those platforms.
+if (process.argv[1]) {
+  try {
+    // Lazy import so this file can still be loaded in environments that
+    // don't have node:url (browsers, Deno test runners in some modes).
+    const { pathToFileURL } = await import("node:url");
+    if (import.meta.url === pathToFileURL(process.argv[1]).href) {
+      runMemoryWorthBenchCli();
+    }
+  } catch {
+    // If the normalization fails for any reason, fall back to the naive
+    // comparison. Worse case: the bench doesn't auto-run; callers can
+    // always invoke `runMemoryWorthBenchCli()` or `runMemoryWorthBench()`
+    // explicitly.
+    if (import.meta.url === `file://${process.argv[1]}`) {
+      runMemoryWorthBenchCli();
+    }
+  }
+}

package/src/memory-worth-filter.test.ts

@@ -0,0 +1,209 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  applyMemoryWorthFilter,
+  buildMemoryWorthCounterMap,
+  type MemoryWorthCounters,
+} from "./memory-worth-filter.js";
+
+/**
+ * Issue #560 PR 4 — recall filter unit tests.
+ *
+ * These pin the behavior the orchestrator relies on when the feature flag
+ * is enabled:
+ *   - Uninstrumented memories are untouched (multiplier 1.0).
+ *   - Memories with many failures sink in the ranking.
+ *   - Memories with many successes rise in the ranking.
+ *   - Stable ordering among ties, including the neutral-prior tie.
+ *   - `buildMemoryWorthCounterMap` only indexes instrumented memories.
+ */
+
+const NOW = new Date("2026-01-01T00:00:00.000Z");
+
+test("uninstrumented memories pass through with multiplier 1.0", () => {
+  const out = applyMemoryWorthFilter(
+    [
+      { path: "a.md", score: 10 },
+      { path: "b.md", score: 5 },
+      { path: "c.md", score: 1 },
+    ],
+    {
+      counters: new Map<string, MemoryWorthCounters>(),
+      now: NOW,
+    },
+  );
+  // All multipliers are 1.0 and scores are unchanged; order preserved.
+  for (const item of out) {
+    assert.equal(item.multiplier, 1);
+    assert.equal(item.score, item.originalScore);
+  }
+  assert.deepEqual(
+    out.map((o) => o.path),
+    ["a.md", "b.md", "c.md"],
+  );
+});
+
+test("memory with heavy failure history sinks below a neutral peer", () => {
+  const counters = new Map<string, MemoryWorthCounters>([
+    // 0 successes, 10 failures. Laplace score ≈ 1/12 ≈ 0.083.
+    // Multiplier ≈ 0.083 / 0.5 ≈ 0.167. Scaled score ≈ 10 * 0.167 = 1.67.
+    ["bad.md", { mw_success: 0, mw_fail: 10 }],
+  ]);
+  const out = applyMemoryWorthFilter(
+    [
+      { path: "bad.md", score: 10 },
+      { path: "neutral.md", score: 5 },
+    ],
+    { counters, now: NOW },
+  );
+  assert.equal(out[0]!.path, "neutral.md", "neutral peer must outrank the bad memory");
+  const bad = out.find((o) => o.path === "bad.md")!;
+  assert.ok(bad.multiplier < 0.5, `bad multiplier ${bad.multiplier} should be < 0.5`);
+  assert.ok(bad.score < 5, `bad scaled score ${bad.score} should be < the neutral 5`);
+});
+
+test("memory with heavy success history rises above a neutral peer", () => {
+  const counters = new Map<string, MemoryWorthCounters>([
+    // 10 successes, 0 failures. Laplace score ≈ 11/12 ≈ 0.917.
+    // Multiplier ≈ 0.917 / 0.5 ≈ 1.833.
+    ["good.md", { mw_success: 10, mw_fail: 0 }],
+  ]);
+  const out = applyMemoryWorthFilter(
+    [
+      { path: "neutral.md", score: 5 },
+      { path: "good.md", score: 3 },
+    ],
+    { counters, now: NOW },
+  );
+  // good.md was ranked second by raw score but should move up once its
+  // history kicks in (3 * ~1.83 ≈ 5.5 > 5).
+  assert.equal(out[0]!.path, "good.md");
+  const good = out.find((o) => o.path === "good.md")!;
+  assert.ok(good.multiplier > 1.5);
+});
+
+test("ordering is stable for ties on the neutral prior", () => {
+  const out = applyMemoryWorthFilter(
+    [
+      { path: "a.md", score: 5 },
+      { path: "b.md", score: 5 },
+      { path: "c.md", score: 5 },
+    ],
+    { counters: new Map<string, MemoryWorthCounters>(), now: NOW },
+  );
+  assert.deepEqual(
+    out.map((o) => o.path),
+    ["a.md", "b.md", "c.md"],
+    "ties must preserve input order (stable sort, CLAUDE.md rule 19)",
+  );
+});
+
+test("reorder:false preserves original order but still applies multiplier", () => {
+  const counters = new Map<string, MemoryWorthCounters>([
+    ["good.md", { mw_success: 10, mw_fail: 0 }],
+  ]);
+  const out = applyMemoryWorthFilter(
+    [
+      { path: "neutral.md", score: 5 },
+      { path: "good.md", score: 3 },
+    ],
+    { counters, now: NOW, reorder: false },
+  );
+  // Order preserved — good.md stays second even though its scaled score
+  // is higher.
+  assert.deepEqual(
+    out.map((o) => o.path),
+    ["neutral.md", "good.md"],
+  );
+  // But the score field still reflects the multiplier.
+  assert.ok(out[1]!.score > out[1]!.originalScore);
+});
+
+test("multiplier is exactly 1.0 for a memory with no counter data", () => {
+  const out = applyMemoryWorthFilter(
+    [{ path: "x.md", score: 7.25 }],
+    {
+      // Entry exists in the map but with empty counters (e.g. after
+      // decay wiped effective counts). Should still hit neutral.
+      counters: new Map<string, MemoryWorthCounters>([["x.md", {}]]),
+      now: NOW,
+    },
+  );
+  assert.equal(out[0]!.multiplier, 1);
+  assert.equal(out[0]!.score, 7.25);
+});
+
+test("worth block is populated for observability", () => {
+  const counters = new Map<string, MemoryWorthCounters>([
+    ["x.md", { mw_success: 3, mw_fail: 1 }],
+  ]);
+  const out = applyMemoryWorthFilter(
+    [{ path: "x.md", score: 10 }],
+    { counters, now: NOW },
+  );
+  const item = out[0]!;
+  assert.ok(item.worth.score > 0 && item.worth.score < 1);
+  assert.equal(item.worth.p_success, item.worth.score);
+  assert.equal(item.worth.confidence, 4);
+});
+
+test("decay is applied when halfLifeMs is supplied", () => {
+  const dayMs = 24 * 60 * 60 * 1000;
+  const tenDaysAgo = new Date(NOW.getTime() - 10 * dayMs).toISOString();
+  const counters = new Map<string, MemoryWorthCounters>([
+    ["stale.md", { mw_success: 10, mw_fail: 0, lastAccessed: tenDaysAgo }],
+  ]);
+  // With a 1-day half-life, 10-day-old outcomes decay by 2^-10 ≈ 1/1024,
+  // dragging the score back to ~prior and the multiplier back to ~1.
+  const decayed = applyMemoryWorthFilter(
+    [{ path: "stale.md", score: 5 }],
+    { counters, now: NOW, halfLifeMs: dayMs },
+  );
+  // And without decay, the same 10/0 counts boost the score.
+  const fresh = applyMemoryWorthFilter(
+    [{ path: "stale.md", score: 5 }],
+    { counters, now: NOW },
+  );
+  assert.ok(
+    decayed[0]!.multiplier < fresh[0]!.multiplier,
+    `decay should reduce multiplier (decayed=${decayed[0]!.multiplier}, fresh=${fresh[0]!.multiplier})`,
+  );
+  assert.ok(Math.abs(decayed[0]!.multiplier - 1) < 0.05, "extremely decayed ⇒ near-prior");
+});
+
+test("buildMemoryWorthCounterMap only indexes memories with counter data", () => {
+  const map = buildMemoryWorthCounterMap([
+    { path: "a.md", frontmatter: { mw_success: 1, mw_fail: 0 } },
+    { path: "b.md", frontmatter: {} }, // legacy, no counters
+    { path: "c.md", frontmatter: { mw_success: undefined, mw_fail: 2 } },
+  ]);
+  assert.equal(map.size, 2, "legacy memories must not bloat the map");
+  assert.ok(map.has("a.md"));
+  assert.ok(map.has("c.md"));
+  assert.ok(!map.has("b.md"));
+});
+
+test("negative candidate scores do not invert filter direction", () => {
+  // Codex P2: if upstream penalties push a score below zero, the
+  // multiplier must not flip sign. A high-worth memory (multiplier > 1)
+  // must NOT move further negative (worse rank), and a low-worth memory
+  // (multiplier < 1) must NOT move toward zero (better rank). Clamping
+  // base at 0 before multiplying preserves "failure-prone sinks".
+  const counters = new Map<string, MemoryWorthCounters>([
+    ["good.md", { mw_success: 10, mw_fail: 0 }],
+    ["bad.md", { mw_success: 0, mw_fail: 10 }],
+  ]);
+  const out = applyMemoryWorthFilter(
+    [
+      { path: "good.md", score: -1 },
+      { path: "bad.md", score: -2 },
+      { path: "neutral.md", score: -3 },
+    ],
+    { counters, now: NOW, reorder: false },
+  );
+  // All scaled scores must be non-negative (bases clamped to 0).
+  for (const item of out) {
+    assert.ok(item.score >= 0, `${item.path} score should be >= 0, got ${item.score}`);
+  }
+});