@remnic/core 1.1.12 → 1.1.13

package/src/memory-worth-outcomes.test.ts

@@ -0,0 +1,316 @@
+import test from "node:test";
+import assert from "node:assert/strict";
+import os from "node:os";
+import path from "node:path";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+
+import { StorageManager } from "./storage.js";
+import {
+  recordMemoryOutcome,
+  memoryWorthOutcomeEligibleCategories,
+} from "./memory-worth-outcomes.js";
+
+/**
+ * Issue #560 PR 3 — tests for the outcome signal pipeline.
+ *
+ * These tests pin the contract `recordMemoryOutcome` exposes to callers:
+ *   - Successful increments land on the right counter and preserve
+ *     unrelated frontmatter fields.
+ *   - Legacy memories (no counters yet) start at 1 / 0 after the first
+ *     success, not at some dedup-polluted value.
+ *   - Non-eligible categories are rejected as ineligible, not silently
+ *     written to.
+ *   - Invalid paths / IDs / outcomes return an `{ok: false}` result, not a
+ *     thrown exception, so a ledger drainer can aggregate failure reasons.
+ */
+
+async function writeFactFile(
+  storage: StorageManager,
+  body: string,
+  extraFrontmatterLines: string[] = [],
+): Promise<{ id: string; filePath: string }> {
+  const today = new Date().toISOString().slice(0, 10);
+  const id = `fact-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+  const lines = [
+    "---",
+    `id: ${id}`,
+    "category: fact",
+    `created: ${new Date().toISOString()}`,
+    `updated: ${new Date().toISOString()}`,
+    "source: extraction",
+    "confidence: 0.8",
+    "confidenceTier: high",
+    "tags: []",
+    ...extraFrontmatterLines,
+    "---",
+  ];
+  const factsDir = path.join((storage as unknown as { baseDir: string }).baseDir, "facts", today);
+  await mkdir(factsDir, { recursive: true });
+  const filePath = path.join(factsDir, `${id}.md`);
+  await writeFile(filePath, `${lines.join("\n")}\n\n${body}\n`, "utf-8");
+  return { id, filePath };
+}
+
+async function writeCorrectionFile(
+  storage: StorageManager,
+  body: string,
+): Promise<{ id: string; filePath: string }> {
+  const id = `correction-${Date.now()}-${Math.random().toString(36).slice(2)}`;
+  const lines = [
+    "---",
+    `id: ${id}`,
+    "category: correction",
+    `created: ${new Date().toISOString()}`,
+    `updated: ${new Date().toISOString()}`,
+    "source: extraction",
+    "confidence: 0.8",
+    "confidenceTier: high",
+    "tags: []",
+    "---",
+  ];
+  const correctionsDir = path.join(
+    (storage as unknown as { baseDir: string }).baseDir,
+    "corrections",
+  );
+  await mkdir(correctionsDir, { recursive: true });
+  const filePath = path.join(correctionsDir, `${id}.md`);
+  await writeFile(filePath, `${lines.join("\n")}\n\n${body}\n`, "utf-8");
+  return { id, filePath };
+}
+
+test("success on a legacy memory increments mw_success from implicit 0", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-success-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const { id, filePath } = await writeFactFile(storage, "Legacy fact pre-dating instrumentation.");
+
+    const result = await recordMemoryOutcome(storage, {
+      memoryPath: filePath,
+      outcome: "success",
+    });
+
+    assert.equal(result.ok, true);
+    if (result.ok) {
+      assert.equal(result.memoryId, id);
+      assert.equal(result.mw_success, 1);
+      assert.equal(result.mw_fail, 0);
+    }
+
+    const after = await storage.getMemoryById(id);
+    assert.ok(after);
+    assert.equal(after!.frontmatter.mw_success, 1);
+    assert.equal(after!.frontmatter.mw_fail, 0);
+    // Unrelated fields must survive the mutation.
+    assert.equal(after!.frontmatter.category, "fact");
+    assert.equal(after!.frontmatter.confidence, 0.8);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("failure on a legacy memory increments mw_fail from implicit 0", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-failure-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const { id, filePath } = await writeFactFile(storage, "Legacy fact destined for a failure.");
+
+    const result = await recordMemoryOutcome(storage, {
+      memoryPath: filePath,
+      outcome: "failure",
+    });
+
+    assert.equal(result.ok, true);
+    if (result.ok) {
+      assert.equal(result.mw_success, 0);
+      assert.equal(result.mw_fail, 1);
+    }
+
+    const after = await storage.getMemoryById(id);
+    assert.equal(after!.frontmatter.mw_success, 0);
+    assert.equal(after!.frontmatter.mw_fail, 1);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("consecutive outcomes accumulate independently", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-accum-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const { id, filePath } = await writeFactFile(storage, "Accumulating outcomes.", [
+      "mw_success: 2",
+      "mw_fail: 0",
+    ]);
+
+    await recordMemoryOutcome(storage, { memoryPath: filePath, outcome: "success" });
+    await recordMemoryOutcome(storage, { memoryPath: filePath, outcome: "failure" });
+    const final = await recordMemoryOutcome(storage, {
+      memoryPath: filePath,
+      outcome: "failure",
+    });
+
+    assert.equal(final.ok, true);
+    if (final.ok) {
+      assert.equal(final.mw_success, 3);
+      assert.equal(final.mw_fail, 2);
+    }
+
+    const after = await storage.getMemoryById(id);
+    assert.equal(after!.frontmatter.mw_success, 3);
+    assert.equal(after!.frontmatter.mw_fail, 2);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("non-fact category returns ineligible_category, does NOT write", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-ineligible-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const { id, filePath } = await writeCorrectionFile(storage, "A correction, not a fact.");
+
+    const result = await recordMemoryOutcome(storage, {
+      memoryPath: filePath,
+      outcome: "success",
+    });
+
+    assert.equal(result.ok, false);
+    if (!result.ok) {
+      assert.equal(result.reason, "ineligible_category");
+    }
+
+    // The frontmatter must be completely untouched.
+    const after = await storage.getMemoryById(id);
+    assert.ok(after);
+    assert.equal(after!.frontmatter.mw_success, undefined);
+    assert.equal(after!.frontmatter.mw_fail, undefined);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("unknown memory id returns not_found, does not throw", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-missing-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+
+    const result = await recordMemoryOutcome(storage, {
+      memoryPath: path.join(dir, "facts", "2026-01-01", "nonexistent-id.md"),
+      outcome: "success",
+    });
+
+    assert.equal(result.ok, false);
+    if (!result.ok) {
+      assert.equal(result.reason, "not_found");
+    }
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("invalid outcome string returns invalid_outcome", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-bad-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const { filePath } = await writeFactFile(storage, "Fact.");
+
+    const result = await recordMemoryOutcome(storage, {
+      memoryPath: filePath,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      outcome: "maybe" as any,
+    });
+
+    assert.equal(result.ok, false);
+    if (!result.ok) {
+      assert.equal(result.reason, "invalid_outcome");
+    }
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("path without .md suffix returns invalid_path", async () => {
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-badpath-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+
+    const result = await recordMemoryOutcome(storage, {
+      memoryPath: "not-a-memory-file",
+      outcome: "success",
+    });
+
+    assert.equal(result.ok, false);
+    if (!result.ok) {
+      assert.equal(result.reason, "invalid_path");
+    }
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("directory-prefixed path without .md suffix also rejected", async () => {
+  // Bugbot/codex P2: previously, conditional .md check let a deep path
+  // like /tmp/facts/2026-01-01/not-a-memory through (basename strips the
+  // dirs, so `basename === memoryPath` was false and the .md guard was
+  // skipped). With the fix the check runs unconditionally on the raw
+  // input.
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-deep-badpath-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const result = await recordMemoryOutcome(storage, {
+      memoryPath: "/tmp/facts/2026-01-01/not-a-memory",
+      outcome: "success",
+    });
+    assert.equal(result.ok, false);
+    if (!result.ok) assert.equal(result.reason, "invalid_path");
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("concurrent outcomes for the same memory do NOT lose updates", async () => {
+  // Codex P1: two concurrent recordMemoryOutcome calls must each persist
+  // a +1 instead of reading the same snapshot and overwriting. The module
+  // enforces per-ID serialization, so 10 parallel successes must land as
+  // mw_success = 10, not something < 10.
+  const dir = await mkdtemp(path.join(os.tmpdir(), "remnic-mw-outcome-concurrent-"));
+  try {
+    const storage = new StorageManager(dir);
+    await storage.ensureDirectories();
+    const { id, filePath } = await writeFactFile(storage, "Concurrency target.");
+
+    const N = 10;
+    const calls = Array.from({ length: N }, () =>
+      recordMemoryOutcome(storage, { memoryPath: filePath, outcome: "success" }),
+    );
+    const results = await Promise.all(calls);
+
+    // Every call must succeed.
+    for (const r of results) assert.equal(r.ok, true);
+
+    // The persisted counter must reflect every increment, not a subset.
+    const after = await storage.getMemoryById(id);
+    assert.ok(after);
+    assert.equal(after!.frontmatter.mw_success, N);
+    assert.equal(after!.frontmatter.mw_fail, 0);
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+});
+
+test("eligible-category set is exactly {fact}", () => {
+  // Pinning this keeps PR 3 in lockstep with the PR 1 doctor allowlist in
+  // operator-toolkit.ts (MEMORY_WORTH_ELIGIBLE_CATEGORIES). If either side
+  // expands, this test and the doctor test must both be updated.
+  const eligible = memoryWorthOutcomeEligibleCategories();
+  assert.equal(eligible.size, 1);
+  assert.ok(eligible.has("fact"));
+});

package/src/memory-worth-outcomes.ts

@@ -0,0 +1,286 @@
+/**
+ * Issue #560 PR 3 — Memory Worth outcome signal pipeline.
+ *
+ * PR 1 added `mw_success` / `mw_fail` fields to MemoryFrontmatter. PR 2 added
+ * a pure scoring helper. This module adds the one piece tying the two
+ * together: a way for callers to record a single outcome observation against
+ * a memory, which increments the appropriate counter in frontmatter.
+ *
+ * The public entry point is `recordMemoryOutcome({ memoryPath, outcome, ... })`.
+ * Callers pass the full path to the memory file (not just the ID) because in
+ * the usual outcome source — the observation ledger — the memory path is
+ * already captured in the event payload, and path-based lookup avoids a
+ * full-corpus scan.
+ *
+ * Intentional properties:
+ *   - Works on a per-memory basis (no bulk API in this slice). Bulk update is
+ *     an easy layer on top of this once a second caller needs it.
+ *   - Reuses the existing `updateMemoryFrontmatter(id, patch)` write path so
+ *     unrelated fields (confidence, importance, lifecycle hooks, etc.) are
+ *     preserved. The PR 1 serializer rejects negative / non-integer counters,
+ *     so we rely on that for defensive validation rather than duplicating it.
+ *   - Only instruments categories in `MEMORY_WORTH_OUTCOME_ELIGIBLE_CATEGORIES`
+ *     (currently `fact`, matching `MEMORY_WORTH_ELIGIBLE_CATEGORIES` in
+ *     operator-toolkit.ts for the doctor audit). Non-eligible memories return
+ *     `{ ok: false, reason: "ineligible_category" }` rather than throwing so
+ *     the caller — typically a ledger consumer draining heterogeneous events
+ *     — doesn't need to pre-filter by category.
+ *   - Missing / unknown memory IDs return `{ ok: false, reason: "not_found" }`
+ *     rather than throwing, because outcome events may reference memories
+ *     that were archived/deleted between the session and the ledger drain.
+ *     That isn't an operator-actionable error.
+ *   - On success, returns the new counter values so observability surfaces
+ *     can report the increment without a second read.
+ *
+ * Out of scope (later PRs):
+ *   - Recall filter reading the counters (PR 4).
+ *   - Benchmark + default flip (PR 5).
+ *   - Automatic increments from extraction or summarization. Only the
+ *     explicit `MEM_OUTCOME` ledger tag or an MCP tool call drives writes.
+ */
+
+import path from "node:path";
+
+import type { StorageManager } from "./storage.js";
+import type { MemoryFrontmatter } from "./types.js";
+
+/**
+ * Per-memory-ID serialization of the read-modify-write increment.
+ *
+ * Codex P1: without this, concurrent calls for the same memory race. Both
+ * callers read the same `mw_*` values, each writes +1, and one increment
+ * is lost — silently undercounting outcomes. We chain an async promise
+ * per memory ID so read-then-write is atomic with respect to other calls
+ * into this module.
+ *
+ * This serializes only THIS module. Direct writers (e.g. hand-rolled
+ * frontmatter patches) still bypass the lock, but the documented surface
+ * for Memory Worth increments is this function, and the bench only uses
+ * this path.
+ *
+ * The lock map is WeakRef-free by design: the chain self-cleans once the
+ * last pending write resolves (we delete the entry inside the `.finally`
+ * of the tail), so memory use stays bounded even for long-running
+ * processes that touch many different memory IDs.
+ */
+const outcomeLocks = new Map<string, Promise<unknown>>();
+
+function runSerialized<T>(key: string, task: () => Promise<T>): Promise<T> {
+  const prev = outcomeLocks.get(key) ?? Promise.resolve();
+  // Swallow upstream errors so a failed outcome for memory A doesn't
+  // permanently poison the chain for memory A. We still propagate our own
+  // result/errors to the caller.
+  const next = prev.catch(() => undefined).then(task);
+  outcomeLocks.set(key, next);
+  // Chain the cleanup off `next` BUT attach a `.catch` so the cleanup
+  // promise itself never surfaces as an unhandled rejection. The caller
+  // gets `next` back and handles it their own way; this detached cleanup
+  // chain has no observer, so without the .catch Node would crash the
+  // process on a task rejection even when the caller awaited `next` and
+  // handled the error correctly.
+  next
+    .then(
+      () => {
+        if (outcomeLocks.get(key) === next) outcomeLocks.delete(key);
+      },
+      () => {
+        if (outcomeLocks.get(key) === next) outcomeLocks.delete(key);
+      },
+    )
+    .catch(() => {
+      // Defensive no-op in case a hook inside the cleanup branches throws.
+    });
+  return next;
+}
+
+/**
+ * Categories currently instrumented for Memory Worth counters. Must be kept
+ * in sync with `MEMORY_WORTH_ELIGIBLE_CATEGORIES` in `operator-toolkit.ts`.
+ * Declared here (not imported) to avoid a circular dep with operator-toolkit,
+ * which imports from this file's peers. The two constants are validated by a
+ * test to stay in lockstep.
+ */
+const MEMORY_WORTH_OUTCOME_ELIGIBLE_CATEGORIES: ReadonlySet<MemoryFrontmatter["category"]> =
+  new Set(["fact"]);
+
+/**
+ * Exported so downstream tests / operators can query the allowlist without
+ * re-declaring it. Returned as a frozen copy so consumers cannot mutate the
+ * module-internal set.
+ */
+export function memoryWorthOutcomeEligibleCategories(): ReadonlySet<MemoryFrontmatter["category"]> {
+  return new Set(MEMORY_WORTH_OUTCOME_ELIGIBLE_CATEGORIES);
+}
+
+/**
+ * The direction of an outcome — whether the session that consumed this
+ * memory succeeded or failed. Restricted to a string literal union so
+ * callers in TypeScript land can't pass arbitrary tags.
+ */
+export type MemoryOutcomeKind = "success" | "failure";
+
+/**
+ * Arguments to `recordMemoryOutcome`.
+ *
+ * `memoryPath` is the filesystem path to the memory; we derive the ID from
+ * the basename, matching how the operator-toolkit and recall layers map
+ * paths to IDs.
+ */
+export interface RecordMemoryOutcomeInput {
+  /**
+   * Absolute or repo-relative path to the memory file. Typically the value
+   * of `MemoryFile.path` for a memory returned by `readAllMemories`.
+   */
+  memoryPath: string;
+  /** Outcome direction — "success" bumps mw_success; "failure" bumps mw_fail. */
+  outcome: MemoryOutcomeKind;
+  /**
+   * Optional observation timestamp for audit / telemetry. This PR doesn't
+   * persist the timestamp (PR 4/5 will use `lastAccessed`, which already
+   * covers the recency-decay requirement), but accepting it here keeps the
+   * call shape stable so future ledger integrations don't need a breaking
+   * change.
+   */
+  timestamp?: Date | string;
+}
+
+/**
+ * Outcome of a `recordMemoryOutcome` call.
+ *
+ * `ok: true` means the counter was incremented and flushed. The returned
+ * values reflect the post-increment state, so callers can log
+ * `"fact-xyz: 4/1 → 5/1"` without re-reading.
+ *
+ * `ok: false` carries a short machine-readable `reason` so a ledger drainer
+ * can aggregate metrics ("how many events hit not_found this hour?"). The
+ * human-readable `message` is a friendlier version for logs.
+ */
+export type RecordMemoryOutcomeResult =
+  | {
+      ok: true;
+      memoryId: string;
+      /** New value of `mw_success` after the increment. */
+      mw_success: number;
+      /** New value of `mw_fail` after the increment. */
+      mw_fail: number;
+    }
+  | {
+      ok: false;
+      reason:
+        | "not_found"
+        | "ineligible_category"
+        | "invalid_outcome"
+        | "invalid_path";
+      message: string;
+    };
+
+/**
+ * Extract the memory ID from a file path. Memory files are stored as
+ * `<id>.md`, matching the rest of the system (see `getMemoryById`, which
+ * infers paths from IDs).
+ *
+ * Returns `null` for any path that does not end in `.md`. The check is
+ * unconditional — `path.basename` strips directory components before the
+ * suffix check, so previous conditional logic silently accepted
+ * directory-prefixed paths like `/tmp/facts/2026-01-01/not-a-memory`
+ * (the basename `not-a-memory` is not equal to the input, so the `.md`
+ * guard was skipped). The new check looks at the raw input once.
+ */
+function memoryIdFromPath(memoryPath: string): string | null {
+  if (!memoryPath || typeof memoryPath !== "string") return null;
+  if (!memoryPath.endsWith(".md")) return null;
+  const basename = path.basename(memoryPath, ".md");
+  if (!basename) return null;
+  return basename;
+}
+
+/**
+ * Record a single outcome observation against a memory. Increments
+ * `mw_success` or `mw_fail` on the memory's frontmatter (preserving all
+ * other fields) via the existing `updateMemoryFrontmatter` write path.
+ *
+ * See the top-of-file doc comment for policy details (eligible categories,
+ * error semantics, and what is intentionally out of scope).
+ */
+export async function recordMemoryOutcome(
+  storage: StorageManager,
+  input: RecordMemoryOutcomeInput,
+): Promise<RecordMemoryOutcomeResult> {
+  if (input.outcome !== "success" && input.outcome !== "failure") {
+    return {
+      ok: false,
+      reason: "invalid_outcome",
+      message: `outcome must be "success" or "failure"; got ${JSON.stringify(input.outcome)}`,
+    };
+  }
+
+  const memoryId = memoryIdFromPath(input.memoryPath);
+  if (memoryId === null) {
+    return {
+      ok: false,
+      reason: "invalid_path",
+      message: `memoryPath must end in .md; got ${JSON.stringify(input.memoryPath)}`,
+    };
+  }
+
+  // Serialize the read-modify-write per memory ID so concurrent callers
+  // (parallel ledger drain, multiple MCP clients, etc.) can't each read
+  // the same snapshot and race to write `+1`, losing an increment.
+  return runSerialized(memoryId, async () => {
+    const memory = await storage.getMemoryById(memoryId);
+    if (!memory) {
+      return {
+        ok: false,
+        reason: "not_found",
+        message: `no memory with id ${memoryId}`,
+      };
+    }
+
+    if (!MEMORY_WORTH_OUTCOME_ELIGIBLE_CATEGORIES.has(memory.frontmatter.category)) {
+      return {
+        ok: false,
+        reason: "ineligible_category",
+        message: `category ${memory.frontmatter.category} is not instrumented for Memory Worth`,
+      };
+    }
+
+    // Absent counters are treated as 0 (matching the PR 1 semantics — legacy
+    // memories implicitly have Beta(1,1) priors). Increments land on the
+    // actual counter regardless of whether the field was previously set.
+    const currentSuccess = memory.frontmatter.mw_success ?? 0;
+    const currentFail = memory.frontmatter.mw_fail ?? 0;
+
+    const nextSuccess = input.outcome === "success" ? currentSuccess + 1 : currentSuccess;
+    const nextFail = input.outcome === "failure" ? currentFail + 1 : currentFail;
+
+    // `updateMemoryFrontmatter` goes through `writeMemoryFrontmatter`, which
+    // routes through the PR 1 serializer — so a stored corrupt value (e.g., a
+    // pre-existing float) would be rejected here. That is intentional: we
+    // refuse to layer a fresh increment on top of garbage. Operators should
+    // use `remnic doctor` to find and fix those rows before PR 4 ships.
+    //
+    // `updateMemoryFrontmatter` returns `false` when the memory is no
+    // longer present (archived / deleted between `getMemoryById` and the
+    // write). Surface that as `not_found` so callers see an honest result
+    // — silently returning `ok: true` with a counter that never landed
+    // would skew downstream worth scoring.
+    const updated = await storage.updateMemoryFrontmatter(memoryId, {
+      mw_success: nextSuccess,
+      mw_fail: nextFail,
+    });
+    if (!updated) {
+      return {
+        ok: false,
+        reason: "not_found",
+        message: `memory ${memoryId} disappeared between read and write`,
+      };
+    }
+
+    return {
+      ok: true,
+      memoryId,
+      mw_success: nextSuccess,
+      mw_fail: nextFail,
+    };
+  });
+}