@remnic/core 9.7.15 → 9.9.0

package/src/testing/lifecycle-matrix.test.ts

@@ -0,0 +1,212 @@
+/**
+ * Self-test for the lifecycle scenario-matrix harness (issue #1993, PR1).
+ *
+ * Proves the harness contract against a TOY subject without leaving a red test
+ * in the suite: a capturing registrar collects the registered `(name, fn)`
+ * pairs, so we can assert (a) exactly nine rows register with row-identifying
+ * names, (b) a healthy toy subject's rows all pass, (c) a deliberately
+ * sabotaged row REJECTS when run and its failure names the row — and only that
+ * row — and (d) `appliesTo` skips register honestly instead of silently
+ * passing.
+ */
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  MATRIX_ROWS,
+  type LifecycleSubject,
+  type MatrixRow,
+  type MatrixRowId,
+  lifecycleTestName,
+  runLifecycleMatrix,
+} from "./lifecycle-matrix.js";
+
+interface Captured {
+  readonly tests: Array<{ name: string; fn: () => void | Promise<void> }>;
+  readonly skipped: Array<{ name: string; reason: string }>;
+}
+
+function capture(): Captured & {
+  register: (name: string, fn: () => void | Promise<void>) => void;
+  registerSkipped: (name: string, reason: string) => void;
+} {
+  const tests: Captured["tests"] = [];
+  const skipped: Captured["skipped"] = [];
+  return {
+    tests,
+    skipped,
+    register: (name, fn) => {
+      tests.push({ name, fn });
+    },
+    registerSkipped: (name, reason) => {
+      skipped.push({ name, reason });
+    },
+  };
+}
+
+/** In-memory toy subject. `sabotageRow` makes exactly one row's invariants throw. */
+function toySubject(sabotageRow?: MatrixRowId): LifecycleSubject<{ seen: string[] }> {
+  return {
+    async setup() {
+      return { seen: [] };
+    },
+    async exercise(state, row) {
+      state.seen.push(row.id);
+    },
+    async invariants(state, row) {
+      assert.equal(state.seen[0], row.id, "exercise must run before invariants");
+      if (row.id === sabotageRow) {
+        throw new Error(`sabotaged invariant for row ${row.id}`);
+      }
+    },
+    async teardown(state) {
+      state.seen.length = 0;
+    },
+  };
+}
+
+test("runLifecycleMatrix registers one named test per canonical row", () => {
+  const cap = capture();
+  runLifecycleMatrix("toy-healthy", toySubject(), {
+    register: cap.register,
+    registerSkipped: cap.registerSkipped,
+  });
+
+  assert.equal(cap.tests.length, MATRIX_ROWS.length);
+  assert.equal(cap.tests.length, 9, "AGENTS.md defines exactly nine session/retrieval/cache rows");
+  assert.equal(cap.skipped.length, 0);
+
+  for (const row of MATRIX_ROWS) {
+    const expected = lifecycleTestName("toy-healthy", row);
+    const match = cap.tests.find((t) => t.name === expected);
+    assert.ok(match, `row ${row.id} must register a test named ${expected}`);
+    assert.match(match.name, new RegExp(row.id), "the test name must carry the row identity");
+  }
+});
+
+test("a healthy toy subject passes every row", async () => {
+  const cap = capture();
+  runLifecycleMatrix("toy-healthy", toySubject(), {
+    register: cap.register,
+    registerSkipped: cap.registerSkipped,
+  });
+  for (const t of cap.tests) {
+    await t.fn();
+  }
+});
+
+test("a sabotaged row rejects and names the row — and ONLY that row", async () => {
+  const sabotaged: MatrixRowId = "before-reset";
+  const cap = capture();
+  runLifecycleMatrix("toy-sabotaged", toySubject(sabotaged), {
+    register: cap.register,
+    registerSkipped: cap.registerSkipped,
+  });
+
+  const sabotagedTest = cap.tests.find((t) => t.name.includes(sabotaged));
+  assert.ok(sabotagedTest, "the sabotaged row must still register");
+  await assert.rejects(
+    async () => sabotagedTest.fn(),
+    (err: unknown) => err instanceof Error && err.message.includes(sabotaged),
+    "the sabotaged row must reject with the row identity in its failure",
+  );
+
+  const others = cap.tests.filter((t) => !t.name.includes(sabotaged));
+  assert.equal(others.length, 8);
+  for (const t of others) {
+    await t.fn();
+  }
+});
+
+test("appliesTo=false registers an honest skip, never a silent pass", () => {
+  const cap = capture();
+  const subject: LifecycleSubject<{ seen: string[] }> = {
+    ...toySubject(),
+    appliesTo(row: MatrixRow) {
+      return row.id === "compaction-flush" ? "no compaction surface in this toy" : true;
+    },
+  };
+  runLifecycleMatrix("toy-partial", subject, {
+    register: cap.register,
+    registerSkipped: cap.registerSkipped,
+  });
+
+  assert.equal(cap.tests.length, 8);
+  assert.equal(cap.skipped.length, 1);
+  assert.match(cap.skipped[0].name, /compaction-flush/);
+  assert.equal(cap.skipped[0].reason, "no compaction surface in this toy");
+});
+
+test("teardown runs when a post-setup phase throws, and is skipped (no crash) when setup throws", async () => {
+  // (a) exercise throws AFTER a successful setup → teardown MUST still run.
+  let toreDown = 0;
+  const exerciseThrows: LifecycleSubject<{ id: string }> = {
+    async setup() {
+      return { id: "fixture" };
+    },
+    async exercise() {
+      throw new Error("exercise boom");
+    },
+    async invariants() {},
+    async teardown(state) {
+      assert.equal(state.id, "fixture", "teardown receives the fixture setup produced");
+      toreDown += 1;
+    },
+  };
+  const cap1 = capture();
+  runLifecycleMatrix("toy-exercise-throws", exerciseThrows, {
+    register: cap1.register,
+    registerSkipped: cap1.registerSkipped,
+    rows: [MATRIX_ROWS[0]],
+  });
+  await assert.rejects(async () => cap1.tests[0].fn(), /exercise boom/);
+  assert.equal(toreDown, 1, "teardown must run even when a post-setup phase throws");
+
+  // (b) setup throws → teardown MUST NOT be called for a fixture that never existed.
+  let toreDownAfterSetupFail = 0;
+  const setupThrows: LifecycleSubject<{ id: string }> = {
+    async setup() {
+      throw new Error("setup boom");
+    },
+    async exercise() {},
+    async invariants() {},
+    async teardown() {
+      toreDownAfterSetupFail += 1;
+    },
+  };
+  const cap2 = capture();
+  runLifecycleMatrix("toy-setup-throws", setupThrows, {
+    register: cap2.register,
+    registerSkipped: cap2.registerSkipped,
+    rows: [MATRIX_ROWS[0]],
+  });
+  await assert.rejects(async () => cap2.tests[0].fn(), /setup boom/);
+  assert.equal(toreDownAfterSetupFail, 0, "teardown must not run on a fixture setup never produced");
+});
+
+test("teardown runs for a void/undefined-fixture subject that completed setup", async () => {
+  // A subject whose fixture type is void (does setup work but returns nothing)
+  // must still be torn down — the sentinel is setup COMPLETION, not the value.
+  let toreDown = 0;
+  const voidSubject: LifecycleSubject<void> = {
+    async setup() {
+      // setup work with no returned handle (e.g. process-global state).
+    },
+    async exercise() {
+      throw new Error("exercise boom");
+    },
+    async invariants() {},
+    async teardown() {
+      toreDown += 1;
+    },
+  };
+  const cap = capture();
+  runLifecycleMatrix("toy-void", voidSubject, {
+    register: cap.register,
+    registerSkipped: cap.registerSkipped,
+    rows: [MATRIX_ROWS[0]],
+  });
+  await assert.rejects(async () => cap.tests[0].fn(), /exercise boom/);
+  assert.equal(toreDown, 1, "a void-fixture subject that completed setup must still be torn down");
+});

package/src/testing/lifecycle-matrix.ts

@@ -0,0 +1,271 @@
+/**
+ * Lifecycle scenario-matrix harness (issue #1993, umbrella #1988 phase 5).
+ *
+ * AGENTS.md mandates a scenario matrix for stateful subsystem changes
+ * (session/retrieval/cache work): the nine canonical rows a reviewer will
+ * otherwise probe by hand — provider identity, sparse metadata ± remembered
+ * binding, provider rebinding, restart/reload recovery, compaction flush,
+ * `before_reset`, `session_end`, and dedupe/replay. Historically that matrix
+ * lived only as prose (see the "Why Stateful PRs Churn" section) and nobody
+ * instantiated it, so PR #1923 burned 53 review threads on exactly those
+ * adjacent invariants.
+ *
+ * This module makes the matrix EXECUTABLE. A subsystem adapter implements
+ * `LifecycleSubject<S>`; `runLifecycleMatrix` registers one named test per
+ * `MATRIX_ROWS` entry. Adding a row here is a one-file change that
+ * automatically extends every adopter — the whole point of the harness.
+ *
+ * Test-support only: this file lives under `src/testing/` (a subdirectory, so
+ * tsup — which entry-points only top-level `src/*.ts` and the public exports —
+ * never bundles it) and is intentionally absent from package.json `exports`.
+ * It imports `node:test` and must NEVER be imported by runtime code.
+ */
+
+import test from "node:test";
+
+/** The nine canonical AGENTS.md session/retrieval/cache matrix rows. */
+export type MatrixRowId =
+  | "explicit-provider-identity"
+  | "sparse-metadata-with-binding"
+  | "sparse-metadata-without-binding"
+  | "provider-rebinding"
+  | "restart-reload-recovery"
+  | "compaction-flush"
+  | "before-reset"
+  | "session-end"
+  | "dedupe-replay";
+
+/**
+ * Typed scenario axes a subject can branch on instead of re-parsing the row
+ * id. These encode the real dimensions the AGENTS.md matrix probes.
+ */
+export interface MatrixRowDimensions {
+  /** How the session's provider/identity binding is established. */
+  readonly providerIdentity: "explicit" | "sparse" | "rebound" | "none";
+  /** Whether a previously remembered binding is available to reuse. */
+  readonly rememberedBinding: boolean;
+  /** Whether the scenario exercises a fresh instance over persisted state. */
+  readonly restart: boolean;
+  /** The forced-flush lifecycle transition under test, if any. */
+  readonly flush: "before_reset" | "session_end" | "compaction" | "none";
+  /** Whether the scenario replays/duplicates content through the dedupe path. */
+  readonly dedupeOrReplay: boolean;
+}
+
+export interface MatrixRow {
+  readonly id: MatrixRowId;
+  /** Human-readable row title (matches the AGENTS.md bullet). */
+  readonly title: string;
+  readonly dimensions: MatrixRowDimensions;
+}
+
+/**
+ * The canonical matrix. Order and membership mirror AGENTS.md
+ * "Minimum scenario matrix for session/retrieval/cache work". Frozen so an
+ * adopter cannot mutate the shared set.
+ */
+export const MATRIX_ROWS: readonly MatrixRow[] = Object.freeze([
+  {
+    id: "explicit-provider-identity",
+    title: "explicit provider identity",
+    dimensions: {
+      providerIdentity: "explicit",
+      rememberedBinding: false,
+      restart: false,
+      flush: "none",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "sparse-metadata-with-binding",
+    title: "sparse metadata with remembered binding",
+    dimensions: {
+      providerIdentity: "sparse",
+      rememberedBinding: true,
+      restart: false,
+      flush: "none",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "sparse-metadata-without-binding",
+    title: "sparse metadata without remembered binding",
+    dimensions: {
+      providerIdentity: "sparse",
+      rememberedBinding: false,
+      restart: false,
+      flush: "none",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "provider-rebinding",
+    title: "provider rebinding",
+    dimensions: {
+      providerIdentity: "rebound",
+      rememberedBinding: true,
+      restart: false,
+      flush: "none",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "restart-reload-recovery",
+    title: "restart/reload recovery",
+    dimensions: {
+      providerIdentity: "explicit",
+      rememberedBinding: true,
+      restart: true,
+      flush: "none",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "compaction-flush",
+    title: "compaction flush",
+    dimensions: {
+      providerIdentity: "explicit",
+      rememberedBinding: false,
+      restart: false,
+      flush: "compaction",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "before-reset",
+    title: "before_reset",
+    dimensions: {
+      providerIdentity: "explicit",
+      rememberedBinding: false,
+      restart: false,
+      flush: "before_reset",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "session-end",
+    title: "session_end",
+    dimensions: {
+      providerIdentity: "explicit",
+      rememberedBinding: false,
+      restart: false,
+      flush: "session_end",
+      dedupeOrReplay: false,
+    },
+  },
+  {
+    id: "dedupe-replay",
+    title: "dedupe/replay behavior",
+    dimensions: {
+      providerIdentity: "explicit",
+      rememberedBinding: false,
+      restart: false,
+      flush: "none",
+      dedupeOrReplay: true,
+    },
+  },
+] as const);
+
+/**
+ * A subsystem adapter. Each row runs `setup → exercise → invariants`. `setup`
+ * is transactional — it cleans up its own partial state if it throws — and once
+ * it returns a fixture, `teardown` always runs (even when `exercise` or
+ * `invariants` throws), so a leaked temp dir or orchestrator cannot poison a
+ * sibling row.
+ *
+ * A row a subject cannot honestly realize must NOT be faked: return a skip
+ * reason from `appliesTo` and the harness registers it as an explicit skipped
+ * test (a mock-only pass is a finding about the code, not the harness — #1993).
+ */
+export interface LifecycleSubject<S> {
+  /** Build the row-specific fixture; MUST clean up its own partial state if it throws. */
+  setup(row: MatrixRow): Promise<S>;
+  /** Drive the behavior under test. */
+  exercise(subject: S, row: MatrixRow): Promise<void>;
+  /** Row-specific assertions against observable effects. */
+  invariants(subject: S, row: MatrixRow): Promise<void>;
+  /** Always-run cleanup. */
+  teardown(subject: S, row: MatrixRow): Promise<void>;
+  /**
+   * Optional: return a non-empty string to SKIP a row with that reason, or
+   * `false`. Omitted / `true` means the row applies. A skip is honest coverage
+   * documentation, not a silent pass.
+   */
+  appliesTo?(row: MatrixRow): boolean | string;
+}
+
+/** Minimal test registrar seam (defaults to `node:test`), injectable for self-test. */
+export type MatrixTestRegistrar = (
+  name: string,
+  fn: () => void | Promise<void>,
+) => void;
+
+export type MatrixSkipRegistrar = (name: string, reason: string) => void;
+
+export interface RunLifecycleMatrixOptions {
+  /** Override the test registrar (self-test injects a capturing fake). */
+  readonly register?: MatrixTestRegistrar;
+  /** Override the skip registrar. */
+  readonly registerSkipped?: MatrixSkipRegistrar;
+  /** Override the row set (self-test only; production adopters use MATRIX_ROWS). */
+  readonly rows?: readonly MatrixRow[];
+}
+
+/** Deterministic, greppable test name — the coverage manifest and CI depend on it. */
+export function lifecycleTestName(subjectName: string, row: MatrixRow): string {
+  return `lifecycle-matrix[${subjectName}] :: ${row.id} — ${row.title}`;
+}
+
+const defaultRegister: MatrixTestRegistrar = (name, fn) => {
+  test(name, fn);
+};
+
+const defaultRegisterSkipped: MatrixSkipRegistrar = (name, reason) => {
+  test(name, { skip: reason }, () => {});
+};
+
+/**
+ * Register one test per matrix row for `subject`, named with the row identity
+ * so a failing row is unambiguous in CI output. Runs `setup → exercise →
+ * invariants`; `teardown` runs whenever `setup` produced a fixture.
+ */
+export function runLifecycleMatrix<S>(
+  subjectName: string,
+  subject: LifecycleSubject<S>,
+  options: RunLifecycleMatrixOptions = {},
+): void {
+  const register = options.register ?? defaultRegister;
+  const registerSkipped = options.registerSkipped ?? defaultRegisterSkipped;
+  const rows = options.rows ?? MATRIX_ROWS;
+
+  for (const row of rows) {
+    const name = lifecycleTestName(subjectName, row);
+    const applicability = subject.appliesTo ? subject.appliesTo(row) : true;
+    if (applicability !== true) {
+      const reason =
+        typeof applicability === "string" && applicability.length > 0
+          ? applicability
+          : "not applicable to this subsystem";
+      registerSkipped(name, reason);
+      continue;
+    }
+
+    register(name, async () => {
+      let built: S | undefined;
+      let setupDone = false;
+      try {
+        built = await subject.setup(row);
+        setupDone = true;
+        await subject.exercise(built, row);
+        await subject.invariants(built, row);
+      } finally {
+        // Teardown runs whenever setup COMPLETED — even when a subject's fixture
+        // type is `void`/undefined — and is skipped only when setup itself threw
+        // (a transactional setup cleans up its own partial state on throw). The
+        // sentinel is a completion flag, NOT the fixture value, so a stateless or
+        // void subject that did setup work is still torn down.
+        if (setupDone) await subject.teardown(built as S, row);
+      }
+    });
+  }
+}