@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2

package/src/cli/commands/memory/__tests__/memory-v2.test.ts

@@ -69,6 +69,7 @@ mock.module("../../../../util/logger.js", () => ({
 // ---------------------------------------------------------------------------
 
 const { registerMemoryV2Command } = await import("../memory-v2.js");
+const { registerMemoryValidateCommand } = await import("../memory-validate.js");
 
 // ---------------------------------------------------------------------------
 // Test helpers
@@ -88,6 +89,7 @@ function buildProgram(): Command {
   const memory = program.command("memory");
   applyCommandHelp(memory, memoryHelp);
   registerMemoryV2Command(memory);
+  registerMemoryValidateCommand(memory);
   return program;
 }
 
@@ -287,7 +289,7 @@ describe("memory v2 validate", () => {
       result: {
         pageCount: 0,
         edgeCount: 0,
-        missingEdgeEndpoints: [],
+        danglingLinks: [],
         oversizedPages: [],
         parseFailures: [],
       },
@@ -306,7 +308,7 @@ describe("memory v2 validate", () => {
       result: {
         pageCount: 49,
         edgeCount: 166,
-        missingEdgeEndpoints: [],
+        danglingLinks: [],
         oversizedPages: [],
         parseFailures: [],
       },
@@ -318,7 +320,7 @@ describe("memory v2 validate", () => {
     expect(logOutput.some((line) => line.includes("Pages: 49"))).toBe(true);
     expect(logOutput.some((line) => line.includes("Edges: 166"))).toBe(true);
     expect(
-      logOutput.some((line) => line.includes("Missing edge endpoints: none")),
+      logOutput.some((line) => line.includes("Dangling links: none")),
     ).toBe(true);
     expect(
       logOutput.some((line) => line.includes("Oversized pages: none")),
@@ -334,7 +336,9 @@ describe("memory v2 validate", () => {
       result: {
         pageCount: 10,
         edgeCount: 20,
-        missingEdgeEndpoints: [{ from: "people/alice", to: "people/missing" }],
+        danglingLinks: [
+          { from: "people/alice", to: "people/missing", kind: "links" },
+        ],
         oversizedPages: [{ slug: "arcs/big-day", chars: 12345 }],
         parseFailures: [
           { slug: "people/broken", error: "missing frontmatter" },
@@ -345,9 +349,11 @@ describe("memory v2 validate", () => {
     const { exitCode } = await runCommand(["memory", "v2", "validate"]);
 
     expect(exitCode).toBe(1);
-    expect(logOutput.some((line) => line.includes("people/missing"))).toBe(
-      true,
-    );
+    expect(
+      logOutput.some((line) =>
+        line.includes("people/alice → people/missing (links)"),
+      ),
+    ).toBe(true);
     expect(logOutput.some((line) => line.includes("arcs/big-day"))).toBe(true);
     expect(logOutput.some((line) => line.includes("people/broken"))).toBe(true);
   });
@@ -360,3 +366,47 @@ describe("memory v2 validate", () => {
     expect(exitCode).toBe(1);
   });
 });
+
+describe("memory validate (top-level, same report)", () => {
+  test("sends memory_v2_validate and prints the report", async () => {
+    mockIpcResult = {
+      ok: true,
+      result: {
+        pageCount: 49,
+        edgeCount: 166,
+        danglingLinks: [],
+        oversizedPages: [],
+        parseFailures: [],
+      },
+    };
+
+    const { exitCode } = await runCommand(["memory", "validate"]);
+
+    expect(exitCode).toBe(0);
+    expect(lastIpcCall!.method).toBe("memory_v2_validate");
+    expect(logOutput.some((line) => line.includes("Pages: 49"))).toBe(true);
+    expect(
+      logOutput.some((line) => line.includes("Dangling links: none")),
+    ).toBe(true);
+  });
+
+  test("exits non-zero on violations, like the v2 spelling", async () => {
+    mockIpcResult = {
+      ok: true,
+      result: {
+        pageCount: 1,
+        edgeCount: 0,
+        danglingLinks: [{ from: "a", to: "missing", kind: "wikilink" }],
+        oversizedPages: [],
+        parseFailures: [],
+      },
+    };
+
+    const { exitCode } = await runCommand(["memory", "validate"]);
+
+    expect(exitCode).toBe(1);
+    expect(
+      logOutput.some((line) => line.includes("a → missing (wikilink)")),
+    ).toBe(true);
+  });
+});

package/src/cli/commands/memory/__tests__/memory-v3.test.ts

@@ -2,12 +2,13 @@
  * Tests for the `assistant memory v3` CLI subgroup (section-lane model).
  *
  * Validates:
- *   - Subcommand registration (rebuild-index, backfill-sections) under
- *     `memory v3`.
- *   - Each subcommand maps to the right `cliIpcCall` method.
+ *   - Subcommand registration (rebuild-index, backfill-sections, gate-stats)
+ *     under `memory v3`.
+ *   - `rebuild-index` and `backfill-sections` route through IPC to the daemon.
  *   - `backfill-sections` passes a long IPC timeout (the one-time full-corpus
- *     embed easily outlasts the default 60s), while `rebuild-index` does not.
- *   - IPC error paths return a non-zero exit code without throwing.
+ *     embed easily outlasts the default 60s).
+ *   - `gate-stats` calls handleMemoryV3GateStats directly — no daemon required.
+ *   - Error paths return a non-zero exit code without throwing.
  */
 
 import { beforeEach, describe, expect, mock, test } from "bun:test";
@@ -39,6 +40,47 @@ let mockIpcResult: {
 /** Captured log output for assertion. */
 let logOutput: string[] = [];
 
+/** Shared fake gate-stats payload used across gate-stats tests. */
+const fakeGateStats = {
+  lookbackDays: 30,
+  totalRuns: 10,
+  buckets: [
+    {
+      pageCountRange: "0–9",
+      total: 2,
+      scored: 1,
+      passed: 2,
+      scoredPassRate: 1,
+      reasons: { dense_pass: 1, dense_disabled: 1 },
+    },
+    {
+      pageCountRange: "10–49",
+      total: 5,
+      scored: 4,
+      passed: 3,
+      scoredPassRate: 0.75,
+      reasons: { dense_pass: 3, fail_no_signal: 1 },
+    },
+    {
+      pageCountRange: "50–199",
+      total: 2,
+      scored: 2,
+      passed: 1,
+      scoredPassRate: 0.5,
+      reasons: { dense_pass: 1, fail_no_signal: 1 },
+    },
+    {
+      pageCountRange: "200+",
+      total: 1,
+      scored: 1,
+      passed: 0,
+      scoredPassRate: 0,
+      reasons: { fail_no_signal: 1 },
+    },
+  ],
+  unknownPageCount: { total: 0, passed: 0, reasons: {} },
+};
+
 // ---------------------------------------------------------------------------
 // Mocks
 // ---------------------------------------------------------------------------
@@ -68,6 +110,33 @@ mock.module("../../../../util/logger.js", () => ({
   getCliLogger: () => fakeLogger,
 }));
 
+// ---------------------------------------------------------------------------
+// gate-stats direct-invocation mocks
+// ---------------------------------------------------------------------------
+
+/** Args captured from the last handleMemoryV3GateStats call. */
+let lastGateStatsArgs: { lookbackDays: number; db: unknown } | null = null;
+
+/** What handleMemoryV3GateStats returns, or an error to throw. */
+let gateStatsImpl: (() => unknown) | null = null;
+
+mock.module(
+  "../../../../plugins/defaults/memory/src/memory-v3-routes.js",
+  () => ({
+    handleMemoryV3GateStats: (lookbackDays: number, db: unknown) => {
+      lastGateStatsArgs = { lookbackDays, db };
+      if (gateStatsImpl) {
+        return gateStatsImpl();
+      }
+      return fakeGateStats;
+    },
+  }),
+);
+
+mock.module("../../../../persistence/db-connection.js", () => ({
+  getTelemetrySqlite: () => ({}),
+}));
+
 // ---------------------------------------------------------------------------
 // Import modules under test (after mocks)
 // ---------------------------------------------------------------------------
@@ -135,6 +204,8 @@ beforeEach(() => {
     ok: true,
     result: { articles: 3, sections: 12, failures: 0 },
   };
+  lastGateStatsArgs = null;
+  gateStatsImpl = null;
   logOutput = [];
   process.exitCode = 0;
 });
@@ -155,6 +226,7 @@ describe("subcommand registration", () => {
       "backfill-sections",
       "eval",
       "eval-tally",
+      "gate-stats",
       "rebuild-index",
     ]);
   });
@@ -236,3 +308,54 @@ describe("memory v3 backfill-sections", () => {
     expect(exitCode).toBe(1);
   });
 });
+
+// ---------------------------------------------------------------------------
+// gate-stats (direct invocation — no daemon required)
+// ---------------------------------------------------------------------------
+
+describe("memory v3 gate-stats", () => {
+  test("calls handleMemoryV3GateStats directly with default lookback (no IPC)", async () => {
+    const { exitCode } = await runCommand(["memory", "v3", "gate-stats"]);
+
+    expect(exitCode).toBe(0);
+    // Must NOT have gone through IPC.
+    expect(lastIpcCall).toBeNull();
+    // Must have called the handler directly with the default lookback.
+    expect(lastGateStatsArgs!.lookbackDays).toBe(30);
+  });
+
+  test("forwards --lookback-days directly to handleMemoryV3GateStats", async () => {
+    await runCommand(["memory", "v3", "gate-stats", "--lookback-days", "7"]);
+
+    expect(lastIpcCall).toBeNull();
+    expect(lastGateStatsArgs!.lookbackDays).toBe(7);
+  });
+
+  test("prints formatted table on success (no --json)", async () => {
+    await runCommand(["memory", "v3", "gate-stats"]);
+
+    expect(logOutput.some((l) => l.includes("Gate-stats"))).toBe(true);
+    expect(logOutput.some((l) => l.includes("10–49"))).toBe(true);
+    expect(logOutput.some((l) => l.includes("75.0%"))).toBe(true);
+  });
+
+  test("emits raw JSON with --json", async () => {
+    await runCommand(["memory", "v3", "gate-stats", "--json"]);
+
+    const raw = logOutput.join("");
+    const parsed = JSON.parse(raw) as typeof fakeGateStats;
+    expect(parsed.totalRuns).toBe(10);
+    expect(parsed.buckets).toHaveLength(4);
+  });
+
+  test("exits with code 1 when handleMemoryV3GateStats throws", async () => {
+    gateStatsImpl = () => {
+      throw new Error("telemetry DB locked");
+    };
+
+    const { exitCode } = await runCommand(["memory", "v3", "gate-stats"]);
+
+    expect(exitCode).toBe(1);
+    expect(logOutput.some((l) => l.includes("telemetry DB locked"))).toBe(true);
+  });
+});

package/src/cli/commands/memory/index.help.ts

@@ -23,7 +23,7 @@ The 'items' subgroup exposes full CRUD over individual memory items
 
 The memory subsystem retrieves concept pages two ways: the v2 concept-page
 activation model (prose pages with directed edges) and the v3 section-lane
-model (section-grain lanes cached as live shadow state). Each subgroup exposes
+model (section-grain lanes held in memory inside the assistant). Each subgroup exposes
 operator-facing maintenance verbs — reindexing, backfills, validation, and evals.
 
 Examples:
@@ -34,7 +34,7 @@ Examples:
   $ assistant memory items list --search "coffee"
   $ assistant memory items update 9f2c4f3a-3f1a-41e4-88e7-abc123 --statement "Prefers tea"
   $ assistant memory items delete 9f2c4f3a-3f1a-41e4-88e7-abc123
-  $ assistant memory v2 validate
+  $ assistant memory validate
   $ assistant memory v3 rebuild-index
   $ assistant memory ingest --dir .mv3/staging --dry-run`,
   subcommands: [
@@ -43,8 +43,9 @@ Examples:
       description:
         "Content-based list, delete, and update of memory graph nodes",
       helpText: `
-Memory nodes are raw graph records (content, type, fidelity) produced by the
-memory v2 subsystem. Unlike 'memory items', which addresses nodes by UUID, these
+Memory nodes are raw graph records (content, type, fidelity) in the memory
+graph store, which every memory tier writes to. Unlike 'memory items', which
+addresses nodes by UUID, these
 commands address nodes by content text — matching the way an operator refers to
 a remembered fact without first looking up its ID.
 
@@ -390,7 +391,6 @@ except reembed-skills which runs synchronously inside the assistant.
 Read-only subcommands print diagnostic reports without mutating state.
 
 Examples:
-  $ assistant memory v2 validate
   $ assistant memory v2 reembed
   $ assistant memory v2 reembed-skills
   $ assistant memory v2 activation`,
@@ -400,9 +400,10 @@ Examples:
           description:
             "Refresh dense + sparse vectors for every concept page in Qdrant",
           helpText: `
-Fans out an embed_concept_page job per concept page slug (plus the four
-reserved meta-file slugs) so each page's dense and sparse vectors get
-recomputed against the current embedding backend. Useful after upgrading
+Fans out an embed_concept_page job per concept page slug so each page's
+dense and sparse vectors get recomputed against the current embedding
+backend (the direct-injected memory/*.md meta files are not embedded and are
+not enqueued). Useful after upgrading
 the embedding model or recovering a corrupted Qdrant collection.
 
 The fan-out runs on the background memory worker — this command returns
@@ -449,20 +450,13 @@ Examples:
         {
           name: "validate",
           description:
-            "Print a diagnostic report of v2 workspace state (read-only)",
+            "Same report as 'memory validate' (read-only concept-page check)",
           helpText: `
-Walks the v2 concept-page tree on disk and reports:
-  - Page count
-  - Edge count (total and unique outgoing targets)
-  - Missing outgoing edge targets (orphan edges)
-  - Oversized pages (over the per-folder size cap)
-  - Parse failures (missing or malformed frontmatter)
-
-Read-only — does not mutate the workspace. Exits non-zero if any
-violations are reported.
+Identical to 'assistant memory validate'; prefer that spelling. The report
+covers the concept-page store, which is the same under memory v2 and v3.
 
 Examples:
-  $ assistant memory v2 validate`,
+  $ assistant memory validate`,
         },
         {
           name: "ema",
@@ -600,19 +594,21 @@ Examples:
       name: "v3",
       description: "Memory v3 live-lane maintenance (section-lane model)",
       helpText: `
-The v3 memory subsystem retrieves concept pages over section-grain lanes and
-caches them as live shadow lanes inside the assistant. These commands maintain
-that live state safely.
+The v3 memory subsystem retrieves concept pages over section-grain lanes held
+in memory inside the assistant. These commands maintain that live state
+safely.
 
 Examples:
   $ assistant memory v3 rebuild-index
-  $ assistant memory v3 backfill-sections`,
+  $ assistant memory v3 backfill-sections
+  $ assistant memory v3 gate-stats
+  $ assistant memory v3 gate-stats --lookback-days 7 --json`,
       subcommands: [
         {
           name: "rebuild-index",
           description: "Invalidate the v3 lanes so the next turn rebuilds",
           helpText: `
-Drops the assistant's cached v3 shadow lanes so the section index is rebuilt
+Drops the assistant's in-memory v3 lanes so the section index is rebuilt
 from the current on-disk state on the next turn. Useful after editing concept
 pages out-of-band.
 
@@ -633,10 +629,10 @@ Examples:
 Embeds EVERY concept page's sections — including synthetic skill and CLI
 capability rows — into the section dense store in one pass, then advances
 the maintain checkpoint so the next incremental pass only re-embeds future
-edits. Use this once on an existing install before the section-lane A/B and
-cutover: the dense collection starts empty, and the periodic maintenance
-pass only re-embeds pages edited since its last run (and never the synthetic
-rows), so most of the corpus would otherwise never be embedded.
+edits. Use this once on an install whose section dense store is empty or
+predates v3: the periodic maintenance pass only re-embeds pages edited since
+its last run (and never the synthetic rows), so most of the corpus would
+otherwise never be embedded.
 
 Idempotent and safe to re-run. Runs inside the assistant so it uses the live
 configuration and advances the checkpoint the assistant reads.
@@ -716,6 +712,46 @@ Examples:
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --turns-file .mv3/eval/key.json
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --exclude-conversation <migration-conv-id>`,
+        },
+        {
+          name: "gate-stats",
+          description:
+            "Show injection gate pass rates bucketed by corpus size (read-only)",
+          options: [
+            {
+              flags: "--lookback-days <n>",
+              description: "Days of telemetry to aggregate (1–90, default 30)",
+              defaultValue: "30",
+            },
+            {
+              flags: "--json",
+              description: "Emit raw JSON instead of a formatted table",
+            },
+          ],
+          helpText: `
+Reads the memory v3 injection gate telemetry outbox and prints pass rates
+and reason distributions grouped by concept page count bucket
+(0–9 / 10–49 / 50–199 / 200+). Scored runs (dense lane was available and
+actually weighed scores) are reported separately from pass-open shortcuts
+(dense disabled / unavailable / gate threw) so scoredPassRate reflects
+only contested gate decisions — the signal relevant for threshold calibration.
+
+Coverage is limited to runs still pending platform flush. In a healthy system
+the outbox holds only the last few minutes to hours of events, so
+--lookback-days is an upper bound, not a guarantee; long-window aggregation
+lives on the platform side on top of flushed watchdog events.
+
+Reads the telemetry database directly — the assistant does not need to be
+running.
+
+Intended use: verify that gate firing rates shift as expected after a
+config or threshold change, and check whether the bm25-auto-calibration
+flag is worth enabling for large corpora.
+
+Examples:
+  $ assistant memory v3 gate-stats
+  $ assistant memory v3 gate-stats --lookback-days 7
+  $ assistant memory v3 gate-stats --json | jq '.buckets[] | select(.pageCountRange == "200+")'`,
         },
         {
           name: "eval-tally",
@@ -762,6 +798,26 @@ Example:
         },
       ],
     },
+    {
+      name: "validate",
+      description:
+        "Check the concept-page store for dangling links, oversized pages, and parse failures (read-only)",
+      helpText: `
+Walks memory/concepts/ on disk and reports:
+  - Page count and edge count
+  - Dangling links: an edges: entry, a links: entry, or an inline
+    [[wikilink]] whose target page does not exist (targets under skills/
+    and cli-commands/ resolve against the registered capability catalog)
+  - Oversized pages (over the configured per-page character cap)
+  - Parse failures (malformed frontmatter)
+
+Read-only. Exits non-zero if anything is reported. Works on every memory
+tier, including before memory.v3.live or memory.v2.enabled is set, so it
+doubles as the pre-flip check for a migration.
+
+Examples:
+  $ assistant memory validate`,
+    },
     {
       name: "ingest",
       description:

package/src/cli/commands/memory/index.ts

@@ -8,6 +8,7 @@ import { registerMemoryIngestCommand } from "./memory-ingest.js";
 import { registerMemoryRetrospectiveCommand } from "./memory-retrospective.js";
 import { registerMemoryV2Command } from "./memory-v2.js";
 import { registerMemoryV3Command } from "./memory-v3.js";
+import { registerMemoryValidateCommand } from "./memory-validate.js";
 import { registerMemoryNodesCommand } from "./nodes.js";
 import { registerMemoryWorkerCommand } from "./worker.js";
 
@@ -24,6 +25,7 @@ export function registerMemoryCommand(program: Command): void {
       registerMemoryV2Command(memory);
       registerMemoryV3Command(memory);
       registerMemoryIngestCommand(memory);
+      registerMemoryValidateCommand(memory);
       registerMemoryRetrospectiveCommand(memory);
       registerMemoryWorkerCommand(memory);
     },

package/src/cli/commands/memory/memory-v2.ts

@@ -12,8 +12,9 @@
  *     from the current skill set.
  *   - `activation` — refresh persisted activation state for every
  *     conversation that has a stored row.
- *   - `validate` — print a diagnostic report (page count, edge count, and
- *     violation lists). Does not mutate the workspace.
+ *   - `validate`: {@link runMemoryValidate}, also registered as the top-level
+ *     `memory validate` (`memory-validate.ts`); kept here while this
+ *     namespace exists.
  */
 
 import type { Command } from "commander";
@@ -59,6 +60,60 @@ async function runBackfillOp(op: MemoryV2BackfillOp): Promise<void> {
   log.info(`Queued ${op} job: ${result.result!.jobId}`);
 }
 
+/**
+ * Print the validate report; sets a non-zero exit code on any violation.
+ * Registered as `memory validate` (`memory-validate.ts`) and `memory v2
+ * validate`.
+ */
+export async function runMemoryValidate(): Promise<void> {
+  const result = await cliIpcCall<MemoryV2ValidateResult>(
+    "memory_v2_validate",
+    { body: {} },
+  );
+
+  if (!result.ok) {
+    log.error(result.error ?? "Failed to validate memory state");
+    process.exitCode = 1;
+    return;
+  }
+
+  const report = result.result!;
+  log.info(`Pages: ${report.pageCount}`);
+  log.info(`Edges: ${report.edgeCount}`);
+  log.info(
+    `Dangling links: ${
+      report.danglingLinks.length === 0 ? "none" : report.danglingLinks.length
+    }`,
+  );
+  for (const d of report.danglingLinks) {
+    log.info(`  - ${d.from} → ${d.to} (${d.kind})`);
+  }
+  log.info(
+    `Oversized pages: ${
+      report.oversizedPages.length === 0 ? "none" : report.oversizedPages.length
+    }`,
+  );
+  for (const p of report.oversizedPages) {
+    log.info(`  - ${p.slug}: ${p.chars} chars`);
+  }
+  log.info(
+    `Parse failures: ${
+      report.parseFailures.length === 0 ? "none" : report.parseFailures.length
+    }`,
+  );
+  for (const p of report.parseFailures) {
+    log.info(`  - ${p.slug}: ${p.error}`);
+  }
+
+  if (
+    report.danglingLinks.length > 0 ||
+    report.oversizedPages.length > 0 ||
+    report.parseFailures.length > 0
+  ) {
+    process.exitCode = 1;
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Registration
 // ---------------------------------------------------------------------------
@@ -97,58 +152,7 @@ export function registerMemoryV2Command(memory: Command): void {
 
   // ── validate ──────────────────────────────────────────────────────────
 
-  subcommand(v2, "validate").action(async () => {
-    const result = await cliIpcCall<MemoryV2ValidateResult>(
-      "memory_v2_validate",
-      { body: {} },
-    );
-
-    if (!result.ok) {
-      log.error(result.error ?? "Failed to validate memory v2 state");
-      process.exitCode = 1;
-      return;
-    }
-
-    const report = result.result!;
-    log.info(`Pages: ${report.pageCount}`);
-    log.info(`Edges: ${report.edgeCount}`);
-    log.info(
-      `Missing edge endpoints: ${
-        report.missingEdgeEndpoints.length === 0
-          ? "none"
-          : report.missingEdgeEndpoints.length
-      }`,
-    );
-    for (const m of report.missingEdgeEndpoints) {
-      log.info(`  - ${m.from} → ${m.to}`);
-    }
-    log.info(
-      `Oversized pages: ${
-        report.oversizedPages.length === 0
-          ? "none"
-          : report.oversizedPages.length
-      }`,
-    );
-    for (const p of report.oversizedPages) {
-      log.info(`  - ${p.slug}: ${p.chars} chars`);
-    }
-    log.info(
-      `Parse failures: ${
-        report.parseFailures.length === 0 ? "none" : report.parseFailures.length
-      }`,
-    );
-    for (const p of report.parseFailures) {
-      log.info(`  - ${p.slug}: ${p.error}`);
-    }
-
-    if (
-      report.missingEdgeEndpoints.length > 0 ||
-      report.oversizedPages.length > 0 ||
-      report.parseFailures.length > 0
-    ) {
-      process.exitCode = 1;
-    }
-  });
+  subcommand(v2, "validate").action(runMemoryValidate);
 
   // ── ema ───────────────────────────────────────────────────────────────