@vellumai/assistant 0.8.5 → 0.8.6

package/src/cli/commands/memory-v3.ts

@@ -13,10 +13,13 @@
  *     marking shared-DAG re-entries.
  *   - `simulate` — dry-run the v3 retrieval loop against an ad-hoc query and
  *     print the per-pass descent trace plus the lane-grouped selection.
+ *   - `shadow-diff` — compare logged v3 shadow selections against the live v2
+ *     router selections, turn-for-turn, and print the overlap / v3-only /
+ *     v2-only breakdown by provenance lane.
  *
- * All are read-only: they mutate nothing. `validate`/`tree` run no LLM;
- * `simulate` invokes the loop (filter + gate LLM calls) but persists nothing.
- * `--json` emits the raw daemon payload for any subcommand.
+ * All are read-only: they mutate nothing. `validate`/`tree`/`shadow-diff` run
+ * no LLM; `simulate` invokes the loop (filter + gate LLM calls) but persists
+ * nothing. `--json` emits the raw daemon payload for any subcommand.
  */
 
 import { mkdirSync, writeFileSync } from "node:fs";
@@ -29,17 +32,38 @@ import type {
   MemoryV3SimulateResult,
   MemoryV3TreeResult,
   MemoryV3ValidateResult,
+  SeedCoretrievalResult,
+  ShadowDiffResult,
 } from "../../runtime/routes/memory-v3-routes.js";
 import { registerCommand } from "../lib/register-command.js";
 import { log } from "../logger.js";
 import {
   renderLlmCalls,
+  renderSeedEdges,
+  renderShadowDiff,
   renderSimulation,
   renderTree,
   renderValidationReport,
   reportHasDefects,
 } from "./memory-v3-render.js";
 
+/**
+ * Parse a positive numeric CLI flag. Returns `{ value }` when valid (or omitted,
+ * with `value` undefined) and `{ error: true }` when present but not a positive
+ * number — or not an integer when `int` is set.
+ */
+function parsePositiveFlag(
+  raw: string | undefined,
+  opts: { int: boolean },
+): { value?: number; error?: boolean } {
+  if (raw === undefined) return {};
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n <= 0 || (opts.int && !Number.isInteger(n))) {
+    return { error: true };
+  }
+  return { value: n };
+}
+
 /** Valid lane names accepted by `--lanes` (matches memory.v3.lanes keys). */
 const V3_LANE_NAMES = ["hot", "sparse", "dense", "tree", "edges"] as const;
 
@@ -72,7 +96,8 @@ Examples:
   $ assistant memory v3 validate
   $ assistant memory v3 tree
   $ assistant memory v3 tree --json | jq '.nodes | length'
-  $ assistant memory v3 simulate -q "what should we ship next"`,
+  $ assistant memory v3 simulate -q "what should we ship next"
+  $ assistant memory v3 shadow-diff`,
       );
 
       // ── validate ──────────────────────────────────────────────────────────
@@ -311,6 +336,232 @@ Examples:
             }
           },
         );
+
+      // ── shadow-diff ─────────────────────────────────────────────────────────
+
+      v3.command("shadow-diff")
+        .description(
+          "Compare v3 shadow selections against live v2 router selections (read-only)",
+        )
+        .option(
+          "--since-days <n>",
+          "Only consider shadow rows newer than N days (default: all)",
+        )
+        .option(
+          "--tolerance-sec <n>",
+          "Max seconds between a shadow row and its paired router row (default: 10)",
+        )
+        .option(
+          "--limit <n>",
+          "Cap per-turn detail rows shown, newest first (default: 50)",
+        )
+        .option("--json", "Emit raw JSON instead of a formatted report")
+        .addHelpText(
+          "after",
+          `
+Pairs each v3 shadow-mode selection (mode='v3_shadow' rows) with the live v2
+router selection (mode='router') from the same turn and reports where they
+agree, what v3 surfaced that v2 missed, and what v2 had that v3 dropped — the
+v3-only / v2-only sets — broken down by v3 provenance lane.
+
+Pairing is by timestamp within --tolerance-sec, not by turn number (the shadow
+and router rows count turns with different counters). The v2 comparand is the
+router's fresh per-turn pick (status='injected'), not its accumulated
+in-context set, so it is comparable to v3's fresh per-turn selection.
+
+Requires that v3 shadow mode has been running (memory.v3.enabled +
+memory.v3.shadow) so v3_shadow rows exist. Read-only — runs no LLM, writes
+nothing.
+
+Examples:
+  $ assistant memory v3 shadow-diff
+  $ assistant memory v3 shadow-diff --since-days 2
+  $ assistant memory v3 shadow-diff --json | jq '.agg.v3OnlyByLane'`,
+        )
+        .action(
+          async (opts: {
+            sinceDays?: string;
+            toleranceSec?: string;
+            limit?: string;
+            json?: boolean;
+          }) => {
+            const sinceDays = parsePositiveFlag(opts.sinceDays, {
+              int: false,
+            });
+            if (sinceDays.error) {
+              log.error(
+                `--since-days must be a positive number (got "${opts.sinceDays}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+            const toleranceSec = parsePositiveFlag(opts.toleranceSec, {
+              int: false,
+            });
+            if (toleranceSec.error) {
+              log.error(
+                `--tolerance-sec must be a positive number (got "${opts.toleranceSec}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+            const limit = parsePositiveFlag(opts.limit, { int: true });
+            if (limit.error) {
+              log.error(
+                `--limit must be a positive integer (got "${opts.limit}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+
+            const result = await cliIpcCall<ShadowDiffResult>(
+              "memory_v3_shadow_diff",
+              {
+                body: {
+                  ...(sinceDays.value !== undefined
+                    ? { sinceDays: sinceDays.value }
+                    : {}),
+                  ...(toleranceSec.value !== undefined
+                    ? { toleranceSec: toleranceSec.value }
+                    : {}),
+                  ...(limit.value !== undefined ? { limit: limit.value } : {}),
+                },
+              },
+            );
+
+            if (!result.ok) {
+              log.error(result.error ?? "Failed to diff v3 shadow selections");
+              process.exitCode = 1;
+              return;
+            }
+
+            const payload = result.result!;
+            if (opts.json === true) {
+              log.info(JSON.stringify(payload, null, 2));
+              return;
+            }
+            log.info(renderShadowDiff(payload));
+          },
+        );
+
+      // ── seed-edges ────────────────────────────────────────────────────────
+
+      v3.command("seed-edges")
+        .description(
+          "Seed the learned co-retrieval edge graph from v2 router history",
+        )
+        .option(
+          "--min-count <n>",
+          "Min co-occurrence turns for an edge (positive integer, default: 5)",
+        )
+        .option(
+          "--top-k <n>",
+          "Neighbors kept per source node (positive integer, default: 20)",
+        )
+        .option(
+          "--max-neighbor-freq-ratio <n>",
+          "Exclude neighbors selected on more than this fraction of turns (0-1, default: 0.4)",
+        )
+        .option(
+          "--seed-weight <n>",
+          "Weight of a perfect-NPMI (1.0) edge; each edge stored at this × its NPMI (positive, default: 2.0)",
+        )
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .addHelpText(
+          "after",
+          `
+Builds an NPMI-scored co-retrieval graph from the v2 router's per-turn
+selections (memory_v2_activation_logs, status='injected') and writes it into
+memory_v3_auto_edges, warm-starting the learned edge graph. This is the only
+'memory v3' subcommand that WRITES. Idempotent — re-running refreshes the
+seeded edges to the seed weight. Runs no LLM.
+
+The edge-expansion lane only merges the learned graph once
+memory.v3.edges.learnedAdjacencyThreshold is set above 0 (default off); seed
+first, then enable the threshold to A/B via 'memory v3 simulate'.
+
+Examples:
+  $ assistant memory v3 seed-edges
+  $ assistant memory v3 seed-edges --min-count 8 --top-k 15
+  $ assistant memory v3 seed-edges --json | jq '.edgesWritten'`,
+        )
+        .action(
+          async (opts: {
+            minCount?: string;
+            topK?: string;
+            maxNeighborFreqRatio?: string;
+            seedWeight?: string;
+            json?: boolean;
+          }) => {
+            const minCount = parsePositiveFlag(opts.minCount, { int: true });
+            if (minCount.error) {
+              log.error(
+                `--min-count must be a positive integer (got "${opts.minCount}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+            const topK = parsePositiveFlag(opts.topK, { int: true });
+            if (topK.error) {
+              log.error(
+                `--top-k must be a positive integer (got "${opts.topK}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+            const ratio = parsePositiveFlag(opts.maxNeighborFreqRatio, {
+              int: false,
+            });
+            if (ratio.error || (ratio.value !== undefined && ratio.value > 1)) {
+              log.error(
+                `--max-neighbor-freq-ratio must be a number in (0, 1] (got "${opts.maxNeighborFreqRatio}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+            const seedWeight = parsePositiveFlag(opts.seedWeight, {
+              int: false,
+            });
+            if (seedWeight.error) {
+              log.error(
+                `--seed-weight must be a positive number (got "${opts.seedWeight}")`,
+              );
+              process.exitCode = 1;
+              return;
+            }
+
+            const result = await cliIpcCall<SeedCoretrievalResult>(
+              "memory_v3_seed_edges",
+              {
+                body: {
+                  ...(minCount.value !== undefined
+                    ? { minCount: minCount.value }
+                    : {}),
+                  ...(topK.value !== undefined ? { topK: topK.value } : {}),
+                  ...(ratio.value !== undefined
+                    ? { maxNeighborFreqRatio: ratio.value }
+                    : {}),
+                  ...(seedWeight.value !== undefined
+                    ? { seedWeight: seedWeight.value }
+                    : {}),
+                },
+              },
+            );
+
+            if (!result.ok) {
+              log.error(result.error ?? "Failed to seed co-retrieval edges");
+              process.exitCode = 1;
+              return;
+            }
+
+            const payload = result.result!;
+            if (opts.json === true) {
+              log.info(JSON.stringify(payload, null, 2));
+              return;
+            }
+            log.info(renderSeedEdges(payload));
+          },
+        );
     },
   });
 }