@vellumai/assistant 0.9.0 → 0.10.0-dev.202606192226.8ed8a6a

package/src/cli/commands/{__tests__ → memory/__tests__}/memory-v3.test.ts

@@ -40,7 +40,7 @@ let logOutput: string[] = [];
 // Mocks
 // ---------------------------------------------------------------------------
 
-mock.module("../../../ipc/cli-client.js", () => ({
+mock.module("../../../../ipc/cli-client.js", () => ({
   cliIpcCall: async (
     method: string,
     params?: any,
@@ -60,7 +60,7 @@ const fakeLogger = {
   error: capture,
   debug: () => {},
 };
-mock.module("../../../util/logger.js", () => ({
+mock.module("../../../../util/logger.js", () => ({
   getLogger: () => fakeLogger,
   getCliLogger: () => fakeLogger,
 }));
@@ -82,7 +82,8 @@ function buildProgram(): Command {
     writeErr: () => {},
     writeOut: () => {},
   });
-  registerMemoryV3Command(program);
+  const memory = program.command("memory");
+  registerMemoryV3Command(memory);
   return program;
 }
 
@@ -144,7 +145,7 @@ describe("subcommand registration", () => {
     const v3 = memory!.commands.find((c) => c.name() === "v3");
     expect(v3).toBeDefined();
     const subcommandNames = v3!.commands.map((c) => c.name()).sort();
-    expect(subcommandNames).toEqual(["backfill-sections", "rebuild-index"]);
+    expect(subcommandNames).toEqual(["backfill-sections", "eval", "rebuild-index"]);
   });
 });
 

package/src/cli/commands/memory/index.ts

@@ -0,0 +1,30 @@
+import type { Command } from "commander";
+
+import { registerCommand } from "../../lib/register-command.js";
+import { registerMemoryV2Command } from "./memory-v2.js";
+import { registerMemoryV3Command } from "./memory-v3.js";
+
+export function registerMemoryCommand(program: Command): void {
+  registerCommand(program, {
+    name: "memory",
+    transport: "ipc",
+    description: "Inspect and maintain the assistant memory subsystem",
+    build: (memory) => {
+      memory.addHelpText(
+        "after",
+        `
+The memory subsystem retrieves concept pages two ways: the v2 concept-page
+activation model (prose pages with directed edges) and the v3 section-lane
+model (section-grain lanes cached as live shadow state). Each subgroup exposes
+operator-facing maintenance verbs — reindexing, backfills, validation, and evals.
+
+Examples:
+  $ assistant memory v2 validate
+  $ assistant memory v3 rebuild-index`,
+      );
+
+      registerMemoryV2Command(memory);
+      registerMemoryV3Command(memory);
+    },
+  });
+}

package/src/cli/commands/{memory-v2-compare-render.ts → memory/memory-v2-compare-render.ts}

@@ -9,7 +9,7 @@
  * type-only imports but forbids pulling in daemon runtime modules.
  */
 
-import type { ComparisonReport } from "../../memory/v2/harness/runner.js";
+import type { ComparisonReport } from "../../../memory/v2/harness/runner.js";
 
 function sortedKs(report: ComparisonReport): number[] {
   return [...report.ks].sort((a, b) => a - b);

package/src/cli/commands/{memory-v2.ts → memory/memory-v2.ts}

@@ -18,8 +18,8 @@
 
 import type { Command } from "commander";
 
-import { cliIpcCall } from "../../ipc/cli-client.js";
-import type { ComparisonReport } from "../../memory/v2/harness/runner.js";
+import { cliIpcCall } from "../../../ipc/cli-client.js";
+import type { ComparisonReport } from "../../../memory/v2/harness/runner.js";
 import type {
   MemoryV2BackfillOp,
   MemoryV2BackfillResult,
@@ -27,9 +27,9 @@ import type {
   MemoryV2ReembedSkillsResult,
   MemoryV2SimulateRouterResult,
   MemoryV2ValidateResult,
-} from "../../runtime/routes/memory-v2-routes.js";
-import { registerCommand } from "../lib/register-command.js";
-import { log } from "../logger.js";
+} from "../../../runtime/routes/memory-v2-routes.js";
+import { registerCommand } from "../../lib/register-command.js";
+import { log } from "../../logger.js";
 import {
   renderComparisonReport,
   renderTurnTrace,
@@ -63,16 +63,7 @@ async function runBackfillOp(op: MemoryV2BackfillOp): Promise<void> {
 // Registration
 // ---------------------------------------------------------------------------
 
-export function registerMemoryV2Command(program: Command): void {
-  // Reuse an existing `memory` parent if some other registrar attached to it
-  // first; otherwise create one. This keeps the registration order between
-  // sibling memory registrars unconstrained.
-  const memory =
-    program.commands.find((c) => c.name() === "memory") ??
-    program
-      .command("memory")
-      .description("Manage the v2 memory subsystem (concept-page model)");
-
+export function registerMemoryV2Command(memory: Command): void {
   registerCommand(memory, {
     name: "v2",
     transport: "ipc",

package/src/cli/commands/{memory-v3.ts → memory/memory-v3.ts}

@@ -17,13 +17,14 @@
 
 import type { Command } from "commander";
 
-import { cliIpcCall } from "../../ipc/cli-client.js";
+import { cliIpcCall } from "../../../ipc/cli-client.js";
+import type { MemoryEvalRunResult } from "../../../runtime/routes/memory-eval-routes.js";
 import type {
   MemoryV3BackfillSectionsResult,
   MemoryV3RebuildIndexResult,
-} from "../../runtime/routes/memory-v3-routes.js";
-import { registerCommand } from "../lib/register-command.js";
-import { log } from "../logger.js";
+} from "../../../runtime/routes/memory-v3-routes.js";
+import { registerCommand } from "../../lib/register-command.js";
+import { log } from "../../logger.js";
 
 /**
  * IPC timeout for `backfill-sections`. The one-time full-corpus section embed
@@ -34,14 +35,14 @@ import { log } from "../logger.js";
  */
 const BACKFILL_IPC_TIMEOUT_MS = 30 * 60 * 1000;
 
-export function registerMemoryV3Command(program: Command): void {
-  // Reuse an existing `memory` parent if some other registrar attached to it
-  // first; otherwise create one. This keeps the registration order between
-  // sibling memory registrars unconstrained.
-  const memory =
-    program.commands.find((c) => c.name() === "memory") ??
-    program.command("memory").description("Manage the memory subsystem");
+/**
+ * IPC timeout for `eval`. Embedding both corpora's sections runs through the
+ * embedder and easily outlasts the default 60s, so allow the same generous
+ * ceiling as the backfill.
+ */
+const EVAL_IPC_TIMEOUT_MS = 30 * 60 * 1000;
 
+export function registerMemoryV3Command(memory: Command): void {
   registerCommand(memory, {
     name: "v3",
     transport: "ipc",
@@ -134,6 +135,91 @@ Examples:
                 : "."),
           );
         });
+
+      // ── eval ──────────────────────────────────────────────────────────────
+
+      v3.command("eval")
+        .description(
+          "Build blinded A/B retrieval-eval packets (snapshot corpus vs staged wiki)",
+        )
+        .requiredOption(
+          "--staging <dir>",
+          "Staged v3 wiki dir (relative to the workspace, or absolute)",
+        )
+        .requiredOption(
+          "--snapshot <dir>",
+          "Read-only v2 snapshot dir (relative to the workspace, or absolute)",
+        )
+        .requiredOption("--out <dir>", "Output dir for packets.json + key.json")
+        .option("--turns <n>", "Number of recent turns to mine", "30")
+        .option("--k <n>", "Pages per memory set", "8")
+        .option(
+          "--seed <n>",
+          "Blinding seed (reproducible A/B assignment)",
+          "1",
+        )
+        .option(
+          "--no-dense",
+          "Needle-only: skip section embedding (fast, cheaper, lower fidelity)",
+        )
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .addHelpText(
+          "after",
+          `
+Mines recent user turns, retrieves the top pages from each corpus per turn, and
+writes blinded A/B packets (plus a separate unblinding key) for a blind-judge
+workflow. Both corpora are read in memory — nothing in the live lanes or Qdrant
+is touched. With the dense lane on (default) it embeds every section of both
+corpora, which can take a while on a large corpus; use --no-dense for a fast
+lexical-only pass.
+
+Examples:
+  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval
+  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --turns 50 --no-dense`,
+        )
+        .action(
+          async (opts: {
+            staging: string;
+            snapshot: string;
+            out: string;
+            turns: string;
+            k: string;
+            seed: string;
+            dense: boolean;
+            json?: boolean;
+          }) => {
+            const result = await cliIpcCall<MemoryEvalRunResult>(
+              "memory_eval_run",
+              {
+                body: {
+                  stagingDir: opts.staging,
+                  snapshotDir: opts.snapshot,
+                  outDir: opts.out,
+                  turns: Number(opts.turns),
+                  k: Number(opts.k),
+                  seed: Number(opts.seed),
+                  dense: opts.dense,
+                },
+              },
+              { timeoutMs: EVAL_IPC_TIMEOUT_MS },
+            );
+            if (!result.ok) {
+              log.error(result.error ?? "Failed to build eval packets");
+              process.exitCode = 1;
+              return;
+            }
+            const payload = result.result!;
+            if (opts.json === true) {
+              log.info(JSON.stringify(payload, null, 2));
+              return;
+            }
+            log.info(
+              `Wrote ${payload.packetsWritten} packets from ${payload.turnsMined} turns ` +
+                `(snapshot ${payload.snapshotPages} pages vs staged ${payload.stagingPages} pages, ` +
+                `dense=${payload.dense}).\n  packets: ${payload.packetsPath}\n  key:     ${payload.keyPath}`,
+            );
+          },
+        );
     },
   });
 }

package/src/cli/commands/oauth/status.test.ts

@@ -0,0 +1,36 @@
+import { afterEach, describe, expect, test } from "bun:test";
+
+import { noConnectionsMessage } from "./status.js";
+
+const original = process.env.__RESOLVED_MODEL;
+
+afterEach(() => {
+  if (original === undefined) delete process.env.__RESOLVED_MODEL;
+  else process.env.__RESOLVED_MODEL = original;
+});
+
+describe("noConnectionsMessage", () => {
+  test("capable models get the terse default", () => {
+    process.env.__RESOLVED_MODEL = "claude-opus-4-8";
+    const msg = noConnectionsMessage("google");
+    expect(msg).toBe(
+      "No active connections for google.\n" +
+        "Connect with `assistant oauth connect google`.\n",
+    );
+  });
+
+  test("no resolved model falls back to the terse default", () => {
+    delete process.env.__RESOLVED_MODEL;
+    expect(noConnectionsMessage("google")).toContain(
+      "Connect with `assistant oauth connect google`.",
+    );
+  });
+
+  test("weak open models are steered to the oauth_connect surface", () => {
+    process.env.__RESOLVED_MODEL = "accounts/fireworks/models/minimax-m3";
+    const msg = noConnectionsMessage("google");
+    expect(msg).toContain('surface_type "oauth_connect"');
+    expect(msg).toContain('data.providerKey "google"');
+    expect(msg).not.toContain("assistant oauth connect google");
+  });
+});

package/src/cli/commands/oauth/status.ts

@@ -1,8 +1,30 @@
 import type { Command } from "commander";
 
 import { cliIpcCall, exitFromIpcResult } from "../../../ipc/cli-client.js";
+import { isWeakOpenModel } from "../../../providers/weak-open-model.js";
 import { shouldOutputJson, writeOutput } from "../../output.js";
 
+/**
+ * Message shown when a provider has no active connections. Weak open models
+ * loop through redundant discovery commands (channel checks, `oauth providers
+ * get`, loading the OAuth setup skill) before rendering the connect button, so
+ * they get an explicit single next action: render the core `oauth_connect`
+ * surface directly. Capable models keep the terse default.
+ */
+export function noConnectionsMessage(provider: string): string {
+  const base = `No active connections for ${provider}.`;
+  if (isWeakOpenModel(process.env.__RESOLVED_MODEL)) {
+    return (
+      `${base}\nTo let the user connect, render the connect button: call ` +
+      `\`ui_show\` with surface_type "oauth_connect" and ` +
+      `data.providerKey "${provider}". That surface is always available — do ` +
+      `not run further \`oauth\`/\`channels\` commands or load a setup skill ` +
+      `just to display it.\n`
+    );
+  }
+  return `${base}\nConnect with \`assistant oauth connect ${provider}\`.\n`;
+}
+
 // ---------------------------------------------------------------------------
 // Types
 // ---------------------------------------------------------------------------
@@ -91,9 +113,7 @@ Examples:
 
           // Text output
           if (connections.length === 0) {
-            process.stdout.write(
-              `No active connections for ${provider}.\nConnect with \`assistant oauth connect ${provider}\`.\n`,
-            );
+            process.stdout.write(noConnectionsMessage(provider));
             return;
           }