@vellumai/assistant 0.10.0 → 0.10.1-staging.1

package/src/cli/commands/memory/memory-v3.ts

@@ -15,10 +15,15 @@
  *     starts empty and the incremental maintain pass only re-embeds deltas.
  */
 
+import { readFileSync, writeFileSync } from "node:fs";
+
 import type { Command } from "commander";
 
 import { cliIpcCall } from "../../../ipc/cli-client.js";
-import type { MemoryEvalRunResult } from "../../../runtime/routes/memory-eval-routes.js";
+import type {
+  MemoryEvalRunResult,
+  MemoryEvalTallyResult,
+} from "../../../runtime/routes/memory-eval-routes.js";
 import type {
   MemoryV3BackfillSectionsResult,
   MemoryV3RebuildIndexResult,
@@ -42,6 +47,56 @@ const BACKFILL_IPC_TIMEOUT_MS = 30 * 60 * 1000;
  */
 const EVAL_IPC_TIMEOUT_MS = 30 * 60 * 1000;
 
+/** Commander accumulator for a repeatable `--exclude-conversation <id>` flag. */
+function collectRepeatable(value: string, acc: string[]): string[] {
+  return [...acc, value];
+}
+
+/**
+ * Read the `turn` ids from a prior run's `key.json` or `packets.json` (both are
+ * arrays of objects carrying a `turn` field). Used to pin `--turns-file` so a
+ * re-judge measures the exact same turn set.
+ */
+function readTurnIdsFromFile(path: string): string[] {
+  const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
+  if (!Array.isArray(parsed)) {
+    throw new Error(
+      `--turns-file ${path} must be a JSON array (a key.json or packets.json)`,
+    );
+  }
+  const ids = parsed
+    .map((e) => (e as { turn?: unknown }).turn)
+    .filter((t): t is string => typeof t === "string");
+  if (ids.length === 0) {
+    throw new Error(`--turns-file ${path} contained no turn ids`);
+  }
+  return ids;
+}
+
+/** Read and parse a JSON file, asserting it is an array. */
+function readJsonArray<T>(path: string, label: string): T[] {
+  const parsed = JSON.parse(readFileSync(path, "utf8")) as unknown;
+  if (!Array.isArray(parsed)) {
+    throw new Error(`--${label} ${path} must be a JSON array`);
+  }
+  return parsed as T[];
+}
+
+/** Render an eval-tally result as a human-readable summary block. */
+function formatTally(r: MemoryEvalTallyResult): string {
+  const lines = [
+    `Eval gate: ${r.gate.toUpperCase()} (${r.verdict})`,
+    `  turns judged:   ${r.turns} (panel ${r.panel.min}-${r.panel.max} votes/turn, mean ${r.panel.mean.toFixed(1)})`,
+    `  snapshot (v2):  ${r.snapshotWins} wins, mean score ${r.meanSnapshot.toFixed(2)}`,
+    `  staging (wiki): ${r.stagingWins} wins, mean score ${r.meanStaging.toFixed(2)}`,
+    `  ties:           ${r.ties}`,
+    `  sign-test p:    ${r.signTestP.toFixed(3)} over ${r.decided} decided turns`,
+    `  confident:      ${r.confident ? "yes" : "no"}`,
+  ];
+  for (const note of r.notes) lines.push(`  note: ${note}`);
+  return lines.join("\n");
+}
+
 export function registerMemoryV3Command(memory: Command): void {
   registerCommand(memory, {
     name: "v3",
@@ -162,6 +217,16 @@ Examples:
           "--no-dense",
           "Needle-only: skip section embedding (fast, cheaper, lower fidelity)",
         )
+        .option(
+          "--turns-file <path>",
+          "Pin the exact turns from a prior key.json/packets.json (reproducible re-judge); overrides --turns",
+        )
+        .option(
+          "--exclude-conversation <id>",
+          "Conversation id to omit from mining (repeatable; e.g. the migration's own chat)",
+          collectRepeatable,
+          [],
+        )
         .option("--json", "Emit raw JSON instead of a formatted summary")
         .addHelpText(
           "after",
@@ -173,9 +238,17 @@ is touched. With the dense lane on (default) it embeds every section of both
 corpora, which can take a while on a large corpus; use --no-dense for a fast
 lexical-only pass.
 
+To iterate on the staged wiki reproducibly, mine the turns ONCE and pin them on
+every re-run with --turns-file (pointing at the first run's key.json), so the
+comparison stays fixed while only the staged corpus changes. Re-runs that
+re-mine drift onto a different turn set and are not comparable. Likewise, do not
+compare a --no-dense run against a dense one, and check eval-meta.json's
+embedding identity is the same across runs.
+
 Examples:
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval
-  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --turns 50 --no-dense`,
+  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --turns-file .mv3/eval/key.json
+  $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --exclude-conversation <migration-conv-id>`,
         )
         .action(
           async (opts: {
@@ -186,8 +259,20 @@ Examples:
             k: string;
             seed: string;
             dense: boolean;
+            turnsFile?: string;
+            excludeConversation: string[];
             json?: boolean;
           }) => {
+            let turnIds: string[] | undefined;
+            if (opts.turnsFile !== undefined) {
+              try {
+                turnIds = readTurnIdsFromFile(opts.turnsFile);
+              } catch (err) {
+                log.error(err instanceof Error ? err.message : String(err));
+                process.exitCode = 1;
+                return;
+              }
+            }
             const result = await cliIpcCall<MemoryEvalRunResult>(
               "memory_eval_run",
               {
@@ -199,6 +284,10 @@ Examples:
                   k: Number(opts.k),
                   seed: Number(opts.seed),
                   dense: opts.dense,
+                  ...(turnIds ? { turnIds } : {}),
+                  ...(opts.excludeConversation.length > 0
+                    ? { excludeConversationIds: opts.excludeConversation }
+                    : {}),
                 },
               },
               { timeoutMs: EVAL_IPC_TIMEOUT_MS },
@@ -213,11 +302,94 @@ Examples:
               log.info(JSON.stringify(payload, null, 2));
               return;
             }
+            const emb = payload.embedding;
+            const embStr = payload.dense
+              ? `${emb.provider}/${emb.model}/${emb.dims ?? "?"}d`
+              : "needle-only (no dense)";
             log.info(
-              `Wrote ${payload.packetsWritten} packets from ${payload.turnsMined} turns ` +
+              `Wrote ${payload.packetsWritten} packets from ${payload.turnsMined}/${payload.turnsRequested} turns ` +
                 `(snapshot ${payload.snapshotPages} pages vs staged ${payload.stagingPages} pages, ` +
-                `dense=${payload.dense}).\n  packets: ${payload.packetsPath}\n  key:     ${payload.keyPath}`,
+                `dense=${payload.dense}, embedding=${embStr}).\n` +
+                `  packets: ${payload.packetsPath}\n  key:     ${payload.keyPath}\n  meta:    ${payload.metaPath}\n` +
+                `  Re-judge reproducibly with: --turns-file ${payload.keyPath}`,
+            );
+            if (payload.turnsMined < payload.turnsRequested) {
+              log.warn(
+                `Only ${payload.turnsMined} of ${payload.turnsRequested} requested turns were mined ` +
+                  `(pinned turns may have been deleted, or there are too few recent user turns).`,
+              );
+            }
+          },
+        );
+
+      v3.command("eval-tally")
+        .description(
+          "Unblind + tally blind-judge verdicts against key.json with a noise-aware win/tie/loss verdict",
+        )
+        .requiredOption(
+          "--verdicts <path>",
+          "JSON file: array of { turn, winner, scoreA, scoreB } (one or more per turn for a panel)",
+        )
+        .requiredOption(
+          "--key <path>",
+          "key.json from `eval` — the per-turn A/B → snapshot/staging unblinding map",
+        )
+        .option(
+          "--alpha <p>",
+          "Significance threshold for the sign test (the wiki only FAILS on a significant snapshot lead)",
+          "0.05",
+        )
+        .option("--out <path>", "Also write the full tally JSON to this path")
+        .option("--json", "Emit raw JSON instead of a formatted summary")
+        .addHelpText(
+          "after",
+          `
+Joins the blind-judge verdicts to the unblinding key (A/B is shuffled PER TURN,
+so the winner must be mapped turn-by-turn — a global A-vs-B count is wrong) and
+applies a two-sided sign test: the wiki only FAILS when the snapshot's win lead
+is statistically significant. A within-noise difference is a tie, which passes
+the win-or-tie gate. Pass a judge PANEL (multiple verdicts per turn, e.g. from
+re-judging under several seeds) to control single-vote noise.
+
+Example:
+  $ assistant memory v3 eval-tally --verdicts .mv3/eval/verdicts.json --key .mv3/eval/key.json`,
+        )
+        .action(
+          async (opts: {
+            verdicts: string;
+            key: string;
+            alpha: string;
+            out?: string;
+            json?: boolean;
+          }) => {
+            let verdicts: unknown[];
+            let key: unknown[];
+            try {
+              verdicts = readJsonArray<unknown>(opts.verdicts, "verdicts");
+              key = readJsonArray<unknown>(opts.key, "key");
+            } catch (err) {
+              log.error(err instanceof Error ? err.message : String(err));
+              process.exitCode = 1;
+              return;
+            }
+            const result = await cliIpcCall<MemoryEvalTallyResult>(
+              "memory_eval_tally",
+              { body: { verdicts, key, alpha: Number(opts.alpha) } },
             );
+            if (!result.ok) {
+              log.error(result.error ?? "Failed to tally verdicts");
+              process.exitCode = 1;
+              return;
+            }
+            const payload = result.result!;
+            if (opts.out !== undefined) {
+              writeFileSync(opts.out, JSON.stringify(payload, null, 2));
+            }
+            if (opts.json === true) {
+              log.info(JSON.stringify(payload, null, 2));
+              return;
+            }
+            log.info(formatTally(payload));
           },
         );
     },

package/src/cli/commands/plugins.ts

@@ -24,17 +24,32 @@ import {
 import {
   DEFAULT_PLUGIN_REF,
   installPlugin,
+  type InstallPluginOptions,
   InvalidPluginNameError,
   PluginAlreadyInstalledError,
   PluginNotFoundError,
 } from "../lib/install-from-github.js";
 import { listInstalledPlugins } from "../lib/list-installed-plugins.js";
 import type { FingerprintComparison } from "../lib/plugin-fingerprint.js";
+import {
+  DEFAULT_PIN_HISTORY_LIMIT,
+  listPinHistory,
+  type PluginPinHistoryEntry,
+  PluginPinHistoryError,
+  resolvePinToMarketplaceCommit,
+} from "../lib/plugin-pin-history.js";
 import { registerCommand } from "../lib/register-command.js";
 import {
   InvalidSearchPatternError,
   searchPlugins,
 } from "../lib/search-plugins.js";
+import {
+  disablePlugin,
+  enablePlugin,
+  InvalidPluginNameError as ToggleInvalidPluginNameError,
+  PluginAlreadyInStateException,
+  PluginDirectoryNotFoundError,
+} from "../lib/toggle-plugin.js";
 import {
   PluginNotInstalledError,
   uninstallPlugin,
@@ -65,6 +80,9 @@ Examples:
   $ assistant plugins install example
   $ assistant plugins install example --force
   $ assistant plugins install example --ref my-feature-branch
+  $ assistant plugins versions example
+  $ assistant plugins versions example --json
+  $ assistant plugins install example --pin <sha> --force
   $ assistant plugins list
   $ assistant plugins list --json
   $ assistant plugins inspect example
@@ -79,7 +97,9 @@ Examples:
   $ assistant plugins search example
   $ assistant plugins search "^example"
   $ assistant plugins search example --json
-  $ assistant plugins uninstall example`,
+  $ assistant plugins uninstall example
+  $ assistant plugins enable example
+  $ assistant plugins disable example`,
       );
 
       plugins
@@ -90,19 +110,35 @@ Examples:
         .option("--force", "Overwrite an existing install")
         .option(
           "--ref <ref>",
-          `Git ref to fetch from (default: ${DEFAULT_PLUGIN_REF})`,
+          `Marketplace manifest revision to read the pin from (default: ${DEFAULT_PLUGIN_REF})`,
+        )
+        .option(
+          "--pin <sha>",
+          "Install a specific reviewed marketplace pin (full commit SHA); run `plugins versions <name>` to list them",
+        )
+        .option(
+          "--allow-unreviewed",
+          "With --pin, install a SHA that is not in the reviewed marketplace history (advanced; the curated adapter may not match)",
         )
         .action(
-          async (name: string, opts: { force?: boolean; ref?: string }) => {
+          async (
+            name: string,
+            opts: {
+              force?: boolean;
+              ref?: string;
+              pin?: string;
+              allowUnreviewed?: boolean;
+            },
+          ) => {
             try {
-              const result = await installPlugin(
-                {
-                  name,
-                  force: opts.force ?? false,
-                  ref: opts.ref ?? DEFAULT_PLUGIN_REF,
-                },
-                { fetch: globalThis.fetch.bind(globalThis) },
-              );
+              const installOpts = await resolveInstallOptions(name, opts);
+              if (installOpts === null) {
+                process.exitCode = 1;
+                return;
+              }
+              const result = await installPlugin(installOpts, {
+                fetch: globalThis.fetch.bind(globalThis),
+              });
               log.info(
                 {
                   name: result.name,
@@ -131,6 +167,14 @@ Examples:
                 process.exitCode = 1;
                 return;
               }
+              if (
+                err instanceof InvalidPluginNameError ||
+                err instanceof PluginPinHistoryError
+              ) {
+                console.error(err.message);
+                process.exitCode = 1;
+                return;
+              }
               const message = err instanceof Error ? err.message : String(err);
               console.error(`Plugin install failed: ${message}`);
               process.exitCode = 1;
@@ -138,6 +182,61 @@ Examples:
           },
         );
 
+      plugins
+        .command("versions <name>")
+        .description(
+          "List the recent reviewed marketplace pins for a plugin, newest first. Install an older one with `plugins install <name> --pin <sha>`",
+        )
+        .option("--json", "Emit machine-readable JSON instead of a table")
+        .option(
+          "--limit <n>",
+          `Maximum number of pins to show (default: ${DEFAULT_PIN_HISTORY_LIMIT})`,
+        )
+        .action(
+          async (name: string, opts: { json?: boolean; limit?: string }) => {
+            let limit: number | undefined;
+            if (opts.limit !== undefined) {
+              limit = Number.parseInt(opts.limit, 10);
+              if (!Number.isInteger(limit) || limit < 1) {
+                console.error("--limit must be a positive integer.");
+                process.exitCode = 1;
+                return;
+              }
+            }
+            try {
+              const history = await listPinHistory(
+                name,
+                { fetch: globalThis.fetch.bind(globalThis) },
+                limit !== undefined ? { limit } : {},
+              );
+
+              if (opts.json) {
+                process.stdout.write(JSON.stringify(history, null, 2) + "\n");
+                return;
+              }
+
+              // Logged after the JSON early-return so the logger's stdout
+              // writes never corrupt --json output.
+              log.info({ name, count: history.length }, "plugin versions");
+              for (const line of formatVersions(name, history)) {
+                console.log(line);
+              }
+            } catch (err) {
+              if (
+                err instanceof InvalidPluginNameError ||
+                err instanceof PluginPinHistoryError
+              ) {
+                console.error(err.message);
+                process.exitCode = 1;
+                return;
+              }
+              const message = err instanceof Error ? err.message : String(err);
+              console.error(`Plugin versions failed: ${message}`);
+              process.exitCode = 1;
+            }
+          },
+        );
+
       plugins
         .command("list")
         .description("List plugins installed under <workspaceDir>/plugins/")
@@ -399,6 +498,61 @@ Examples:
           }
         });
 
+      plugins
+        .command("disable <name>")
+        .description(
+          "Disable a plugin by creating a .disabled sentinel file. Works for both user-installed and default plugins. Restart the assistant for the change to take effect.",
+        )
+        .action((name: string) => {
+          try {
+            const result = disablePlugin(name);
+            log.info({ name: result.name }, "plugin disabled");
+            console.log(
+              `Disabled plugin "${result.name}". Restart the assistant for the change to take effect.`,
+            );
+          } catch (err) {
+            if (
+              err instanceof PluginAlreadyInStateException ||
+              err instanceof ToggleInvalidPluginNameError ||
+              err instanceof PluginDirectoryNotFoundError
+            ) {
+              console.error(err.message);
+              process.exitCode = 1;
+              return;
+            }
+            const message = err instanceof Error ? err.message : String(err);
+            console.error(`Plugin disable failed: ${message}`);
+            process.exitCode = 1;
+          }
+        });
+
+      plugins
+        .command("enable <name>")
+        .description(
+          "Re-enable a disabled plugin by removing the .disabled sentinel file. Restart the assistant for the change to take effect.",
+        )
+        .action((name: string) => {
+          try {
+            const result = enablePlugin(name);
+            log.info({ name: result.name }, "plugin enabled");
+            console.log(
+              `Enabled plugin "${result.name}". Restart the assistant for the change to take effect.`,
+            );
+          } catch (err) {
+            if (
+              err instanceof PluginAlreadyInStateException ||
+              err instanceof ToggleInvalidPluginNameError
+            ) {
+              console.error(err.message);
+              process.exitCode = 1;
+              return;
+            }
+            const message = err instanceof Error ? err.message : String(err);
+            console.error(`Plugin enable failed: ${message}`);
+            process.exitCode = 1;
+          }
+        });
+
       plugins
         .command("upgrade <name>")
         .description(
@@ -483,6 +637,109 @@ Examples:
 }
 
 /** Abbreviate a commit SHA for display; passes through non-SHA / null values. */
+/** Full Git commit SHA — 40 hex (SHA-1) or 64 (SHA-256). */
+const FULL_SHA_RE = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i;
+
+/**
+ * Resolve `plugins install` flags into {@link InstallPluginOptions}, or `null`
+ * when the flag combination is invalid (a message is printed in that case, and
+ * the caller exits non-zero). Network / marketplace failures raised while
+ * resolving a `--pin` propagate to the caller's catch.
+ *
+ * `--pin <sha>` installs a specific reviewed marketplace pin: the SHA is looked
+ * up in the plugin's pin history and installed from the marketplace commit that
+ * introduced it, so the curated adapter stub of that era comes along. With
+ * `--allow-unreviewed` the SHA is materialized directly (adapter from `main`),
+ * bypassing the history check — the advanced escape hatch.
+ */
+async function resolveInstallOptions(
+  name: string,
+  opts: {
+    force?: boolean;
+    ref?: string;
+    pin?: string;
+    allowUnreviewed?: boolean;
+  },
+): Promise<InstallPluginOptions | null> {
+  const force = opts.force ?? false;
+
+  if (!opts.pin) {
+    if (opts.allowUnreviewed) {
+      console.error("--allow-unreviewed only applies together with --pin.");
+      return null;
+    }
+    return { name, force, ref: opts.ref ?? DEFAULT_PLUGIN_REF };
+  }
+
+  const pin = opts.pin.trim();
+  if (!FULL_SHA_RE.test(pin)) {
+    console.error(
+      `--pin must be a full commit SHA (40 or 64 hex chars); got ${JSON.stringify(opts.pin)}.`,
+    );
+    return null;
+  }
+  if (opts.ref) {
+    console.error(
+      "--ref and --pin cannot be combined; --pin already selects the revision.",
+    );
+    return null;
+  }
+
+  if (opts.allowUnreviewed) {
+    // Materialize the exact SHA from the plugin's current-manifest repo,
+    // bypassing the reviewed-history check. The adapter stub still comes from
+    // the default ref, so an adapted plugin may not reproduce faithfully.
+    return { name, force, ref: DEFAULT_PLUGIN_REF, commitOverride: pin };
+  }
+
+  const entry = await resolvePinToMarketplaceCommit(name, pin, {
+    fetch: globalThis.fetch.bind(globalThis),
+  });
+  if (!entry) {
+    console.error(
+      `"${pin}" is not a reviewed marketplace pin for "${name}".\n` +
+        `Run \`assistant plugins versions ${name}\` to see available pins, ` +
+        "or pass --allow-unreviewed to install it anyway.",
+    );
+    return null;
+  }
+  return { name, force, ref: entry.marketplaceCommit };
+}
+
+/**
+ * Render a plugin's marketplace-pin history as a table: the pinned commit (short
+ * SHA), when it was promoted, and a marker for the pin currently active. An
+ * empty history reports that none was found.
+ */
+function formatVersions(
+  name: string,
+  history: readonly PluginPinHistoryEntry[],
+): string[] {
+  if (history.length === 0) {
+    return [`No marketplace pin history found for "${name}".`];
+  }
+  const rows = history.map((entry) => ({
+    pin: shortSha(entry.pin),
+    promoted: formatTimestamp(entry.promotedAt),
+    marker: entry.current ? "(current)" : "",
+  }));
+  const pinW = Math.max(3, ...rows.map((r) => r.pin.length));
+  const promotedW = Math.max(8, ...rows.map((r) => r.promoted.length));
+  const pad = (s: string, w: number) => s + " ".repeat(w - s.length);
+  const lines = [
+    `${pad("PIN", pinW)}  ${pad("PROMOTED", promotedW)}  `.trimEnd(),
+  ];
+  for (const r of rows) {
+    lines.push(
+      `${pad(r.pin, pinW)}  ${pad(r.promoted, promotedW)}  ${r.marker}`.trimEnd(),
+    );
+  }
+  lines.push("");
+  lines.push("Install an older pin with:");
+  lines.push(`  assistant plugins install ${name} --pin <sha> --force`);
+  return lines;
+}
+
 function shortSha(sha: string | null): string {
   return sha ? sha.slice(0, 7) : "—";
 }

package/src/cli/lib/__tests__/install-from-github.test.ts

@@ -32,6 +32,7 @@ import {
   PluginPostinstallError,
   PluginSourceUnavailableError,
   type PostinstallRunner,
+  readInstallMeta,
   sanitizePluginName,
 } from "../install-from-github.js";
 
@@ -292,6 +293,36 @@ describe("installPlugin — install lifecycle", () => {
     expect(existsSync(join(target, "README.md"))).toBe(true);
   });
 
+  test("commitOverride materializes a specific commit, keeping repo from the manifest", async () => {
+    // GIVEN a manifest pinning the current SHA, but an override to an older one
+    const OLD_SHA = "1".repeat(40);
+    const fetch = makeContentsFetch({ tree: {}, manifest: CAVEMAN_MANIFEST });
+    const calls: string[][] = [];
+    const runGit = fakeGitRunner({
+      tree: { "package.json": '{"name":"caveman"}' },
+      commit: OLD_SHA,
+      calls,
+    });
+
+    // WHEN we install with the commit override
+    const result = await installPlugin(
+      { name: "caveman", force: false, ref: "main", commitOverride: OLD_SHA },
+      { fetch, runGit, workspacePluginsDir: pluginsDir },
+    );
+
+    // THEN the clone fetched the override commit, not the manifest's pin
+    expect(calls.find((c) => c[0] === "fetch")?.at(-1)).toBe(OLD_SHA);
+    expect(result.commit).toBe(OLD_SHA);
+    expect(result.ref).toBe(OLD_SHA);
+
+    // AND provenance records the override as the installed ref while still
+    // resolving owner/repo from the manifest entry
+    const meta = readInstallMeta(join(pluginsDir, "caveman"));
+    expect(meta?.source.ref).toBe(OLD_SHA);
+    expect(meta?.source.repo).toBe("caveman");
+    expect(meta?.source.owner).toBe("JuliusBrussee");
+  });
+
   test("--force preserves the existing install when the clone fails", async () => {
     // GIVEN a working copy already on disk
     const target = join(pluginsDir, "caveman");
@@ -823,4 +854,13 @@ describe("sanitizePluginName", () => {
     expect(sanitizePluginName("plugin_2")).toBe("plugin_2");
     expect(sanitizePluginName("a")).toBe("a");
   });
+
+  test.each([
+    "default-advisor",
+    "default-memory-retrieval",
+    "default-",
+    "default-x",
+  ])("rejects reserved prefix name %p", (reserved) => {
+    expect(() => sanitizePluginName(reserved)).toThrow(InvalidPluginNameError);
+  });
 });