@vellumai/assistant 0.11.3-dev.202608111333.7065e2a → 0.11.3-dev.202608111523.8a0fdc8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.3-dev.202608111333.7065e2a",
+  "version": "0.11.3-dev.202608111523.8a0fdc8",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/mcp-list-plugin-servers.test.ts

@@ -204,12 +204,12 @@ describe("internal_mcp_list, plugin-declared servers", () => {
     expect(workspace.hasStaticAuth).toBe(true);
   });
 
-  test("plugin servers default to high risk", async () => {
+  test("plugin servers default to low risk", async () => {
     writePlugin("unabyss", unabyssManifest());
 
     const servers = await listServers();
     expect(servers.find((s) => s.id === "unabyss")!.defaultRiskLevel).toEqual(
-      "high",
+      "low",
     );
   });
 

package/src/cli/commands/memory/__tests__/memory-v3.test.ts

@@ -2,12 +2,13 @@
  * Tests for the `assistant memory v3` CLI subgroup (section-lane model).
  *
  * Validates:
- *   - Subcommand registration (rebuild-index, backfill-sections) under
- *     `memory v3`.
- *   - Each subcommand maps to the right `cliIpcCall` method.
+ *   - Subcommand registration (rebuild-index, backfill-sections, gate-stats)
+ *     under `memory v3`.
+ *   - `rebuild-index` and `backfill-sections` route through IPC to the daemon.
  *   - `backfill-sections` passes a long IPC timeout (the one-time full-corpus
- *     embed easily outlasts the default 60s), while `rebuild-index` does not.
- *   - IPC error paths return a non-zero exit code without throwing.
+ *     embed easily outlasts the default 60s).
+ *   - `gate-stats` calls handleMemoryV3GateStats directly — no daemon required.
+ *   - Error paths return a non-zero exit code without throwing.
  */
 
 import { beforeEach, describe, expect, mock, test } from "bun:test";
@@ -39,6 +40,47 @@ let mockIpcResult: {
 /** Captured log output for assertion. */
 let logOutput: string[] = [];
 
+/** Shared fake gate-stats payload used across gate-stats tests. */
+const fakeGateStats = {
+  lookbackDays: 30,
+  totalRuns: 10,
+  buckets: [
+    {
+      pageCountRange: "0–9",
+      total: 2,
+      scored: 1,
+      passed: 2,
+      scoredPassRate: 1,
+      reasons: { dense_pass: 1, dense_disabled: 1 },
+    },
+    {
+      pageCountRange: "10–49",
+      total: 5,
+      scored: 4,
+      passed: 3,
+      scoredPassRate: 0.75,
+      reasons: { dense_pass: 3, fail_no_signal: 1 },
+    },
+    {
+      pageCountRange: "50–199",
+      total: 2,
+      scored: 2,
+      passed: 1,
+      scoredPassRate: 0.5,
+      reasons: { dense_pass: 1, fail_no_signal: 1 },
+    },
+    {
+      pageCountRange: "200+",
+      total: 1,
+      scored: 1,
+      passed: 0,
+      scoredPassRate: 0,
+      reasons: { fail_no_signal: 1 },
+    },
+  ],
+  unknownPageCount: { total: 0, passed: 0, reasons: {} },
+};
+
 // ---------------------------------------------------------------------------
 // Mocks
 // ---------------------------------------------------------------------------
@@ -68,6 +110,33 @@ mock.module("../../../../util/logger.js", () => ({
   getCliLogger: () => fakeLogger,
 }));
 
+// ---------------------------------------------------------------------------
+// gate-stats direct-invocation mocks
+// ---------------------------------------------------------------------------
+
+/** Args captured from the last handleMemoryV3GateStats call. */
+let lastGateStatsArgs: { lookbackDays: number; db: unknown } | null = null;
+
+/** What handleMemoryV3GateStats returns, or an error to throw. */
+let gateStatsImpl: (() => unknown) | null = null;
+
+mock.module(
+  "../../../../plugins/defaults/memory/src/memory-v3-routes.js",
+  () => ({
+    handleMemoryV3GateStats: (lookbackDays: number, db: unknown) => {
+      lastGateStatsArgs = { lookbackDays, db };
+      if (gateStatsImpl) {
+        return gateStatsImpl();
+      }
+      return fakeGateStats;
+    },
+  }),
+);
+
+mock.module("../../../../persistence/db-connection.js", () => ({
+  getTelemetrySqlite: () => ({}),
+}));
+
 // ---------------------------------------------------------------------------
 // Import modules under test (after mocks)
 // ---------------------------------------------------------------------------
@@ -135,6 +204,8 @@ beforeEach(() => {
     ok: true,
     result: { articles: 3, sections: 12, failures: 0 },
   };
+  lastGateStatsArgs = null;
+  gateStatsImpl = null;
   logOutput = [];
   process.exitCode = 0;
 });
@@ -155,6 +226,7 @@ describe("subcommand registration", () => {
       "backfill-sections",
       "eval",
       "eval-tally",
+      "gate-stats",
       "rebuild-index",
     ]);
   });
@@ -236,3 +308,54 @@ describe("memory v3 backfill-sections", () => {
     expect(exitCode).toBe(1);
   });
 });
+
+// ---------------------------------------------------------------------------
+// gate-stats (direct invocation — no daemon required)
+// ---------------------------------------------------------------------------
+
+describe("memory v3 gate-stats", () => {
+  test("calls handleMemoryV3GateStats directly with default lookback (no IPC)", async () => {
+    const { exitCode } = await runCommand(["memory", "v3", "gate-stats"]);
+
+    expect(exitCode).toBe(0);
+    // Must NOT have gone through IPC.
+    expect(lastIpcCall).toBeNull();
+    // Must have called the handler directly with the default lookback.
+    expect(lastGateStatsArgs!.lookbackDays).toBe(30);
+  });
+
+  test("forwards --lookback-days directly to handleMemoryV3GateStats", async () => {
+    await runCommand(["memory", "v3", "gate-stats", "--lookback-days", "7"]);
+
+    expect(lastIpcCall).toBeNull();
+    expect(lastGateStatsArgs!.lookbackDays).toBe(7);
+  });
+
+  test("prints formatted table on success (no --json)", async () => {
+    await runCommand(["memory", "v3", "gate-stats"]);
+
+    expect(logOutput.some((l) => l.includes("Gate-stats"))).toBe(true);
+    expect(logOutput.some((l) => l.includes("10–49"))).toBe(true);
+    expect(logOutput.some((l) => l.includes("75.0%"))).toBe(true);
+  });
+
+  test("emits raw JSON with --json", async () => {
+    await runCommand(["memory", "v3", "gate-stats", "--json"]);
+
+    const raw = logOutput.join("");
+    const parsed = JSON.parse(raw) as typeof fakeGateStats;
+    expect(parsed.totalRuns).toBe(10);
+    expect(parsed.buckets).toHaveLength(4);
+  });
+
+  test("exits with code 1 when handleMemoryV3GateStats throws", async () => {
+    gateStatsImpl = () => {
+      throw new Error("telemetry DB locked");
+    };
+
+    const { exitCode } = await runCommand(["memory", "v3", "gate-stats"]);
+
+    expect(exitCode).toBe(1);
+    expect(logOutput.some((l) => l.includes("telemetry DB locked"))).toBe(true);
+  });
+});

package/src/cli/commands/memory/index.help.ts

@@ -606,7 +606,9 @@ that live state safely.
 
 Examples:
   $ assistant memory v3 rebuild-index
-  $ assistant memory v3 backfill-sections`,
+  $ assistant memory v3 backfill-sections
+  $ assistant memory v3 gate-stats
+  $ assistant memory v3 gate-stats --lookback-days 7 --json`,
       subcommands: [
         {
           name: "rebuild-index",
@@ -716,6 +718,46 @@ Examples:
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --turns-file .mv3/eval/key.json
   $ assistant memory v3 eval --snapshot .mv3/snapshot/concepts --staging .mv3/staging --out .mv3/eval --exclude-conversation <migration-conv-id>`,
+        },
+        {
+          name: "gate-stats",
+          description:
+            "Show injection gate pass rates bucketed by corpus size (read-only)",
+          options: [
+            {
+              flags: "--lookback-days <n>",
+              description: "Days of telemetry to aggregate (1–90, default 30)",
+              defaultValue: "30",
+            },
+            {
+              flags: "--json",
+              description: "Emit raw JSON instead of a formatted table",
+            },
+          ],
+          helpText: `
+Reads the memory v3 injection gate telemetry outbox and prints pass rates
+and reason distributions grouped by concept page count bucket
+(0–9 / 10–49 / 50–199 / 200+). Scored runs (dense lane was available and
+actually weighed scores) are reported separately from pass-open shortcuts
+(dense disabled / unavailable / gate threw) so scoredPassRate reflects
+only contested gate decisions — the signal relevant for threshold calibration.
+
+Coverage is limited to runs still pending platform flush. In a healthy system
+the outbox holds only the last few minutes to hours of events, so
+--lookback-days is an upper bound, not a guarantee; long-window aggregation
+lives on the platform side on top of flushed watchdog events.
+
+Reads the telemetry database directly — the assistant does not need to be
+running.
+
+Intended use: verify that gate firing rates shift as expected after a
+config or threshold change, and check whether the bm25-auto-calibration
+flag is worth enabling for large corpora.
+
+Examples:
+  $ assistant memory v3 gate-stats
+  $ assistant memory v3 gate-stats --lookback-days 7
+  $ assistant memory v3 gate-stats --json | jq '.buckets[] | select(.pageCountRange == "200+")'`,
         },
         {
           name: "eval-tally",

package/src/cli/commands/memory/memory-v3.ts

@@ -25,6 +25,7 @@ import type {
   MemoryEvalTallyResult,
 } from "../../../plugins/defaults/memory/src/memory-eval-routes.js";
 import type {
+  GateStatsResponse,
   MemoryV3BackfillSectionsResult,
   MemoryV3RebuildIndexResult,
 } from "../../../plugins/defaults/memory/src/memory-v3-routes.js";
@@ -52,6 +53,36 @@ function collectRepeatable(value: string, acc: string[]): string[] {
   return [...acc, value];
 }
 
+/** Render gate-stats as a human-readable table. */
+function formatGateStats(r: GateStatsResponse): string {
+  const lines: string[] = [
+    `Gate-stats (last ${r.lookbackDays} day${r.lookbackDays === 1 ? "" : "s"}, ${r.totalRuns} run${r.totalRuns === 1 ? "" : "s"})`,
+    "",
+    `Bucket     Total  Scored  Passed  ScoredPassRate  Top reasons`,
+  ];
+  for (const b of r.buckets) {
+    const rate =
+      b.scoredPassRate !== null
+        ? `${(b.scoredPassRate * 100).toFixed(1)}%`
+        : "n/a";
+    const topReasons = Object.entries(b.reasons)
+      .sort(([, a], [, bv]) => (bv as number) - (a as number))
+      .slice(0, 3)
+      .map(([k, v]) => `${k}: ${v}`)
+      .join(", ");
+    lines.push(
+      `${b.pageCountRange.padEnd(10)} ${String(b.total).padStart(5)}  ${String(b.scored).padStart(6)}  ${String(b.passed).padStart(6)}  ${rate.padStart(14)}  ${topReasons}`,
+    );
+  }
+  if (r.unknownPageCount.total > 0) {
+    lines.push("");
+    lines.push(
+      `Unknown page count: ${r.unknownPageCount.total} total, ${r.unknownPageCount.passed} passed`,
+    );
+  }
+  return lines.join("\n");
+}
+
 /**
  * Read the `turn` ids from a prior run's `key.json` or `packets.json` (both are
  * arrays of objects carrying a `turn` field). Used to pin `--turns-file` so a
@@ -271,4 +302,37 @@ export function registerMemoryV3Command(memory: Command): void {
       log.info(formatTally(payload));
     },
   );
+
+  // ── gate-stats ────────────────────────────────────────────────────────
+  // Reads the telemetry DB directly — no daemon required.
+
+  subcommand(v3, "gate-stats").action(
+    async (opts: { lookbackDays: string; json?: boolean }) => {
+      const lookbackDays = Number(opts.lookbackDays);
+      const [{ handleMemoryV3GateStats }, { getTelemetrySqlite }] =
+        await Promise.all([
+          import("../../../plugins/defaults/memory/src/memory-v3-routes.js"),
+          import("../../../persistence/db-connection.js"),
+        ]);
+      let payload;
+      try {
+        payload = handleMemoryV3GateStats(
+          Number.isFinite(lookbackDays) ? lookbackDays : 30,
+          getTelemetrySqlite(),
+        );
+      } catch (err) {
+        log.error(
+          "Failed to read gate stats: %s",
+          err instanceof Error ? err.message : String(err),
+        );
+        process.exitCode = 1;
+        return;
+      }
+      if (opts.json === true) {
+        log.info(JSON.stringify(payload, null, 2));
+        return;
+      }
+      log.info(formatGateStats(payload));
+    },
+  );
 }

package/src/cli/lib/bundled-marketplace.json

@@ -422,6 +422,19 @@
       "license": "MIT",
       "icon": "✈️"
     },
+    {
+      "name": "unabyss",
+      "source": {
+        "source": "github",
+        "repo": "Unabyss/unabyss-vellum",
+        "ref": "9f0bb0753dad09dace8578e242961bc2da912af3"
+      },
+      "description": "Personal and company context from Unabyss, available to your agent through the Unabyss MCP server.",
+      "category": "memory",
+      "homepage": "https://unabyss.com",
+      "license": "MIT",
+      "icon": "🌊"
+    },
     {
       "name": "vellum-client-qa",
       "source": {

package/src/config/schemas/mcp.ts

@@ -109,3 +109,26 @@ export const McpConfigSchema = z
 export type McpTransport = z.infer<typeof McpTransportSchema>;
 export type McpServerConfig = z.infer<typeof McpServerConfigSchema>;
 export type McpConfig = z.infer<typeof McpConfigSchema>;
+
+/**
+ * Who declared a server: the workspace `config.json`, which the user owns,
+ * or a plugin's `mcp.json`, which its author owns.
+ *
+ * Deliberately not a schema field. It is resolved from where the entry was
+ * read, never parsed from a file, so nothing on disk can claim to be
+ * workspace-owned. What it gates is credential access: only a workspace
+ * server resolves `mcp:<serverId>:*` from the credential store, because a
+ * plugin controls both its server key and its URL and would otherwise
+ * receive a workspace credential at an endpoint it chose.
+ */
+export type McpServerSource = "workspace" | "plugin";
+
+/** A server config with its origin resolved. */
+export interface ResolvedMcpServerConfig extends McpServerConfig {
+  readonly source: McpServerSource;
+}
+
+/** The MCP config the daemon runs: both origins, every server attributed. */
+export interface ResolvedMcpConfig extends Omit<McpConfig, "servers"> {
+  readonly servers: Record<string, ResolvedMcpServerConfig>;
+}

package/src/daemon/__tests__/plugin-mcp-reconcile.test.ts

@@ -0,0 +1,82 @@
+/**
+ * The plugin source reconcile drives MCP reload.
+ *
+ * Without this, installing a plugin leaves its `mcp.json` servers
+ * unconnected and its tools unregistered until an explicit MCP reload, a
+ * config edit, or a restart — and, worse in the other direction, an
+ * uninstalled or disabled plugin's connected client and registered tools
+ * stay callable. The reconcile is the seam every install / uninstall /
+ * upgrade / enable / disable already funnels through.
+ *
+ * The guard is equally load-bearing: a reconcile also fires for plugin
+ * edits that touch no `mcp.json`, and reloading on those would tear down
+ * every healthy workspace connection to rebuild it for nothing.
+ */
+
+import { beforeEach, describe, expect, jest, mock, test } from "bun:test";
+
+let pluginServersChanged = true;
+
+const managerStop = jest.fn(async () => {});
+const managerStart = jest.fn(async () => []);
+
+mock.module("../../mcp/effective-config.js", () => ({
+  buildEffectiveMcpConfig: () => ({
+    servers: {
+      unabyss: {
+        transport: { type: "streamable-http", url: "https://mcp.example/x" },
+        enabled: true,
+        defaultRiskLevel: "low",
+        maxTools: 20,
+        source: "plugin",
+      },
+    },
+    globalMaxTools: 50,
+  }),
+  pluginMcpServersChangedSinceLastBuild: () => pluginServersChanged,
+}));
+
+mock.module("../../mcp/manager.js", () => ({
+  getMcpServerManager: () => ({ stop: managerStop, start: managerStart }),
+}));
+
+mock.module("../../mcp/mcp-header-store.js", () => ({
+  migrateLegacyMcpHeaders: async () => {},
+}));
+
+mock.module("../../config/loader.js", () => ({
+  getConfig: () => ({ mcp: { servers: {}, globalMaxTools: 50 } }),
+  invalidateConfigCache: () => {},
+}));
+
+mock.module("../../tools/registry.js", () => ({
+  registerMcpTools: () => [],
+  unregisterAllMcpTools: () => {},
+}));
+
+const { reconcilePluginMcpServers } = await import("../mcp-reload-service.js");
+
+describe("reconcilePluginMcpServers", () => {
+  beforeEach(() => {
+    managerStop.mockClear();
+    managerStart.mockClear();
+  });
+
+  test("reloads when the plugin-declared set changed", async () => {
+    pluginServersChanged = true;
+
+    await reconcilePluginMcpServers();
+
+    expect(managerStop).toHaveBeenCalledTimes(1);
+    expect(managerStart).toHaveBeenCalledTimes(1);
+  });
+
+  test("does nothing when no plugin changed the set", async () => {
+    pluginServersChanged = false;
+
+    await reconcilePluginMcpServers();
+
+    expect(managerStop).not.toHaveBeenCalled();
+    expect(managerStart).not.toHaveBeenCalled();
+  });
+});

package/src/daemon/mcp-reload-service.ts

@@ -6,6 +6,10 @@
  */
 
 import { getConfig, invalidateConfigCache } from "../config/loader.js";
+import {
+  buildEffectiveMcpConfig,
+  pluginMcpServersChangedSinceLastBuild,
+} from "../mcp/effective-config.js";
 import { getMcpServerManager } from "../mcp/manager.js";
 import { migrateLegacyMcpHeaders } from "../mcp/mcp-header-store.js";
 import { createMcpToolsFromServer } from "../tools/mcp/mcp-tool-factory.js";
@@ -52,6 +56,29 @@ export function reloadMcpServers(): Promise<McpReloadResult> {
   return reloadInProgress;
 }
 
+/**
+ * Reload MCP servers if — and only if — the plugin-declared set moved.
+ *
+ * Called by the plugin source reconcile, which every install / uninstall /
+ * upgrade / enable / disable already funnels through, so a plugin's MCP
+ * tools come up and go down with the plugin instead of lingering until an
+ * explicit reload, a config edit, or a restart.
+ *
+ * The guard matters: a reconcile also fires for plugin edits that touch no
+ * `mcp.json` at all, and an unconditional reload would tear down every
+ * healthy workspace connection to re-establish it for nothing.
+ *
+ * Never throws — the reconcile is best-effort and must not fail an install
+ * whose files already landed on disk.
+ */
+export async function reconcilePluginMcpServers(): Promise<void> {
+  if (!pluginMcpServersChangedSinceLastBuild()) {
+    return;
+  }
+  log.info("Plugin-declared MCP servers changed; reloading");
+  await reloadMcpServers();
+}
+
 async function doReload(): Promise<McpReloadResult> {
   try {
     const manager = getMcpServerManager();
@@ -74,17 +101,20 @@ async function doReload(): Promise<McpReloadResult> {
     // 2. Stop existing MCP servers + unregister their tools
     await manager.stop();
     unregisterAllMcpTools();
-    const serverIds = config.mcp?.servers
-      ? Object.keys(config.mcp.servers)
-      : [];
+
+    // Plugins are re-read here too: installing or removing one changes the
+    // server set exactly like editing config.json does, and both arrive
+    // through this same reload.
+    const mcpConfig = buildEffectiveMcpConfig(config.mcp);
+    const serverIds = Object.keys(mcpConfig.servers);
 
     // 3. Restart MCP servers
     let serverCount = 0;
     let toolCount = 0;
     const servers: McpReloadServerResult[] = [];
 
-    if (config.mcp?.servers && Object.keys(config.mcp.servers).length > 0) {
-      const serverToolInfos = await manager.start(config.mcp);
+    if (serverIds.length > 0) {
+      const serverToolInfos = await manager.start(mcpConfig);
       for (const { serverId, serverConfig, tools } of serverToolInfos) {
         const mcpTools = createMcpToolsFromServer(
           tools,
@@ -105,7 +135,7 @@ async function doReload(): Promise<McpReloadResult> {
       // Include servers that were configured but failed to connect or are disabled
       for (const id of serverIds) {
         if (!servers.some((s) => s.id === id)) {
-          const serverConfig = config.mcp!.servers![id];
+          const serverConfig = mcpConfig.servers[id];
           const isDisabled = serverConfig?.enabled === false;
           servers.push({
             id,

package/src/daemon/providers-setup.ts

@@ -1,6 +1,7 @@
 import { maybeDefaultSpeechToManaged } from "../config/managed-speech-defaults.js";
 import { rehydratePlatformCredentials } from "../config/platform-rehydration.js";
 import type { AssistantConfig } from "../config/types.js";
+import { buildEffectiveMcpConfig } from "../mcp/effective-config.js";
 import { getMcpServerManager } from "../mcp/manager.js";
 import { gmailMessagingProvider } from "../messaging/providers/gmail/adapter.js";
 import { outlookMessagingProvider } from "../messaging/providers/outlook/adapter.js";
@@ -61,11 +62,13 @@ export async function initializeProvidersAndTools(
     );
   }
 
-  // Start MCP servers and register their tools
-  if (config.mcp?.servers && Object.keys(config.mcp.servers).length > 0) {
+  // Start MCP servers — workspace-configured and plugin-declared alike —
+  // and register their tools.
+  const mcpConfig = buildEffectiveMcpConfig(config.mcp);
+  if (Object.keys(mcpConfig.servers).length > 0) {
     const manager = getMcpServerManager();
     try {
-      const serverToolInfos = await manager.start(config.mcp);
+      const serverToolInfos = await manager.start(mcpConfig);
       for (const { serverId, serverConfig, tools } of serverToolInfos) {
         const mcpTools = createMcpToolsFromServer(
           tools,