@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/runtime/routes/memory-eval-routes.ts

@@ -16,6 +16,10 @@ import { getConfig } from "../../config/loader.js";
 import type { AssistantConfig } from "../../config/types.js";
 import { getDb } from "../../memory/db-connection.js";
 import { runMemoryEval } from "../../memory/v3-eval/eval-packets.js";
+import {
+  type TallyResult,
+  tallyVerdicts,
+} from "../../memory/v3-eval/eval-tally.js";
 import { getLogger } from "../../util/logger.js";
 import { getWorkspaceDir } from "../../util/platform.js";
 import { ACTOR_PRINCIPALS, type RoutePolicy } from "../auth/route-policy.js";
@@ -35,16 +39,30 @@ const MemoryEvalRunParamsSchema = z.object({
   k: z.number().int().positive().optional(),
   dense: z.boolean().optional(),
   seed: z.number().int().optional(),
+  /** Pin the exact turns to re-mine (turn ids from a prior key.json/packets.json). */
+  turnIds: z.array(z.string()).optional(),
+  /** Conversations to omit from recency mining (e.g. the migration's own chat). */
+  excludeConversationIds: z.array(z.string()).optional(),
 });
 
 const MemoryEvalRunResultSchema = z.object({
   turnsMined: z.number(),
+  turnsRequested: z.number(),
   packetsWritten: z.number(),
   packetsPath: z.string(),
   keyPath: z.string(),
+  metaPath: z.string(),
   snapshotPages: z.number(),
   stagingPages: z.number(),
   dense: z.boolean(),
+  seed: z.number(),
+  k: z.number(),
+  embedding: z.object({
+    provider: z.string(),
+    model: z.string(),
+    dims: z.number().nullable(),
+  }),
+  turnIds: z.array(z.string()),
 });
 export type MemoryEvalRunResult = z.infer<typeof MemoryEvalRunResultSchema>;
 
@@ -62,10 +80,72 @@ export async function handleMemoryEvalRun(
     workspaceDir: getWorkspaceDir(),
     db: getDb(),
   });
+  if (result.turnsMined < result.turnsRequested) {
+    log.warn(
+      { requested: result.turnsRequested, mined: result.turnsMined },
+      "Fewer eval turns mined than requested — pinned turns may have been " +
+        "deleted, or there are too few recent user turns",
+    );
+  }
   log.info(result, "memory eval packets written");
   return result;
 }
 
+const MemoryEvalTallyParamsSchema = z.object({
+  /** Judge verdicts (one or more per turn for a panel); winner derived from scores if absent. */
+  verdicts: z.array(
+    z.object({
+      turn: z.string(),
+      winner: z.string().optional(),
+      scoreA: z.number(),
+      scoreB: z.number(),
+    }),
+  ),
+  /** The per-turn A/B → snapshot/staging unblinding map from `eval` (key.json). */
+  key: z.array(
+    z.object({
+      turn: z.string(),
+      a: z.enum(["snapshot", "staging"]),
+      b: z.enum(["snapshot", "staging"]),
+    }),
+  ),
+  /** Sign-test significance threshold (default 0.05). */
+  alpha: z.number().positive().optional(),
+});
+
+const MemoryEvalTallyResultSchema = z.object({
+  turns: z.number(),
+  verdictsCounted: z.number(),
+  unmatchedVerdicts: z.number(),
+  panel: z.object({ min: z.number(), max: z.number(), mean: z.number() }),
+  snapshotWins: z.number(),
+  stagingWins: z.number(),
+  ties: z.number(),
+  decided: z.number(),
+  meanSnapshot: z.number(),
+  meanStaging: z.number(),
+  signTestP: z.number(),
+  verdict: z.enum(["wiki-wins", "tie", "wiki-loses"]),
+  gate: z.enum(["pass", "fail"]),
+  confident: z.boolean(),
+  notes: z.array(z.string()),
+});
+export type MemoryEvalTallyResult = z.infer<typeof MemoryEvalTallyResultSchema>;
+
+/**
+ * Pure: join the judge verdicts to the unblinding key and return the noise-aware
+ * gate verdict. The CLI reads the verdicts/key files and passes the arrays.
+ */
+export function handleMemoryEvalTally(body: unknown): MemoryEvalTallyResult {
+  const params = MemoryEvalTallyParamsSchema.parse(body ?? {});
+  const result: TallyResult = tallyVerdicts(
+    params.verdicts,
+    params.key,
+    params.alpha !== undefined ? { alpha: params.alpha } : {},
+  );
+  return result;
+}
+
 const WRITE_POLICY: RoutePolicy = {
   requiredScopes: ["settings.write"],
   allowedPrincipalTypes: ACTOR_PRINCIPALS,
@@ -84,4 +164,16 @@ export const ROUTES: RouteDefinition[] = [
     requestBody: MemoryEvalRunParamsSchema,
     responseBody: MemoryEvalRunResultSchema,
   },
+  {
+    operationId: "memory_eval_tally",
+    method: "POST",
+    policy: WRITE_POLICY,
+    endpoint: "memory/eval/tally",
+    handler: ({ body }) => handleMemoryEvalTally(body),
+    summary:
+      "Unblind + tally blind-judge verdicts against the key with a noise-aware win/tie/loss verdict",
+    tags: ["memory"],
+    requestBody: MemoryEvalTallyParamsSchema,
+    responseBody: MemoryEvalTallyResultSchema,
+  },
 ];

package/src/runtime/routes/memory-item-routes.test.ts

@@ -66,7 +66,11 @@ mock.module("../../memory/qdrant-circuit-breaker.js", () => ({
 
 import { eq } from "drizzle-orm";
 
-import { getDb } from "../../memory/db-connection.js";
+import {
+  getDb,
+  getMemoryDb,
+  getMemorySqlite,
+} from "../../memory/db-connection.js";
 import { initializeDb } from "../../memory/db-init.js";
 import { memoryGraphNodes, memoryJobs } from "../../memory/schema.js";
 import { BadRequestError, ConflictError, NotFoundError } from "./errors.js";
@@ -176,8 +180,8 @@ function insertItem(opts: {
 // ---------------------------------------------------------------------------
 
 describe("Memory Item Routes", () => {
-  beforeAll(() => {
-    initializeDb();
+  beforeAll(async () => {
+    await initializeDb();
   });
 
   beforeEach(() => {
@@ -186,7 +190,7 @@ describe("Memory Item Routes", () => {
     db.run("DELETE FROM memory_graph_triggers");
     db.run("DELETE FROM memory_graph_edges");
     db.run("DELETE FROM memory_graph_nodes");
-    db.run("DELETE FROM memory_jobs");
+    getMemorySqlite()!.run("DELETE FROM memory_jobs");
   });
 
   // =========================================================================
@@ -729,8 +733,7 @@ describe("Memory Item Routes", () => {
       });
 
       // Verify a memory job was enqueued
-      const db = getDb();
-      const jobs = db.select().from(memoryJobs).all();
+      const jobs = getMemoryDb()!.select().from(memoryJobs).all();
       const embedJobs = jobs.filter(
         (j) => j.type === "embed_graph_node" && j.status === "pending",
       );
@@ -835,15 +838,14 @@ describe("Memory Item Routes", () => {
       });
 
       // Clear jobs first
-      getDb().run("DELETE FROM memory_jobs");
+      getMemorySqlite()!.run("DELETE FROM memory_jobs");
 
       await callHandler(route, {
         pathParams: { id: "i1" },
         body: { statement: "new statement" },
       });
 
-      const db = getDb();
-      const jobs = db.select().from(memoryJobs).all();
+      const jobs = getMemoryDb()!.select().from(memoryJobs).all();
       const embedJobs = jobs.filter(
         (j) => j.type === "embed_graph_node" && j.status === "pending",
       );
@@ -897,8 +899,7 @@ describe("Memory Item Routes", () => {
       expect(res.status).toBe(204);
 
       // Verify a delete_qdrant_vectors job was enqueued with graph_node targetType
-      const db = getDb();
-      const jobs = db.select().from(memoryJobs).all();
+      const jobs = getMemoryDb()!.select().from(memoryJobs).all();
       const deleteJobs = jobs.filter(
         (j) => j.type === "delete_qdrant_vectors" && j.status === "pending",
       );

package/src/runtime/routes/migration-routes.ts

@@ -24,7 +24,12 @@ import { getPlatformAssistantId } from "../../config/env.js";
 import { invalidateConfigCache } from "../../config/loader.js";
 import { getAssistantName } from "../../daemon/identity-helpers.js";
 import { runAsyncSqlite } from "../../memory/db-async-query.js";
-import { getDb, resetDb } from "../../memory/db-connection.js";
+import {
+  getDb,
+  getLogsSqlite,
+  getMemorySqlite,
+  resetDb,
+} from "../../memory/db-connection.js";
 import { validateMigrationState } from "../../memory/migrations/validate-migration-state.js";
 import { credentialKey } from "../../security/credential-key.js";
 import {
@@ -151,6 +156,49 @@ export async function reconcileVellumMetadataFromCes(warningSink: {
 
 const log = getLogger("migration-routes");
 
+/**
+ * Flush each database file's WAL before an export copies them. The bundle
+ * walker includes `*.db` but skips `*.db-wal`, so committed frames still sitting
+ * in a WAL would be missing from the bundle.
+ *
+ * The main DB is flushed through `runAsyncSqlite` so the (potentially multi-GB)
+ * flush runs in a sqlite3 subprocess where available rather than stalling the
+ * event loop. The dedicated logs/memory connections are checkpointed in-process
+ * on their own handles — they are small and the daemon already holds them open.
+ * FULL (not TRUNCATE) writes committed frames back to the main file, which is
+ * all the copy needs.
+ */
+async function checkpointDbsForExport(): Promise<void> {
+  const mainResult = await runAsyncSqlite("PRAGMA wal_checkpoint(FULL)");
+  if (!mainResult.ok) {
+    log.warn(
+      { error: mainResult.error, backend: mainResult.backend, db: "main" },
+      "WAL checkpoint failed — exporting without checkpoint",
+    );
+  }
+
+  for (const [label, sqlite] of [
+    ["logs", getLogsSqlite()],
+    ["memory", getMemorySqlite()],
+  ] as const) {
+    if (!sqlite) {
+      log.warn(
+        { db: label },
+        "Dedicated database unavailable — exporting without checkpoint",
+      );
+      continue;
+    }
+    try {
+      sqlite.exec("PRAGMA wal_checkpoint(FULL)");
+    } catch (err) {
+      log.warn(
+        { err, db: label },
+        "WAL checkpoint failed — exporting without checkpoint",
+      );
+    }
+  }
+}
+
 /**
  * Fields the export pipeline must populate on the v1 manifest.
  *
@@ -353,28 +401,7 @@ export async function handleMigrationExport(
       ...manifestInputs,
       secretsRedacted,
       credentials,
-      checkpoint: async () => {
-        // Dispatch through `runAsyncSqlite` so the WAL checkpoint runs
-        // in a sqlite3 subprocess on hosts where it's available. A WAL
-        // flush on a multi-GB WAL file can otherwise stall the daemon's
-        // event loop for the full duration of the flush.
-        //
-        // FULL (not TRUNCATE): FULL blocks until every committed frame is
-        // written back to the main `.db` file, which is all the export
-        // needs since it copies that file. TRUNCATE additionally restarts
-        // the WAL and truncates it to zero bytes — extra work and blocking
-        // this path does not need.
-        const result = await runAsyncSqlite("PRAGMA wal_checkpoint(FULL)");
-        if (!result.ok) {
-          // Best-effort: if the DB can't be checkpointed (e.g. not a valid
-          // SQLite file, missing WAL, etc.) we still proceed with the export
-          // using whatever is on disk.
-          log.warn(
-            { error: result.error, backend: result.backend },
-            "WAL checkpoint failed — exporting without checkpoint",
-          );
-        }
-      },
+      checkpoint: checkpointDbsForExport,
     });
 
     cleanup = result.cleanup;
@@ -582,23 +609,7 @@ export async function handleMigrationExportToGcs({ body }: RouteHandlerArgs) {
           ...manifestInputs,
           secretsRedacted,
           credentials: collected.credentials,
-          checkpoint: async () => {
-            // Dispatch through `runAsyncSqlite` so the WAL checkpoint runs
-            // in a sqlite3 subprocess on hosts where it's available. A
-            // WAL flush on a multi-GB WAL file can otherwise stall the
-            // daemon's event loop for the full duration of the flush.
-            //
-            // FULL (not TRUNCATE): see the disk-export checkpoint above
-            // for rationale. assistant/AGENTS.md "SQLite WAL
-            // checkpointing".
-            const result = await runAsyncSqlite("PRAGMA wal_checkpoint(FULL)");
-            if (!result.ok) {
-              log.warn(
-                { error: result.error, backend: result.backend },
-                "WAL checkpoint failed — exporting without checkpoint",
-              );
-            }
-          },
+          checkpoint: checkpointDbsForExport,
         });
 
         cleanup = result.cleanup;

package/src/runtime/routes/plugins-routes.ts

@@ -50,6 +50,12 @@ import {
   getPluginDetails,
   PluginDetailsNotFoundError,
 } from "../../cli/lib/plugin-details.js";
+import {
+  DEFAULT_PIN_HISTORY_LIMIT,
+  listPinHistory,
+  PluginPinHistoryError,
+  resolvePinToMarketplaceCommit,
+} from "../../cli/lib/plugin-pin-history.js";
 import {
   assertValidSearchPattern,
   filterPluginCatalog,
@@ -73,6 +79,7 @@ import {
   ConflictError,
   InternalError,
   NotFoundError,
+  RouteError,
   ServiceUnavailableError,
 } from "./errors.js";
 import type { RouteDefinition, RouteHandlerArgs } from "./types.js";
@@ -236,6 +243,12 @@ const pluginInstallRequestSchema = z.object({
     .boolean()
     .optional()
     .describe("Overwrite an existing install in place. Defaults to false."),
+  pin: z
+    .string()
+    .optional()
+    .describe(
+      "Install a specific reviewed marketplace pin (full commit SHA) instead of the current one. The pin is validated against the plugin's marketplace pin history (`GET /v1/plugins/:name/versions`) and installed from the marketplace revision that introduced it; an unreviewed SHA is rejected. Use this to roll a plugin back to an older reviewed version.",
+    ),
 });
 
 const pluginInstallResponseSchema = z.object({
@@ -250,6 +263,32 @@ const pluginInstallResponseSchema = z.object({
   ref: z.string().describe("Git ref the plugin was fetched from."),
 });
 
+const pluginPinHistoryEntrySchema = z.object({
+  pin: z
+    .string()
+    .describe("Plugin commit SHA pinned at this point in marketplace history."),
+  marketplaceCommit: z
+    .string()
+    .describe(
+      "Marketplace-manifest commit to install this pin from (the newest commit carrying it).",
+    ),
+  promotedAt: z
+    .string()
+    .nullable()
+    .describe(
+      "ISO-8601 committer date (UTC) of the marketplace commit that promoted this pin; null when unreadable.",
+    ),
+  current: z
+    .boolean()
+    .describe("True for the pin currently active on the default branch."),
+});
+
+const pluginVersionsResponseSchema = z
+  .array(pluginPinHistoryEntrySchema)
+  .describe(
+    "Distinct marketplace pins a plugin has been promoted to, newest first; the first entry is the current pin. Empty when the plugin has no resolvable history.",
+  );
+
 const fingerprintComparisonSchema = z
   .object({
     modified: z
@@ -771,23 +810,52 @@ async function handleGetPluginDetails({
 // Handler — install
 // ---------------------------------------------------------------------------
 
+/**
+ * Resolve the marketplace ref an install should read from. With no `pin`, that
+ * is the default reviewed branch. With a `pin`, it is the marketplace commit
+ * that introduced that pin — but only when the pin appears in the plugin's
+ * reviewed history; an unreviewed SHA is rejected as a bad request so the route
+ * never installs an unvetted revision.
+ */
+async function resolveInstallMarketplaceRef(
+  name: string,
+  pin: string | undefined,
+): Promise<string> {
+  if (!pin) return DEFAULT_PLUGIN_REF;
+  const entry = await resolvePinToMarketplaceCommit(name, pin, {
+    fetch: globalThis.fetch.bind(globalThis),
+  });
+  if (!entry) {
+    throw new BadRequestError(
+      `"${pin}" is not a reviewed marketplace pin for "${name}". ` +
+        `Use GET /v1/plugins/${name}/versions to list installable pins.`,
+    );
+  }
+  return entry.marketplaceCommit;
+}
+
 async function handleInstallPlugin({ body = {} }: RouteHandlerArgs) {
   const name = typeof body.name === "string" ? body.name : "";
   if (!name) {
     throw new BadRequestError("`name` is required");
   }
   const force = typeof body.force === "boolean" ? body.force : undefined;
+  const pin = typeof body.pin === "string" ? body.pin : undefined;
 
-  // The ref is pinned to the curated `DEFAULT_PLUGIN_REF` rather than taken
-  // from the request: a caller-supplied ref would let any `settings.write`
-  // principal install from an unreviewed revision (a PR branch, fork ref,
-  // ...) whose marketplace manifest could carry attacker code that the loader
-  // then dynamically imports. Installs over HTTP therefore only ever resolve
-  // against the reviewed catalog on the default ref. Operators who need
-  // another ref use the local CLI's `assistant plugins install --ref`.
+  // The marketplace ref is never taken raw from the request: a caller-supplied
+  // ref would let any `settings.write` principal install from an unreviewed
+  // revision (a PR branch, fork ref, ...) whose manifest could carry attacker
+  // code the loader then dynamically imports. Installs over HTTP therefore
+  // resolve only against reviewed history. A `pin` is honored by mapping it —
+  // server-side — to the marketplace commit that introduced it, but only if it
+  // appears in the plugin's reviewed pin history; an unreviewed SHA is refused.
+  // The default install reads the current catalog on `DEFAULT_PLUGIN_REF`.
+  // Operators who need an unreviewed revision use the local CLI's
+  // `assistant plugins install --pin <sha> --allow-unreviewed`.
   try {
+    const marketplaceRef = await resolveInstallMarketplaceRef(name, pin);
     const result = await installPlugin(
-      { name, ref: DEFAULT_PLUGIN_REF, force },
+      { name, ref: marketplaceRef, force },
       { fetch: globalThis.fetch.bind(globalThis) },
     );
     return {
@@ -798,6 +866,11 @@ async function handleInstallPlugin({ body = {} }: RouteHandlerArgs) {
       ref: result.ref,
     };
   } catch (err) {
+    // Pin resolution already maps unreviewed/bad-pin cases to a RouteError;
+    // re-throw those verbatim rather than masking them as a 500.
+    if (err instanceof RouteError) {
+      throw err;
+    }
     if (err instanceof InvalidPluginNameError) {
       throw new BadRequestError(err.message);
     }
@@ -812,6 +885,10 @@ async function handleInstallPlugin({ body = {} }: RouteHandlerArgs) {
     if (err instanceof PluginSourceUnavailableError) {
       throw new ServiceUnavailableError(err.message);
     }
+    // The pin-history read hits GitHub too; treat its failures as retryable.
+    if (err instanceof PluginPinHistoryError) {
+      throw new ServiceUnavailableError(err.message);
+    }
     throw new InternalError(
       err instanceof Error ? err.message : "plugin install failed",
     );
@@ -846,6 +923,45 @@ async function handleInspectPlugin({ pathParams = {} }: RouteHandlerArgs) {
   }
 }
 
+// ---------------------------------------------------------------------------
+// Handler — versions (marketplace pin history)
+// ---------------------------------------------------------------------------
+
+async function handlePluginVersions({
+  pathParams = {},
+  queryParams = {},
+}: RouteHandlerArgs) {
+  const rawName = pathParams.name ?? "";
+
+  let limit: number | undefined;
+  if (queryParams.limit !== undefined) {
+    const parsed = Number.parseInt(queryParams.limit, 10);
+    if (!Number.isInteger(parsed) || parsed < 1) {
+      throw new BadRequestError("`limit` must be a positive integer.");
+    }
+    limit = parsed;
+  }
+
+  try {
+    return await listPinHistory(
+      rawName,
+      { fetch: globalThis.fetch.bind(globalThis) },
+      limit !== undefined ? { limit } : {},
+    );
+  } catch (err) {
+    if (err instanceof InvalidPluginNameError) {
+      throw new BadRequestError(err.message);
+    }
+    // The history read hits GitHub; its failures are retryable upstream errors.
+    if (err instanceof PluginPinHistoryError) {
+      throw new ServiceUnavailableError(err.message);
+    }
+    throw new InternalError(
+      err instanceof Error ? err.message : "plugin versions failed",
+    );
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Handler — diff
 // ---------------------------------------------------------------------------
@@ -1150,6 +1266,46 @@ export const ROUTES: RouteDefinition[] = [
     },
     handler: handleInspectPlugin,
   },
+  {
+    operationId: "plugins_versions",
+    endpoint: "plugins/:name/versions",
+    method: "GET",
+    policy: {
+      requiredScopes: ["settings.read"],
+      allowedPrincipalTypes: ACTOR_PRINCIPALS,
+    },
+    summary: "List a plugin's reviewed marketplace pins",
+    description: `Report the distinct marketplace pins a plugin has been promoted to over time, newest first (the first entry is the current pin), capped at \`limit\` (default ${DEFAULT_PIN_HISTORY_LIMIT}). The curated \`marketplace.json\` stores only the current pin, so this is reconstructed from the manifest's own commit history on the default branch — every entry is therefore a reviewed, known-good revision. Pair with \`POST /v1/plugins/install\`'s \`pin\` field to roll a plugin back to an older reviewed version. An unknown name (never in the manifest) returns an empty array, not 404.`,
+    tags: ["plugins"],
+    pathParams: [
+      {
+        name: "name",
+        type: "string",
+        description:
+          "Install name. Must match the kebab-case name accepted by `assistant plugins install`.",
+      },
+    ],
+    queryParams: [
+      {
+        name: "limit",
+        type: "string",
+        required: false,
+        description: `Maximum number of pins to return (positive integer; default ${DEFAULT_PIN_HISTORY_LIMIT}).`,
+      },
+    ],
+    responseBody: pluginVersionsResponseSchema,
+    additionalResponses: {
+      "400": {
+        description:
+          "The plugin name failed sanitization, or `limit` was not a positive integer.",
+      },
+      "503": {
+        description:
+          "The marketplace pin history could not be read from GitHub (rate-limited or upstream error); retryable.",
+      },
+    },
+    handler: handlePluginVersions,
+  },
   {
     operationId: "plugins_diff",
     endpoint: "plugins/:name/diff",

package/src/runtime/routes/schedule-routes.ts

@@ -1061,6 +1061,7 @@ async function handleRunScheduleNow(id: string) {
         conversationId: schedule.wakeConversationId,
         hint: schedule.message,
         source: "defer",
+        persistTriggerAsEvent: true,
         ...(schedule.inferenceProfile
           ? { forceOverrideProfile: schedule.inferenceProfile }
           : {}),

package/src/runtime/routes/subagents-routes.ts

@@ -39,6 +39,8 @@ export interface SubagentDetailResult {
     toolName?: string;
     isError?: boolean;
     messageId?: string;
+    toolUseId?: string;
+    input?: Record<string, unknown>;
   }>;
 }
 
@@ -112,6 +114,8 @@ export function parseSubagentMessages(
           type: "tool_use",
           content: JSON.stringify(input),
           toolName: name,
+          toolUseId: id || undefined,
+          input,
         });
         if (id) pendingTools.set(id, name);
       } else if (
@@ -147,6 +151,7 @@ export function parseSubagentMessages(
           content: resultContent,
           toolName: toolName ?? "unknown",
           isError,
+          toolUseId: toolUseId || undefined,
         });
       }
     }