@vellumai/assistant 0.10.12 → 0.11.0-staging.2

package/src/workflows/leaf-runner.ts

@@ -24,7 +24,7 @@
  *   `tool_choice` provider call whose synthetic tool's input schema is the
  *   caller's Zod schema. The returned tool input is Zod-validated and returned
  *   as `output`. Copies the proven pattern from `memory/v2/router.ts` and
- *   `memory/graph/extraction.ts`.
+ *   `memory/v1/graph/extraction.ts`.
  * - **Tool path** (`tools` provided, no `schema`): a real {@link AgentLoop} with
  *   a minimal task-scoped system prompt and a tool-executor restricted to the
  *   supplied registry tools. No conversation is bootstrapped — the loop runs
@@ -42,11 +42,13 @@ import { randomUUID } from "node:crypto";
 import { z } from "zod";
 
 import { type AgentEvent, AgentLoop } from "../agent/loop.js";
+import type { AssistantEvent } from "../api/index.js";
 import { getEffectiveProfile } from "../config/default-profile-catalog.js";
 import { getConfig } from "../config/loader.js";
-import type { ServerMessage } from "../daemon/message-protocol.js";
+import { isMemoryEnabled } from "../config/memory-v3-gate.js";
 import { isPersonalMemoryAllowed } from "../daemon/trust-context.js";
 import type { TrustContext } from "../daemon/trust-context-types.js";
+import { isOutOfWorkspaceFileInvocation } from "../permissions/workspace-policy.js";
 import { ConversationGraphMemory } from "../plugins/defaults/memory/graph/conversation-graph-memory.js";
 import { buildSystemPrompt } from "../prompts/system-prompt.js";
 import {
@@ -128,21 +130,31 @@ async function resolveLeafContext(
  * persona leaf still gets the assistant's identity system prompt but NO personal
  * memory. Without this, a workflow launched by an untrusted actor whose manifest
  * grants `persona` could exfiltrate private memory in its output.
+ *
+ * On a v3-live assistant a persona leaf receives no injected memory at all:
+ * `prepareMemory`'s v2 route is gated on `isV2InjectionEngineActive` (v3 live
+ * suppresses it), the v1 retriever short-circuits to empty under the
+ * concept-page substrate, and v3 orchestration is conversation-turn-scoped so
+ * it does not run for workflow leaves.
  */
 async function injectPersonaMemory(
   opts: RunLeafOptions,
   messages: Message[],
 ): Promise<Message[]> {
-  if (!isPersonalMemoryAllowed(opts.trustContext)) return messages;
+  if (!isPersonalMemoryAllowed(opts.trustContext)) {
+    return messages;
+  }
 
   const config = getConfig();
-  if (config.memory?.enabled === false) return messages;
+  if (!isMemoryEnabled(config)) {
+    return messages;
+  }
 
   const ephemeralConversationId = `workflow-leaf:${randomUUID()}`;
   const graphMemory = new ConversationGraphMemory(ephemeralConversationId);
   // The memory pipeline broadcasts retrieval progress to the shared event hub
   // (matching the main-agent hook); a leaf has no per-turn event callback.
-  const onEvent = (msg: ServerMessage): void => {
+  const onEvent = (msg: AssistantEvent): void => {
     broadcastMessage(msg);
   };
   // `prepareMemory` requires a non-aborting signal; reuse the caller's when
@@ -525,6 +537,17 @@ async function executeLeafTool(
     };
   }
 
+  // Leaf tool calls bypass the ToolExecutor's permission lane, so the file
+  // tools' out-of-workspace host fallback must not be reachable from here:
+  // the manifest consent covers workspace-scoped operation only, and leaves
+  // reject host tools outright. Keep the leaf workspace-bound.
+  if (isOutOfWorkspaceFileInvocation(name, input, ctx.workingDir)) {
+    return {
+      content: `Tool "${name}" may only access paths inside the workspace when invoked from a workflow leaf.`,
+      isError: true,
+    };
+  }
+
   const toolContext: ToolContext = {
     conversationId: ctx.ephemeralConversationId,
     workingDir: ctx.workingDir,
@@ -552,7 +575,9 @@ async function executeLeafTool(
 function finalAssistantText(history: Message[]): string {
   for (let i = history.length - 1; i >= 0; i--) {
     const msg = history[i];
-    if (msg.role !== "assistant") continue;
+    if (msg.role !== "assistant") {
+      continue;
+    }
     const text = msg.content
       .filter(
         (b): b is Extract<typeof b, { type: "text" }> => b.type === "text",

package/src/workspace/migrations/070-memory-v2-summary-schema-rebuild.ts

@@ -4,8 +4,8 @@ import type { WorkspaceMigration } from "./types.js";
  * Audit-only entry for the v2 concept-page schema upgrade introduced in PR
  * #29823 (summary_dense / summary_sparse named vectors). The destructive
  * collection rebuild and reembed enqueue both run inside the daemon at
- * Qdrant init time — see `maybeRebuildMemoryV2Concepts` in
- * `assistant/src/plugins/defaults/memory/v2/memory-v2-startup.ts`. The "exactly-once" fence is
+ * Qdrant init time — see `maybeRebuildConceptCollection` in
+ * `assistant/src/plugins/defaults/memory/substrate/boot-maintenance.ts`. The "exactly-once" fence is
  * per-collection schema introspection, not per-workspace checkpoint, so
  * users who wipe Qdrant separately still get re-rebuilt without resetting
  * any workspace flag.

package/src/workspace/migrations/134-image-generation-mode-to-provider.ts

@@ -0,0 +1,128 @@
+import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import type { WorkspaceMigration } from "./types.js";
+
+/**
+ * Fold `services["image-generation"].mode` into the provider field.
+ *
+ * Configs written by older versions carry two axes: `mode: "managed"` routes
+ * generation through the Vellum platform runtime proxy while `provider`
+ * names the backend the proxy fronts (gemini or openai). The schema treats
+ * `provider` as the only axis, with `"vellum"` meaning managed and the
+ * backend derived from the selected model's prefix at request time, so this
+ * migration rewrites every managed service to `provider: "vellum"`. There is
+ * no per-provider exemption: the stored backend value under managed mode is
+ * itself derived from the model on every model write, so replacing it loses
+ * nothing — `model` is preserved verbatim and keeps routing the managed
+ * request to the same backend.
+ *
+ * Idempotent: a config with no `mode` key is left untouched.
+ */
+export const imageGenerationModeToProviderMigration: WorkspaceMigration = {
+  id: "134-image-generation-mode-to-provider",
+  description:
+    "Fold services.image-generation mode into provider (managed -> provider: vellum)",
+  run(workspaceDir: string): void {
+    const configPath = join(workspaceDir, "config.json");
+    if (!existsSync(configPath)) {
+      return;
+    }
+
+    let config: Record<string, unknown>;
+    try {
+      const raw = JSON.parse(readFileSync(configPath, "utf-8"));
+      if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+        return;
+      }
+      config = raw as Record<string, unknown>;
+    } catch {
+      return; // Malformed JSON — skip
+    }
+
+    const services = readObj(config, "services");
+    if (!services) {
+      return;
+    }
+
+    const service = readObj(services, "image-generation");
+    if (!service || !("mode" in service)) {
+      return;
+    }
+
+    if (service.mode === "managed") {
+      service.provider = "vellum";
+    } else if (!("provider" in service)) {
+      // An explicit non-managed mode with no persisted provider resolves to
+      // the gemini schema default. Pin that choice: with `mode` gone, an
+      // absent provider leaf would otherwise be context-filled to "vellum"
+      // on platform deployments, silently flipping a your-own config to
+      // managed billing.
+      service.provider = "gemini";
+    }
+    delete service.mode;
+
+    // Write-then-rename keeps the migration rerunnable: a crash mid-write
+    // must not leave a truncated config.json that a retry reads as malformed
+    // and "completes" past, quarantining the user's unrelated settings.
+    const tmpPath = `${configPath}.tmp`;
+    writeFileSync(tmpPath, JSON.stringify(config, null, 2) + "\n");
+    renameSync(tmpPath, configPath);
+  },
+  down(workspaceDir: string): void {
+    const configPath = join(workspaceDir, "config.json");
+    if (!existsSync(configPath)) {
+      return;
+    }
+
+    let config: Record<string, unknown>;
+    try {
+      const raw = JSON.parse(readFileSync(configPath, "utf-8"));
+      if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+        return;
+      }
+      config = raw as Record<string, unknown>;
+    } catch {
+      return;
+    }
+
+    const services = readObj(config, "services");
+    if (!services) {
+      return;
+    }
+
+    const service = readObj(services, "image-generation");
+    if (!service || "mode" in service) {
+      return;
+    }
+
+    // Schemas that predate this migration accept provider "vellum" alongside
+    // mode "managed", so the managed pair round-trips exactly. The backend
+    // provider a managed service holds before `run()` is not recoverable —
+    // `run()` overwrites it with "vellum". That value is derived from the
+    // model prefix, so it re-derives on the next model write.
+    service.mode = service.provider === "vellum" ? "managed" : "your-own";
+
+    // Write-then-rename keeps the migration rerunnable: a crash mid-write
+    // must not leave a truncated config.json that a retry reads as malformed
+    // and "completes" past, quarantining the user's unrelated settings.
+    const tmpPath = `${configPath}.tmp`;
+    writeFileSync(tmpPath, JSON.stringify(config, null, 2) + "\n");
+    renameSync(tmpPath, configPath);
+  },
+};
+
+// ---------------------------------------------------------------------------
+// Helpers (self-contained per migration AGENTS.md)
+// ---------------------------------------------------------------------------
+
+function readObj(
+  parent: Record<string, unknown>,
+  key: string,
+): Record<string, unknown> | undefined {
+  const value = parent[key];
+  if (value == null || typeof value !== "object" || Array.isArray(value)) {
+    return undefined;
+  }
+  return value as Record<string, unknown>;
+}

package/src/workspace/migrations/135-copy-substrate-tunables.ts

@@ -0,0 +1,248 @@
+import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+
+import { getLogger } from "../../util/logger.js";
+import type { WorkspaceMigration } from "./types.js";
+
+const log = getLogger("workspace-migration-135");
+
+/**
+ * Marks the two optional `memory.v2` keys that ship with no schema default
+ * (`min_sparse_spread`, `full_sparse_spread`) — for those, raw presence IS
+ * user intent, so they copy on presence alone.
+ */
+const NO_DEFAULT: unique symbol = Symbol("no-default");
+
+/**
+ * The fifteen substrate tunables shared between the historical `memory.v2`
+ * namespace and the new `memory.substrate` namespace, each paired with the
+ * shipped `memory.v2` schema default (from `config/schemas/memory-v2.ts`).
+ * `memory.v2.k` and `memory.v2.hops` were renamed to `spread_k` /
+ * `spread_hops` on the substrate side; every other key keeps its name.
+ *
+ * The defaults matter because raw presence in config.json is NOT user intent:
+ * the config loader serializes the fully-parsed configuration for
+ * normally-created workspaces, so defaulted `memory.v2` leaves are persisted
+ * for most existing assistants. Copying those seeded values would populate
+ * the override-only `memory.substrate` namespace with today's defaults and
+ * permanently pin those assistants when substrate defaults are later retuned
+ * — the same default-pinning problem migration 119 solves for `memory.v3`
+ * tuning, distinguished the same way (value-vs-shipped-default comparison,
+ * table inlined per the migrations self-containment rule).
+ */
+const SUBSTRATE_KEY_PAIRS: ReadonlyArray<{
+  v2Key: string;
+  substrateKey: string;
+  shippedDefault: number | boolean | string | null | typeof NO_DEFAULT;
+  /**
+   * Additional values the loader has seeded as this key's default in earlier
+   * releases (multi-default keys, mirroring migration 119). A persisted value
+   * matching any of them is seeded, not a user override.
+   */
+  priorDefaults?: ReadonlyArray<number | boolean | string | null>;
+}> = [
+  {
+    v2Key: "sweep_enabled",
+    substrateKey: "sweep_enabled",
+    shippedDefault: false,
+  },
+  { v2Key: "dense_weight", substrateKey: "dense_weight", shippedDefault: 0.85 },
+  {
+    v2Key: "sparse_weight",
+    substrateKey: "sparse_weight",
+    shippedDefault: 0.15,
+  },
+  {
+    v2Key: "min_sparse_spread",
+    substrateKey: "min_sparse_spread",
+    shippedDefault: NO_DEFAULT,
+  },
+  {
+    v2Key: "full_sparse_spread",
+    substrateKey: "full_sparse_spread",
+    shippedDefault: NO_DEFAULT,
+  },
+  { v2Key: "bm25_k1", substrateKey: "bm25_k1", shippedDefault: 1.2 },
+  {
+    v2Key: "bm25_b",
+    substrateKey: "bm25_b",
+    shippedDefault: 0.4,
+    // Workspaces seeded before migration 075 carry the earlier 0.75 default.
+    priorDefaults: [0.75],
+  },
+  {
+    v2Key: "consolidation_interval_hours",
+    substrateKey: "consolidation_interval_hours",
+    shippedDefault: 8,
+  },
+  {
+    v2Key: "consolidation_max_buffer_lines",
+    substrateKey: "consolidation_max_buffer_lines",
+    shippedDefault: 100,
+  },
+  {
+    v2Key: "consolidation_max_entries_per_run",
+    substrateKey: "consolidation_max_entries_per_run",
+    shippedDefault: 150,
+  },
+  {
+    v2Key: "max_page_chars",
+    substrateKey: "max_page_chars",
+    shippedDefault: 5000,
+  },
+  {
+    v2Key: "consolidation_prompt_path",
+    substrateKey: "consolidation_prompt_path",
+    shippedDefault: null,
+  },
+  { v2Key: "k", substrateKey: "spread_k", shippedDefault: 0.5 },
+  { v2Key: "hops", substrateKey: "spread_hops", shippedDefault: 2 },
+  {
+    v2Key: "ann_candidate_limit",
+    substrateKey: "ann_candidate_limit",
+    shippedDefault: null,
+  },
+];
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return value != null && typeof value === "object" && !Array.isArray(value);
+}
+
+/**
+ * Copy user-overridden substrate tunables from `memory.v2` to
+ * `memory.substrate` so the substrate namespace carries the user's tuned
+ * values itself, independent of the runtime resolver's substrate→v2 fallback.
+ *
+ * Value-based, not presence-based: a key is copied only when it is present
+ * under `memory.v2`, absent under `memory.substrate`, AND its value differs
+ * from the shipped v2 schema default. The config loader persists the
+ * fully-parsed configuration for normally-created workspaces, so a defaulted
+ * `memory.v2` leaf in config.json is seeded, not user intent — copying it
+ * would pin the assistant to today's value when substrate defaults are later
+ * retuned (migration 119's rationale). A persisted value equal to its shipped
+ * default is treated as seeded even if a user typed it deliberately — the
+ * acceptable trade-off 119 already makes — so `memory.substrate` stays an
+ * override-only surface. The two no-default keys (`min_sparse_spread`,
+ * `full_sparse_spread`) copy on presence alone.
+ *
+ * That rule is narrower than "explicitly present", and the gap is deliberate:
+ * a value equal to a shipped default never copies, even when the user typed it
+ * on purpose. For every key whose shipped default is the current one that costs
+ * nothing — `memory.substrate` plus the shipped defaults resolve to exactly
+ * what the resolver's fallback produces. `bm25_b`'s retired `0.75` is the sole
+ * exception: it is skipped as seeded yet differs from today's `0.4`, so on a
+ * workspace carrying it the value survives only through the resolver's
+ * `memory.v2` fallback. That is the one case where dropping the fallback would
+ * change behavior — see the equivalence proof in this migration's test.
+ *
+ * `memory.v2.*` is never modified — the v2 injection engine still reads it.
+ * An already-present `memory.substrate` key is never clobbered. `memory.substrate` is only
+ * written at all when at least one key copies, so a fresh config never gains
+ * an empty object.
+ *
+ * Weight-pair safety: copying cannot create an invalid config. The v2
+ * schema's own refinement validates `dense_weight + sparse_weight` on the
+ * PERSISTED values (absent leaves resolve to defaults before the check), so a
+ * valid config carrying a non-default weight necessarily carries its
+ * non-default twin — both differ from their defaults, both copy, and the
+ * copied substrate pair is exactly the pair v2 already accepted.
+ */
+export const copySubstrateTunablesMigration: WorkspaceMigration = {
+  id: "135-copy-substrate-tunables",
+  description:
+    "Copy memory.v2 substrate tunables that differ from their shipped defaults to memory.substrate (k→spread_k, hops→spread_hops) without touching memory.v2, clobbering existing memory.substrate values, or pinning loader-seeded defaults",
+  // run() only ever adds missing memory.substrate keys and never clobbers
+  // present ones, so retrying after a failed write attempt is safe.
+  retryFailedCheckpoint: true,
+
+  run(workspaceDir: string): void {
+    const configPath = join(workspaceDir, "config.json");
+    if (!existsSync(configPath)) {
+      return;
+    }
+
+    // A filesystem read failure propagates so the failed checkpoint retries
+    // on the next boot; only malformed JSON is a permanent no-op (the config
+    // loader owns surfacing that error).
+    const rawText = readFileSync(configPath, "utf-8");
+    let config: Record<string, unknown>;
+    try {
+      const raw = JSON.parse(rawText);
+      if (!isPlainObject(raw)) {
+        return;
+      }
+      config = raw;
+    } catch {
+      return;
+    }
+
+    const memory = config.memory;
+    if (!isPlainObject(memory)) {
+      return;
+    }
+    const v2 = memory.v2;
+    if (!isPlainObject(v2)) {
+      return;
+    }
+    // A malformed (non-object) persisted memory.substrate is left alone — the
+    // config loader surfaces that error; this migration must not clobber it.
+    if (memory.substrate !== undefined && !isPlainObject(memory.substrate)) {
+      return;
+    }
+
+    // Staged separately so memory.substrate is only attached when a key
+    // actually copies — never introduce an empty object.
+    const substrate: Record<string, unknown> = isPlainObject(memory.substrate)
+      ? memory.substrate
+      : {};
+
+    let copied = false;
+    for (const {
+      v2Key,
+      substrateKey,
+      shippedDefault,
+      priorDefaults,
+    } of SUBSTRATE_KEY_PAIRS) {
+      if (!(v2Key in v2) || substrateKey in substrate) {
+        continue;
+      }
+      // A persisted value equal to a shipped default (current or from an
+      // earlier release) is loader-seeded, not a user override — skip it so
+      // the substrate namespace stays override-only. Strict equality
+      // suffices: every tunable value is a primitive or null.
+      const value = v2[v2Key];
+      if (shippedDefault !== NO_DEFAULT && value === shippedDefault) {
+        continue;
+      }
+      if (priorDefaults?.some((seeded) => seeded === value)) {
+        continue;
+      }
+      substrate[substrateKey] = v2[v2Key];
+      copied = true;
+    }
+    if (!copied) {
+      return;
+    }
+    memory.substrate = substrate;
+
+    // Write-then-rename keeps the migration rerunnable: a crash mid-write
+    // must not leave a truncated config.json that a retry reads as malformed
+    // and "completes" past, stranding the user's v2 overrides. A write or
+    // rename failure propagates to the runner, which checkpoints this
+    // migration as failed and (via retryFailedCheckpoint) retries it on the
+    // next boot instead of recording it as completed.
+    const tmpPath = `${configPath}.tmp`;
+    writeFileSync(tmpPath, JSON.stringify(config, null, 2) + "\n");
+    renameSync(tmpPath, configPath);
+    log.info(
+      "Copied non-default memory.v2 substrate tunables to memory.substrate",
+    );
+  },
+
+  down(_workspaceDir: string): void {
+    // Forward-only: a copied value is indistinguishable from one the user set
+    // under memory.substrate directly, so removal could destroy deliberate
+    // overrides. The copy is also behavior-preserving, so there is nothing to
+    // unwind.
+  },
+};