@vellumai/assistant 0.12.4-staging.1 → 0.12.4-staging.3

package/openapi.yaml

@@ -12060,7 +12060,9 @@ paths:
     post:
       operationId: dictation_post
       summary: Process dictation
-      description: Classify voice input as dictation or action, clean up text, and apply user style preferences.
+      description:
+        Preserve dictated words with explicit dictionary and snippet replacements, classify actions, or edit
+        selected text when instructed.
       tags:
         - diagnostics
       requestBody:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.12.4-staging.1",
+  "version": "0.12.4-staging.3",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/config-loader-backfill.test.ts

@@ -1869,6 +1869,7 @@ describe("seedInferenceProfiles BYOK-mode default profiles", () => {
       "quality-optimized-backup",
       "cost-optimized-backup",
       "latency-optimized-backup",
+      "jev-managed",
       "my-custom",
     ]);
   });

package/src/__tests__/default-profile-catalog-fallback.test.ts

@@ -116,6 +116,7 @@ describe("managed-column-only scoping", () => {
     expect(Object.keys(MANAGED_PROFILE_TEMPLATES)).toEqual([
       ...DEFAULT_PROFILE_KEYS,
       ...BACKUP_PROFILE_KEYS,
+      "jev-managed",
     ]);
   });
 

package/src/__tests__/inference-profile-validation-jev.test.ts

@@ -0,0 +1,17 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import { validateInferenceProfileKey } from "../config/inference-profile-validation.js";
+import { setConfig } from "./helpers/set-config.js";
+
+describe("validateInferenceProfileKey and the managed Jev profile", () => {
+  beforeEach(() => {
+    setConfig("llm", { profiles: {} });
+  });
+
+  test("a schedule or subagent turn cannot run on the verdict model", () => {
+    expect(validateInferenceProfileKey("balanced")).toBeNull();
+    expect(validateInferenceProfileKey("jev-managed")).toContain(
+      "is not defined in llm.profiles",
+    );
+  });
+});

package/src/__tests__/llm-catalog-parity.test.ts

@@ -235,6 +235,10 @@ describe("LLM catalog parity: daemon vs client", () => {
 
   test("jev-latest opts out of chat text generation", () => {
     expect(catalogModelSupportsText("typesafe", "jev-latest")).toBe(false);
+    // A managed Jev profile is configured as `vellum`; the routing identity
+    // owns no catalog models, so the lookup must judge by the catalog owner.
+    expect(catalogModelSupportsText("vellum", "jev-latest")).toBe(false);
+    expect(catalogModelSupportsText("vellum", "claude-opus-5")).toBe(true);
     expect(catalogModelSupportsText("anthropic", "claude-opus-4-8")).toBe(true);
     expect(catalogModelSupportsText("openai-compatible", "local-model")).toBe(
       true,

package/src/__tests__/llm-resolver-override-or-default.test.ts

@@ -225,6 +225,49 @@ describe("composeCallSiteTweak", () => {
 });
 
 describe("shipped call-site tuning", () => {
+  test("the escalation judge uses managed Jev by default", () => {
+    const resolved = resolveCallSiteConfig(
+      "voiceEscalationJudge",
+      LLMSchema.parse({}),
+    );
+
+    expect(resolved.provider).toBe("vellum");
+    expect(resolved.provider_connection).toBeUndefined();
+    expect(resolved.model).toBe("jev-latest");
+  });
+
+  test("the escalation judge falls through on a BYOK default provider", () => {
+    const resolved = resolveCallSiteConfig(
+      "voiceEscalationJudge",
+      LLMSchema.parse(anthropicDp),
+    );
+
+    expect(resolved.provider).toBe("anthropic");
+    expect(resolved.model).not.toBe("jev-latest");
+  });
+
+  test("a BYOK Jev call-site override replaces the managed default", () => {
+    const llm = LLMSchema.parse({
+      ...anthropicDp,
+      profiles: {
+        jev: {
+          ...completeCustom,
+          provider: "typesafe",
+          provider_connection: "typesafe-personal",
+          model: "jev-latest",
+        },
+      },
+      callSites: {
+        voiceEscalationJudge: { profile: "jev" },
+      },
+    });
+    const resolved = resolveCallSiteConfig("voiceEscalationJudge", llm);
+
+    expect(resolved.provider).toBe("typesafe");
+    expect(resolved.provider_connection).toBe("typesafe-personal");
+    expect(resolved.model).toBe("jev-latest");
+  });
+
   // A workspace entry that only repoints the profile must not cost the call
   // site the tuning it ships with. `recall` carries a full set: a token
   // budget, effort, sampling, thinking, and cache posture.

package/src/__tests__/openai-responses-prompt-cache.test.ts

@@ -446,7 +446,7 @@ describe("OpenAIResponsesProvider explicit prompt caching (GPT-5.6+)", () => {
     expect(JSON.stringify(messages)).not.toContain("prompt_cache_breakpoint");
   });
 
-  test("catalog flags GPT-5.6 and GPT-6 Astra direct-openai rows", () => {
+  test("catalog flags GPT-5.6 and GPT-6 direct-openai rows", () => {
     const openai = PROVIDER_CATALOG.find((p) => p.id === "openai");
     const flagged = (openai?.models ?? [])
       .filter((m) => m.supportsPromptCacheBreakpoints)
@@ -457,6 +457,8 @@ describe("OpenAIResponsesProvider explicit prompt caching (GPT-5.6+)", () => {
       "gpt-5.6-sol",
       "gpt-5.6-terra",
       "gpt-6-astra",
+      "gpt-6-luna",
+      "gpt-6-sol",
     ]);
   });
 });

package/src/__tests__/plugin-api-model-profiles.test.ts

@@ -40,6 +40,13 @@ describe("getModelProfiles", () => {
     writeFixtureConfig({});
   });
 
+  test("omits the managed Jev profile, whose model returns verdicts rather than text", () => {
+    writeFixtureConfig({});
+    const keys = getModelProfiles().map((p) => p.key);
+    expect(keys).toContain("balanced");
+    expect(keys).not.toContain("jev-managed");
+  });
+
   test("orders profiles by profileOrder then the remaining keys alphabetically", () => {
     // GIVEN a workspace whose profileOrder names some keys, with a duplicate
     // ("balanced") and a key that resolves to no profile ("ghost").

package/src/__tests__/pricing.test.ts

@@ -135,6 +135,39 @@ describe("resolvePricing", () => {
       expect(result.estimatedCostUsd).toBeCloseTo(0.5 + 0.625 + 0.05, 10);
     });
 
+    test("bills GPT-6 Sol at published short-context rates", () => {
+      const result = resolvePricingForUsage("openai", "gpt-6-sol", {
+        directInputTokens: 50_000,
+        outputTokens: 50_000,
+        cacheCreationInputTokens: 50_000,
+        cacheReadInputTokens: 50_000,
+        anthropicCacheCreation: null,
+      });
+
+      expect(result.pricingStatus).toBe("priced");
+      // 0.05M x $2 direct + 0.05M x $2.5 write + 0.05M x $0.2 read
+      // + 0.05M x $10 output.
+      expect(result.estimatedCostUsd).toBeCloseTo(0.1 + 0.125 + 0.01 + 0.5, 10);
+    });
+
+    test("bills GPT-6 Luna at the long-context tier above 272k", () => {
+      const result = resolvePricingForUsage("openai", "gpt-6-luna", {
+        directInputTokens: 500_000,
+        outputTokens: 1_000_000,
+        cacheCreationInputTokens: 500_000,
+        cacheReadInputTokens: 1_000_000,
+        anthropicCacheCreation: null,
+      });
+
+      expect(result.pricingStatus).toBe("priced");
+      // Tier rates: 0.5M x $0.2 direct + 0.5M x $0.25 write + 1M x $0.02 read
+      // + 1M x $0.75 output.
+      expect(result.estimatedCostUsd).toBeCloseTo(
+        0.1 + 0.125 + 0.02 + 0.75,
+        10,
+      );
+    });
+
     test("bills GPT-6 Astra at the long-context tier above 272k", () => {
       const result = resolvePricingForUsage("openai", "gpt-6-astra", {
         directInputTokens: 500_000,

package/src/__tests__/skill-load-tool.test.ts

@@ -1,4 +1,4 @@
-import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
+import { mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
@@ -781,16 +781,48 @@ describe("skill_load tool", () => {
     expect(result.content).toContain("Deploy the application to production");
     expect(result.content).toContain("Rollback to previous version");
 
-    // Should list parameters with types and required/optional markers
-    expect(result.content).toContain(
-      "- environment (string, required): Target environment",
+    const schemas = [
+      ...result.content.matchAll(/```json\n([\s\S]*?)\n```/g),
+    ].map((match) => JSON.parse(match[1]));
+    expect(schemas[0].required).toEqual(["environment"]);
+    expect(schemas[0].properties.environment).toEqual({
+      type: "string",
+      description: "Target environment",
+    });
+    expect(schemas[0].properties.force.type).toBe("boolean");
+    expect(schemas[1].required).toEqual(["version"]);
+  });
+
+  test("discloses the complete annotation mark schema before execution", async () => {
+    const manifest = JSON.parse(
+      readFileSync(
+        join(
+          import.meta.dir,
+          "../config/bundled-skills/screen-annotation/TOOLS.json",
+        ),
+        "utf-8",
+      ),
     );
-    expect(result.content).toContain(
-      "- force (boolean, optional): Force deploy even if checks fail",
+    writeSkill(
+      "annotation-schema",
+      "Annotation Schema",
+      "Point at a control",
+      "Name the control.",
     );
-    expect(result.content).toContain(
-      "- version (string, required): Version to rollback to",
+    writeToolsJson("annotation-schema", manifest.tools);
+
+    const result = await executeSkillLoad(
+      { skill: "annotation-schema" },
+      "macos",
     );
+    expect(result.isError).toBe(false);
+    const schemaMatch = result.content.match(/```json\n([\s\S]*?)\n```/);
+    expect(schemaMatch).not.toBeNull();
+    const schema = JSON.parse(schemaMatch![1]);
+    expect(schema).toEqual(manifest.tools[0].input_schema);
+    expect(schema.properties.marks.items.oneOf).toHaveLength(2);
+    expect(schema.properties.marks.items.properties.target.minLength).toBe(1);
+    expect(schema.properties.marks.items.properties.caption.maxLength).toBe(80);
   });
 
   test("skill without TOOLS.json does not include tool schemas section", async () => {
@@ -862,9 +894,8 @@ describe("skill_load tool", () => {
     // The child skill's tool schemas should appear (#### level under ### Tools from …)
     expect(result.content).toContain("#### child_action");
     expect(result.content).toContain("A child tool action");
-    expect(result.content).toContain(
-      "- target (string, required): Action target",
-    );
+    const schemaMatch = result.content.match(/```json\n([\s\S]*?)\n```/);
+    expect(JSON.parse(schemaMatch![1]).required).toEqual(["target"]);
     expect(result.content).toContain(
       "Use `skill_execute` to call these tools.",
     );

package/src/__tests__/skill-tool-factory.test.ts

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
 
-import type { SkillToolEntry } from "../config/skills.js";
+import { getBundledSkillsDir, type SkillToolEntry } from "../config/skills.js";
 import { RiskLevel } from "../permissions/types.js";
 import { computeSkillVersionHash } from "../skills/version-hash.js";
 import { resolveSubagentRole } from "../subagent/role-resolution.js";
@@ -261,6 +261,54 @@ describe("createSkillTool — unknown parameter validation", () => {
     expect(result.content).toContain('Unknown parameter "bar"');
   });
 
+  test.each([
+    { description: "Filters" },
+    { marks: [{ description: "Filters" }] },
+    { marks: [{}] },
+    { marks: [{ target: 42 }] },
+    { marks: [{ x: 0.1, y: 0.2 }] },
+    { marks: [{ target: "Filters", x: 0.1, y: 0.2, width: 0.3, height: 0.4 }] },
+  ])(
+    "an annotation validation error provides enough schema for one corrected retry: %j",
+    async (invalid) => {
+      const skillDir = join(getBundledSkillsDir(), "screen-annotation");
+      const manifest = JSON.parse(
+        readFileSync(join(skillDir, "TOOLS.json"), "utf-8"),
+      );
+      const entry = manifest.tools.find(
+        (tool: SkillToolEntry) => tool.name === "screen_point_at",
+      );
+      const tool = createSkillTool(
+        entry,
+        skillDir,
+        computeSkillVersionHash(skillDir),
+        BUNDLED,
+      );
+      const calls: unknown[] = [];
+      const context = makeContext({
+        proxyToolResolver: async (name, input) => {
+          calls.push({ name, input });
+          return { content: "Drew 1 mark", isError: false };
+        },
+      });
+
+      const rejected = await tool.execute(invalid, context);
+      expect(rejected.isError).toBe(true);
+      expect(calls).toHaveLength(0);
+      expect(rejected.content).toContain("skill_load");
+      const schemaMatch = rejected.content.match(/```json\n([\s\S]*?)\n```/);
+      expect(schemaMatch).not.toBeNull();
+      expect(JSON.parse(schemaMatch![1])).toEqual(entry.input_schema);
+
+      const input = {
+        marks: [{ target: "Filters", caption: "Click Filters" }],
+      };
+      const corrected = await tool.execute(input, context);
+      expect(corrected.isError).toBe(false);
+      expect(calls).toEqual([{ name: "computer_use_point_at", input }]);
+    },
+  );
+
   test("allows input with only known parameters", async () => {
     const hash = computeSkillVersionHash(tempDir);
     const tool = createSkillTool(
@@ -471,7 +519,7 @@ describe("createSkillTool — required/type/enum validation", () => {
 
     expect(result.isError).toBe(true);
     expect(result.content).toContain(
-      "auto_open must be a boolean — pass true or false as a JSON boolean, not a string",
+      "auto_open must be a boolean: pass true or false as a JSON boolean, not a string",
     );
   });
 

package/src/__tests__/validate-input.test.ts

@@ -6,6 +6,79 @@ import {
   validateInputAgainstSchema,
 } from "../skills/validate-input.js";
 
+describe("nested input validation", () => {
+  const schema = {
+    properties: {
+      options: {
+        type: "object",
+        properties: { label: { type: "string" } },
+        required: ["label"],
+      },
+    },
+  };
+
+  test("reports nested required fields and types with their path", () => {
+    expect(
+      validateInputAgainstSchema("example", { options: {} }, schema),
+    ).toEqual({
+      ok: false,
+      errors: ["options.label is required"],
+    });
+    expect(
+      validateInputAgainstSchema("example", { options: { label: 42 } }, schema),
+    ).toEqual({
+      ok: false,
+      errors: ["options.label must be a string"],
+    });
+  });
+
+  test("preserves open nested objects and only rejects explicitly closed extra fields", () => {
+    const input = { options: { label: "Example", extra: true } };
+    expect(validateInputAgainstSchema("example", input, schema)).toEqual({
+      ok: true,
+    });
+    expect(
+      validateInputAgainstSchema("example", input, {
+        properties: {
+          options: {
+            ...schema.properties.options,
+            additionalProperties: false,
+          },
+        },
+      }),
+    ).toEqual({
+      ok: false,
+      errors: ['Unknown parameter "options.extra". Supported: "label"'],
+    });
+  });
+
+  test("does not treat a richer oneOf as a presence-only union", () => {
+    expect(
+      validateInputAgainstSchema(
+        "example",
+        { options: { label: "Example" } },
+        {
+          properties: {
+            options: {
+              ...schema.properties.options,
+              oneOf: [
+                {
+                  required: ["label"],
+                  properties: { label: { const: "Example" } },
+                },
+                {
+                  required: ["label"],
+                  properties: { label: { const: "Other" } },
+                },
+              ],
+            },
+          },
+        },
+      ),
+    ).toEqual({ ok: true });
+  });
+});
+
 // ---------------------------------------------------------------------------
 // required
 // ---------------------------------------------------------------------------

package/src/__tests__/workspace-migration-159-rename-colliding-jev-profile-name.test.ts

@@ -0,0 +1,197 @@
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { Database } from "bun:sqlite";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { WORKSPACE_MIGRATIONS } from "../workspace/migrations/registry.js";
+import { assertNotLiveDb } from "./assert-not-live-db.js";
+
+const migration = WORKSPACE_MIGRATIONS.find(
+  (entry) => entry.id === "159-rename-colliding-jev-profile-name",
+)!;
+
+let workspaceDir: string;
+
+function writeConfig(data: Record<string, unknown>): void {
+  writeFileSync(
+    join(workspaceDir, "config.json"),
+    JSON.stringify(data, null, 2) + "\n",
+  );
+}
+
+function readConfig(): Record<string, any> {
+  return JSON.parse(readFileSync(join(workspaceDir, "config.json"), "utf-8"));
+}
+
+function seedProfilePinTables(rows: {
+  conversations: string[];
+  cronJobs: string[];
+}): void {
+  mkdirSync(join(workspaceDir, "data", "db"), { recursive: true });
+  const db = new Database(join(workspaceDir, "data", "db", "assistant.db"));
+  db.run(
+    `CREATE TABLE conversations (id TEXT PRIMARY KEY, inference_profile TEXT)`,
+  );
+  db.run(
+    `CREATE TABLE cron_jobs (id TEXT PRIMARY KEY, inference_profile TEXT)`,
+  );
+  rows.conversations.forEach((profile, index) => {
+    db.query(
+      `INSERT INTO conversations (id, inference_profile) VALUES (?, ?)`,
+    ).run(`conv-${index}`, profile);
+  });
+  rows.cronJobs.forEach((profile, index) => {
+    db.query(`INSERT INTO cron_jobs (id, inference_profile) VALUES (?, ?)`).run(
+      `job-${index}`,
+      profile,
+    );
+  });
+  db.close();
+}
+
+function readPins(table: string): string[] {
+  const db = new Database(join(workspaceDir, "data", "db", "assistant.db"));
+  try {
+    return (
+      db
+        .query(`SELECT inference_profile AS p FROM ${table} ORDER BY id`)
+        .all() as Array<{ p: string }>
+    ).map((row) => row.p);
+  } finally {
+    db.close();
+  }
+}
+
+beforeEach(() => {
+  workspaceDir = join(
+    tmpdir(),
+    `vellum-migration-159-test-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+  mkdirSync(workspaceDir, { recursive: true });
+});
+
+afterEach(() => {
+  if (existsSync(workspaceDir)) {
+    assertNotLiveDb(workspaceDir);
+    rmSync(workspaceDir, { recursive: true, force: true });
+  }
+});
+
+describe("159-rename-colliding-jev-profile-name migration", () => {
+  test("is registered under its id with checkpoint retry", () => {
+    expect(migration).toBeDefined();
+    expect(migration.retryFailedCheckpoint).toBe(true);
+  });
+
+  test("renames a colliding user profile and rewrites every reference", async () => {
+    writeConfig({
+      llm: {
+        defaultProvider: { provider: "vellum" },
+        profiles: {
+          "jev-managed": {
+            source: "user",
+            provider: "anthropic",
+            model: "claude-opus-4-7",
+            label: "Mine",
+          },
+          scratch: { source: "user", fallbackProfile: "jev-managed" },
+          armA: { source: "user", speed: "fast" },
+          blend: {
+            source: "user",
+            mix: [
+              { profile: "jev-managed", weight: 1 },
+              { profile: "armA", weight: 1 },
+            ],
+          },
+        },
+        profileOrder: ["balanced", "jev-managed", "blend"],
+        activeProfile: "jev-managed",
+        advisorProfile: "jev-managed",
+        callSites: { recall: { profile: "jev-managed", maxTokens: 4096 } },
+      },
+    });
+    seedProfilePinTables({
+      conversations: ["jev-managed", "balanced"],
+      cronJobs: ["jev-managed"],
+    });
+
+    await migration.run(workspaceDir);
+
+    const llm = readConfig().llm;
+    expect(llm.profiles["jev-managed"]).toBeUndefined();
+    expect(llm.profiles["jev-managed-custom"]).toEqual({
+      source: "user",
+      provider: "anthropic",
+      model: "claude-opus-4-7",
+      label: "Mine",
+    });
+    expect(Object.keys(llm.profiles)).toEqual([
+      "jev-managed-custom",
+      "scratch",
+      "armA",
+      "blend",
+    ]);
+    expect(llm.activeProfile).toBe("jev-managed-custom");
+    expect(llm.advisorProfile).toBe("jev-managed-custom");
+    expect(llm.profileOrder).toEqual([
+      "balanced",
+      "jev-managed-custom",
+      "blend",
+    ]);
+    expect(llm.callSites.recall.profile).toBe("jev-managed-custom");
+    expect(llm.profiles.scratch.fallbackProfile).toBe("jev-managed-custom");
+    expect(llm.profiles.blend.mix[0].profile).toBe("jev-managed-custom");
+    expect(readPins("conversations")).toEqual([
+      "jev-managed-custom",
+      "balanced",
+    ]);
+    expect(readPins("cron_jobs")).toEqual(["jev-managed-custom"]);
+  });
+
+  test("keeps suffixing past taken and reserved names", async () => {
+    writeConfig({
+      llm: {
+        profiles: {
+          "jev-managed": {
+            source: "user",
+            provider: "openai",
+            model: "gpt-5.5",
+          },
+          "jev-managed-custom": { source: "user", provider: "openai" },
+        },
+      },
+    });
+
+    await migration.run(workspaceDir);
+
+    const keys = Object.keys(readConfig().llm.profiles);
+    expect(keys).toEqual(["jev-managed-custom-2", "jev-managed-custom"]);
+  });
+
+  test("leaves a managed stub alone and is idempotent", async () => {
+    const original = {
+      llm: {
+        profiles: {
+          "jev-managed": { source: "managed", status: "disabled" },
+          balanced: { source: "managed" },
+        },
+      },
+    };
+    writeConfig(original);
+
+    await migration.run(workspaceDir);
+    await migration.run(workspaceDir);
+
+    expect(readConfig()).toEqual(original);
+    expect(existsSync(join(workspaceDir, "data", "db", "assistant.db"))).toBe(
+      false,
+    );
+  });
+});

package/src/api/requests/dictation.ts

@@ -1,8 +1,8 @@
 /**
  * Wire contract for the dictation REST endpoint (`POST /dictation`).
  *
- * The body web/macOS sends when asking the daemon to clean up a raw voice
- * transcript and classify it as dictation vs. action. `context` is its own
+ * The body web/macOS sends to apply explicit dictation replacements, classify
+ * voice input, or edit selected text. `context` is its own
  * named schema/type so consumers (web client) can import `DictationContext`
  * directly instead of inferring it from the generated client.
  *