@vellumai/assistant 0.12.4-staging.2 → 0.12.4-staging.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.12.4-staging.2",
+  "version": "0.12.4-staging.4",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/openai-responses-prompt-cache.test.ts

@@ -446,7 +446,7 @@ describe("OpenAIResponsesProvider explicit prompt caching (GPT-5.6+)", () => {
     expect(JSON.stringify(messages)).not.toContain("prompt_cache_breakpoint");
   });
 
-  test("catalog flags GPT-5.6 and GPT-6 Astra direct-openai rows", () => {
+  test("catalog flags GPT-5.6 and GPT-6 direct-openai rows", () => {
     const openai = PROVIDER_CATALOG.find((p) => p.id === "openai");
     const flagged = (openai?.models ?? [])
       .filter((m) => m.supportsPromptCacheBreakpoints)
@@ -457,6 +457,8 @@ describe("OpenAIResponsesProvider explicit prompt caching (GPT-5.6+)", () => {
       "gpt-5.6-sol",
       "gpt-5.6-terra",
       "gpt-6-astra",
+      "gpt-6-luna",
+      "gpt-6-sol",
     ]);
   });
 });

package/src/__tests__/pricing.test.ts

@@ -135,6 +135,39 @@ describe("resolvePricing", () => {
       expect(result.estimatedCostUsd).toBeCloseTo(0.5 + 0.625 + 0.05, 10);
     });
 
+    test("bills GPT-6 Sol at published short-context rates", () => {
+      const result = resolvePricingForUsage("openai", "gpt-6-sol", {
+        directInputTokens: 50_000,
+        outputTokens: 50_000,
+        cacheCreationInputTokens: 50_000,
+        cacheReadInputTokens: 50_000,
+        anthropicCacheCreation: null,
+      });
+
+      expect(result.pricingStatus).toBe("priced");
+      // 0.05M x $2 direct + 0.05M x $2.5 write + 0.05M x $0.2 read
+      // + 0.05M x $10 output.
+      expect(result.estimatedCostUsd).toBeCloseTo(0.1 + 0.125 + 0.01 + 0.5, 10);
+    });
+
+    test("bills GPT-6 Luna at the long-context tier above 272k", () => {
+      const result = resolvePricingForUsage("openai", "gpt-6-luna", {
+        directInputTokens: 500_000,
+        outputTokens: 1_000_000,
+        cacheCreationInputTokens: 500_000,
+        cacheReadInputTokens: 1_000_000,
+        anthropicCacheCreation: null,
+      });
+
+      expect(result.pricingStatus).toBe("priced");
+      // Tier rates: 0.5M x $0.2 direct + 0.5M x $0.25 write + 1M x $0.02 read
+      // + 1M x $0.75 output.
+      expect(result.estimatedCostUsd).toBeCloseTo(
+        0.1 + 0.125 + 0.02 + 0.75,
+        10,
+      );
+    });
+
     test("bills GPT-6 Astra at the long-context tier above 272k", () => {
       const result = resolvePricingForUsage("openai", "gpt-6-astra", {
         directInputTokens: 500_000,

package/src/__tests__/skill-load-tool.test.ts

@@ -1,4 +1,4 @@
-import { mkdirSync, symlinkSync, writeFileSync } from "node:fs";
+import { mkdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
@@ -781,16 +781,48 @@ describe("skill_load tool", () => {
     expect(result.content).toContain("Deploy the application to production");
     expect(result.content).toContain("Rollback to previous version");
 
-    // Should list parameters with types and required/optional markers
-    expect(result.content).toContain(
-      "- environment (string, required): Target environment",
+    const schemas = [
+      ...result.content.matchAll(/```json\n([\s\S]*?)\n```/g),
+    ].map((match) => JSON.parse(match[1]));
+    expect(schemas[0].required).toEqual(["environment"]);
+    expect(schemas[0].properties.environment).toEqual({
+      type: "string",
+      description: "Target environment",
+    });
+    expect(schemas[0].properties.force.type).toBe("boolean");
+    expect(schemas[1].required).toEqual(["version"]);
+  });
+
+  test("discloses the complete annotation mark schema before execution", async () => {
+    const manifest = JSON.parse(
+      readFileSync(
+        join(
+          import.meta.dir,
+          "../config/bundled-skills/screen-annotation/TOOLS.json",
+        ),
+        "utf-8",
+      ),
     );
-    expect(result.content).toContain(
-      "- force (boolean, optional): Force deploy even if checks fail",
+    writeSkill(
+      "annotation-schema",
+      "Annotation Schema",
+      "Point at a control",
+      "Name the control.",
     );
-    expect(result.content).toContain(
-      "- version (string, required): Version to rollback to",
+    writeToolsJson("annotation-schema", manifest.tools);
+
+    const result = await executeSkillLoad(
+      { skill: "annotation-schema" },
+      "macos",
     );
+    expect(result.isError).toBe(false);
+    const schemaMatch = result.content.match(/```json\n([\s\S]*?)\n```/);
+    expect(schemaMatch).not.toBeNull();
+    const schema = JSON.parse(schemaMatch![1]);
+    expect(schema).toEqual(manifest.tools[0].input_schema);
+    expect(schema.properties.marks.items.oneOf).toHaveLength(2);
+    expect(schema.properties.marks.items.properties.target.minLength).toBe(1);
+    expect(schema.properties.marks.items.properties.caption.maxLength).toBe(80);
   });
 
   test("skill without TOOLS.json does not include tool schemas section", async () => {
@@ -862,9 +894,8 @@ describe("skill_load tool", () => {
     // The child skill's tool schemas should appear (#### level under ### Tools from …)
     expect(result.content).toContain("#### child_action");
     expect(result.content).toContain("A child tool action");
-    expect(result.content).toContain(
-      "- target (string, required): Action target",
-    );
+    const schemaMatch = result.content.match(/```json\n([\s\S]*?)\n```/);
+    expect(JSON.parse(schemaMatch![1]).required).toEqual(["target"]);
     expect(result.content).toContain(
       "Use `skill_execute` to call these tools.",
     );

package/src/__tests__/skill-tool-factory.test.ts

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
 
-import type { SkillToolEntry } from "../config/skills.js";
+import { getBundledSkillsDir, type SkillToolEntry } from "../config/skills.js";
 import { RiskLevel } from "../permissions/types.js";
 import { computeSkillVersionHash } from "../skills/version-hash.js";
 import { resolveSubagentRole } from "../subagent/role-resolution.js";
@@ -261,6 +261,54 @@ describe("createSkillTool — unknown parameter validation", () => {
     expect(result.content).toContain('Unknown parameter "bar"');
   });
 
+  test.each([
+    { description: "Filters" },
+    { marks: [{ description: "Filters" }] },
+    { marks: [{}] },
+    { marks: [{ target: 42 }] },
+    { marks: [{ x: 0.1, y: 0.2 }] },
+    { marks: [{ target: "Filters", x: 0.1, y: 0.2, width: 0.3, height: 0.4 }] },
+  ])(
+    "an annotation validation error provides enough schema for one corrected retry: %j",
+    async (invalid) => {
+      const skillDir = join(getBundledSkillsDir(), "screen-annotation");
+      const manifest = JSON.parse(
+        readFileSync(join(skillDir, "TOOLS.json"), "utf-8"),
+      );
+      const entry = manifest.tools.find(
+        (tool: SkillToolEntry) => tool.name === "screen_point_at",
+      );
+      const tool = createSkillTool(
+        entry,
+        skillDir,
+        computeSkillVersionHash(skillDir),
+        BUNDLED,
+      );
+      const calls: unknown[] = [];
+      const context = makeContext({
+        proxyToolResolver: async (name, input) => {
+          calls.push({ name, input });
+          return { content: "Drew 1 mark", isError: false };
+        },
+      });
+
+      const rejected = await tool.execute(invalid, context);
+      expect(rejected.isError).toBe(true);
+      expect(calls).toHaveLength(0);
+      expect(rejected.content).toContain("skill_load");
+      const schemaMatch = rejected.content.match(/```json\n([\s\S]*?)\n```/);
+      expect(schemaMatch).not.toBeNull();
+      expect(JSON.parse(schemaMatch![1])).toEqual(entry.input_schema);
+
+      const input = {
+        marks: [{ target: "Filters", caption: "Click Filters" }],
+      };
+      const corrected = await tool.execute(input, context);
+      expect(corrected.isError).toBe(false);
+      expect(calls).toEqual([{ name: "computer_use_point_at", input }]);
+    },
+  );
+
   test("allows input with only known parameters", async () => {
     const hash = computeSkillVersionHash(tempDir);
     const tool = createSkillTool(
@@ -471,7 +519,7 @@ describe("createSkillTool — required/type/enum validation", () => {
 
     expect(result.isError).toBe(true);
     expect(result.content).toContain(
-      "auto_open must be a boolean — pass true or false as a JSON boolean, not a string",
+      "auto_open must be a boolean: pass true or false as a JSON boolean, not a string",
     );
   });
 

package/src/__tests__/validate-input.test.ts

@@ -6,6 +6,79 @@ import {
   validateInputAgainstSchema,
 } from "../skills/validate-input.js";
 
+describe("nested input validation", () => {
+  const schema = {
+    properties: {
+      options: {
+        type: "object",
+        properties: { label: { type: "string" } },
+        required: ["label"],
+      },
+    },
+  };
+
+  test("reports nested required fields and types with their path", () => {
+    expect(
+      validateInputAgainstSchema("example", { options: {} }, schema),
+    ).toEqual({
+      ok: false,
+      errors: ["options.label is required"],
+    });
+    expect(
+      validateInputAgainstSchema("example", { options: { label: 42 } }, schema),
+    ).toEqual({
+      ok: false,
+      errors: ["options.label must be a string"],
+    });
+  });
+
+  test("preserves open nested objects and only rejects explicitly closed extra fields", () => {
+    const input = { options: { label: "Example", extra: true } };
+    expect(validateInputAgainstSchema("example", input, schema)).toEqual({
+      ok: true,
+    });
+    expect(
+      validateInputAgainstSchema("example", input, {
+        properties: {
+          options: {
+            ...schema.properties.options,
+            additionalProperties: false,
+          },
+        },
+      }),
+    ).toEqual({
+      ok: false,
+      errors: ['Unknown parameter "options.extra". Supported: "label"'],
+    });
+  });
+
+  test("does not treat a richer oneOf as a presence-only union", () => {
+    expect(
+      validateInputAgainstSchema(
+        "example",
+        { options: { label: "Example" } },
+        {
+          properties: {
+            options: {
+              ...schema.properties.options,
+              oneOf: [
+                {
+                  required: ["label"],
+                  properties: { label: { const: "Example" } },
+                },
+                {
+                  required: ["label"],
+                  properties: { label: { const: "Other" } },
+                },
+              ],
+            },
+          },
+        },
+      ),
+    ).toEqual({ ok: true });
+  });
+});
+
 // ---------------------------------------------------------------------------
 // required
 // ---------------------------------------------------------------------------

package/src/cli/commands/__tests__/plugins.test.ts

@@ -66,6 +66,20 @@ let platformInstallCalls: Array<{ name: string; force?: boolean }> = [];
 let upgradePluginCalls: UpgradePluginOptions[] = [];
 let catalogMatches: PluginSearchMatch[] = [];
 let catalogError: Error | null = null;
+const CATALOG_COMMIT = "a".repeat(40);
+
+function githubCatalogMatch(name: string): PluginSearchMatch {
+  return {
+    name,
+    path: `github:example-org/${name}@${CATALOG_COMMIT}`,
+    category: null,
+    source: {
+      kind: "github",
+      repo: `example-org/${name}`,
+      ref: CATALOG_COMMIT,
+    },
+  };
+}
 
 /**
  * Queued daemon IPC responses. The default (empty queue) is a transport
@@ -269,7 +283,10 @@ beforeEach(() => {
   installPluginCalls = [];
   platformInstallCalls = [];
   upgradePluginCalls = [];
-  catalogMatches = [];
+  catalogMatches = [
+    githubCatalogMatch("example"),
+    githubCatalogMatch("imessage"),
+  ];
   catalogError = null;
   ipcResults = [];
   inspectResult = null;
@@ -485,6 +502,17 @@ describe("plugins install - declared-schedules consent", () => {
     ]);
     expect(r.exitCode).toBe(0);
   });
+
+  test("does not use the outage fallback for a gated integration", async () => {
+    catalogError = new Error("catalog unavailable");
+
+    const r = await runCommand(["plugins", "install", "gamma"]);
+
+    expect(installPluginCalls).toHaveLength(0);
+    expect(platformInstallCalls).toHaveLength(0);
+    expect(r.stderr).toContain("not in the marketplace catalog");
+    expect(r.exitCode).toBe(1);
+  });
 });
 
 describe("plugins upgrade - declared-schedules consent (local fallback)", () => {

package/src/cli/commands/plugins.ts

@@ -238,7 +238,15 @@ export function registerPluginsCommand(program: Command): void {
               const platformEnabled =
                 libs.catalogLocal.arePlatformFeaturesEnabled();
               if (platformEnabled) {
+                if (!libs.catalogLocal.isPluginCatalogEntryVisible(nameOrUrl)) {
+                  console.error(
+                    `Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
+                  );
+                  process.exitCode = 1;
+                  return;
+                }
                 let match;
+                let catalogResolved = false;
                 try {
                   match = (
                     await libs.catalogCache.getPluginCatalog(
@@ -246,10 +254,18 @@ export function registerPluginsCommand(program: Command): void {
                       { fetch: globalThis.fetch.bind(globalThis) },
                     )
                   ).matches.find((candidate) => candidate.name === nameOrUrl);
+                  catalogResolved = true;
                 } catch {
                   // The install endpoint remains authoritative and can succeed
                   // independently when catalog discovery is unavailable.
                 }
+                if (!match && catalogResolved) {
+                  console.error(
+                    `Plugin "${nameOrUrl}" is not in the marketplace catalog.`,
+                  );
+                  process.exitCode = 1;
+                  return;
+                }
                 if (match?.source.kind === "local") {
                   result = await libs.installGitHub.installPlugin(
                     {

package/src/cli/lib/__tests__/inspect-plugin.test.ts

@@ -306,6 +306,35 @@ describe("inspectPlugin", () => {
     expect(result.remote?.commit).toBe(SHA_B);
   });
 
+  test("hides an uninstalled gated integration without fetching the marketplace", async () => {
+    let fetchCalls = 0;
+    const fetch = (async () => {
+      fetchCalls += 1;
+      return new Response("unexpected request", { status: 500 });
+    }) as FetchLike;
+
+    await expect(
+      inspectPlugin(
+        { name: "gamma" },
+        { fetch, workspacePluginsDir: workspace },
+      ),
+    ).rejects.toBeInstanceOf(PluginInspectNotFoundError);
+    expect(fetchCalls).toBe(0);
+  });
+
+  test("keeps an installed gated integration inspectable", async () => {
+    installPlugin(workspace, "gamma", { sidecar: null });
+    const fetch = makeFetch({ marketplace: manifestWith("gamma", SHA_B) });
+
+    const result = await inspectPlugin(
+      { name: "gamma" },
+      { fetch, workspacePluginsDir: workspace },
+    );
+
+    expect(result.installed).toBe(true);
+    expect(result.local?.description).toBe("Installed copy.");
+  });
+
   test("previews an uninstalled bundled provider offline without fetching GitHub", async () => {
     const savedDisablePlatform = process.env.VELLUM_DISABLE_PLATFORM;
     const savedIsPlatform = process.env.IS_PLATFORM;

package/src/cli/lib/__tests__/plugin-catalog-cache.test.ts

@@ -19,6 +19,7 @@ import {
 import type { FetchLike } from "../fetch-like.js";
 import {
   getPluginCatalog,
+  getPluginCatalogForInstalledMetadata,
   invalidatePluginCatalogCache,
   mergePlatformCatalogWithBundledLocals,
   PLUGIN_CATALOG_CACHE_TTL_MS,
@@ -102,7 +103,7 @@ describe("getPluginCatalog", () => {
 
     // THEN the platform is fetched exactly once — the second call is cached
     expect(calls()).toBe(1);
-    expect(first).toBe(second);
+    expect(first).toEqual(second);
     expect(githubNames(first)).toEqual(["a"]);
     expect(first.ref).toBe("main");
   });
@@ -161,7 +162,10 @@ describe("getPluginCatalog", () => {
     delete process.env.IS_PLATFORM;
 
     const { fetch, calls } = platformFetch(["ignored"]);
-    const deps: SearchPluginsDeps = { fetch };
+    const deps: SearchPluginsDeps = {
+      fetch,
+      featureFlagEnabled: () => true,
+    };
 
     // WHEN we request the catalog
     const result = await getPluginCatalog("main", deps);
@@ -173,6 +177,32 @@ describe("getPluginCatalog", () => {
     // The requested ref is echoed onto the wire contract.
     expect(result.ref).toBe("main");
   });
+
+  test("applies feature-flag visibility to bundled catalog entries", async () => {
+    process.env.VELLUM_DISABLE_PLATFORM = "true";
+    delete process.env.IS_PLATFORM;
+
+    const { fetch } = platformFetch(["ignored"]);
+    const result = await getPluginCatalog("main", {
+      fetch,
+      featureFlagEnabled: () => false,
+    });
+
+    expect(result.matches.map((match) => match.name)).not.toContain("gamma");
+    expect(result.matches.map((match) => match.name)).toContain("fathom");
+  });
+
+  test("preserves hidden entries for installed-plugin metadata", async () => {
+    process.env.VELLUM_DISABLE_PLATFORM = "true";
+    delete process.env.IS_PLATFORM;
+
+    const { fetch } = platformFetch(["ignored"]);
+    const result = await getPluginCatalogForInstalledMetadata("main", {
+      fetch,
+    });
+
+    expect(result.matches.map((match) => match.name)).toContain("gamma");
+  });
 });
 
 describe("mergePlatformCatalogWithBundledLocals", () => {

package/src/cli/lib/__tests__/plugin-catalog-local.test.ts

@@ -13,6 +13,7 @@ import bundledManifest from "../bundled-marketplace.json" with { type: "json" };
 import {
   buildBundledPluginCatalog,
   readBundledPluginCatalog,
+  resolveBundledPluginSource,
 } from "../plugin-catalog-local.js";
 import { marketplaceManifestSchema } from "../plugin-marketplace.js";
 import { marketplaceMatch } from "../search-plugins.js";
@@ -96,3 +97,17 @@ describe("readBundledPluginCatalog", () => {
     expect(readBundledPluginCatalog()).toBe(readBundledPluginCatalog());
   });
 });
+
+describe("resolveBundledPluginSource", () => {
+  test("hides a QA integration while its feature flag is off", () => {
+    expect(resolveBundledPluginSource("gamma", () => false)).toBeNull();
+  });
+
+  test("resolves a QA integration while its feature flag is on", () => {
+    expect(resolveBundledPluginSource("gamma", () => true)).toEqual({
+      kind: "local",
+      path: "plugins/mcp-catalog/gamma",
+      version: "1.0.2",
+    });
+  });
+});

package/src/cli/lib/__tests__/plugin-catalog-resolve.test.ts

@@ -195,6 +195,27 @@ describe("catalog-backed resolvers (bundled, offline)", () => {
       await resolvePluginSourceFromCatalog("no-such-plugin", rejectingDeps),
     ).toBeNull();
   });
+
+  test("does not resolve a QA integration while its feature flag is off", async () => {
+    expect(
+      await resolvePluginSourceFromCatalog("gamma", {
+        ...rejectingDeps,
+        featureFlagEnabled: () => false,
+      }),
+    ).toBeNull();
+  });
+
+  test("resolves a QA integration while its feature flag is on", async () => {
+    expect(
+      await resolvePluginSourceFromCatalog("gamma", {
+        ...rejectingDeps,
+        featureFlagEnabled: () => true,
+      }),
+    ).toMatchObject({
+      kind: "local",
+      path: "plugins/mcp-catalog/gamma",
+    });
+  });
 });
 
 describe("catalog-backed resolvers (platform enabled)", () => {

package/src/cli/lib/__tests__/plugin-catalog-visibility.test.ts

@@ -0,0 +1,65 @@
+import { describe, expect, test } from "bun:test";
+
+import {
+  filterPluginCatalogByFeatureFlags,
+  MCP_CATALOG_QA_INTEGRATIONS_FLAG,
+} from "../plugin-catalog-visibility.js";
+import type { PluginCatalog, PluginSearchMatch } from "../search-plugins.js";
+
+function match(name: string): PluginSearchMatch {
+  return {
+    name,
+    path: `local:plugins/mcp-catalog/${name}@1.0.0`,
+    category: null,
+    source: {
+      kind: "local",
+      path: `plugins/mcp-catalog/${name}`,
+      version: "1.0.0",
+    },
+  };
+}
+
+function catalog(names: string[]): PluginCatalog {
+  return { ref: "main", matches: names.map(match) };
+}
+
+describe("filterPluginCatalogByFeatureFlags", () => {
+  test("hides QA integrations while the flag is off", () => {
+    const input = catalog([
+      "fathom",
+      "gamma",
+      "guru",
+      "intercom",
+      "ramp",
+      "semrush",
+      "typeform",
+      "notion",
+    ]);
+
+    const result = filterPluginCatalogByFeatureFlags(input, (key) => {
+      expect(key).toBe(MCP_CATALOG_QA_INTEGRATIONS_FLAG);
+      return false;
+    });
+
+    expect(result.matches.map((entry) => entry.name)).toEqual([
+      "fathom",
+      "notion",
+    ]);
+  });
+
+  test("returns the full catalog while the flag is on", () => {
+    const input = catalog([
+      "fathom",
+      "gamma",
+      "guru",
+      "intercom",
+      "ramp",
+      "semrush",
+      "typeform",
+    ]);
+
+    const result = filterPluginCatalogByFeatureFlags(input, () => true);
+
+    expect(result).toBe(input);
+  });
+});

package/src/cli/lib/__tests__/plugin-details.test.ts

@@ -513,6 +513,49 @@ describe("getPluginDetails (bundled catalog, offline)", () => {
     expect(details.ref).toBe(historicalRef);
   });
 
+  test("hides a gated integration at an explicit historical ref", async () => {
+    const historicalRef = "a".repeat(40);
+    let fetchCalls = 0;
+    const fetch = (async () => {
+      fetchCalls += 1;
+      return new Response("unexpected request", { status: 500 });
+    }) as FetchLike;
+
+    await expect(
+      getPluginDetails(
+        { name: "gamma", ref: historicalRef },
+        { fetch, workspacePluginsDir: workspace },
+      ),
+    ).rejects.toBeInstanceOf(PluginDetailsNotFoundError);
+    expect(fetchCalls).toBe(0);
+  });
+
+  test("keeps an installed gated integration visible at an explicit historical ref", async () => {
+    const target = join(workspace, "gamma");
+    mkdirSync(target, { recursive: true });
+    writeFileSync(join(target, "README.md"), "# Installed Gamma");
+    writeFileSync(
+      join(target, "package.json"),
+      JSON.stringify({ version: "1.0.0", description: "local gamma" }),
+    );
+    let fetchCalls = 0;
+    const fetch = (async () => {
+      fetchCalls += 1;
+      return new Response("unexpected request", { status: 500 });
+    }) as FetchLike;
+
+    const details = await getPluginDetails(
+      { name: "gamma", ref: "a".repeat(40) },
+      { fetch, workspacePluginsDir: workspace },
+    );
+
+    expect(details.installed).toBe(true);
+    expect(details.readme).toBe("# Installed Gamma");
+    expect(details.description).toBe("local gamma");
+    expect(details.source).toBeNull();
+    expect(fetchCalls).toBe(0);
+  });
+
   test("degrades gracefully when the historical marketplace fetch fails", async () => {
     // GIVEN an installed copy and a historical ref whose marketplace fetch errors
     const historicalRef = "a".repeat(40);