@vellumai/assistant 0.11.3 → 0.11.4-dev.202608190019.b94dbf2

package/src/runtime/routes/__tests__/inference-profiles-routes.test.ts

@@ -150,6 +150,50 @@ describe("POST inference/profiles (create) validation", () => {
     ).rejects.toBeInstanceOf(BadRequestError);
   });
 
+  test("creates an entry-backed profile, validating the model by the row's kind", async () => {
+    seedConnection("anthropic-work", "anthropic", {
+      type: "api_key",
+      credential: "credential/anthropic-work/api_key",
+    });
+    const result = (await call("inference_profiles_create", {
+      body: {
+        name: "work-profile",
+        provider: "anthropic-work",
+        model: "claude-opus-4-8",
+      },
+    })) as { entry: Record<string, unknown> };
+    expect(result.entry.provider).toBe("anthropic-work");
+    expect(result.entry.model).toBe("claude-opus-4-8");
+  });
+
+  test("rejects an entry-backed profile whose model the row's kind cannot serve", async () => {
+    seedConnection("anthropic-work", "anthropic", {
+      type: "api_key",
+      credential: "credential/anthropic-work/api_key",
+    });
+    await expect(
+      call("inference_profiles_create", {
+        body: {
+          name: "work-profile",
+          provider: "anthropic-work",
+          model: "gpt-5.5",
+        },
+      }),
+    ).rejects.toBeInstanceOf(BadRequestError);
+  });
+
+  test("rejects a provider naming no known vendor or entry", async () => {
+    await expect(
+      call("inference_profiles_create", {
+        body: {
+          name: "ghost-profile",
+          provider: "no-such-entry",
+          model: "claude-opus-4-8",
+        },
+      }),
+    ).rejects.toBeInstanceOf(BadRequestError);
+  });
+
   test("creates a vellum profile for a managed-routable model", async () => {
     seedVellumConnection();
     const result = (await call("inference_profiles_create", {

package/src/runtime/routes/__tests__/inference-provider-connection-routes.test.ts

@@ -232,6 +232,61 @@ describe("POST inference/provider-connections (create)", () => {
     expect((err as BadRequestError).message).toContain("vellum");
   });
 
+  test("rejects a chatgpt identity row under a non-canonical name", async () => {
+    const err = await call(
+      findHandler("inference_provider_connections_create"),
+      {
+        body: {
+          name: "my-chatgpt",
+          provider: "chatgpt",
+          auth: { type: "oauth_subscription", credential: "credential/x" },
+        },
+      },
+    ).catch((e: unknown) => e);
+
+    expect(err).toBeInstanceOf(BadRequestError);
+    expect((err as BadRequestError).message).toContain("chatgpt-subscription");
+  });
+
+  test("rejects key auth on the chatgpt provider, derived or explicit", async () => {
+    for (const body of [
+      {
+        name: "chatgpt-subscription",
+        provider: "chatgpt",
+        auth: { type: "api_key", credential: "vault/x" },
+      },
+      // No explicit auth: derivation would fall through to api_key.
+      {
+        name: "chatgpt-subscription",
+        provider: "chatgpt",
+        credential: "vault/x",
+      },
+    ]) {
+      const err = await call(
+        findHandler("inference_provider_connections_create"),
+        { body },
+      ).catch((e: unknown) => e);
+      expect(err).toBeInstanceOf(BadRequestError);
+      expect((err as BadRequestError).message).toContain("sign-in");
+    }
+  });
+
+  test("rejects subscription auth on a non-chatgpt provider", async () => {
+    const err = await call(
+      findHandler("inference_provider_connections_create"),
+      {
+        body: {
+          name: "keyed-oauth",
+          provider: "openai",
+          auth: { type: "oauth_subscription", credential: "credential/x" },
+        },
+      },
+    ).catch((e: unknown) => e);
+
+    expect(err).toBeInstanceOf(BadRequestError);
+    expect((err as BadRequestError).message).toContain("chatgpt");
+  });
+
   test("rejects key auth on the vellum provider", async () => {
     const err = await call(
       findHandler("inference_provider_connections_create"),
@@ -396,7 +451,7 @@ describe("POST inference/provider-connections (create)", () => {
           models: [{ id: "my-model" }],
         },
       }),
-    ).rejects.toThrow(/belongs to a built-in provider/);
+    ).rejects.toThrow(/reserved as a provider id/);
 
     await call(findHandler("inference_provider_connections_create"), {
       body: {
@@ -677,6 +732,52 @@ describe("PATCH inference/provider-connections/:name (update)", () => {
 
   // The guard must not trap a legacy row in its mismatched state: an auth
   // change that repairs the pairing is exactly what should be allowed.
+  // The canonical subscription row owns the "chatgpt" identity: writing
+  // subscription auth to it stamps the provider (the CLI login-chatgpt path
+  // PATCHes auth through this route on a row the identity migration
+  // deliberately skipped).
+  test("stamps the chatgpt identity when subscription auth lands on the canonical row", async () => {
+    seedConnection({
+      name: "chatgpt-subscription",
+      provider: "openai",
+      auth: { type: "api_key", credential: "vault/openai/key" },
+    });
+
+    const result = (await call(
+      findHandler("inference_provider_connections_update"),
+      {
+        pathParams: { name: "chatgpt-subscription" },
+        body: {
+          auth: {
+            type: "oauth_subscription",
+            credential: "credential/chatgpt/access_token",
+          },
+        },
+      },
+    )) as { provider: string; auth: { type: string } };
+    expect(result.provider).toBe("chatgpt");
+    expect(result.auth.type).toBe("oauth_subscription");
+  });
+
+  test("does not stamp the identity for non-subscription auth on that name", async () => {
+    seedConnection({
+      name: "chatgpt-subscription",
+      provider: "openai",
+      auth: { type: "api_key", credential: "vault/openai/key" },
+    });
+
+    const result = (await call(
+      findHandler("inference_provider_connections_update"),
+      {
+        pathParams: { name: "chatgpt-subscription" },
+        body: {
+          auth: { type: "api_key", credential: "vault/openai/other-key" },
+        },
+      },
+    )) as { provider: string };
+    expect(result.provider).toBe("openai");
+  });
+
   test("allows repairing a legacy mismatched row with a valid auth change", async () => {
     seedConnection({
       name: "legacy-managed-anthropic",

package/src/runtime/routes/__tests__/plugins-routes.test.ts

@@ -38,9 +38,12 @@
  *     `plugins:list` tag via the canonical resource-sync publisher (enable and
  *     disable emit the SAME invalidation)
  *   - Threads `x-vellum-client-id` into the published event's `originClientId`
- *   - Pokes the plugin-declared schedule reconcile on a successful toggle (and
- *     not on a failed one), so the plugin's rows disarm / re-arm immediately
- *     instead of at the reconciler's next backstop sweep
+ *   - Enable pokes the imperative plugin source reconcile, which activates a
+ *     plugin the boot scan skipped for its sentinel and converges its
+ *     schedules and MCP servers
+ *   - Disable pokes the plugin-declared schedule reconcile on a successful
+ *     toggle (and not on a failed one), so the plugin's rows disarm
+ *     immediately instead of at the reconciler's next backstop sweep
  *   - A broadcast failure does not fail a successful toggle (the publisher
  *     swallows hub errors)
  *   - Maps `InvalidPluginNameError` → BadRequestError (400)
@@ -109,6 +112,7 @@ import {
 } from "../../../cli/lib/uninstall-plugin.js";
 import {
   PluginMergeBaselineError,
+  PluginNotCuratedError,
   PluginNotUpgradableError,
   type PluginUpgradeResult,
   type UpgradePluginDeps,
@@ -252,6 +256,7 @@ const upgradeSpy = mock(
 
 mock.module("../../../cli/lib/upgrade-plugin.js", () => ({
   PluginMergeBaselineError,
+  PluginNotCuratedError,
   PluginNotUpgradableError,
   upgradePlugin: upgradeSpy,
 }));
@@ -314,8 +319,8 @@ mock.module("../../../cli/lib/toggle-plugin.js", () => ({
   enablePlugin: enablePluginSpy,
 }));
 
-// Spy on the plugin-schedule reconcile the toggle handlers poke. The handlers
-// import it lazily at call time, so this mock intercepts the dynamic import
+// Spy on the plugin-schedule reconcile the disable handler pokes. The handler
+// imports it lazily at call time, so this mock intercepts the dynamic import
 // and keeps the reconciler's persistence graph out of the test.
 const reconcilePluginSchedulesSpy = mock(() => Promise.resolve());
 
@@ -323,6 +328,21 @@ mock.module("../../../schedule/plugin-schedule-reconciler.js", () => ({
   reconcilePluginSchedules: reconcilePluginSchedulesSpy,
 }));
 
+// Spy on the imperative plugin source reconcile so the enable route's
+// activation poke is assertable without scanning a real plugins tree. Only
+// that one export is swapped; the rest of the module passes through real,
+// because other modules in this route's graph import their own bindings from
+// it.
+const reconcilePluginSourcesNowSpy = mock(() => Promise.resolve());
+
+const actualMtimeCache: typeof import("../../../plugins/mtime-cache.js") =
+  await import("../../../plugins/mtime-cache.js");
+
+mock.module("../../../plugins/mtime-cache.js", () => ({
+  ...actualMtimeCache,
+  reconcilePluginSourcesNow: reconcilePluginSourcesNowSpy,
+}));
+
 // Spy on broadcastMessage so we can assert the sync_changed invalidation the
 // enable/disable handlers emit. The handlers publish through the canonical
 // `publishPluginsChanged` → `publishSyncInvalidation` path (both left real), so
@@ -2127,6 +2147,7 @@ describe("POST /v1/plugins/:name/upgrade", () => {
       name: "level-up",
       dryRun: false,
       strategy: undefined,
+      marketplaceOnly: false,
     });
   });
 
@@ -2148,6 +2169,7 @@ describe("POST /v1/plugins/:name/upgrade", () => {
       name: "level-up",
       dryRun: undefined,
       strategy: "ours",
+      marketplaceOnly: false,
     });
   });
 
@@ -2175,6 +2197,7 @@ describe("POST /v1/plugins/:name/upgrade", () => {
       name: "level-up",
       dryRun: undefined,
       strategy: "assistant",
+      marketplaceOnly: false,
     });
   });
 
@@ -2300,6 +2323,10 @@ describe("POST /v1/plugins/:name/upgrade", () => {
     expect(upgradeSpy.mock.calls[0]?.[0]).toEqual({
       name: "level-up",
       dryRun: undefined,
+      strategy: undefined,
+      // Unlike dryRun, an absent flag here is a definite "no": the untrusted
+      // direct path stays available unless a caller opts out of it.
+      marketplaceOnly: false,
     });
   });
 
@@ -2343,6 +2370,41 @@ describe("POST /v1/plugins/:name/upgrade", () => {
     ).rejects.toBeInstanceOf(ConflictError);
   });
 
+  test("marketplaceOnly is forwarded so the lib enforces it, not the caller", async () => {
+    // The lib re-inspects the catalog inside the call, so the flag has to
+    // travel with the request: a caller's own pre-check cannot bind a decision
+    // the lib makes afterwards.
+    upgradeSpy.mockImplementation(async () => upgradeResult());
+
+    await invokeUpgrade({
+      pathParams: { name: "level-up" },
+      body: { marketplaceOnly: true },
+    });
+
+    expect(upgradeSpy.mock.calls[0]?.[0]).toEqual({
+      name: "level-up",
+      dryRun: undefined,
+      strategy: undefined,
+      marketplaceOnly: true,
+    });
+  });
+
+  test("PluginNotCuratedError → ConflictError (409)", async () => {
+    // A marketplaceOnly caller asked about an install the catalog does not
+    // claim. Retrying changes nothing: only a human upgrading it without the
+    // flag can move it.
+    upgradeSpy.mockImplementation(async () => {
+      throw new PluginNotCuratedError("level-up");
+    });
+
+    await expect(
+      invokeUpgrade({
+        pathParams: { name: "level-up" },
+        body: { marketplaceOnly: true },
+      }),
+    ).rejects.toBeInstanceOf(ConflictError);
+  });
+
   test("PluginNotFoundError → NotFoundError (404)", async () => {
     upgradeSpy.mockImplementation(async () => {
       throw new PluginNotFoundError("level-up", "main", "vellum-ai/level-up");
@@ -2529,8 +2591,10 @@ function toggleResult(
   };
 }
 
-function invokeEnable(args: RouteHandlerArgs = {}): { ok: boolean } {
-  return enableHandler(args) as { ok: boolean };
+async function invokeEnable(
+  args: RouteHandlerArgs = {},
+): Promise<{ ok: boolean }> {
+  return (await enableHandler(args)) as { ok: boolean };
 }
 
 function invokeDisable(args: RouteHandlerArgs = {}): { ok: boolean } {
@@ -2538,18 +2602,22 @@ function invokeDisable(args: RouteHandlerArgs = {}): { ok: boolean } {
 }
 
 /**
- * Wait for the toggle handlers' fire-and-forget reconcile poke to land. The
- * lazy `import()` resolves off the microtask queue, so the assertion cannot
- * run on the handler's synchronous return.
+ * Wait for a toggle handler's fire-and-forget reconcile poke to land. The poke
+ * runs off the microtask queue (the disable path also resolves a lazy
+ * `import()` first), so the assertion cannot run on the handler's synchronous
+ * return.
  */
-async function waitForScheduleReconcile(): Promise<void> {
+async function waitForReconcile(
+  spy: { mock: { calls: unknown[] } },
+  label: string,
+): Promise<void> {
   for (let i = 0; i < 50; i++) {
-    if (reconcilePluginSchedulesSpy.mock.calls.length > 0) {
+    if (spy.mock.calls.length > 0) {
       return;
     }
     await new Promise((resolve) => setTimeout(resolve, 2));
   }
-  throw new Error("plugin schedule reconcile was never triggered");
+  throw new Error(`${label} was never triggered`);
 }
 
 /** Assert the spy received exactly one sync_changed carrying `plugins:list`. */
@@ -2565,30 +2633,50 @@ describe("POST /v1/plugins/:name/enable", () => {
     enablePluginSpy.mockReset();
     broadcastMessageSpy.mockReset();
     reconcilePluginSchedulesSpy.mockClear();
+    reconcilePluginSourcesNowSpy.mockClear();
   });
 
-  test("reconciles plugin-declared schedules so the plugin's rows re-arm now", async () => {
+  test("awaits the source reconcile so a boot-disabled plugin activates before the route returns", async () => {
     enablePluginSpy.mockImplementation((name) => toggleResult(name, "enable"));
 
-    invokeEnable({ pathParams: { name: "simple-memory" } });
+    await invokeEnable({ pathParams: { name: "simple-memory" } });
+
+    // The boot scan skipped this plugin for its sentinel, so clearing the
+    // sentinel only brings its hooks, tools, MCP servers and schedules up if
+    // the source reconcile runs. The route awaits it, so by the time the
+    // response lands the reconcile has completed; it converges the schedules
+    // itself, so the route does not poke them separately.
+    expect(reconcilePluginSourcesNowSpy).toHaveBeenCalledTimes(1);
+    expect(reconcilePluginSchedulesSpy).not.toHaveBeenCalled();
+  });
+
+  test("a failed toggle does not run the source reconcile", async () => {
+    enablePluginSpy.mockImplementation((name) => {
+      throw new PluginDirectoryNotFoundError(name);
+    });
 
-    await waitForScheduleReconcile();
+    await expect(
+      invokeEnable({ pathParams: { name: "ghost" } }),
+    ).rejects.toThrow(NotFoundError);
+    expect(reconcilePluginSourcesNowSpy).not.toHaveBeenCalled();
   });
 
-  test("enables the plugin and broadcasts sync_changed(plugins:list)", () => {
+  test("enables the plugin and broadcasts sync_changed(plugins:list)", async () => {
     enablePluginSpy.mockImplementation((name) => toggleResult(name, "enable"));
 
-    const result = invokeEnable({ pathParams: { name: "simple-memory" } });
+    const result = await invokeEnable({
+      pathParams: { name: "simple-memory" },
+    });
 
     expect(result).toEqual({ ok: true });
     expect(enablePluginSpy.mock.calls[0]?.[0]).toBe("simple-memory");
     expectPluginsListBroadcast();
   });
 
-  test("threads x-vellum-client-id into the published event's originClientId", () => {
+  test("threads x-vellum-client-id into the published event's originClientId", async () => {
     enablePluginSpy.mockImplementation((name) => toggleResult(name, "enable"));
 
-    invokeEnable({
+    await invokeEnable({
       pathParams: { name: "simple-memory" },
       headers: { "x-vellum-client-id": "client-abc" },
     });
@@ -2602,7 +2690,7 @@ describe("POST /v1/plugins/:name/enable", () => {
     });
   });
 
-  test("a broadcast failure does not fail a successful toggle", () => {
+  test("a broadcast failure does not fail a successful toggle", async () => {
     enablePluginSpy.mockImplementation((name) => toggleResult(name, "enable"));
     // The sentinel was already flipped; a hub throw AFTER that must not surface
     // as a 500 — the canonical publisher swallows broadcast errors.
@@ -2610,40 +2698,42 @@ describe("POST /v1/plugins/:name/enable", () => {
       throw new Error("hub unavailable");
     });
 
-    const result = invokeEnable({ pathParams: { name: "simple-memory" } });
+    const result = await invokeEnable({
+      pathParams: { name: "simple-memory" },
+    });
     expect(result).toEqual({ ok: true });
   });
 
-  test("PluginAlreadyInStateException → ConflictError (409), no broadcast", () => {
+  test("PluginAlreadyInStateException → ConflictError (409), no broadcast", async () => {
     enablePluginSpy.mockImplementation((name) => {
       throw new PluginAlreadyInStateException(name, "enable");
     });
 
-    expect(() =>
+    await expect(
       invokeEnable({ pathParams: { name: "simple-memory" } }),
-    ).toThrow(ConflictError);
+    ).rejects.toThrow(ConflictError);
     // A no-op toggle must not fan out a spurious invalidation.
     expect(broadcastMessageSpy).not.toHaveBeenCalled();
   });
 
-  test("PluginDirectoryNotFoundError → NotFoundError (404)", () => {
+  test("PluginDirectoryNotFoundError → NotFoundError (404)", async () => {
     enablePluginSpy.mockImplementation((name) => {
       throw new PluginDirectoryNotFoundError(name);
     });
 
-    expect(() => invokeEnable({ pathParams: { name: "ghost" } })).toThrow(
-      NotFoundError,
-    );
+    await expect(
+      invokeEnable({ pathParams: { name: "ghost" } }),
+    ).rejects.toThrow(NotFoundError);
   });
 
-  test("InvalidPluginNameError → BadRequestError (400)", () => {
+  test("InvalidPluginNameError → BadRequestError (400)", async () => {
     enablePluginSpy.mockImplementation(() => {
       throw new ToggleInvalidPluginNameError("../escape");
     });
 
-    expect(() => invokeEnable({ pathParams: { name: "../escape" } })).toThrow(
-      BadRequestError,
-    );
+    await expect(
+      invokeEnable({ pathParams: { name: "../escape" } }),
+    ).rejects.toThrow(BadRequestError);
   });
 });
 
@@ -2652,6 +2742,7 @@ describe("POST /v1/plugins/:name/disable", () => {
     disablePluginSpy.mockReset();
     broadcastMessageSpy.mockReset();
     reconcilePluginSchedulesSpy.mockClear();
+    reconcilePluginSourcesNowSpy.mockClear();
   });
 
   test("reconciles plugin-declared schedules so the plugin's rows disarm now", async () => {
@@ -2662,8 +2753,14 @@ describe("POST /v1/plugins/:name/disable", () => {
     invokeDisable({ pathParams: { name: "simple-memory" } });
 
     // Without this poke the rows stay armed until the reconciler's next
-    // backstop sweep.
-    await waitForScheduleReconcile();
+    // backstop sweep. Disable stays on the schedule poke: the sentinel filters
+    // the plugin out at read time, and a source reconcile here would run its
+    // shutdown hooks.
+    await waitForReconcile(
+      reconcilePluginSchedulesSpy,
+      "plugin schedule reconcile",
+    );
+    expect(reconcilePluginSourcesNowSpy).not.toHaveBeenCalled();
   });
 
   test("a failed toggle does not poke the schedule reconcile", async () => {