@vellumai/assistant 0.10.1-staging.1 → 0.10.1-staging.2

package/openapi.yaml

@@ -8378,6 +8378,36 @@ paths:
                           anyOf:
                             - type: string
                             - type: "null"
+                        error:
+                          anyOf:
+                            - type: object
+                              properties:
+                                name:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                message:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                code:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                provider:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                statusCode:
+                                  anyOf:
+                                    - type: number
+                                    - type: "null"
+                                retryAfterMs:
+                                  anyOf:
+                                    - type: number
+                                    - type: "null"
+                              additionalProperties: false
+                            - type: "null"
                       required:
                         - id
                         - createdAt
@@ -15278,6 +15308,36 @@ paths:
                     anyOf:
                       - type: string
                       - type: "null"
+                  error:
+                    anyOf:
+                      - type: object
+                        properties:
+                          name:
+                            anyOf:
+                              - type: string
+                              - type: "null"
+                          message:
+                            anyOf:
+                              - type: string
+                              - type: "null"
+                          code:
+                            anyOf:
+                              - type: string
+                              - type: "null"
+                          provider:
+                            anyOf:
+                              - type: string
+                              - type: "null"
+                          statusCode:
+                            anyOf:
+                              - type: number
+                              - type: "null"
+                          retryAfterMs:
+                            anyOf:
+                              - type: number
+                              - type: "null"
+                        additionalProperties: false
+                      - type: "null"
                 required:
                   - id
                   - createdAt
@@ -17705,6 +17765,36 @@ paths:
                           anyOf:
                             - type: string
                             - type: "null"
+                        error:
+                          anyOf:
+                            - type: object
+                              properties:
+                                name:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                message:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                code:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                provider:
+                                  anyOf:
+                                    - type: string
+                                    - type: "null"
+                                statusCode:
+                                  anyOf:
+                                    - type: number
+                                    - type: "null"
+                                retryAfterMs:
+                                  anyOf:
+                                    - type: number
+                                    - type: "null"
+                              additionalProperties: false
+                            - type: "null"
                       required:
                         - id
                         - createdAt

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.10.1-staging.1",
+  "version": "0.10.1-staging.2",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/memory/embedding-gemini.test.ts

@@ -244,7 +244,9 @@ describe("GeminiEmbeddingBackend", () => {
       });
       globalThis.fetch = mockFetch as unknown as typeof fetch;
 
-      const backend = new GeminiEmbeddingBackend("test-key", "test-model");
+      const backend = new GeminiEmbeddingBackend("test-key", "test-model", {
+        interCallDelayMs: 0,
+      });
       const result = await backend.embed(["hello", "world"]);
 
       expect(mockFetch).toHaveBeenCalledTimes(2);

package/src/memory/embedding-gemini.ts

@@ -18,6 +18,11 @@ export interface GeminiEmbeddingOptions {
   dimensions?: number;
   /** When set, routes requests through the managed proxy at this base URL. */
   managedBaseUrl?: string;
+  /**
+   * Milliseconds to sleep between sequential embed calls to yield to the
+   * event loop. Defaults to 5000 in production; set to 0 in tests.
+   */
+  interCallDelayMs?: number;
 }
 
 export class GeminiEmbeddingBackend implements EmbeddingBackend {
@@ -27,6 +32,7 @@ export class GeminiEmbeddingBackend implements EmbeddingBackend {
   private readonly taskType?: EmbeddingTaskType;
   private readonly dimensions?: number;
   private readonly managedBaseUrl?: string;
+  private readonly interCallDelayMs: number;
 
   constructor(apiKey: string, model: string, options?: GeminiEmbeddingOptions) {
     this.apiKey = apiKey;
@@ -34,6 +40,7 @@ export class GeminiEmbeddingBackend implements EmbeddingBackend {
     this.taskType = options?.taskType;
     this.dimensions = options?.dimensions;
     this.managedBaseUrl = options?.managedBaseUrl;
+    this.interCallDelayMs = options?.interCallDelayMs ?? 5000;
   }
 
   /** True when requests route through the managed platform proxy. */
@@ -46,9 +53,18 @@ export class GeminiEmbeddingBackend implements EmbeddingBackend {
     options?: EmbeddingRequestOptions,
   ): Promise<number[][]> {
     const vectors: number[][] = [];
-    for (const input of inputs) {
-      const values = await this.embedSingle(input, options);
+    for (let i = 0; i < inputs.length; i++) {
+      const values = await this.embedSingle(inputs[i], options);
       vectors.push(values);
+      // Yield to the event loop between sequential embed calls so the
+      // daemon can serve HTTP requests, health checks, and cron ticks
+      // while a large batch (e.g. startup skill reseed / concept-page
+      // reembed) is in flight. Without this, 68+ sequential Gemini
+      // round-trips starve the event loop for minutes at a time.
+      // TODO: replace with full backgrounding (worker thread / subprocess).
+      if (i < inputs.length - 1 && this.interCallDelayMs > 0) {
+        await Bun.sleep(this.interCallDelayMs);
+      }
     }
     return vectors;
   }

package/src/messaging/providers/gmail/client.ts

@@ -17,14 +17,13 @@ import type {
 const GMAIL_BATCH_URL = "https://www.googleapis.com/batch/gmail/v1";
 
 /**
- * Minimum Google OAuth scope a connection must carry to be usable for Gmail.
+ * Minimum Google OAuth scope a connection must carry for Gmail read access.
  *
  * The managed `google` OAuth app bundles Gmail + Calendar + Drive, but a
  * connection can be granted a narrow subset (e.g. the onboarding check-in flow
- * requests Calendar-only). Every Gmail read/search/send call needs at least
- * `gmail.readonly`, so a connection lacking it cannot serve Gmail at all —
- * resolving against this scope turns a downstream 403 into an actionable
- * "reconnect Google and grant Gmail" error at resolution time.
+ * requests Calendar-only). Resolving against Gmail read access turns a
+ * downstream 403 into an actionable "reconnect Google and grant Gmail" error
+ * at resolution time when the selected connection cannot read Gmail.
  */
 export const GMAIL_REQUIRED_SCOPES = [
   "https://www.googleapis.com/auth/gmail.readonly",

package/src/oauth/connection-resolver.test.ts

@@ -302,6 +302,7 @@ describe("resolveOAuthConnection", () => {
 
 describe("resolveOAuthConnection scope-awareness", () => {
   const GMAIL_SCOPE = "https://www.googleapis.com/auth/gmail.readonly";
+  const GMAIL_FULL_ACCESS_SCOPE = "https://mail.google.com/";
   const CALENDAR_ONLY = [
     "https://www.googleapis.com/auth/calendar.events",
     "https://www.googleapis.com/auth/userinfo.email",
@@ -343,6 +344,21 @@ describe("resolveOAuthConnection scope-awareness", () => {
     expect(result).toBeInstanceOf(PlatformOAuthConnection);
   });
 
+  test("managed: treats full Gmail access as covering Gmail read access", async () => {
+    mockPlatformClient = clientReturning([
+      {
+        id: "full-gmail-access",
+        account_label: null,
+        scopes_granted: [GMAIL_FULL_ACCESS_SCOPE],
+      },
+    ]);
+
+    const result = await resolveOAuthConnection("google", {
+      requiredScopes: [GMAIL_SCOPE],
+    });
+    expect(result).toBeInstanceOf(PlatformOAuthConnection);
+  });
+
   test("managed: unknown scope data never blocks (back-compat)", async () => {
     // Older connections report no scopes_granted — must not be rejected.
     mockPlatformClient = clientReturning([
@@ -391,6 +407,18 @@ describe("resolveOAuthConnection scope-awareness", () => {
     ).rejects.toThrow(/missing required access/);
   });
 
+  test("BYO: treats full Gmail access as covering Gmail read access", async () => {
+    (mockConfig.services as Record<string, unknown>)["google-oauth"] = {
+      mode: "your-own",
+    };
+    mockConnection!.grantedScopes = JSON.stringify([GMAIL_FULL_ACCESS_SCOPE]);
+
+    const result = await resolveOAuthConnection("google", {
+      requiredScopes: [GMAIL_SCOPE],
+    });
+    expect(result).toBeInstanceOf(BYOOAuthConnection);
+  });
+
   test("BYO: unknown granted scopes never block", async () => {
     (mockConfig.services as Record<string, unknown>)["google-oauth"] = {
       mode: "your-own",

package/src/oauth/scope-utils.ts

@@ -16,6 +16,24 @@ export function scopeDifference(
   required: string[],
   granted: string[],
 ): string[] {
-  const grantedSet = new Set(granted);
-  return required.filter((s) => !grantedSet.has(s));
+  return required.filter(
+    (requiredScope) =>
+      !granted.some((grantedScope) =>
+        grantedScopeCoversRequiredScope(grantedScope, requiredScope),
+      ),
+  );
+}
+
+const GMAIL_FULL_ACCESS_SCOPE = "https://mail.google.com/";
+const GMAIL_READONLY_SCOPE = "https://www.googleapis.com/auth/gmail.readonly";
+
+function grantedScopeCoversRequiredScope(
+  grantedScope: string,
+  requiredScope: string,
+): boolean {
+  if (grantedScope === requiredScope) return true;
+  return (
+    grantedScope === GMAIL_FULL_ACCESS_SCOPE &&
+    requiredScope === GMAIL_READONLY_SCOPE
+  );
 }

package/src/plugins/defaults/image-fallback/__tests__/image-fallback.test.ts

@@ -1,9 +1,12 @@
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
 import type {
+  ContentBlock,
   ImageContent,
   Message,
   ModelProfileInfo,
+  PostToolUseContext,
+  ToolResultContent,
   UserPromptSubmitContext,
 } from "@vellumai/plugin-api";
 
@@ -27,20 +30,24 @@ const fakeProvider = {
 // Mock @vellumai/plugin-api — only the runtime handles the plugin imports.
 // `extractAllText` stays real (imported from the relative path, not plugin-api).
 mock.module("@vellumai/plugin-api", () => ({
-  doesSupportVision: (profile: ModelProfileInfo) => visionProfiles.has(profile.key),
+  doesSupportVision: (profile: ModelProfileInfo) =>
+    visionProfiles.has(profile.key),
   getModelProfiles: () => mockProfiles,
   getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
 }));
 
 // Mock the image-persist module to avoid filesystem side effects in tests.
-let mockPersistPath: string | null = "/workspace/data/attachments/mock-hash.png";
+let mockPersistPath: string | null =
+  "/workspace/data/attachments/mock-hash.png";
 mock.module("../src/image-persist.js", () => ({
   persistImage: () => mockPersistPath,
 }));
 
 // ─── Imports (after mocks are registered) ───────────────────────────────────
 
-const userPromptSubmit = (await import("../hooks/user-prompt-submit.js")).default;
+const userPromptSubmit = (await import("../hooks/user-prompt-submit.js"))
+  .default;
+const postToolUse = (await import("../hooks/post-tool-use.js")).default;
 const { findVisionProfile } = await import("../src/vision-caption.js");
 const { resetCaptionCacheForTests } = await import("../src/caption-cache.js");
 
@@ -101,6 +108,30 @@ function makeCtx(
   } as unknown as UserPromptSubmitContext;
 }
 
+function toolResult(contentBlocks?: ContentBlock[]): ToolResultContent {
+  return {
+    type: "tool_result",
+    tool_use_id: "tu1",
+    content: "Took a screenshot.",
+    ...(contentBlocks ? { contentBlocks } : {}),
+  };
+}
+
+function makeToolCtx(
+  overrides: Partial<PostToolUseContext> = {},
+): PostToolUseContext {
+  return {
+    conversationId: "c1",
+    toolResponse: toolResult(),
+    messages: [],
+    additionalContext: null,
+    model: "text-only-model",
+    maxInputTokens: 100_000,
+    logger,
+    ...overrides,
+  } as unknown as PostToolUseContext;
+}
+
 // ─── Setup ──────────────────────────────────────────────────────────────────
 
 beforeEach(() => {
@@ -133,9 +164,9 @@ describe("image-fallback user-prompt-submit hook", () => {
     const ctx = makeCtx({ latestMessages: messages, isNonInteractive: true });
     await userPromptSubmit(ctx);
     expect(ctx.latestMessages[0].content[0].type).toBe("text");
-    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
-      "[Image:",
-    );
+    expect(
+      (ctx.latestMessages[0].content[0] as { text: string }).text,
+    ).toContain("[Image auto-described");
   });
 
   test("replaces image blocks with captions when active model is text-only", async () => {
@@ -143,27 +174,27 @@ describe("image-fallback user-prompt-submit hook", () => {
     const ctx = makeCtx({ latestMessages: messages });
     await userPromptSubmit(ctx);
     expect(ctx.latestMessages[0].content[0].type).toBe("text");
-    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
-      "[Image: A red chart showing Q3 revenue.]",
+    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toBe(
+      "[Image auto-described for text-only model: A red chart showing Q3 revenue.]",
     );
   });
 
-  test("references the saved image path in the caption text", async () => {
+  test("caption states the model can't view images and the text is derived", async () => {
     const messages = [imageMsg("img1")];
     const ctx = makeCtx({ latestMessages: messages });
     await userPromptSubmit(ctx);
     const text = (ctx.latestMessages[0].content[0] as { text: string }).text;
-    expect(text).toContain("(saved to /workspace/data/attachments/");
+    expect(text).toContain("text-only model");
+    expect(text).toContain("auto-described");
   });
 
-  test("works without a saved path when persist fails", async () => {
-    mockPersistPath = null;
+  test("does not embed the saved image path in the caption text", async () => {
     const messages = [imageMsg("img1")];
     const ctx = makeCtx({ latestMessages: messages });
     await userPromptSubmit(ctx);
     const text = (ctx.latestMessages[0].content[0] as { text: string }).text;
-    expect(text).toContain("[Image: A red chart showing Q3 revenue.]");
-    expect(text).not.toContain("(saved to");
+    expect(text).not.toContain("saved to");
+    expect(text).not.toContain("/workspace/data/attachments/");
   });
 
   test("preserves non-image blocks and captions only images", async () => {
@@ -183,9 +214,9 @@ describe("image-fallback user-prompt-submit hook", () => {
       "Look at this:",
     );
     expect(ctx.latestMessages[0].content[1].type).toBe("text");
-    expect((ctx.latestMessages[0].content[1] as { text: string }).text).toContain(
-      "[Image:",
-    );
+    expect(
+      (ctx.latestMessages[0].content[1] as { text: string }).text,
+    ).toContain("[Image auto-described");
     expect((ctx.latestMessages[0].content[2] as { text: string }).text).toBe(
       "What do you see?",
     );
@@ -197,9 +228,9 @@ describe("image-fallback user-prompt-submit hook", () => {
     const ctx = makeCtx({ latestMessages: messages });
     await userPromptSubmit(ctx);
     expect(ctx.latestMessages[0].content[0].type).toBe("text");
-    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
-      "no vision-capable model",
-    );
+    expect(
+      (ctx.latestMessages[0].content[0] as { text: string }).text,
+    ).toContain("no vision-capable model");
   });
 
   test("uses fail-open placeholder when provider resolution returns null", async () => {
@@ -208,9 +239,9 @@ describe("image-fallback user-prompt-submit hook", () => {
     const ctx = makeCtx({ latestMessages: messages });
     await userPromptSubmit(ctx);
     expect(ctx.latestMessages[0].content[0].type).toBe("text");
-    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
-      "captioning failed",
-    );
+    expect(
+      (ctx.latestMessages[0].content[0] as { text: string }).text,
+    ).toContain("auto-description failed");
   });
 
   test("caches captions — second call with same image does not invoke provider", async () => {
@@ -244,7 +275,8 @@ describe("image-fallback user-prompt-submit hook", () => {
     mock.module("@vellumai/plugin-api", () => ({
       doesSupportVision: (p: ModelProfileInfo) => visionProfiles.has(p.key),
       getModelProfiles: () => mockProfiles,
-      getConfiguredProvider: async () => (providerResolves ? fakeProvider : null),
+      getConfiguredProvider: async () =>
+        providerResolves ? fakeProvider : null,
     }));
   });
 
@@ -260,13 +292,13 @@ describe("image-fallback user-prompt-submit hook", () => {
     const ctx = makeCtx({ latestMessages: messages });
     await userPromptSubmit(ctx);
     expect(ctx.latestMessages[0].content[0].type).toBe("text");
-    expect((ctx.latestMessages[0].content[0] as { text: string }).text).toContain(
-      "[Image:",
-    );
+    expect(
+      (ctx.latestMessages[0].content[0] as { text: string }).text,
+    ).toContain("[Image auto-described");
     expect(ctx.latestMessages[2].content[0].type).toBe("text");
-    expect((ctx.latestMessages[2].content[0] as { text: string }).text).toContain(
-      "[Image:",
-    );
+    expect(
+      (ctx.latestMessages[2].content[0] as { text: string }).text,
+    ).toContain("[Image auto-described");
     expect((ctx.latestMessages[2].content[1] as { text: string }).text).toBe(
       "both?",
     );
@@ -300,3 +332,70 @@ describe("findVisionProfile", () => {
     expect(findVisionProfile()).toBeNull();
   });
 });
+
+describe("image-fallback post-tool-use hook", () => {
+  test("captions image blocks nested in a tool result for a text-only model", async () => {
+    const ctx = makeToolCtx({
+      toolResponse: toolResult([imageBlock("shot1")]),
+    });
+    await postToolUse(ctx);
+    const block = ctx.toolResponse.contentBlocks![0];
+    expect(block.type).toBe("text");
+    expect((block as { text: string }).text).toBe(
+      "[Image auto-described for text-only model: A red chart showing Q3 revenue.]",
+    );
+  });
+
+  test("is a no-op when the active model supports vision", async () => {
+    visionProfiles = new Set(["text-only"]); // active profile supports vision
+    const ctx = makeToolCtx({
+      toolResponse: toolResult([imageBlock("shot1")]),
+    });
+    await postToolUse(ctx);
+    expect(ctx.toolResponse.contentBlocks![0].type).toBe("image");
+  });
+
+  test("is a no-op when the tool result has no contentBlocks", async () => {
+    const ctx = makeToolCtx({ toolResponse: toolResult() });
+    await postToolUse(ctx);
+    expect(ctx.toolResponse.contentBlocks).toBeUndefined();
+  });
+
+  test("preserves non-image contentBlocks and captions only images", async () => {
+    const ctx = makeToolCtx({
+      toolResponse: toolResult([
+        { type: "text", text: "page title" },
+        imageBlock("shot1"),
+      ]),
+    });
+    await postToolUse(ctx);
+    const blocks = ctx.toolResponse.contentBlocks!;
+    expect((blocks[0] as { text: string }).text).toBe("page title");
+    expect(blocks[1].type).toBe("text");
+    expect((blocks[1] as { text: string }).text).toContain(
+      "[Image auto-described",
+    );
+  });
+
+  test("uses fail-open placeholder when no vision profile is configured", async () => {
+    visionProfiles = new Set<string>(); // no vision profiles
+    const ctx = makeToolCtx({
+      toolResponse: toolResult([imageBlock("shot1")]),
+    });
+    await postToolUse(ctx);
+    const block = ctx.toolResponse.contentBlocks![0];
+    expect(block.type).toBe("text");
+    expect((block as { text: string }).text).toContain(
+      "no vision-capable model",
+    );
+  });
+
+  test("does not embed the saved image path in the caption text", async () => {
+    const ctx = makeToolCtx({
+      toolResponse: toolResult([imageBlock("shot1")]),
+    });
+    await postToolUse(ctx);
+    const text = (ctx.toolResponse.contentBlocks![0] as { text: string }).text;
+    expect(text).not.toContain("saved to");
+  });
+});

package/src/plugins/defaults/image-fallback/hooks/post-tool-use.ts

@@ -0,0 +1,58 @@
+/**
+ * Default `post-tool-use` hook: when the active model is text-only, captions
+ * the image blocks a tool returns (e.g. a `browser_screenshot`) and
+ * substitutes the caption as a text block so the result stays sendable to a
+ * provider that would otherwise reject the raw image.
+ *
+ * Tool images arrive nested in `toolResponse.contentBlocks` (the rich-content
+ * companion to the tool result's text `content`), so the hook scans there
+ * rather than the top-level message content the `user-prompt-submit` hook
+ * handles. Both share {@link captionImageBlocks}.
+ *
+ * The active model is resolved from the workspace's active profile — the
+ * post-tool-use context carries the running model, and the active profile is
+ * what the loop is executing this turn. If that profile supports vision, the
+ * hook is a no-op and the image reaches the model untouched.
+ */
+
+import {
+  doesSupportVision,
+  getModelProfiles,
+  type PluginHookFn,
+  type PostToolUseContext,
+} from "@vellumai/plugin-api";
+
+import { captionImageBlocks } from "../src/caption-blocks.js";
+import { findVisionProfile } from "../src/vision-caption.js";
+
+const postToolUse: PluginHookFn<PostToolUseContext> = async (ctx) => {
+  const blocks = ctx.toolResponse.contentBlocks;
+  if (blocks == null || blocks.length === 0) return;
+
+  // If the active model already supports vision, leave the image in place.
+  const activeProfile = getModelProfiles().find((p) => p.isActive);
+  if (activeProfile == null) return;
+  if (doesSupportVision(activeProfile)) return;
+
+  // Find a vision-capable profile for captioning.
+  const visionProfileKey = findVisionProfile();
+
+  const imageCount = await captionImageBlocks(
+    blocks,
+    visionProfileKey,
+    ctx.logger,
+  );
+
+  if (imageCount > 0) {
+    ctx.logger.info(
+      {
+        plugin: "image-fallback",
+        toolUseId: ctx.toolResponse.tool_use_id,
+        imageCount,
+      },
+      "Replaced tool-result image blocks with text captions for text-only model",
+    );
+  }
+};
+
+export default postToolUse;