@vellumai/assistant 0.12.2 → 0.12.3-staging.2

package/src/media/openrouter-image-service.ts

@@ -0,0 +1,153 @@
+import {
+  DEFAULT_OPENROUTER_IMAGE_MODEL,
+  qualifyImageModelForOpenRouter,
+  resolveImageModel,
+} from "./image-models.js";
+import {
+  type GeneratedImage,
+  type ImageGenCredentials,
+  type ImageGenerationRequest,
+  type ImageGenerationResult,
+  isImageProviderBillingError,
+  MAX_VARIANTS,
+} from "./types.js";
+
+const OPENROUTER_IMAGES_URL = "https://openrouter.ai/api/v1/images";
+
+/**
+ * Qualify a bare built-in ID or alias for OpenRouter. Call sites that skip
+ * tool-level validation (app-icon, avatar) still reach this boundary.
+ */
+function resolveOpenRouterModel(model: string | undefined): string {
+  const trimmed = model?.trim();
+  if (!trimmed) {
+    return DEFAULT_OPENROUTER_IMAGE_MODEL;
+  }
+  if (resolveImageModel(trimmed)) {
+    return qualifyImageModelForOpenRouter(trimmed);
+  }
+  return trimmed;
+}
+
+const OPENROUTER_BILLING_MESSAGE =
+  "Image generation is unavailable because the OpenRouter account or API key is out of credits. " +
+  "Add funds with the provider or update the key in Settings. Retrying won't help until credits are added.";
+
+class OpenRouterImageError extends Error {
+  constructor(
+    readonly status: number,
+    message: string,
+  ) {
+    super(message);
+    this.name = "OpenRouterImageError";
+  }
+}
+
+export function mapOpenRouterError(error: unknown): string {
+  if (error instanceof OpenRouterImageError) {
+    if (
+      isImageProviderBillingError({
+        status: error.status,
+        message: error.message,
+      })
+    ) {
+      return OPENROUTER_BILLING_MESSAGE;
+    }
+    if (error.status === 400) {
+      return `The OpenRouter image request was invalid: ${error.message}`;
+    }
+    if (error.status === 401 || error.status === 403) {
+      return "Authentication failed. Please check your OpenRouter API key.";
+    }
+    if (error.status === 429) {
+      return "OpenRouter rate limit exceeded. Please wait a moment and try again.";
+    }
+    if (error.status >= 500) {
+      return "The OpenRouter service is temporarily unavailable. Please try again later.";
+    }
+    return `OpenRouter API error (status ${error.status}). Please try again.`;
+  }
+  if (error instanceof Error) {
+    if (isImageProviderBillingError({ message: error.message })) {
+      return OPENROUTER_BILLING_MESSAGE;
+    }
+    return `Image generation failed: ${error.message}`;
+  }
+  return "An unexpected error occurred during image generation.";
+}
+
+function errorMessage(body: unknown): string {
+  if (typeof body !== "object" || body === null) {
+    return "Request failed";
+  }
+  const error = (body as { error?: unknown }).error;
+  if (typeof error === "string") {
+    return error;
+  }
+  if (typeof error === "object" && error !== null) {
+    const message = (error as { message?: unknown }).message;
+    if (typeof message === "string" && message) {
+      return message;
+    }
+  }
+  return "Request failed";
+}
+
+export async function generateImageOpenRouter(
+  credentials: ImageGenCredentials,
+  request: ImageGenerationRequest,
+): Promise<ImageGenerationResult> {
+  if (credentials.type !== "direct") {
+    throw new Error("OpenRouter image generation requires an OpenRouter API key.");
+  }
+
+  const model = resolveOpenRouterModel(request.model);
+  const variants = Math.max(1, Math.min(request.variants ?? 1, MAX_VARIANTS));
+  const inputReferences = request.sourceImages?.map((image) => ({
+    type: "image_url" as const,
+    image_url: {
+      url: `data:${image.mimeType};base64,${image.dataBase64}`,
+    },
+  }));
+
+  const response = await fetch(OPENROUTER_IMAGES_URL, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${credentials.apiKey}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({
+      model,
+      prompt: request.prompt,
+      n: variants,
+      ...(request.mode === "edit" && inputReferences?.length
+        ? { input_references: inputReferences }
+        : {}),
+    }),
+    ...(request.signal ? { signal: request.signal } : {}),
+  });
+
+  const body = (await response.json().catch(() => undefined)) as
+    | {
+        data?: Array<{ b64_json?: string; media_type?: string }>;
+        error?: unknown;
+      }
+    | undefined;
+
+  if (!response.ok) {
+    throw new OpenRouterImageError(response.status, errorMessage(body));
+  }
+
+  const images: GeneratedImage[] = [];
+  for (const entry of body?.data ?? []) {
+    if (!entry.b64_json) {
+      continue;
+    }
+    images.push({
+      mimeType: entry.media_type || "image/png",
+      dataBase64: entry.b64_json,
+    });
+  }
+
+  return { images, text: undefined, resolvedModel: model };
+}

package/src/media/types.ts

@@ -1,4 +1,4 @@
-export type ImageGenProvider = "gemini" | "openai";
+export type ImageGenProvider = "gemini" | "openai" | "openrouter";
 
 export interface DirectCredentials {
   type: "direct";

package/src/messaging/provider-message-metadata.ts

@@ -30,9 +30,9 @@ import { CHANNEL_IDS } from "../channels/types.js";
  * The schema passes through what it does not name, so a provider carries its
  * own fields (Slack's file markers and timezone labels) on this same object
  * and validates them with its own schema. This is how `SourceMetadataSchema`
- * already carries `slackBotMentioned` and the email fields on the wire: one
- * object per row, no per-provider sub-envelope, and no second copy of
- * anything named here.
+ * carries `botMentioned` and the email fields on the wire: one object per
+ * row, no per-provider sub-envelope, and no second copy of anything named
+ * here.
  */
 
 const providerReactionMetadataSchema = z.object({

package/src/messaging/providers/__tests__/transport-dispatch.test.ts

@@ -1,6 +1,7 @@
 import { beforeEach, describe, expect, mock, test } from "bun:test";
 
 import type { ChannelReplyPayload } from "@vellumai/gateway-client";
+import { ChannelDeliveryError } from "@vellumai/gateway-client/http-delivery";
 
 // Replace each channel's provider-API send layer with spies so the dispatcher's
 // routing and sub-operation selection can be asserted without network calls.
@@ -559,25 +560,78 @@ describe("acknowledged provider posts", () => {
   });
 
   test("a delivery with no text acknowledges no post", async () => {
+    const attachments = [
+      {
+        id: "att-1",
+        filename: "a.txt",
+        mimeType: "text/plain",
+        sizeBytes: 1,
+        kind: "file",
+      },
+    ];
     const result = await deliverDirect(
       `${BASE}/deliver/telegram`,
-      payload({
-        attachments: [
-          {
-            id: "att-1",
-            filename: "a.txt",
-            mimeType: "text/plain",
-            sizeBytes: 1,
-            kind: "file",
-          },
-        ],
-      } as Partial<ChannelReplyPayload>),
+      payload({ attachments }),
     );
     expect(result).toEqual({ ok: true, messageIds: [] });
     expect(telegram.sendTelegramReply).not.toHaveBeenCalled();
+    expect(telegram.sendTelegramAttachments).toHaveBeenCalledTimes(1);
+    expect(telegram.sendTelegramAttachments.mock.calls[0]?.slice(0, 2)).toEqual(
+      ["C1", attachments],
+    );
   });
 });
 
+describe("attachment-only delivery failure", () => {
+  const attachments = [
+    {
+      id: "att-1",
+      filename: "a.txt",
+      mimeType: "text/plain",
+      sizeBytes: 1,
+      kind: "file",
+    },
+    {
+      id: "att-2",
+      filename: "b.txt",
+      mimeType: "text/plain",
+      sizeBytes: 1,
+      kind: "file",
+    },
+  ];
+  const allFailed = { allFailed: true, failureCount: 2, totalCount: 2 };
+
+  for (const { channel, sendAttachments } of [
+    { channel: "telegram", sendAttachments: telegram.sendTelegramAttachments },
+    { channel: "whatsapp", sendAttachments: whatsapp.sendWhatsAppAttachments },
+  ]) {
+    test(`${channel} fails with 502 when every attachment fails and there is no text`, async () => {
+      sendAttachments.mockImplementationOnce(() => Promise.resolve(allFailed));
+
+      const delivery = deliverDirect(
+        `${BASE}/deliver/${channel}`,
+        payload({ attachments }),
+      );
+
+      await expect(delivery).rejects.toBeInstanceOf(ChannelDeliveryError);
+      await expect(delivery).rejects.toMatchObject({ statusCode: 502 });
+      expect(sendAttachments).toHaveBeenCalledTimes(1);
+    });
+
+    test(`${channel} succeeds when every attachment fails but the text went out`, async () => {
+      sendAttachments.mockImplementationOnce(() => Promise.resolve(allFailed));
+
+      const result = await deliverDirect(
+        `${BASE}/deliver/${channel}`,
+        payload({ text: "hi", attachments }),
+      );
+
+      expect(result.ok).toBe(true);
+      expect(sendAttachments).toHaveBeenCalledTimes(1);
+    });
+  }
+});
+
 describe("unsupported callback", () => {
   test("throws when no transport owns the callback", async () => {
     await expect(

package/src/messaging/providers/discord/send.ts

@@ -9,11 +9,7 @@ import type {
   ApprovalUIMetadata,
   ChannelDeliveryResult,
 } from "@vellumai/gateway-client";
-import {
-  classifyReactionEmojiSpelling,
-  parseDiscordEmojiMention,
-  type ReactionEmojiIdentity,
-} from "@vellumai/service-contracts/reactions";
+import { parseDiscordEmojiMention } from "@vellumai/service-contracts/reactions";
 
 import { getAttachmentContent } from "../../../persistence/attachments-store.js";
 import type { RuntimeAttachmentMetadata } from "../../../runtime/http-types.js";
@@ -45,8 +41,9 @@ const DISCORD_ALLOWED_MENTIONS = { parse: ["users"] } as const;
 
 /**
  * Upper bound on an outbound attachment. Discord's real limit is the guild's
- * boost tier (10 MiB with no boosts, more above that), which the API does not
- * expose here, so the client-side guard is only the ceiling no tier exceeds:
+ * boost tier (20 MiB per file by default, more above that; see
+ * https://discord.com/developers/docs/reference#uploading-files), which the
+ * API does not expose here, so the client-side guard is only the ceiling no tier exceeds:
  * past it the upload is provably futile and not worth the bandwidth. Anything
  * under it is attempted and, if the guild's own tier rejects it, reported
  * through the same failure notice as any other attachment error.
@@ -433,16 +430,6 @@ function discordReactionPathEmoji(emoji: string): string {
   return encodeURIComponent(custom ? `${custom.name}:${custom.id}` : emoji);
 }
 
-/**
- * What a spelling the assistant reacts with means on Discord: a guild emoji
- * in its mention form, else the character itself.
- */
-export function describeDiscordReactionEmoji(
-  emoji: string,
-): ReactionEmojiIdentity {
-  return classifyReactionEmojiSpelling(emoji);
-}
-
 /**
  * Add or remove the bot's own emoji reaction on a message.
  *

package/src/messaging/providers/discord/transport.ts

@@ -1,4 +1,5 @@
 import { ChannelDeliveryError } from "@vellumai/gateway-client/http-delivery";
+import { classifyReactionEmojiSpelling } from "@vellumai/service-contracts/reactions";
 
 import { getLogger } from "../../../util/logger.js";
 import { directDeliveryContext } from "../callback-routing.js";
@@ -10,7 +11,6 @@ import { isBusyActivityPhase } from "../channel-transport.js";
 import { openDiscordDmChannel } from "./api.js";
 import type { DiscordSendTarget } from "./send.js";
 import {
-  describeDiscordReactionEmoji,
   editDiscordMessage,
   sendDiscordAttachments,
   sendDiscordReaction,
@@ -95,7 +95,7 @@ export const discordTransport: ChannelTransport = {
     return { ok: true };
   },
 
-  describeReactionEmoji: describeDiscordReactionEmoji,
+  describeReactionEmoji: classifyReactionEmojiSpelling,
 
   async react(target) {
     return sendDiscordReaction(

package/src/messaging/providers/outlook/client.ts

@@ -69,28 +69,22 @@ async function request<T>(
   const canRetry = isIdempotent(method);
 
   for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
-    let resp: OAuthConnectionResponse;
-    try {
-      const extraHeaders =
-        options?.headers &&
-        typeof options.headers === "object" &&
-        !Array.isArray(options.headers)
-          ? (options.headers as Record<string, string>)
-          : {};
-      resp = await connection.request({
-        method,
-        path,
-        query,
-        headers: {
-          "Content-Type": "application/json",
-          ...extraHeaders,
-        },
-        body: options?.body ? JSON.parse(options.body as string) : undefined,
-      });
-    } catch (err) {
-      // Network-level errors from connection.request() are not retryable
-      throw err;
-    }
+    const extraHeaders =
+      options?.headers &&
+      typeof options.headers === "object" &&
+      !Array.isArray(options.headers)
+        ? (options.headers as Record<string, string>)
+        : {};
+    const resp: OAuthConnectionResponse = await connection.request({
+      method,
+      path,
+      query,
+      headers: {
+        "Content-Type": "application/json",
+        ...extraHeaders,
+      },
+      body: options?.body ? JSON.parse(options.body as string) : undefined,
+    });
 
     if (resp.status < 200 || resp.status >= 300) {
       if (canRetry && isRetryable(resp.status) && attempt < MAX_RETRIES) {

package/src/messaging/providers/telegram-bot/send.test.ts

@@ -5,20 +5,30 @@ import type {
   ChannelReplyPayload,
 } from "@vellumai/gateway-client";
 
+import type { RuntimeAttachmentMetadata } from "../../../runtime/http-types.js";
 import type { CallbackContext } from "../channel-transport.js";
 
 // Derive the mock signature from the real export so the test cannot drift from
 // the production call signature.
 type CallTelegramBotApi = typeof import("./api.js").callTelegramBotApi;
+type CallTelegramBotApiMultipart =
+  typeof import("./api.js").callTelegramBotApiMultipart;
+type GetAttachmentContent =
+  typeof import("../../../persistence/attachments-store.js").getAttachmentContent;
 
 const callTelegramBotApiMock = mock<CallTelegramBotApi>(
   async () => ({}) as never,
 );
+const callTelegramBotApiMultipartMock = mock<CallTelegramBotApiMultipart>(
+  async () => ({}) as never,
+);
+const getAttachmentContentMock = mock<GetAttachmentContent>(() => null);
 
 mock.module("./api.js", () => ({
   callTelegramBotApi: (method: string, body: Record<string, unknown>) =>
     callTelegramBotApiMock(method, body),
-  callTelegramBotApiMultipart: async () => ({}),
+  callTelegramBotApiMultipart: (method: string, form: FormData) =>
+    callTelegramBotApiMultipartMock(method, form),
   TelegramNonRetryableError: class TelegramNonRetryableError extends Error {
     readonly description: string | undefined;
     constructor(message: string, description?: string) {
@@ -29,9 +39,15 @@ mock.module("./api.js", () => ({
   },
 }));
 
+mock.module("../../../persistence/attachments-store.js", () => ({
+  getAttachmentContent: (attachmentId: string) =>
+    getAttachmentContentMock(attachmentId),
+}));
+
 const { TelegramNonRetryableError } = await import("./api.js");
 const {
   editTelegramMessage,
+  sendTelegramAttachments,
   sendTelegramReaction,
   sendTelegramReply,
   sendTelegramRichReply,
@@ -61,6 +77,10 @@ const callsTo = (method: string) =>
 beforeEach(() => {
   callTelegramBotApiMock.mockReset();
   callTelegramBotApiMock.mockImplementation(async () => ({}) as never);
+  callTelegramBotApiMultipartMock.mockReset();
+  callTelegramBotApiMultipartMock.mockImplementation(async () => ({}) as never);
+  getAttachmentContentMock.mockReset();
+  getAttachmentContentMock.mockImplementation(() => null);
 });
 
 describe("sendTelegramRichReply", () => {
@@ -670,3 +690,218 @@ describe("telegramTransport.streamReply", () => {
     expect(result).toEqual({ ok: false });
   });
 });
+
+describe("sendTelegramAttachments", () => {
+  // The Bot API's multipart upload limits: a photo, and any other file. The
+  // sender skips anything over the second.
+  const MAX_PHOTO_BYTES = 10 * 1024 * 1024;
+  const MAX_ATTACHMENT_BYTES = 50 * 1024 * 1024;
+
+  function attachment(
+    id: string,
+    filename: string,
+    mimeType: string,
+    sizeBytes = 3,
+  ): RuntimeAttachmentMetadata {
+    return { id, filename, mimeType, sizeBytes, kind: "file" };
+  }
+
+  /** Serve these bytes from the attachment store, keyed by attachment id. */
+  function storeHolds(contents: Record<string, Buffer>) {
+    getAttachmentContentMock.mockImplementation((id) => contents[id] ?? null);
+  }
+
+  const multipartCalls = () =>
+    callTelegramBotApiMultipartMock.mock.calls.map(([method, form]) => ({
+      method,
+      form,
+    }));
+
+  const noticeTexts = () =>
+    callsTo("sendMessage").map(([, body]) => body.text as string);
+
+  test("sends an image from the store with sendPhoto", async () => {
+    storeHolds({ "att-1": Buffer.from("png") });
+
+    const result = await sendTelegramAttachments("123", [
+      attachment("att-1", "chart.png", "image/png"),
+    ]);
+
+    const [call] = multipartCalls();
+    expect(multipartCalls()).toHaveLength(1);
+    expect(call?.method).toBe("sendPhoto");
+    expect(call?.form.get("chat_id")).toBe("123");
+    expect(call?.form.get("document")).toBeNull();
+    const photo = call?.form.get("photo") as File;
+    expect(photo.name).toBe("chart.png");
+    expect(photo.type).toBe("image/png");
+    expect(await photo.text()).toBe("png");
+    expect(noticeTexts()).toEqual([]);
+    expect(result).toEqual({
+      allFailed: false,
+      failureCount: 0,
+      totalCount: 1,
+    });
+  });
+
+  test("sends a non-image from the store with sendDocument", async () => {
+    storeHolds({ "att-1": Buffer.from("pdf") });
+
+    await sendTelegramAttachments("123", [
+      attachment("att-1", "report.pdf", "application/pdf"),
+    ]);
+
+    const [call] = multipartCalls();
+    expect(multipartCalls()).toHaveLength(1);
+    expect(call?.method).toBe("sendDocument");
+    expect(call?.form.get("photo")).toBeNull();
+    const document = call?.form.get("document") as File;
+    expect(document.name).toBe("report.pdf");
+    expect(document.type).toBe("application/pdf");
+    expect(await document.text()).toBe("pdf");
+  });
+
+  test("sends an image over the photo limit with sendDocument", async () => {
+    storeHolds({
+      "att-1": Buffer.alloc(MAX_PHOTO_BYTES),
+      "att-2": Buffer.alloc(MAX_PHOTO_BYTES + 1),
+    });
+
+    const result = await sendTelegramAttachments("123", [
+      attachment("att-1", "fits.png", "image/png", MAX_PHOTO_BYTES),
+      attachment("att-2", "scan.png", "image/png", MAX_PHOTO_BYTES + 1),
+    ]);
+
+    expect(multipartCalls().map((c) => c.method)).toEqual([
+      "sendPhoto",
+      "sendDocument",
+    ]);
+    const document = multipartCalls()[1]?.form.get("document") as File;
+    expect(document.name).toBe("scan.png");
+    expect(document.type).toBe("image/png");
+    expect(document.size).toBe(MAX_PHOTO_BYTES + 1);
+    expect(noticeTexts()).toEqual([]);
+    expect(result).toEqual({
+      allFailed: false,
+      failureCount: 0,
+      totalCount: 2,
+    });
+  });
+
+  test("targets the topic on both the upload and the failure notice", async () => {
+    storeHolds({ "att-1": Buffer.from("png") });
+
+    await sendTelegramAttachments(
+      "123",
+      [
+        attachment("att-1", "chart.png", "image/png"),
+        attachment("att-2", "gone.txt", "text/plain"),
+      ],
+      { messageThreadId: "42" },
+    );
+
+    expect(multipartCalls()[0]?.form.get("message_thread_id")).toBe("42");
+    const [notice] = callsTo("sendMessage");
+    expect(notice?.[1].message_thread_id).toBe(42);
+  });
+
+  test("leaves the topic field off a main-chat upload", async () => {
+    storeHolds({ "att-1": Buffer.from("png") });
+
+    await sendTelegramAttachments("123", [
+      attachment("att-1", "chart.png", "image/png"),
+    ]);
+
+    expect(multipartCalls()[0]?.form.has("message_thread_id")).toBe(false);
+  });
+
+  test("skips an attachment whose declared size is over the cap without reading it", async () => {
+    storeHolds({ "att-1": Buffer.from("png") });
+
+    const result = await sendTelegramAttachments("123", [
+      attachment("att-1", "huge.mov", "video/mp4", MAX_ATTACHMENT_BYTES + 1),
+    ]);
+
+    expect(getAttachmentContentMock).not.toHaveBeenCalled();
+    expect(multipartCalls()).toHaveLength(0);
+    expect(noticeTexts()).toEqual([
+      "\u26a0\ufe0f 1 attachment(s) could not be delivered: huge.mov",
+    ]);
+    expect(result).toEqual({ allFailed: true, failureCount: 1, totalCount: 1 });
+  });
+
+  test("skips an attachment whose stored bytes are over the cap", async () => {
+    // The declared size understates what the store actually holds.
+    storeHolds({ "att-1": Buffer.alloc(MAX_ATTACHMENT_BYTES + 1) });
+
+    const result = await sendTelegramAttachments("123", [
+      attachment("att-1", "huge.bin", "application/octet-stream"),
+    ]);
+
+    expect(multipartCalls()).toHaveLength(0);
+    expect(noticeTexts()).toEqual([
+      "\u26a0\ufe0f 1 attachment(s) could not be delivered: huge.bin",
+    ]);
+    expect(result).toEqual({ allFailed: true, failureCount: 1, totalCount: 1 });
+  });
+
+  test("reports an attachment the store has no content for", async () => {
+    const result = await sendTelegramAttachments("123", [
+      attachment("att-1", "missing.txt", "text/plain"),
+    ]);
+
+    expect(getAttachmentContentMock).toHaveBeenCalledWith("att-1");
+    expect(multipartCalls()).toHaveLength(0);
+    expect(noticeTexts()).toEqual([
+      "\u26a0\ufe0f 1 attachment(s) could not be delivered: missing.txt",
+    ]);
+    expect(result).toEqual({ allFailed: true, failureCount: 1, totalCount: 1 });
+  });
+
+  test("keeps sending after one upload fails and names only the failure", async () => {
+    storeHolds({
+      "att-1": Buffer.from("a"),
+      "att-2": Buffer.from("b"),
+      "att-3": Buffer.from("c"),
+    });
+    callTelegramBotApiMultipartMock.mockImplementation(
+      async (_method, form) => {
+        if ((form.get("document") as File | null)?.name === "b.txt") {
+          throw new Error("Bad Request: file is empty");
+        }
+        return {} as never;
+      },
+    );
+
+    const result = await sendTelegramAttachments("123", [
+      attachment("att-1", "a.txt", "text/plain"),
+      attachment("att-2", "b.txt", "text/plain"),
+      attachment("att-3", "c.txt", "text/plain"),
+    ]);
+
+    expect(
+      multipartCalls().map((c) => (c.form.get("document") as File).name),
+    ).toEqual(["a.txt", "b.txt", "c.txt"]);
+    expect(noticeTexts()).toEqual([
+      "\u26a0\ufe0f 1 attachment(s) could not be delivered: b.txt",
+    ]);
+    expect(result).toEqual({
+      allFailed: false,
+      failureCount: 1,
+      totalCount: 3,
+    });
+  });
+
+  test("still returns the result when the failure notice cannot be sent", async () => {
+    callTelegramBotApiMock.mockImplementation(async () => {
+      throw new Error("Forbidden: bot was blocked by the user");
+    });
+
+    const result = await sendTelegramAttachments("123", [
+      attachment("att-1", "missing.txt", "text/plain"),
+    ]);
+
+    expect(callsTo("sendMessage")).toHaveLength(1);
+    expect(result).toEqual({ allFailed: true, failureCount: 1, totalCount: 1 });
+  });
+});