@vellumai/vellum-gateway 0.6.0 → 0.6.2

package/src/http/routes/email-webhook.test.ts

@@ -0,0 +1,393 @@
+import { createHmac } from "node:crypto";
+import { describe, it, expect, mock, beforeEach } from "bun:test";
+import type { GatewayConfig } from "../../config.js";
+import type { CredentialCache } from "../../credential-cache.js";
+import { credentialKey } from "../../credential-key.js";
+
+// --- Mocks ----------------------------------------------------------------
+
+const handleInboundMock = mock(
+  (_config: GatewayConfig, _normalized: unknown, _options?: unknown) =>
+    Promise.resolve({ forwarded: true, rejected: false }),
+);
+
+const resetConversationMock = mock(() => Promise.resolve());
+
+mock.module("../../handlers/handle-inbound.js", () => ({
+  handleInbound: handleInboundMock,
+}));
+
+mock.module("../../runtime/client.js", () => ({
+  resetConversation: resetConversationMock,
+  uploadAttachment: mock(() => Promise.resolve({ id: "att-1" })),
+  AttachmentValidationError: class extends Error {},
+  CircuitBreakerOpenError: class extends Error {},
+}));
+
+// Import after mocks are registered
+const { createEmailWebhookHandler } = await import("./email-webhook.js");
+
+// --- Helpers ---------------------------------------------------------------
+
+const TEST_WEBHOOK_SECRET = "test-webhook-secret-1234";
+
+function computeSignature(body: string, secret: string): string {
+  return (
+    "sha256=" + createHmac("sha256", secret).update(body, "utf8").digest("hex")
+  );
+}
+
+const baseConfig: GatewayConfig = {
+  assistantRuntimeBaseUrl: "http://localhost:7821",
+  defaultAssistantId: "ast-default",
+  gatewayInternalBaseUrl: "http://127.0.0.1:7830",
+  logFile: { dir: undefined, retentionDays: 30 },
+  maxAttachmentBytes: {
+    telegram: 50 * 1024 * 1024,
+    slack: 100 * 1024 * 1024,
+    whatsapp: 16 * 1024 * 1024,
+    default: 50 * 1024 * 1024,
+  },
+  maxAttachmentConcurrency: 3,
+  maxWebhookPayloadBytes: 1024 * 1024,
+  port: 7830,
+  routingEntries: [],
+  runtimeInitialBackoffMs: 500,
+  runtimeMaxRetries: 2,
+  runtimeProxyEnabled: false,
+  runtimeProxyRequireAuth: true,
+  runtimeTimeoutMs: 30000,
+  shutdownDrainMs: 5000,
+  unmappedPolicy: "default",
+  trustProxy: false,
+};
+
+function makeEmailPayload(overrides?: {
+  from?: string;
+  fromName?: string;
+  to?: string;
+  subject?: string;
+  strippedText?: string;
+  bodyText?: string;
+  messageId?: string;
+  inReplyTo?: string;
+  references?: string;
+  conversationId?: string;
+  timestamp?: string;
+}) {
+  return JSON.stringify({
+    from: overrides?.from ?? "sender@example.com",
+    fromName: overrides?.fromName,
+    to: overrides?.to ?? "assistant@vellum.me",
+    subject: overrides?.subject ?? "Hello",
+    strippedText: overrides?.strippedText ?? "Hi, how are you?",
+    bodyText:
+      overrides?.bodyText ??
+      "On Mon, someone wrote:\n> old\n\nHi, how are you?",
+    messageId: overrides?.messageId ?? "<msg-456@example.com>",
+    inReplyTo: overrides?.inReplyTo,
+    references: overrides?.references,
+    conversationId: overrides?.conversationId ?? "conv-abc",
+    timestamp: overrides?.timestamp ?? "2026-04-03T01:00:00.000Z",
+  });
+}
+
+function postRequest(body: string, secret?: string): Request {
+  const sigSecret = secret ?? TEST_WEBHOOK_SECRET;
+  return new Request("http://localhost:7830/webhooks/email/inbound", {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+      "vellum-signature": computeSignature(body, sigSecret),
+    },
+    body,
+  });
+}
+
+function makeCaches(secret?: string) {
+  const credentials = {
+    get: async (key: string) => {
+      if (key === credentialKey("email", "webhook_secret"))
+        return secret ?? TEST_WEBHOOK_SECRET;
+      return undefined;
+    },
+    invalidate: () => {},
+  } as unknown as CredentialCache;
+  return { credentials };
+}
+
+// --- Tests ----------------------------------------------------------------
+
+describe("email-webhook", () => {
+  beforeEach(() => {
+    handleInboundMock.mockClear();
+    handleInboundMock.mockResolvedValue({ forwarded: true, rejected: false });
+  });
+
+  it("rejects non-POST requests with 405", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const req = new Request("http://localhost:7830/webhooks/email/inbound", {
+      method: "GET",
+    });
+    const res = await handler(req);
+    expect(res.status).toBe(405);
+  });
+
+  it("forwards a valid email event to runtime", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload();
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(200);
+
+    const json = (await res.json()) as { ok: boolean };
+    expect(json.ok).toBe(true);
+    expect(handleInboundMock).toHaveBeenCalledTimes(1);
+
+    // Verify the normalized event structure
+    const callArgs = handleInboundMock.mock.calls[0];
+    const event = callArgs[1] as {
+      sourceChannel: string;
+      message: {
+        content: string;
+        conversationExternalId: string;
+        externalMessageId: string;
+      };
+      actor: { actorExternalId: string; displayName: string };
+    };
+    expect(event.sourceChannel).toBe("email");
+    expect(event.message.content).toBe("Hi, how are you?");
+    expect(event.message.conversationExternalId).toBe("conv-abc");
+    expect(event.message.externalMessageId).toBe("<msg-456@example.com>");
+    expect(event.actor.actorExternalId).toBe("sender@example.com");
+  });
+
+  it("acknowledges payloads missing required fields", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = JSON.stringify({ someOtherEvent: true });
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(200);
+    expect(handleInboundMock).not.toHaveBeenCalled();
+  });
+
+  it("deduplicates events by message ID", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload({ messageId: "<dedup-test@example.com>" });
+
+    const res1 = await handler(postRequest(body));
+    expect(res1.status).toBe(200);
+    expect(handleInboundMock).toHaveBeenCalledTimes(1);
+
+    const res2 = await handler(postRequest(body));
+    expect(res2.status).toBe(200);
+    // Second call should be deduped
+    expect(handleInboundMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("rejects payloads exceeding size limit", async () => {
+    const config = { ...baseConfig, maxWebhookPayloadBytes: 10 };
+    const { handler } = createEmailWebhookHandler(config, makeCaches());
+    const body = makeEmailPayload();
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(413);
+  });
+
+  it("rejects invalid JSON with 400", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = "not json";
+    const req = new Request("http://localhost:7830/webhooks/email/inbound", {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "vellum-signature": computeSignature(body, TEST_WEBHOOK_SECRET),
+      },
+      body,
+    });
+    const res = await handler(req);
+    expect(res.status).toBe(400);
+  });
+
+  it("uses fromName as displayName when provided", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload({
+      from: "alice@example.com",
+      fromName: "Alice Smith",
+      messageId: "<display-name@example.com>",
+    });
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(200);
+
+    const callArgs = handleInboundMock.mock.calls[0];
+    const event = callArgs[1] as {
+      actor: { actorExternalId: string; displayName: string };
+    };
+    expect(event.actor.actorExternalId).toBe("alice@example.com");
+    expect(event.actor.displayName).toBe("Alice Smith");
+  });
+
+  it("falls back to email as displayName when fromName is absent", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload({
+      from: "bob@example.com",
+      messageId: "<no-name@example.com>",
+    });
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(200);
+
+    const callArgs = handleInboundMock.mock.calls[0];
+    const event = callArgs[1] as {
+      actor: { displayName: string };
+    };
+    expect(event.actor.displayName).toBe("bob@example.com");
+  });
+
+  it("prefers strippedText over bodyText", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload({
+      strippedText: "New reply here",
+      bodyText: "On Monday, someone wrote:\n> old content\n\nNew reply here",
+      messageId: "<stripped@example.com>",
+    });
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(200);
+
+    const callArgs = handleInboundMock.mock.calls[0];
+    const event = callArgs[1] as { message: { content: string } };
+    expect(event.message.content).toBe("New reply here");
+  });
+
+  it("falls back to bodyText when strippedText is absent", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = JSON.stringify({
+      from: "test@example.com",
+      to: "bot@vellum.me",
+      messageId: "<fallback@example.com>",
+      conversationId: "conv-fallback",
+      bodyText: "Full body content here",
+    });
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(200);
+
+    const callArgs = handleInboundMock.mock.calls[0];
+    const event = callArgs[1] as { message: { content: string } };
+    expect(event.message.content).toBe("Full body content here");
+  });
+
+  it("returns 409 when webhook secret is not configured", async () => {
+    const emptyCaches = {
+      credentials: {
+        get: async () => undefined,
+        invalidate: () => {},
+      } as unknown as CredentialCache,
+    };
+    const { handler } = createEmailWebhookHandler(baseConfig, emptyCaches);
+    const body = makeEmailPayload({ messageId: "<no-secret@example.com>" });
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(409);
+  });
+
+  it("rejects requests with wrong webhook secret (HMAC mismatch)", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload({ messageId: "<wrong-secret@example.com>" });
+    // Sign with a different secret than what the cache returns
+    const req = new Request("http://localhost:7830/webhooks/email/inbound", {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "vellum-signature": computeSignature(body, "wrong-secret"),
+      },
+      body,
+    });
+    const res = await handler(req);
+    expect(res.status).toBe(403);
+  });
+
+  it("rejects requests with missing signature header", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload({
+      messageId: "<missing-header@example.com>",
+    });
+    const req = new Request("http://localhost:7830/webhooks/email/inbound", {
+      method: "POST",
+      headers: { "content-type": "application/json" },
+      body,
+    });
+    const res = await handler(req);
+    expect(res.status).toBe(403);
+  });
+
+  it("passes email subject and threading headers in sourceMetadata", async () => {
+    const { handler } = createEmailWebhookHandler(baseConfig, makeCaches());
+    const body = makeEmailPayload({
+      subject: "Re: Project Update",
+      inReplyTo: "<parent@example.com>",
+      references: "<root@example.com> <parent@example.com>",
+      messageId: "<metadata@example.com>",
+    });
+    const res = await handler(postRequest(body));
+    expect(res.status).toBe(200);
+
+    const callArgs = handleInboundMock.mock.calls[0];
+    const options = callArgs[2] as {
+      sourceMetadata: {
+        emailSubject: string;
+        emailRecipient: string;
+        emailInReplyTo: string;
+        emailReferences: string;
+      };
+    };
+    expect(options.sourceMetadata.emailSubject).toBe("Re: Project Update");
+    expect(options.sourceMetadata.emailRecipient).toBe("assistant@vellum.me");
+    expect(options.sourceMetadata.emailInReplyTo).toBe("<parent@example.com>");
+    expect(options.sourceMetadata.emailReferences).toBe(
+      "<root@example.com> <parent@example.com>",
+    );
+  });
+
+  it("uses messageId as dedup key for event ID", async () => {
+    const { handler, dedupCache } = createEmailWebhookHandler(
+      baseConfig,
+      makeCaches(),
+    );
+    const body = makeEmailPayload({
+      messageId: "<unique-dedup-id@example.com>",
+    });
+    await handler(postRequest(body));
+
+    // The dedup cache should have reserved this message ID
+    const status = dedupCache.reserve("<unique-dedup-id@example.com>");
+    // Should return false because it's already reserved/marked
+    expect(status).toBe(false);
+  });
+
+  it("returns 403 (not 409) when cache miss resolves on force-refresh but signature is invalid", async () => {
+    // Simulate: initial cache miss, force-refresh returns real secret,
+    // but signature doesn't match. Should be 403 (not 409 "not configured").
+    const caches = {
+      credentials: {
+        get: async (_key: string, opts?: { force?: boolean }) => {
+          // First call: cache miss
+          if (!opts?.force) return undefined;
+          // Force-refresh: return real secret
+          return TEST_WEBHOOK_SECRET;
+        },
+        invalidate: () => {},
+      } as unknown as CredentialCache,
+    };
+    const { handler } = createEmailWebhookHandler(baseConfig, caches);
+    const body = makeEmailPayload({
+      messageId: "<stale-var-fix@example.com>",
+    });
+    // Sign with wrong secret
+    const req = new Request("http://localhost:7830/webhooks/email/inbound", {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "vellum-signature": computeSignature(body, "wrong-secret"),
+      },
+      body,
+    });
+    const res = await handler(req);
+    // This was the stale variable bug — it used to return 409 here
+    expect(res.status).toBe(403);
+  });
+});

package/src/http/routes/email-webhook.ts

@@ -0,0 +1,243 @@
+import { buildEmailTransportMetadata } from "../../channels/transport-hints.js";
+import type { ConfigFileCache } from "../../config-file-cache.js";
+import type { GatewayConfig } from "../../config.js";
+import type { CredentialCache } from "../../credential-cache.js";
+import { credentialKey } from "../../credential-key.js";
+import { StringDedupCache } from "../../dedup-cache.js";
+import { normalizeEmailWebhook } from "../../email/normalize.js";
+import { verifyEmailWebhookSignature } from "../../email/verify.js";
+import { handleInbound } from "../../handlers/handle-inbound.js";
+import { getLogger } from "../../logger.js";
+import {
+  resolveAssistant,
+  isRejection,
+} from "../../routing/resolve-assistant.js";
+import {
+  handleCircuitBreakerError,
+  processInboundResult,
+} from "../../webhook-pipeline.js";
+
+const log = getLogger("email-webhook");
+
+export function createEmailWebhookHandler(
+  config: GatewayConfig,
+  caches?: { credentials?: CredentialCache; configFile?: ConfigFileCache },
+) {
+  // 24-hour TTL — Message-IDs are globally unique per RFC 5322
+  const dedupCache = new StringDedupCache(24 * 60 * 60_000);
+
+  const handler = async (req: Request): Promise<Response> => {
+    const traceId = req.headers.get("x-trace-id") ?? undefined;
+    const tlog = traceId ? log.child({ traceId }) : log;
+
+    if (req.method !== "POST") {
+      return Response.json({ error: "Method not allowed" }, { status: 405 });
+    }
+
+    // Payload size guard
+    const contentLength = req.headers.get("content-length");
+    if (
+      contentLength &&
+      Number(contentLength) > config.maxWebhookPayloadBytes
+    ) {
+      tlog.warn({ contentLength }, "Email webhook payload too large");
+      return Response.json({ error: "Payload too large" }, { status: 413 });
+    }
+
+    let rawBody: string;
+    try {
+      rawBody = await req.text();
+    } catch {
+      return Response.json({ error: "Failed to read body" }, { status: 400 });
+    }
+
+    if (Buffer.byteLength(rawBody) > config.maxWebhookPayloadBytes) {
+      tlog.warn(
+        { bodyLength: Buffer.byteLength(rawBody) },
+        "Email webhook payload too large",
+      );
+      return Response.json({ error: "Payload too large" }, { status: 413 });
+    }
+
+    // Resolve webhook secret from credential cache
+    const webhookSecret = caches?.credentials
+      ? await caches.credentials.get(credentialKey("email", "webhook_secret"))
+      : undefined;
+
+    // If the initial cache read returned undefined but a credential cache is available,
+    // attempt one forced refresh before fail-closing — the credential may have been
+    // written after the TTL cache was last populated.
+    let effectiveSecret = webhookSecret;
+    if (!effectiveSecret && caches?.credentials) {
+      effectiveSecret = await caches.credentials.get(
+        credentialKey("email", "webhook_secret"),
+        { force: true },
+      );
+      if (effectiveSecret) {
+        tlog.info(
+          "Email webhook secret resolved after forced credential refresh",
+        );
+      }
+    }
+
+    // Signature validation is required — reject when no secret is configured
+    // rather than silently accepting unauthenticated payloads (fail-closed).
+    if (!effectiveSecret) {
+      tlog.warn("Email webhook secret is not configured — rejecting request");
+      return Response.json(
+        { error: "Webhook secret not configured" },
+        { status: 409 },
+      );
+    }
+
+    let signatureValid = verifyEmailWebhookSignature(
+      req.headers,
+      rawBody,
+      effectiveSecret,
+    );
+
+    // One-shot force retry: if verification failed and caches are available,
+    // force-refresh the webhook secret and retry once.
+    if (!signatureValid && caches?.credentials) {
+      const freshSecret = await caches.credentials.get(
+        credentialKey("email", "webhook_secret"),
+        { force: true },
+      );
+      if (freshSecret) {
+        signatureValid = verifyEmailWebhookSignature(
+          req.headers,
+          rawBody,
+          freshSecret,
+        );
+        if (signatureValid) {
+          tlog.info(
+            "Email webhook signature verified after forced credential refresh",
+          );
+        }
+      }
+    }
+
+    if (!signatureValid) {
+      tlog.warn("Email webhook signature verification failed");
+      return Response.json({ error: "Forbidden" }, { status: 403 });
+    }
+
+    let payload: Record<string, unknown>;
+    try {
+      payload = JSON.parse(rawBody) as Record<string, unknown>;
+    } catch {
+      return Response.json({ error: "Invalid JSON" }, { status: 400 });
+    }
+
+    // Normalize the webhook payload
+    const normalized = normalizeEmailWebhook(payload);
+    if (!normalized) {
+      // Missing required fields — log and acknowledge
+      tlog.debug("Email webhook missing required fields, acknowledging");
+      return Response.json({ ok: true });
+    }
+
+    const { event, eventId, recipientAddress } = normalized;
+
+    // Dedup by event ID
+    if (!dedupCache.reserve(eventId)) {
+      tlog.info({ eventId }, "Duplicate email event ID, ignoring");
+      return Response.json({ ok: true });
+    }
+
+    tlog.info(
+      {
+        source: "email",
+        eventId,
+        from: event.actor.actorExternalId,
+        to: recipientAddress,
+        messageId: event.message.externalMessageId,
+      },
+      "Email webhook received",
+    );
+
+    // Resolve routing using the recipient address as both conversation
+    // and actor ID — the standard routing chain will check explicit
+    // routes first, then fall back to the default assistant.
+    const routing = resolveAssistant(
+      config,
+      event.message.conversationExternalId,
+      event.actor.actorExternalId,
+    );
+
+    if (isRejection(routing)) {
+      tlog.warn(
+        {
+          from: event.actor.actorExternalId,
+          to: recipientAddress,
+          reason: routing.reason,
+        },
+        "Routing rejected inbound email",
+      );
+      // No way to reply to the sender for rejected emails — just log
+      dedupCache.mark(eventId);
+      return Response.json({ ok: true });
+    }
+
+    // Forward to runtime
+    try {
+      const result = await handleInbound(config, event, {
+        transportMetadata: buildEmailTransportMetadata(),
+        replyCallbackUrl: undefined, // Email replies go through the outbound send path (PR 4)
+        traceId,
+        routingOverride: routing,
+        sourceMetadata: {
+          emailSubject: (payload.subject as string | undefined) ?? undefined,
+          emailRecipient: recipientAddress,
+          ...(payload.inReplyTo ? { emailInReplyTo: payload.inReplyTo } : {}),
+          ...(payload.references
+            ? { emailReferences: payload.references }
+            : {}),
+        },
+      });
+
+      const processed = processInboundResult(
+        result,
+        dedupCache,
+        eventId,
+        () => {
+          // No real-time reply mechanism for email — rejection is logged only
+          tlog.warn(
+            { from: event.actor.actorExternalId, to: recipientAddress },
+            "Email routing rejected after forwarding attempt",
+          );
+        },
+        tlog,
+      );
+
+      if (!processed.ok) {
+        return Response.json({ error: "Internal error" }, { status: 500 });
+      }
+
+      dedupCache.mark(eventId);
+
+      if (!result.rejected) {
+        tlog.info(
+          { status: "forwarded", eventId },
+          "Email message forwarded to runtime",
+        );
+      }
+    } catch (err) {
+      const cbResponse = handleCircuitBreakerError(
+        err,
+        dedupCache,
+        eventId,
+        tlog,
+      );
+      if (cbResponse) return cbResponse;
+
+      tlog.error({ err, eventId }, "Failed to process inbound email");
+      dedupCache.unreserve(eventId);
+      return Response.json({ error: "Internal error" }, { status: 500 });
+    }
+
+    return Response.json({ ok: true });
+  };
+
+  return { handler, dedupCache };
+}