@vellumai/vellum-gateway 0.6.0 → 0.6.2

package/src/index.ts

@@ -41,6 +41,7 @@ import {
 } from "./http/routes/twilio-relay-websocket.js";
 import { createWhatsAppWebhookHandler } from "./http/routes/whatsapp-webhook.js";
 import { createWhatsAppDeliverHandler } from "./http/routes/whatsapp-deliver.js";
+import { createEmailWebhookHandler } from "./http/routes/email-webhook.js";
 import { createSlackDeliverHandler } from "./http/routes/slack-deliver.js";
 import { createOAuthCallbackHandler } from "./http/routes/oauth-callback.js";
 import { createPairingProxyHandler } from "./http/routes/pairing-proxy.js";
@@ -67,6 +68,7 @@ import {
 import { createMigrationRollbackProxyHandler } from "./http/routes/migration-rollback-proxy.js";
 import { createWorkspaceCommitProxyHandler } from "./http/routes/workspace-commit-proxy.js";
 import { createBrainGraphProxyHandler } from "./http/routes/brain-graph-proxy.js";
+import { createLogExportHandler } from "./http/routes/log-export.js";
 import {
   createTrustRulesListHandler,
   createTrustRulesAddHandler,
@@ -238,6 +240,11 @@ async function main() {
     credentials: credentialCache,
     configFile: configFileCache,
   });
+  const { handler: handleEmailWebhook, dedupCache: emailDedupCache } =
+    createEmailWebhookHandler(config, {
+      credentials: credentialCache,
+      configFile: configFileCache,
+    });
   // Map: "channel:threadTs" -> { messageTs, expiresAt } for replacing approval
   // messages with the bot's follow-up content after an approval button click.
   const pendingApprovalReplacements = new Map<
@@ -282,6 +289,7 @@ async function main() {
   const migrationRollbackProxy = createMigrationRollbackProxyHandler(config);
   const workspaceCommitProxy = createWorkspaceCommitProxyHandler(config);
   const brainGraphProxy = createBrainGraphProxyHandler(config);
+  const handleLogExport = createLogExportHandler(config);
   const handleFeatureFlagsGet = createFeatureFlagsGetHandler();
   const handleFeatureFlagsPatch = createFeatureFlagsPatchHandler();
   const handlePrivacyConfigPatch = createPrivacyConfigPatchHandler();
@@ -351,6 +359,10 @@ async function main() {
       precondition: requireWhatsApp,
       handler: (req) => handleWhatsAppWebhook(req),
     },
+    {
+      path: "/webhooks/email/inbound",
+      handler: (req) => handleEmailWebhook(req),
+    },
 
     // ── Audio serving (unauthenticated — Twilio fetches these URLs directly) ──
     {
@@ -923,6 +935,15 @@ async function main() {
       handler: (req) => handlePrivacyConfigPatch(req),
     },
 
+    // ── Log export ──
+    {
+      path: "/v1/logs/export",
+      method: "POST",
+      auth: "edge",
+      handler: (req, params, getClientIp) =>
+        handleLogExport(req, params, getClientIp),
+    },
+
     // ── Trust rules ──
     {
       path: "/v1/trust-rules/clear",
@@ -1161,6 +1182,7 @@ async function main() {
   // Start periodic background cleanup for dedup caches
   telegramDedupCache.startCleanup();
   whatsappDedupCache.startCleanup();
+  emailDedupCache.startCleanup();
 
   const telegramCaches = {
     credentials: credentialCache,
@@ -1240,6 +1262,7 @@ async function main() {
               !isCallback
             ) {
               attachmentIds = [];
+              const failedAttachmentNames: string[] = [];
               const maxBytes =
                 config.maxAttachmentBytes.slack ??
                 config.maxAttachmentBytes.default;
@@ -1290,17 +1313,22 @@ async function main() {
                     });
                   }),
                 );
-                for (const result of results) {
+                for (let j = 0; j < results.length; j++) {
+                  const result = results[j];
                   if (result.status === "fulfilled") {
                     attachmentIds.push(result.value.id);
                   } else if (
                     result.reason instanceof AttachmentValidationError
                   ) {
+                    const att = batch[j];
+                    failedAttachmentNames.push(att.fileName || att.fileId);
                     log.warn(
                       { err: result.reason },
                       "Skipping Slack attachment with validation error",
                     );
                   } else {
+                    const att = batch[j];
+                    failedAttachmentNames.push(att.fileName || att.fileId);
                     log.warn(
                       { err: result.reason },
                       "Skipping Slack attachment due to download/upload failure",
@@ -1308,6 +1336,13 @@ async function main() {
                   }
                 }
               }
+
+              if (failedAttachmentNames.length > 0) {
+                const nameList = failedAttachmentNames
+                  .map((n) => `"${n}"`)
+                  .join(", ");
+                normalized.event.message.content += `\n\n[The user attached file(s) that could not be retrieved: ${nameList}. Ask them to re-send if the content is important.]`;
+              }
             }
 
             handleInbound(config, normalized.event, {
@@ -1508,6 +1543,7 @@ async function main() {
     remoteFeatureFlagSync.stop();
     telegramDedupCache.stopCleanup();
     whatsappDedupCache.stopCleanup();
+    emailDedupCache.stopCleanup();
     if (slackSocketClient) {
       slackSocketClient.stop();
       slackSocketClient = null;

package/src/logger.ts

@@ -17,7 +17,7 @@ export type LogFileConfig = {
 
 const LOG_FILE_PREFIX = "gateway-";
 const LOG_FILE_SUFFIX = ".log";
-const LOG_FILE_PATTERN = /^gateway-(\d{4}-\d{2}-\d{2})\.log$/;
+export const LOG_FILE_PATTERN = /^gateway-(\d{4}-\d{2}-\d{2})\.log$/;
 
 function formatDate(date: Date): string {
   const y = date.getUTCFullYear();
@@ -119,10 +119,25 @@ export function initLogger(config: LogFileConfig): void {
   }
 }
 
+/**
+ * Returns a lazy proxy logger that always delegates to the **current**
+ * rootLogger. This is critical because module-level `const log = getLogger(...)`
+ * calls execute before `initLogger()` runs. Without the proxy, those early
+ * child loggers would permanently hold the fallback stdout-only stream and
+ * never write to log files.
+ */
 export function getLogger(name: string): pino.Logger {
-  ensureCurrentDate();
-  if (!rootLogger) {
-    rootLogger = buildLogger(null);
-  }
-  return rootLogger.child({ module: name });
+  const handler: ProxyHandler<pino.Logger> = {
+    get(_target, prop, receiver) {
+      ensureCurrentDate();
+      if (!rootLogger) {
+        rootLogger = buildLogger(null);
+      }
+      const child = rootLogger.child({ module: name });
+      const value = Reflect.get(child, prop, receiver);
+      return typeof value === "function" ? value.bind(child) : value;
+    },
+  };
+  // The proxy target is a throwaway logger — all access is intercepted.
+  return new Proxy({} as pino.Logger, handler);
 }

package/src/remote-feature-flag-sync.ts

@@ -1,20 +1,28 @@
 import type { CredentialCache } from "./credential-cache.js";
 import { credentialKey } from "./credential-key.js";
 import { fetchImpl } from "./fetch.js";
+import { loadFeatureFlagDefaults } from "./feature-flag-defaults.js";
 import { writeRemoteFeatureFlags } from "./feature-flag-remote-store.js";
 import { getLogger } from "./logger.js";
 
 const log = getLogger("remote-feature-flag-sync");
 
 /**
- * Default polling interval: 5 minutes.
+ * Steady-state polling interval: 5 minutes.
  *
  * Configurable via `REMOTE_FF_POLL_INTERVAL_MS` env var for testing or
  * deployment tuning.
  */
 const DEFAULT_POLL_INTERVAL_MS = 5 * 60 * 1000;
 
-function getPollIntervalMs(): number {
+/**
+ * Initial polling interval when the first fetch fails (e.g. CES sidecar
+ * not ready yet). Doubles on each consecutive failure until it reaches
+ * the steady-state interval.
+ */
+const INITIAL_POLL_INTERVAL_MS = 10_000;
+
+function getMaxPollIntervalMs(): number {
   const envVal = process.env.REMOTE_FF_POLL_INTERVAL_MS;
   if (envVal) {
     const parsed = parseInt(envVal, 10);
@@ -26,63 +34,114 @@ function getPollIntervalMs(): number {
 export type RemoteFeatureFlagSyncConfig = {
   /** Credential cache for resolving platform URL, API key, and assistant ID dynamically. */
   credentials: CredentialCache;
+  /** Override the initial poll interval (ms) — useful for testing. Defaults to 10 000. */
+  initialPollIntervalMs?: number;
 };
 
 /**
  * Manages the lifecycle of syncing remote feature flags from the platform.
  *
  * On start, fetches the current flag state and persists it to disk via the
- * remote feature flag store, then polls on a configurable interval. Errors
- * are caught and logged — the system falls through to registry defaults if
- * this fails.
+ * remote feature flag store, then polls with adaptive back-off: starts at
+ * {@link INITIAL_POLL_INTERVAL_MS} and doubles on each failure until it
+ * reaches the steady-state interval. On the first success the interval
+ * snaps to steady-state immediately.
  */
 export class RemoteFeatureFlagSync {
   private started = false;
-  private pollTimer: ReturnType<typeof setInterval> | null = null;
+  private pollTimer: ReturnType<typeof setTimeout> | null = null;
+  private currentIntervalMs: number;
+  private readonly maxIntervalMs: number;
   private readonly credentials: CredentialCache;
 
   constructor(config: RemoteFeatureFlagSyncConfig) {
     this.credentials = config.credentials;
+    this.currentIntervalMs =
+      config.initialPollIntervalMs ?? INITIAL_POLL_INTERVAL_MS;
+    this.maxIntervalMs = getMaxPollIntervalMs();
   }
 
   async start(): Promise<void> {
     this.started = true;
 
+    let ok = false;
     try {
-      await this.fetchAndCache();
+      ok = await this.fetchAndCache();
     } catch (err) {
       log.warn({ err }, "Failed to sync remote feature flags on startup");
     }
 
-    const intervalMs = getPollIntervalMs();
-    this.pollTimer = setInterval(() => {
-      this.fetchAndCache().catch((err) => {
-        log.warn({ err }, "Failed to sync remote feature flags during poll");
-      });
-    }, intervalMs);
+    if (ok) {
+      // First fetch succeeded — jump straight to steady-state polling.
+      this.currentIntervalMs = this.maxIntervalMs;
+    }
 
-    log.info({ intervalMs }, "Remote feature flag polling started");
+    this.scheduleNextPoll();
+    log.info(
+      { intervalMs: this.currentIntervalMs },
+      "Remote feature flag polling started",
+    );
   }
 
   stop(): void {
     this.started = false;
     if (this.pollTimer) {
-      clearInterval(this.pollTimer);
+      clearTimeout(this.pollTimer);
       this.pollTimer = null;
     }
   }
 
-  private async fetchAndCache(): Promise<void> {
+  private scheduleNextPoll(): void {
+    this.pollTimer = setTimeout(() => {
+      this.poll();
+    }, this.currentIntervalMs);
+  }
+
+  private poll(): void {
+    if (!this.started) return;
+    this.fetchAndCache()
+      .then((ok) => {
+        if (ok) {
+          // Success — snap to steady-state interval.
+          this.currentIntervalMs = this.maxIntervalMs;
+        } else {
+          // Failure — double the interval, capped at max.
+          this.currentIntervalMs = Math.min(
+            this.currentIntervalMs * 2,
+            this.maxIntervalMs,
+          );
+        }
+      })
+      .catch((err) => {
+        log.warn({ err }, "Failed to sync remote feature flags during poll");
+        this.currentIntervalMs = Math.min(
+          this.currentIntervalMs * 2,
+          this.maxIntervalMs,
+        );
+      })
+      .finally(() => {
+        if (this.started) {
+          this.scheduleNextPoll();
+        }
+      });
+  }
+
+  /**
+   * Fetch remote flags and write them to the store.
+   * Returns true if flags were successfully written, false otherwise.
+   */
+  private async fetchAndCache(): Promise<boolean> {
     const values = await this.fetchRemoteFeatureFlags();
     if (values === null) {
-      log.debug("Skipping cache write — fetch returned no usable data");
-      return;
+      log.warn("Skipping cache write — fetch returned no usable data");
+      return false;
     }
     writeRemoteFeatureFlags(values);
     log.info(
       { count: Object.keys(values).length },
       "Synced remote feature flags",
     );
+    return true;
   }
 
   private async fetchRemoteFeatureFlags(): Promise<Record<
@@ -153,12 +212,23 @@ export class RemoteFeatureFlagSync {
       return null;
     }
 
-    // Filter to boolean values only (defensive)
+    // Filter to boolean values only (defensive), and prevent the platform
+    // from disabling flags that are already GA (defaultEnabled: true in the
+    // registry). The platform uses a blanket-deny posture, sending false for
+    // every flag it knows about. Without this filter, shipped features get
+    // silently turned off for all users.
+    const registry = loadFeatureFlagDefaults();
     const result: Record<string, boolean> = {};
     for (const [key, value] of Object.entries(body.flags)) {
-      if (typeof value === "boolean") {
-        result[key] = value;
+      if (typeof value !== "boolean") continue;
+      if (!value && registry[key]?.defaultEnabled) {
+        log.debug(
+          { key },
+          "Ignoring remote false for GA flag (defaultEnabled: true)",
+        );
+        continue;
       }
+      result[key] = value;
     }
 
     return result;

package/src/schema.ts

@@ -613,6 +613,95 @@ export function buildSchema(): Record<string, unknown> {
           },
         },
       },
+      "/webhooks/email/inbound": {
+        post: {
+          summary: "Email inbound webhook",
+          description:
+            "Receives inbound email webhook events from the Vellum platform, verifies the HMAC signature, normalizes the message, and forwards it to the assistant runtime.",
+          operationId: "emailInboundWebhook",
+          security: [{ VellumSignature: [] }],
+          requestBody: {
+            required: true,
+            content: {
+              "application/json": {
+                schema: {
+                  type: "object",
+                  description: "Vellum email webhook payload.",
+                },
+              },
+            },
+          },
+          responses: {
+            "200": {
+              description: "Webhook accepted",
+              content: {
+                "application/json": {
+                  schema: {
+                    type: "object",
+                    properties: { ok: { type: "boolean" } },
+                  },
+                },
+              },
+            },
+            "400": {
+              description: "Invalid JSON or unreadable body",
+              content: {
+                "application/json": {
+                  schema: { $ref: "#/components/schemas/ErrorResponse" },
+                },
+              },
+            },
+            "403": {
+              description: "Signature verification failed",
+              content: {
+                "application/json": {
+                  schema: { $ref: "#/components/schemas/ErrorResponse" },
+                },
+              },
+            },
+            "405": {
+              description: "Method not allowed",
+              content: {
+                "application/json": {
+                  schema: { $ref: "#/components/schemas/ErrorResponse" },
+                },
+              },
+            },
+            "413": {
+              description: "Payload too large",
+              content: {
+                "application/json": {
+                  schema: { $ref: "#/components/schemas/ErrorResponse" },
+                },
+              },
+            },
+            "409": {
+              description: "Webhook secret not configured",
+              content: {
+                "application/json": {
+                  schema: { $ref: "#/components/schemas/ErrorResponse" },
+                },
+              },
+            },
+            "500": {
+              description: "Internal error",
+              content: {
+                "application/json": {
+                  schema: { $ref: "#/components/schemas/ErrorResponse" },
+                },
+              },
+            },
+            "503": {
+              description: "Service temporarily unavailable",
+              content: {
+                "application/json": {
+                  schema: { $ref: "#/components/schemas/ErrorResponse" },
+                },
+              },
+            },
+          },
+        },
+      },
       "/v1/audio/{audioId}": {
         get: {
           summary: "Retrieve synthesized audio",
@@ -2354,6 +2443,59 @@ export function buildSchema(): Record<string, unknown> {
           },
         },
       },
+      "/v1/logs/export": {
+        post: {
+          summary: "Export logs from all services",
+          description:
+            "Orchestrates parallel log collection from the gateway, daemon, and CES, " +
+            "returning a tar.gz archive containing logs from all three services.",
+          operationId: "logsExport",
+          security: [{ BearerAuth: [] }],
+          requestBody: {
+            required: false,
+            content: {
+              "application/json": {
+                schema: {
+                  type: "object",
+                  properties: {
+                    startTime: {
+                      type: "number",
+                      description: "Start of time range (epoch ms)",
+                    },
+                    endTime: {
+                      type: "number",
+                      description: "End of time range (epoch ms)",
+                    },
+                    auditLimit: {
+                      type: "number",
+                      description: "Maximum number of audit records to include",
+                    },
+                    conversationId: {
+                      type: "string",
+                      description: "Scope export to a specific conversation",
+                    },
+                  },
+                },
+              },
+            },
+          },
+          responses: {
+            "200": {
+              description: "tar.gz archive of collected logs",
+              content: {
+                "application/gzip": {
+                  schema: { type: "string", format: "binary" },
+                },
+              },
+            },
+            "400": { description: "Invalid JSON body" },
+            "401": {
+              description: "Unauthorized — missing or invalid edge JWT",
+            },
+            "500": { description: "Failed to create export archive" },
+          },
+        },
+      },
       "/v1/trust-rules": {
         get: {
           summary: "List trust rules",
@@ -3409,6 +3551,13 @@ export function buildSchema(): Record<string, unknown> {
           description:
             "HMAC-SHA1 signature computed by Twilio over the request URL and form parameters.",
         },
+        VellumSignature: {
+          type: "apiKey",
+          in: "header",
+          name: "Vellum-Signature",
+          description:
+            "HMAC-SHA256 signature computed by the Vellum platform over the raw request body using the webhook secret. Format: sha256=<hex-digest>.",
+        },
         WhatsAppHubSignature: {
           type: "apiKey",
           in: "header",

package/src/slack/download.test.ts

@@ -1,4 +1,5 @@
 import { afterEach, describe, expect, mock, test } from "bun:test";
+import { ContentMismatchError } from "../download-validation.js";
 import type { SlackFile } from "./normalize.js";
 
 type FetchFn = (
@@ -67,9 +68,7 @@ describe("downloadSlackFile", () => {
 
     expect(result.filename).toBe("test-image.png");
     expect(result.mimeType).toBe("image/png");
-    expect(result.data).toBe(
-      Buffer.from(fileBuffer).toString("base64"),
-    );
+    expect(result.data).toBe(Buffer.from(fileBuffer).toString("base64"));
 
     // Verify the correct URL and auth header were used
     expect(fetchMock).toHaveBeenCalledTimes(1);
@@ -101,21 +100,68 @@ describe("downloadSlackFile", () => {
       url_private: undefined,
     });
 
-    await expect(
-      downloadSlackFile(file, "xoxb-test-token"),
-    ).rejects.toThrow("Slack file F12345 has no download URL");
+    await expect(downloadSlackFile(file, "xoxb-test-token")).rejects.toThrow(
+      "Slack file F12345 has no download URL",
+    );
+  });
+
+  test("uses redirect: manual to prevent auth header stripping", async () => {
+    const fileBuffer = makePngBuffer();
+    fetchMock = mock(async () => new Response(fileBuffer));
+
+    const file = makeSlackFile();
+    await downloadSlackFile(file, "xoxb-test-token");
+
+    const [, init] = fetchMock.mock.calls[0];
+    expect((init as RequestInit).redirect).toBe("manual");
+  });
+
+  test("follows redirect to CDN without auth header", async () => {
+    const fileBuffer = makePngBuffer();
+    let callCount = 0;
+    fetchMock = mock(async () => {
+      callCount++;
+      if (callCount === 1) {
+        return new Response(null, {
+          status: 302,
+          headers: { Location: "https://files-edge.slack.com/signed/F12345" },
+        });
+      }
+      return new Response(fileBuffer);
+    });
+
+    const file = makeSlackFile();
+    const result = await downloadSlackFile(file, "xoxb-test-token");
+
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+    // Second call should be to the redirect URL without auth
+    const [redirectUrl, redirectInit] = fetchMock.mock.calls[1];
+    expect(redirectUrl).toBe("https://files-edge.slack.com/signed/F12345");
+    expect((redirectInit as RequestInit).headers).toBeUndefined();
+    expect(result.mimeType).toBe("image/png");
+  });
+
+  test("throws when redirect has no Location header", async () => {
+    fetchMock = mock(
+      async () => new Response(null, { status: 302 }),
+    );
+
+    const file = makeSlackFile();
+
+    await expect(downloadSlackFile(file, "xoxb-test-token")).rejects.toThrow(
+      "returned 302 redirect with no Location header",
+    );
   });
 
   test("throws on HTTP error response", async () => {
     fetchMock = mock(
-      async () => new Response("Forbidden", { status: 403, statusText: "Forbidden" }),
+      async () =>
+        new Response("Forbidden", { status: 403, statusText: "Forbidden" }),
     );
 
     const file = makeSlackFile();
 
-    await expect(
-      downloadSlackFile(file, "xoxb-test-token"),
-    ).rejects.toThrow(
+    await expect(downloadSlackFile(file, "xoxb-test-token")).rejects.toThrow(
       "Failed to download Slack file F12345: 403 Forbidden",
     );
   });
@@ -180,4 +226,29 @@ describe("downloadSlackFile", () => {
 
     expect(result.filename).toBe("slack_file_F12345");
   });
+
+  test("throws ContentMismatchError when Slack returns HTML error page", async () => {
+    const htmlBuffer = new TextEncoder().encode(
+      "<!DOCTYPE html><html><body><h1>Error</h1></body></html>",
+    ).buffer;
+    fetchMock = mock(async () => new Response(htmlBuffer));
+
+    const file = makeSlackFile({ mimetype: "image/png" });
+
+    await expect(
+      downloadSlackFile(file, "xoxb-test-token"),
+    ).rejects.toBeInstanceOf(ContentMismatchError);
+  });
+
+  test("succeeds with valid image data when validation is active", async () => {
+    const fileBuffer = makePngBuffer();
+    fetchMock = mock(async () => new Response(fileBuffer));
+
+    const file = makeSlackFile({ mimetype: "image/png" });
+    const result = await downloadSlackFile(file, "xoxb-test-token");
+
+    expect(result.filename).toBe("test-image.png");
+    expect(result.mimeType).toBe("image/png");
+    expect(result.data).toBe(Buffer.from(fileBuffer).toString("base64"));
+  });
 });

package/src/slack/download.ts

@@ -1,4 +1,5 @@
 import { fileTypeFromBuffer } from "file-type";
+import { validateDownloadedContent } from "../download-validation.js";
 import { fetchImpl } from "../fetch.js";
 import type { SlackFile } from "./normalize.js";
 
@@ -23,11 +24,30 @@ export async function downloadSlackFile(
     throw new Error(`Slack file ${file.id} has no download URL`);
   }
 
-  const response = await fetchImpl(url, {
+  // Use manual redirect handling — Slack may redirect to a CDN subdomain
+  // (e.g. files-edge.slack.com) and the Fetch spec strips the Authorization
+  // header on cross-origin redirects, causing the CDN request to fail.
+  let response = await fetchImpl(url, {
     headers: { Authorization: `Bearer ${botToken}` },
+    redirect: "manual",
     signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
   });
 
+  if (response.status >= 300 && response.status < 400) {
+    const location = response.headers.get("Location");
+    if (!location) {
+      throw new Error(
+        `Slack file ${file.id} returned ${response.status} redirect with no Location header`,
+      );
+    }
+    // CDN redirect URLs are signed — no Authorization needed.
+    // Resolve against the original URL to handle relative Location headers.
+    const resolvedLocation = new URL(location, url).href;
+    response = await fetchImpl(resolvedLocation, {
+      signal: AbortSignal.timeout(DOWNLOAD_TIMEOUT_MS),
+    });
+  }
+
   if (!response.ok) {
     throw new Error(
       `Failed to download Slack file ${file.id}: ${response.status} ${response.statusText}`,
@@ -43,6 +63,8 @@ export async function downloadSlackFile(
     response.headers.get("Content-Type")?.split(";")[0].trim() ||
     "application/octet-stream";
 
+  await validateDownloadedContent(new Uint8Array(buffer), mimeType, file.id);
+
   const filename = file.name || `slack_file_${file.id}`;
   const data = Buffer.from(buffer).toString("base64");