@vellumai/vellum-gateway 0.5.11 → 0.5.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/vellum-gateway",
-  "version": "0.5.11",
+  "version": "0.5.12",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/credential-watcher-managed-bootstrap.test.ts

@@ -0,0 +1,224 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { spawn, type ChildProcess } from "node:child_process";
+import { mkdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const TEST_SERVICE_TOKEN = "test-ces-service-token";
+
+const testDir = join(tmpdir(), `gw-managed-${Date.now()}-${Math.random()}`);
+
+function metadataRecord(
+  credentialId: string,
+  service: string,
+  field: string,
+): Record<string, unknown> {
+  return {
+    credentialId,
+    service,
+    field,
+    allowedTools: [],
+    allowedDomains: [],
+    createdAt: Date.now(),
+    updatedAt: Date.now(),
+  };
+}
+
+function writeCredentialMetadata(
+  credentials: Record<string, unknown>[] = [
+    metadataRecord("test-bt", "telegram", "bot_token"),
+    metadataRecord("test-ws", "telegram", "webhook_secret"),
+  ],
+): void {
+  const dir = join(testDir, ".vellum", "workspace", "data", "credentials");
+  mkdirSync(dir, { recursive: true });
+  const metadataPath = join(dir, "metadata.json");
+  const tmpPath = join(dir, `.tmp-${Date.now()}-metadata.json`);
+  writeFileSync(
+    tmpPath,
+    JSON.stringify({
+      version: 2,
+      credentials,
+    }),
+  );
+  renameSync(tmpPath, metadataPath);
+}
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const gatewayRoot = join(__dirname, "..", "..");
+const gatewayEntry = join(gatewayRoot, "src", "index.ts");
+
+let gatewayProc: ChildProcess | null = null;
+let gatewayPort = 0;
+let cesPort = 0;
+let cesServer: ReturnType<typeof Bun.serve> | null = null;
+
+function assignPorts(): void {
+  if (gatewayPort !== 0 && cesPort !== 0) return;
+  gatewayPort = 49152 + Math.floor(Math.random() * 8_192);
+  cesPort = gatewayPort + 1;
+}
+
+async function startGateway(): Promise<void> {
+  assignPorts();
+
+  gatewayProc = spawn("bun", ["run", gatewayEntry], {
+    env: {
+      ...process.env,
+      BASE_DATA_DIR: testDir,
+      GATEWAY_PORT: String(gatewayPort),
+      CES_CREDENTIAL_URL: `http://127.0.0.1:${cesPort}`,
+      CES_SERVICE_TOKEN: TEST_SERVICE_TOKEN,
+      TELEGRAM_BOT_TOKEN: "",
+      TELEGRAM_WEBHOOK_SECRET: "",
+    },
+    stdio: ["ignore", "pipe", "pipe"],
+  });
+
+  const deadline = Date.now() + 5_000;
+  while (Date.now() < deadline) {
+    try {
+      const res = await fetch(`http://localhost:${gatewayPort}/healthz`);
+      if (res.ok) return;
+    } catch {
+      // Gateway not ready yet.
+    }
+    await new Promise((resolve) => setTimeout(resolve, 100));
+  }
+  throw new Error("Gateway failed to start within 5 seconds");
+}
+
+function startFakeCes(opts: {
+  accounts?: string[];
+  credentials?: Record<string, string>;
+  resolveValue?: (account: string) => string | undefined;
+}): void {
+  assignPorts();
+  const accounts = opts.accounts ?? Object.keys(opts.credentials ?? {});
+  const credentials = opts.credentials ?? {};
+  cesServer = Bun.serve({
+    port: cesPort,
+    fetch(req) {
+      const authHeader = req.headers.get("authorization");
+      if (authHeader !== `Bearer ${TEST_SERVICE_TOKEN}`) {
+        return Response.json(
+          { error: "Invalid service token" },
+          { status: 403 },
+        );
+      }
+
+      const url = new URL(req.url);
+      if (req.method === "GET" && url.pathname === "/v1/credentials") {
+        return Response.json({ accounts });
+      }
+
+      if (req.method === "GET" && url.pathname.startsWith("/v1/credentials/")) {
+        const account = decodeURIComponent(
+          url.pathname.slice("/v1/credentials/".length),
+        );
+        const value = opts.resolveValue?.(account) ?? credentials[account];
+        if (!value) {
+          return Response.json(
+            { error: "Credential not found", account },
+            { status: 404 },
+          );
+        }
+        return Response.json({ account, value });
+      }
+
+      return new Response("Not Found", { status: 404 });
+    },
+  });
+}
+
+afterEach(() => {
+  cesServer?.stop(true);
+  cesServer = null;
+  gatewayPort = 0;
+  cesPort = 0;
+
+  if (gatewayProc) {
+    gatewayProc.kill();
+    gatewayProc = null;
+  }
+
+  rmSync(testDir, { recursive: true, force: true });
+});
+
+describe("gateway managed credential bootstrap retry", () => {
+  test("reloads Telegram credentials after CES becomes reachable without a metadata rewrite", async () => {
+    mkdirSync(testDir, { recursive: true });
+    writeCredentialMetadata();
+
+    await startGateway();
+
+    const base = `http://localhost:${gatewayPort}`;
+    const before = await fetch(`${base}/webhooks/telegram`, { method: "POST" });
+    expect(before.status).toBe(503);
+
+    startFakeCes({
+      credentials: {
+        "credential/telegram/bot_token": "fake-bot-token:ABC123",
+        "credential/telegram/webhook_secret": "fake-webhook-secret",
+      },
+    });
+
+    const deadline = Date.now() + 5_000;
+    let status = before.status;
+    while (Date.now() < deadline) {
+      const resp = await fetch(`${base}/webhooks/telegram`, {
+        method: "POST",
+      });
+      status = resp.status;
+      if (status === 401) break;
+      await new Promise((resolve) => setTimeout(resolve, 200));
+    }
+
+    expect(status).toBe(401);
+  }, 15_000);
+
+  test("keeps retrying until configured credential reads succeed after CES list is already available", async () => {
+    mkdirSync(testDir, { recursive: true });
+    writeCredentialMetadata();
+
+    let readsReady = false;
+    startFakeCes({
+      accounts: [
+        "credential/telegram/bot_token",
+        "credential/telegram/webhook_secret",
+      ],
+      resolveValue(account) {
+        if (!readsReady) return undefined;
+        if (account === "credential/telegram/bot_token") {
+          return "fake-bot-token:ABC123";
+        }
+        if (account === "credential/telegram/webhook_secret") {
+          return "fake-webhook-secret";
+        }
+        return undefined;
+      },
+    });
+
+    await startGateway();
+
+    const base = `http://localhost:${gatewayPort}`;
+    const before = await fetch(`${base}/webhooks/telegram`, { method: "POST" });
+    expect(before.status).toBe(503);
+
+    readsReady = true;
+
+    const deadline = Date.now() + 5_000;
+    let status = before.status;
+    while (Date.now() < deadline) {
+      const resp = await fetch(`${base}/webhooks/telegram`, {
+        method: "POST",
+      });
+      status = resp.status;
+      if (status === 401) break;
+      await new Promise((resolve) => setTimeout(resolve, 200));
+    }
+
+    expect(status).toBe(401);
+  }, 15_000);
+});

package/src/credential-watcher.ts

@@ -9,7 +9,13 @@
  * causing later credential changes to be missed until restart.
  */
 
-import { mkdirSync, watch, type FSWatcher } from "node:fs";
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  watch,
+  type FSWatcher,
+} from "node:fs";
 import { dirname, join } from "node:path";
 import { getLogger } from "./logger.js";
 import {
@@ -22,6 +28,8 @@ import {
 const log = getLogger("credential-watcher");
 
 const DEBOUNCE_MS = 500;
+const MANAGED_BOOTSTRAP_POLL_MS = 1_000;
+const MANAGED_BOOTSTRAP_TIMEOUT_MS = 1_000;
 
 export type CredentialChangeEvent = {
   /** Map from service name to resolved credentials (null if unavailable) */
@@ -35,6 +43,10 @@ export type CredentialChangeCallback = (event: CredentialChangeEvent) => void;
 export class CredentialWatcher {
   private watchers: FSWatcher[] = [];
   private debounceTimer: ReturnType<typeof setTimeout> | null = null;
+  private managedBootstrapTimer: ReturnType<typeof setInterval> | null = null;
+  private managedBootstrapPollInFlight = false;
+  private lastConfiguredServices = new Set<string>();
+  private lastReadyServices = new Set<string>();
   private lastSerialized: Map<string, string> = new Map();
   private polling = false;
   private pendingPoll = false;
@@ -70,6 +82,8 @@ export class CredentialWatcher {
     // but the encrypted ciphertext will (new IV). Force a full reload so
     // channel listeners restart even when the plaintext values match.
     this.startWatcher(protectedDir, "keys.enc", { forceChanged: true });
+
+    this.startManagedBootstrapRetry();
   }
 
   private startWatcher(
@@ -101,6 +115,11 @@ export class CredentialWatcher {
       clearTimeout(this.debounceTimer);
       this.debounceTimer = null;
     }
+    if (this.managedBootstrapTimer) {
+      clearInterval(this.managedBootstrapTimer);
+      this.managedBootstrapTimer = null;
+    }
+    this.managedBootstrapPollInFlight = false;
     this.pendingPoll = false;
     for (const watcher of this.watchers) {
       watcher.close();
@@ -108,6 +127,63 @@ export class CredentialWatcher {
     this.watchers = [];
   }
 
+  private startManagedBootstrapRetry(): void {
+    const baseUrl = process.env.CES_CREDENTIAL_URL?.trim();
+    const serviceToken = process.env.CES_SERVICE_TOKEN?.trim();
+    if (!baseUrl || !serviceToken) return;
+
+    const poll = (): void => {
+      void this.pollManagedBootstrap(baseUrl, serviceToken);
+    };
+
+    this.managedBootstrapTimer = setInterval(poll, MANAGED_BOOTSTRAP_POLL_MS);
+    this.managedBootstrapTimer.unref?.();
+    poll();
+  }
+
+  private async pollManagedBootstrap(
+    baseUrl: string,
+    serviceToken: string,
+  ): Promise<void> {
+    if (this.managedBootstrapPollInFlight) return;
+    this.managedBootstrapPollInFlight = true;
+    try {
+      const resp = await fetch(`${baseUrl}/v1/credentials`, {
+        method: "GET",
+        headers: {
+          Authorization: `Bearer ${serviceToken}`,
+          Accept: "application/json",
+        },
+        signal: AbortSignal.timeout(MANAGED_BOOTSTRAP_TIMEOUT_MS),
+      });
+      if (resp.status === 401 || resp.status === 403 || resp.status === 404) {
+        if (this.managedBootstrapTimer) {
+          clearInterval(this.managedBootstrapTimer);
+          this.managedBootstrapTimer = null;
+        }
+        log.warn(
+          { status: resp.status },
+          "Stopping managed credential bootstrap retry due to non-retryable CES response",
+        );
+        return;
+      }
+      if (!resp.ok) {
+        return;
+      }
+
+      await this.pollOnce();
+
+      if (this.allConfiguredServicesReady() && this.managedBootstrapTimer) {
+        clearInterval(this.managedBootstrapTimer);
+        this.managedBootstrapTimer = null;
+      }
+    } catch {
+      // CES isn't reachable yet. Keep retrying until the sidecar is ready.
+    } finally {
+      this.managedBootstrapPollInFlight = false;
+    }
+  }
+
   /** Whether the next scheduled poll should treat all services as changed. */
   private pendingForceChanged = false;
 
@@ -135,9 +211,16 @@ export class CredentialWatcher {
     this.polling = true;
     try {
       const credentials = new Map<string, Record<string, string> | null>();
+      const configuredServices = this.loadConfiguredServices();
       for (const spec of ALL_CREDENTIAL_SPECS) {
         credentials.set(spec.service, await readServiceCredentials(spec));
       }
+      this.lastConfiguredServices = configuredServices;
+      this.lastReadyServices = new Set(
+        [...credentials.entries()]
+          .filter(([, creds]) => creds !== null)
+          .map(([service]) => service),
+      );
 
       const changedServices = new Set<string>();
       for (const [service, creds] of credentials) {
@@ -166,4 +249,42 @@ export class CredentialWatcher {
       }
     }
   }
+
+  private loadConfiguredServices(): Set<string> {
+    if (!existsSync(this.metadataPath)) return new Set();
+
+    try {
+      const raw = readFileSync(this.metadataPath, "utf-8");
+      const data = JSON.parse(raw) as {
+        credentials?: Array<{ service?: string; field?: string }>;
+      };
+      if (!Array.isArray(data.credentials)) return new Set();
+
+      const configured = new Set<string>();
+      for (const spec of ALL_CREDENTIAL_SPECS) {
+        const hasAllRequiredFields = spec.requiredFields.every((field) =>
+          data.credentials?.some(
+            (credential) =>
+              credential.service === spec.service && credential.field === field,
+          ),
+        );
+        if (hasAllRequiredFields) {
+          configured.add(spec.service);
+        }
+      }
+
+      return configured;
+    } catch {
+      return new Set();
+    }
+  }
+
+  private allConfiguredServicesReady(): boolean {
+    for (const service of this.lastConfiguredServices) {
+      if (!this.lastReadyServices.has(service)) {
+        return false;
+      }
+    }
+    return true;
+  }
 }

package/src/feature-flag-registry.json

@@ -255,7 +255,7 @@
       "key": "managed-google-oauth",
       "label": "Managed Google OAuth",
       "description": "Show the Google OAuth service card in Models & Services settings",
-      "defaultEnabled": false
+      "defaultEnabled": true
     },
     {
       "id": "settings-embedding-provider",

package/src/http/routes/slack-deliver.ts

@@ -330,6 +330,10 @@ export function createSlackDeliverHandler(
   config: GatewayConfig,
   onThreadReply?: (threadTs: string) => void,
   caches?: { credentials?: CredentialCache; configFile?: ConfigFileCache },
+  pendingApprovalReplacements?: Map<
+    string,
+    { messageTs: string; expiresAt: number }
+  >,
 ) {
   return async (req: Request): Promise<Response> => {
     const traceId = req.headers.get("x-trace-id") ?? undefined;
@@ -541,8 +545,31 @@ export function createSlackDeliverHandler(
 
     // Support threading via query param
     const threadTs = new URL(req.url).searchParams.get("threadTs") ?? undefined;
-    const messageTs = body.messageTs ?? updateTs;
-    const isUpdate = typeof messageTs === "string" && messageTs.length > 0;
+    let messageTs = body.messageTs ?? updateTs;
+    let isUpdate = typeof messageTs === "string" && messageTs.length > 0;
+
+    // Check for pending approval message replacement: if this is a new message
+    // (not already an update) to a thread with a pending approval replacement,
+    // convert it to an update of the approval message so the follow-up content
+    // replaces the original approval prompt.
+    let isApprovalReplacement = false;
+    if (threadTs && !isUpdate && !isEphemeral && !chatAction && text) {
+      const replacementKey = `${chatId}:${threadTs}`;
+      const pending = pendingApprovalReplacements?.get(replacementKey);
+      if (pending && pending.expiresAt > Date.now()) {
+        messageTs = pending.messageTs;
+        isUpdate = true;
+        isApprovalReplacement = true;
+        pendingApprovalReplacements!.delete(replacementKey);
+        tlog.info(
+          { chatId, threadTs, approvalMessageTs: messageTs },
+          "Converting delivery to approval message replacement",
+        );
+      } else if (pending) {
+        // Expired — clean up stale entry
+        pendingApprovalReplacements!.delete(replacementKey);
+      }
+    }
 
     // Resolve Block Kit blocks: use provided blocks, approval prompt, or auto-format text
     const blocks: Block[] =
@@ -804,7 +831,7 @@ export function createSlackDeliverHandler(
             text,
             ts: messageTs,
           };
-          if (blocks.length > 0) {
+          if (blocks.length > 0 || isApprovalReplacement) {
             updateBody.blocks = blocks;
           }
           result = await callSlackApiWithRetries(

package/src/http/routes/telegram-webhook.ts

@@ -436,6 +436,27 @@ export function createTelegramWebhookHandler(
           });
         } else {
           tlog.info({ status: "forwarded" }, "Forwarded /start to runtime");
+
+          // Fallback: if the runtime denied the message and could not
+          // deliver the rejection reply via callback, send it directly.
+          const startRuntimeResp = result.runtimeResponse;
+          if (startRuntimeResp?.denied && startRuntimeResp.replyText) {
+            sendTelegramReply(
+              config,
+              normalized.message.conversationExternalId,
+              startRuntimeResp.replyText,
+              undefined,
+              {
+                credentials: caches?.credentials,
+                configFile: caches?.configFile,
+              },
+            ).catch((err) => {
+              tlog.error(
+                { err, chatId: normalized.message.conversationExternalId },
+                "Failed to send ACL denial fallback reply",
+              );
+            });
+          }
         }
       } catch (err) {
         if (err instanceof CircuitBreakerOpenError) {
@@ -703,6 +724,21 @@ export function createTelegramWebhookHandler(
 
       tlog.info({ status: "forwarded" }, "Forwarded to runtime");
 
+      // Fallback: if the runtime denied the message and could not
+      // deliver the rejection reply via callback, send it directly.
+      const runtimeResp = result.runtimeResponse;
+      if (runtimeResp?.denied && runtimeResp.replyText) {
+        sendTelegramReply(config, chatId, runtimeResp.replyText, undefined, {
+          credentials: caches?.credentials,
+          configFile: caches?.configFile,
+        }).catch((err) => {
+          tlog.error(
+            { err, chatId },
+            "Failed to send ACL denial fallback reply",
+          );
+        });
+      }
+
       // Acknowledge the callback query to clear the button spinner in the
       // Telegram client. Best-effort — log errors but don't fail the flow.
       if (isCallback)

package/src/index.ts

@@ -241,12 +241,28 @@ async function main() {
     credentials: credentialCache,
     configFile: configFileCache,
   });
+  // Map: "channel:threadTs" -> { messageTs, expiresAt } for replacing approval
+  // messages with the bot's follow-up content after an approval button click.
+  const pendingApprovalReplacements = new Map<
+    string,
+    { messageTs: string; expiresAt: number }
+  >();
+
+  // Clean up expired entries every 60s
+  setInterval(() => {
+    const now = Date.now();
+    for (const [key, entry] of pendingApprovalReplacements) {
+      if (entry.expiresAt <= now) pendingApprovalReplacements.delete(key);
+    }
+  }, 60_000);
+
   const handleSlackDeliver = createSlackDeliverHandler(
     config,
     (threadTs) => {
       slackSocketClient?.trackThread(threadTs);
     },
     { credentials: credentialCache, configFile: configFileCache },
+    pendingApprovalReplacements,
   );
   const handleOAuthCallback = createOAuthCallbackHandler(config);
   const pairingProxy = createPairingProxyHandler(config);
@@ -1193,6 +1209,18 @@ async function main() {
         } else {
           forward();
         }
+
+        // When an approval button is clicked, store the approval message ts
+        // so the next outbound delivery to this thread replaces the approval
+        // message instead of posting a new one.
+        const callbackData = normalized.event.message.callbackData;
+        if (callbackData?.startsWith("apr:") && messageTs && threadTs) {
+          const key = `${channel}:${threadTs}`;
+          pendingApprovalReplacements.set(key, {
+            messageTs,
+            expiresAt: Date.now() + 60_000, // 60s TTL
+          });
+        }
       },
     );
 

package/src/runtime/client.ts

@@ -179,6 +179,14 @@ export type RuntimeInboundResponse = {
     timestamp: string;
     attachments: RuntimeAttachmentMeta[];
   };
+  /** When true, the runtime denied the inbound message (e.g. ACL rejection). */
+  denied?: boolean;
+  /**
+   * A user-facing rejection message that the runtime could not deliver via
+   * the callback URL (e.g. due to auth failure). When present, the gateway
+   * should deliver it directly to the channel.
+   */
+  replyText?: string;
 };
 
 export type ForwardOptions = {

package/src/schema.ts

@@ -1738,7 +1738,7 @@ export function buildSchema(): Record<string, unknown> {
               required: true,
               schema: { type: "string" },
               description:
-                "OAuth provider key to filter by, for example `integration:google`.",
+                "OAuth provider key to filter by, for example `google`.",
             },
           ],
           security: [{ BearerAuth: [] }],

package/src/telegram/webhook-manager.ts

@@ -148,7 +148,21 @@ export async function reconcileTelegramWebhook(
     return;
   }
 
-  const expectedUrl = await resolveExpectedTelegramWebhookUrl(caches);
+  let expectedUrl: string | undefined;
+  try {
+    expectedUrl = await resolveExpectedTelegramWebhookUrl(caches);
+  } catch (err) {
+    // Managed callback route registration failed — this is a platform-side
+    // issue. Do not suggest ngrok or other tunnel options; they are not
+    // usable in containerized deployments.
+    const detail = err instanceof Error ? err.message : String(err);
+    log.error(
+      { err },
+      `Telegram webhook registration failed: managed platform callback route could not be registered. ` +
+        `Please contact support. (${detail})`,
+    );
+    return;
+  }
   if (!expectedUrl) {
     log.debug(
       "Skipping webhook reconciliation: no public ingress or managed callback route available",