@agent-native/core 0.31.0 → 0.31.2

package/dist/integrations/pending-tasks-retry-job.js.map

@@ -1 +1 @@
-{"version":3,"file":"pending-tasks-retry-job.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-retry-job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,8EAA8E;AAC9E,MAAM,sBAAsB,GAAG,MAAM,CAAC;AACtC,4EAA4E;AAC5E,MAAM,iCAAiC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACxD,MAAM,oCAAoC,GAAG,MAAM,CAAC;AACpD,mEAAmE;AACnE,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,MAAM,cAAc,GAAG,GAAG,sBAAsB,4BAA4B,CAAC;AAE7E,IAAI,aAAa,GAA0C,IAAI,CAAC;AAChE,IAAI,YAAY,GAAyC,IAAI,CAAC;AAC9D,IAAI,oBAAwC,CAAC;AAC7C;;;GAGG;AACH,IAAI,WAAW,GAAmB,IAAI,CAAC;AAQvC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,cAAuB;IAEvB,MAAM,OAAO,GAAG,cAAc,IAAI,oBAAoB,CAAC;IACvD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,aAAa,GAAG,GAAG,GAAG,sBAAsB,CAAC;IACnD,MAAM,gBAAgB,GAAG,GAAG,GAAG,yBAAyB,EAAE,CAAC;IAE3D,IAAI,SAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YACpC,GAAG,EAAE;;;;;OAKJ;YACD,IAAI,EAAE,CAAC,aAAa,EAAE,gBAAgB,CAAC;SACxC,CAAC,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,EAAE,EAAE,CAAC,CAAC,EAAY;YAClB,MAAM,EAAE,CAAC,CAAC,MAAgB;YAC1B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;SAClC,CAAC,CAAC,CAAC;QACJ,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,yEAAyE;QACzE,sEAAsE;QACtE,iCAAiC;QACjC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,WAAW,GAAG,KAAK,CAAC;YACpB,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CACT,yEAAyE,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,kEAAkE;YAClE,0CAA0C;YAC1C,IAAI,GAAG,CAAC,QAAQ,IAAI,YAAY,EAAE,CAAC;gBACjC,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,GAAG,EAAE;;;;;;;WAOJ;oBACD,IAAI,EAAE;wBACJ,IAAI,CAAC,GAAG,EAAE;wBACV,uBAAuB,YAAY,WAAW;wBAC9C,GAAG,CAAC,EAAE;wBACN,GAAG,CAAC,MAAM;qBACX;iBACF,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CACV,+BAA+B,GAAG,CAAC,EAAE,aAAa,YAAY,4BAA4B,CAC3F,CAAC;gBACF,SAAS;YACX,CAAC;YAED,qEAAqE;YACrE,+DAA+D;YAC/D,oDAAoD;YACpD,yEAAyE;YACzE,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;YACvE,MAAM,MAAM,CAAC,OAAO,CAAC;gBACnB,GAAG,EAAE;;;;;SAKJ;gBACD,IAAI,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC;aAClD,CAAC,CAAC;YAEH,MAAM,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CACX,+CAA+C,GAAG,CAAC,EAAE,GAAG,EACxD,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,IACE,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,UAAU,IAAI,UAAU,EACxB,CAAC;QACD,OAAO,oCAAoC,CAAC;IAC9C,CAAC;IACD,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,MAAc,EACd,cAAkC;IAElC,MAAM,OAAO,GACX,cAAc;QACd,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAC5B,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG;QACf,OAAO,CAAC,GAAG,CAAC,UAAU;QACtB,oBAAoB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;IAEjD,MAAM,GAAG,GAAG,GAAG,yBAAyB,CAAC,OAAO,CAAC,GAAG,cAAc,EAAE,CAAC;IAErE,2EAA2E;IAC3E,0EAA0E;IAC1E,yEAAyE;IACzE,wEAAwE;IACxE,sBAAsB;IACtB,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;KACnC,CAAC;IACF,IAAI,CAAC;QACH,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;IACnE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC1C,OAAO,CAAC,KAAK,CACX,4CAA4C,MAAM,gCAAgC;gBAChF,mDAAmD,CACtD,CAAC;YACF,OAAO;QACT,CAAC;QACD,sEAAsE;QACtE,6EAA6E;QAC7E,IAAI,GAAG,YAAY,KAAK,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CACX,4DAA4D,MAAM,GAAG,EACrE,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,sEAAsE;IACtE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE;YACf,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;YAChC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAEzC;IACC,IAAI,aAAa;QAAE,OAAO;IAC1B,oBAAoB,GAAG,OAAO,EAAE,cAAc,CAAC;IAE/C,6EAA6E;IAC7E,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QAC7B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACX,UAAU,CAAC,YAAY,CAAC,CAAC;IAEzB,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QAC/B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACtB,UAAU,CAAC,aAAa,CAAC,CAAC;IAE1B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,yDACE,iBAAiB,GAAG,IACtB,IAAI,CACL,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,wBAAwB;IACtC,IAAI,YAAY,EAAE,CAAC;QACjB,YAAY,CAAC,YAAY,CAAC,CAAC;QAC3B,YAAY,GAAG,IAAI,CAAC;IACtB,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,aAAa,CAAC,aAAa,CAAC,CAAC;QAC7B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,oBAAoB,GAAG,SAAS,CAAC;AACnC,CAAC;AAED,SAAS,UAAU,CAAC,KAAqC;IACtD,KAA2C,CAAC,KAAK,EAAE,EAAE,CAAC;AACzD,CAAC","sourcesContent":["import { getDbExec } from \"../db/client.js\";\nimport { FRAMEWORK_ROUTE_PREFIX } from \"../server/core-routes-plugin.js\";\nimport { withConfiguredAppBasePath } from \"../server/app-base-path.js\";\nimport { signInternalToken } from \"./internal-token.js\";\n\n/**\n * Retries stuck integration webhook tasks.\n *\n * The integration webhook flow enqueues work into `integration_pending_tasks`\n * (see `pending-tasks-store.ts`) and then fires a self-webhook to the\n * `/_agent-native/integrations/process-task` endpoint to drain the queue.\n * If that fire-and-forget dispatch fails (e.g. transient network blip), the\n * row stays in `pending` forever. Likewise, if the processor is killed mid-\n * processing (function timeout, container shutdown), a row can remain in\n * `processing` forever.\n *\n * This job runs every 60s and re-fires the processor endpoint for tasks that\n * look stuck:\n *   - status='pending' AND created_at older than 90s (initial dispatch lost)\n *   - status='processing' AND updated_at older than the host-specific\n *     function budget (75s on serverless, 5min elsewhere)\n *\n * Retries are capped at MAX_ATTEMPTS attempts; after that the row is marked\n * `failed` permanently so it stops being retried.\n *\n * If the `integration_pending_tasks` table does not yet exist (e.g. older\n * deploy that hasn't run the new webhook flow), this job no-ops silently\n * rather than spamming logs.\n */\n\nconst RETRY_INTERVAL_MS = 60_000;\n/** Tasks pending longer than this are considered stuck on initial dispatch */\nconst PENDING_STUCK_AFTER_MS = 90_000;\n/** Tasks \"processing\" longer than this are considered killed mid-flight. */\nconst DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000;\nconst SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000;\n/** After this many attempts we give up and mark the task failed */\nconst MAX_ATTEMPTS = 3;\n\nconst PROCESSOR_PATH = `${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`;\n\nlet retryInterval: ReturnType<typeof setInterval> | null = null;\nlet initialTimer: ReturnType<typeof setTimeout> | null = null;\nlet activeWebhookBaseUrl: string | undefined;\n/**\n * Whether the table exists. Cached after first probe so we don't log every\n * minute when the queue isn't in use yet on a given deployment.\n */\nlet tableExists: boolean | null = null;\n\ninterface StuckTaskRow {\n  id: string;\n  status: string;\n  attempts: number;\n}\n\n/**\n * One pass: find stuck tasks and re-fire the processor for each.\n * Exported for tests and for manual triggers.\n */\nexport async function retryStuckPendingTasks(\n  webhookBaseUrl?: string,\n): Promise<void> {\n  const baseUrl = webhookBaseUrl ?? activeWebhookBaseUrl;\n  const client = getDbExec();\n  const now = Date.now();\n  const pendingCutoff = now - PENDING_STUCK_AFTER_MS;\n  const processingCutoff = now - getProcessingStuckAfterMs();\n\n  let stuckRows: StuckTaskRow[];\n  try {\n    const { rows } = await client.execute({\n      sql: `\n        SELECT id, status, attempts\n          FROM integration_pending_tasks\n         WHERE (status = 'pending' AND created_at <= ?)\n            OR (status = 'processing' AND updated_at <= ?)\n      `,\n      args: [pendingCutoff, processingCutoff],\n    });\n    stuckRows = rows.map((r) => ({\n      id: r.id as string,\n      status: r.status as string,\n      attempts: Number(r.attempts ?? 0),\n    }));\n    tableExists = true;\n  } catch (err) {\n    // Most common case: the table hasn't been created yet because no inbound\n    // integration webhook has been processed on this deployment. Silently\n    // no-op until the table appears.\n    if (tableExists !== false) {\n      tableExists = false;\n      if (process.env.DEBUG) {\n        console.log(\n          \"[integrations] pending-tasks retry job: table not present yet, skipping\",\n        );\n      }\n    }\n    return;\n  }\n\n  if (stuckRows.length === 0) return;\n\n  for (const row of stuckRows) {\n    try {\n      // Cap retries — mark failed and move on so the row stops bouncing\n      // between pending and processing forever.\n      if (row.attempts >= MAX_ATTEMPTS) {\n        await client.execute({\n          sql: `\n            UPDATE integration_pending_tasks\n               SET status = 'failed',\n                   updated_at = ?,\n                   error_message = COALESCE(error_message, ?)\n             WHERE id = ?\n               AND status = ?\n          `,\n          args: [\n            Date.now(),\n            `Retry job: exceeded ${MAX_ATTEMPTS} attempts`,\n            row.id,\n            row.status,\n          ],\n        });\n        console.warn(\n          `[integrations] Pending task ${row.id} exceeded ${MAX_ATTEMPTS} attempts — marking failed`,\n        );\n        continue;\n      }\n\n      // Reset stuck `processing` rows back to `pending` so the processor's\n      // atomic claim (which only matches pending) can re-acquire it.\n      // Without this, processing rows stay stuck forever.\n      // For pending rows, just touch updated_at to avoid re-firing every tick.\n      const newStatus = row.status === \"processing\" ? \"pending\" : row.status;\n      await client.execute({\n        sql: `\n          UPDATE integration_pending_tasks\n             SET status = ?, updated_at = ?\n           WHERE id = ?\n             AND status = ?\n        `,\n        args: [newStatus, Date.now(), row.id, row.status],\n      });\n\n      await refireProcessor(row.id, baseUrl);\n    } catch (err) {\n      console.error(\n        `[integrations] Failed to retry pending task ${row.id}:`,\n        err,\n      );\n    }\n  }\n}\n\nfunction getProcessingStuckAfterMs(): number {\n  if (\n    process.env.NETLIFY ||\n    process.env.AWS_LAMBDA_FUNCTION_NAME ||\n    process.env.VERCEL ||\n    \"__cf_env\" in globalThis\n  ) {\n    return SERVERLESS_PROCESSING_STUCK_AFTER_MS;\n  }\n  return DEFAULT_PROCESSING_STUCK_AFTER_MS;\n}\n\n/**\n * Fire-and-forget POST to the processor endpoint for a single task id.\n * Mirrors the original dispatch from the webhook handler, including the\n * short-lived HMAC bearer token bound to this taskId.\n */\nasync function refireProcessor(\n  taskId: string,\n  webhookBaseUrl: string | undefined,\n): Promise<void> {\n  const baseUrl =\n    webhookBaseUrl ||\n    process.env.WEBHOOK_BASE_URL ||\n    process.env.APP_URL ||\n    process.env.URL ||\n    process.env.DEPLOY_URL ||\n    `http://localhost:${process.env.PORT || 3000}`;\n\n  const url = `${withConfiguredAppBasePath(baseUrl)}${PROCESSOR_PATH}`;\n\n  // Sign with HMAC if A2A_SECRET is configured. In production we MUST sign —\n  // an unsigned dispatch in production lets attackers re-trigger any queued\n  // task with a guessable id (C3 in the webhook security audit). In dev we\n  // fall back to unsigned so contributors can iterate without configuring\n  // A2A_SECRET locally.\n  const headers: Record<string, string> = {\n    \"Content-Type\": \"application/json\",\n  };\n  try {\n    headers[\"Authorization\"] = `Bearer ${signInternalToken(taskId)}`;\n  } catch (err) {\n    if (process.env.NODE_ENV === \"production\") {\n      console.error(\n        `[integrations] Refusing to dispatch task ${taskId} — A2A_SECRET not configured. ` +\n          \"Set A2A_SECRET to enable signed retry dispatches.\",\n      );\n      return;\n    }\n    // Dev: proceed unsigned. Log the underlying error path so a malformed\n    // secret (different from \"not set\") doesn't fail silently (L5 in the audit).\n    if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) {\n      console.error(\n        `[integrations] signInternalToken failed unexpectedly for ${taskId}:`,\n        err,\n      );\n    }\n  }\n\n  // Don't await the body — we just want the request to leave the box.\n  // A short timeout avoids tying up the retry loop on a hung processor.\n  const controller = new AbortController();\n  const timer = setTimeout(() => controller.abort(), 5_000);\n\n  try {\n    await fetch(url, {\n      method: \"POST\",\n      headers,\n      body: JSON.stringify({ taskId }),\n      signal: controller.signal,\n    });\n  } finally {\n    clearTimeout(timer);\n  }\n}\n\n/**\n * Start the periodic retry loop. Safe to call multiple times — second call\n * is a no-op.\n */\nexport function startPendingTasksRetryJob(options?: {\n  webhookBaseUrl?: string;\n}): void {\n  if (retryInterval) return;\n  activeWebhookBaseUrl = options?.webhookBaseUrl;\n\n  // Stagger the first run a bit so we don't hammer the DB immediately on boot.\n  initialTimer = setTimeout(() => {\n    void retryStuckPendingTasks().catch((err) => {\n      console.error(\"[integrations] Pending-tasks retry job error:\", err);\n    });\n  }, 10_000);\n  unrefTimer(initialTimer);\n\n  retryInterval = setInterval(() => {\n    void retryStuckPendingTasks().catch((err) => {\n      console.error(\"[integrations] Pending-tasks retry job error:\", err);\n    });\n  }, RETRY_INTERVAL_MS);\n  unrefTimer(retryInterval);\n\n  if (process.env.DEBUG) {\n    console.log(\n      `[integrations] Pending-tasks retry job started (every ${\n        RETRY_INTERVAL_MS / 1000\n      }s)`,\n    );\n  }\n}\n\n/** Stop the retry loop. */\nexport function stopPendingTasksRetryJob(): void {\n  if (initialTimer) {\n    clearTimeout(initialTimer);\n    initialTimer = null;\n  }\n  if (retryInterval) {\n    clearInterval(retryInterval);\n    retryInterval = null;\n  }\n  activeWebhookBaseUrl = undefined;\n}\n\nfunction unrefTimer(timer: ReturnType<typeof setInterval>): void {\n  (timer as unknown as { unref?: () => void }).unref?.();\n}\n"]}
+{"version":3,"file":"pending-tasks-retry-job.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-retry-job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AACzE,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAExD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,8EAA8E;AAC9E,MAAM,sBAAsB,GAAG,MAAM,CAAC;AACtC,4EAA4E;AAC5E,MAAM,iCAAiC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACxD,MAAM,oCAAoC,GAAG,MAAM,CAAC;AACpD,mEAAmE;AACnE,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,MAAM,cAAc,GAAG,GAAG,sBAAsB,4BAA4B,CAAC;AAE7E,IAAI,aAAa,GAA0C,IAAI,CAAC;AAChE,IAAI,YAAY,GAAyC,IAAI,CAAC;AAC9D,IAAI,oBAAwC,CAAC;AAC7C;;;GAGG;AACH,IAAI,WAAW,GAAmB,IAAI,CAAC;AAQvC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,cAAuB;IAEvB,MAAM,OAAO,GAAG,cAAc,IAAI,oBAAoB,CAAC;IACvD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,aAAa,GAAG,GAAG,GAAG,sBAAsB,CAAC;IACnD,MAAM,gBAAgB,GAAG,GAAG,GAAG,yBAAyB,EAAE,CAAC;IAE3D,IAAI,SAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YACpC,GAAG,EAAE;;;;;OAKJ;YACD,wEAAwE;YACxE,qEAAqE;YACrE,wEAAwE;YACxE,wEAAwE;YACxE,IAAI,EAAE,CAAC,aAAa,EAAE,aAAa,EAAE,gBAAgB,CAAC;SACvD,CAAC,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,EAAE,EAAE,CAAC,CAAC,EAAY;YAClB,MAAM,EAAE,CAAC,CAAC,MAAgB;YAC1B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;SAClC,CAAC,CAAC,CAAC;QACJ,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,yEAAyE;QACzE,sEAAsE;QACtE,iCAAiC;QACjC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,WAAW,GAAG,KAAK,CAAC;YACpB,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CACT,yEAAyE,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,kEAAkE;YAClE,0CAA0C;YAC1C,IAAI,GAAG,CAAC,QAAQ,IAAI,YAAY,EAAE,CAAC;gBACjC,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,GAAG,EAAE;;;;;;;WAOJ;oBACD,IAAI,EAAE;wBACJ,IAAI,CAAC,GAAG,EAAE;wBACV,uBAAuB,YAAY,WAAW;wBAC9C,GAAG,CAAC,EAAE;wBACN,GAAG,CAAC,MAAM;qBACX;iBACF,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CACV,+BAA+B,GAAG,CAAC,EAAE,aAAa,YAAY,4BAA4B,CAC3F,CAAC;gBACF,SAAS;YACX,CAAC;YAED,qEAAqE;YACrE,+DAA+D;YAC/D,oDAAoD;YACpD,yEAAyE;YACzE,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;YACvE,MAAM,MAAM,CAAC,OAAO,CAAC;gBACnB,GAAG,EAAE;;;;;SAKJ;gBACD,IAAI,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC;aAClD,CAAC,CAAC;YAEH,MAAM,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CACX,+CAA+C,GAAG,CAAC,EAAE,GAAG,EACxD,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,IACE,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,UAAU,IAAI,UAAU,EACxB,CAAC;QACD,OAAO,oCAAoC,CAAC;IAC9C,CAAC;IACD,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,MAAc,EACd,cAAkC;IAElC,MAAM,OAAO,GACX,cAAc;QACd,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAC5B,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG;QACf,OAAO,CAAC,GAAG,CAAC,UAAU;QACtB,oBAAoB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;IAEjD,MAAM,GAAG,GAAG,GAAG,yBAAyB,CAAC,OAAO,CAAC,GAAG,cAAc,EAAE,CAAC;IAErE,2EAA2E;IAC3E,0EAA0E;IAC1E,yEAAyE;IACzE,wEAAwE;IACxE,sBAAsB;IACtB,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;KACnC,CAAC;IACF,IAAI,CAAC;QACH,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;IACnE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC1C,OAAO,CAAC,KAAK,CACX,4CAA4C,MAAM,gCAAgC;gBAChF,mDAAmD,CACtD,CAAC;YACF,OAAO;QACT,CAAC;QACD,sEAAsE;QACtE,6EAA6E;QAC7E,IAAI,GAAG,YAAY,KAAK,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CACX,4DAA4D,MAAM,GAAG,EACrE,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,sEAAsE;IACtE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE;YACf,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;YAChC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAEzC;IACC,IAAI,aAAa;QAAE,OAAO;IAC1B,oBAAoB,GAAG,OAAO,EAAE,cAAc,CAAC;IAE/C,6EAA6E;IAC7E,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QAC7B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACX,UAAU,CAAC,YAAY,CAAC,CAAC;IAEzB,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QAC/B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACtB,UAAU,CAAC,aAAa,CAAC,CAAC;IAE1B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,yDACE,iBAAiB,GAAG,IACtB,IAAI,CACL,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,wBAAwB;IACtC,IAAI,YAAY,EAAE,CAAC;QACjB,YAAY,CAAC,YAAY,CAAC,CAAC;QAC3B,YAAY,GAAG,IAAI,CAAC;IACtB,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,aAAa,CAAC,aAAa,CAAC,CAAC;QAC7B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,oBAAoB,GAAG,SAAS,CAAC;AACnC,CAAC;AAED,SAAS,UAAU,CAAC,KAAqC;IACtD,KAA2C,CAAC,KAAK,EAAE,EAAE,CAAC;AACzD,CAAC","sourcesContent":["import { getDbExec } from \"../db/client.js\";\nimport { FRAMEWORK_ROUTE_PREFIX } from \"../server/core-routes-plugin.js\";\nimport { withConfiguredAppBasePath } from \"../server/app-base-path.js\";\nimport { signInternalToken } from \"./internal-token.js\";\n\n/**\n * Retries stuck integration webhook tasks.\n *\n * The integration webhook flow enqueues work into `integration_pending_tasks`\n * (see `pending-tasks-store.ts`) and then fires a self-webhook to the\n * `/_agent-native/integrations/process-task` endpoint to drain the queue.\n * If that fire-and-forget dispatch fails (e.g. transient network blip), the\n * row stays in `pending` forever. Likewise, if the processor is killed mid-\n * processing (function timeout, container shutdown), a row can remain in\n * `processing` forever.\n *\n * This job runs every 60s and re-fires the processor endpoint for tasks that\n * look stuck:\n *   - status='pending' AND created_at older than 90s (initial dispatch lost)\n *   - status='processing' AND updated_at older than the host-specific\n *     function budget (75s on serverless, 5min elsewhere)\n *\n * Retries are capped at MAX_ATTEMPTS attempts; after that the row is marked\n * `failed` permanently so it stops being retried.\n *\n * If the `integration_pending_tasks` table does not yet exist (e.g. older\n * deploy that hasn't run the new webhook flow), this job no-ops silently\n * rather than spamming logs.\n */\n\nconst RETRY_INTERVAL_MS = 60_000;\n/** Tasks pending longer than this are considered stuck on initial dispatch */\nconst PENDING_STUCK_AFTER_MS = 90_000;\n/** Tasks \"processing\" longer than this are considered killed mid-flight. */\nconst DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000;\nconst SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000;\n/** After this many attempts we give up and mark the task failed */\nconst MAX_ATTEMPTS = 3;\n\nconst PROCESSOR_PATH = `${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`;\n\nlet retryInterval: ReturnType<typeof setInterval> | null = null;\nlet initialTimer: ReturnType<typeof setTimeout> | null = null;\nlet activeWebhookBaseUrl: string | undefined;\n/**\n * Whether the table exists. Cached after first probe so we don't log every\n * minute when the queue isn't in use yet on a given deployment.\n */\nlet tableExists: boolean | null = null;\n\ninterface StuckTaskRow {\n  id: string;\n  status: string;\n  attempts: number;\n}\n\n/**\n * One pass: find stuck tasks and re-fire the processor for each.\n * Exported for tests and for manual triggers.\n */\nexport async function retryStuckPendingTasks(\n  webhookBaseUrl?: string,\n): Promise<void> {\n  const baseUrl = webhookBaseUrl ?? activeWebhookBaseUrl;\n  const client = getDbExec();\n  const now = Date.now();\n  const pendingCutoff = now - PENDING_STUCK_AFTER_MS;\n  const processingCutoff = now - getProcessingStuckAfterMs();\n\n  let stuckRows: StuckTaskRow[];\n  try {\n    const { rows } = await client.execute({\n      sql: `\n        SELECT id, status, attempts\n          FROM integration_pending_tasks\n         WHERE (status = 'pending' AND created_at <= ? AND updated_at <= ?)\n            OR (status = 'processing' AND updated_at <= ?)\n      `,\n      // `updated_at` is initialized to `created_at` on insert, so a genuinely\n      // stuck pending row still matches on the first sweep. The retry path\n      // below touches `updated_at`, which (with this predicate) keeps the row\n      // from being re-selected — and re-firing the processor — on every tick.\n      args: [pendingCutoff, pendingCutoff, processingCutoff],\n    });\n    stuckRows = rows.map((r) => ({\n      id: r.id as string,\n      status: r.status as string,\n      attempts: Number(r.attempts ?? 0),\n    }));\n    tableExists = true;\n  } catch (err) {\n    // Most common case: the table hasn't been created yet because no inbound\n    // integration webhook has been processed on this deployment. Silently\n    // no-op until the table appears.\n    if (tableExists !== false) {\n      tableExists = false;\n      if (process.env.DEBUG) {\n        console.log(\n          \"[integrations] pending-tasks retry job: table not present yet, skipping\",\n        );\n      }\n    }\n    return;\n  }\n\n  if (stuckRows.length === 0) return;\n\n  for (const row of stuckRows) {\n    try {\n      // Cap retries — mark failed and move on so the row stops bouncing\n      // between pending and processing forever.\n      if (row.attempts >= MAX_ATTEMPTS) {\n        await client.execute({\n          sql: `\n            UPDATE integration_pending_tasks\n               SET status = 'failed',\n                   updated_at = ?,\n                   error_message = COALESCE(error_message, ?)\n             WHERE id = ?\n               AND status = ?\n          `,\n          args: [\n            Date.now(),\n            `Retry job: exceeded ${MAX_ATTEMPTS} attempts`,\n            row.id,\n            row.status,\n          ],\n        });\n        console.warn(\n          `[integrations] Pending task ${row.id} exceeded ${MAX_ATTEMPTS} attempts — marking failed`,\n        );\n        continue;\n      }\n\n      // Reset stuck `processing` rows back to `pending` so the processor's\n      // atomic claim (which only matches pending) can re-acquire it.\n      // Without this, processing rows stay stuck forever.\n      // For pending rows, just touch updated_at to avoid re-firing every tick.\n      const newStatus = row.status === \"processing\" ? \"pending\" : row.status;\n      await client.execute({\n        sql: `\n          UPDATE integration_pending_tasks\n             SET status = ?, updated_at = ?\n           WHERE id = ?\n             AND status = ?\n        `,\n        args: [newStatus, Date.now(), row.id, row.status],\n      });\n\n      await refireProcessor(row.id, baseUrl);\n    } catch (err) {\n      console.error(\n        `[integrations] Failed to retry pending task ${row.id}:`,\n        err,\n      );\n    }\n  }\n}\n\nfunction getProcessingStuckAfterMs(): number {\n  if (\n    process.env.NETLIFY ||\n    process.env.AWS_LAMBDA_FUNCTION_NAME ||\n    process.env.VERCEL ||\n    \"__cf_env\" in globalThis\n  ) {\n    return SERVERLESS_PROCESSING_STUCK_AFTER_MS;\n  }\n  return DEFAULT_PROCESSING_STUCK_AFTER_MS;\n}\n\n/**\n * Fire-and-forget POST to the processor endpoint for a single task id.\n * Mirrors the original dispatch from the webhook handler, including the\n * short-lived HMAC bearer token bound to this taskId.\n */\nasync function refireProcessor(\n  taskId: string,\n  webhookBaseUrl: string | undefined,\n): Promise<void> {\n  const baseUrl =\n    webhookBaseUrl ||\n    process.env.WEBHOOK_BASE_URL ||\n    process.env.APP_URL ||\n    process.env.URL ||\n    process.env.DEPLOY_URL ||\n    `http://localhost:${process.env.PORT || 3000}`;\n\n  const url = `${withConfiguredAppBasePath(baseUrl)}${PROCESSOR_PATH}`;\n\n  // Sign with HMAC if A2A_SECRET is configured. In production we MUST sign —\n  // an unsigned dispatch in production lets attackers re-trigger any queued\n  // task with a guessable id (C3 in the webhook security audit). In dev we\n  // fall back to unsigned so contributors can iterate without configuring\n  // A2A_SECRET locally.\n  const headers: Record<string, string> = {\n    \"Content-Type\": \"application/json\",\n  };\n  try {\n    headers[\"Authorization\"] = `Bearer ${signInternalToken(taskId)}`;\n  } catch (err) {\n    if (process.env.NODE_ENV === \"production\") {\n      console.error(\n        `[integrations] Refusing to dispatch task ${taskId} — A2A_SECRET not configured. ` +\n          \"Set A2A_SECRET to enable signed retry dispatches.\",\n      );\n      return;\n    }\n    // Dev: proceed unsigned. Log the underlying error path so a malformed\n    // secret (different from \"not set\") doesn't fail silently (L5 in the audit).\n    if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) {\n      console.error(\n        `[integrations] signInternalToken failed unexpectedly for ${taskId}:`,\n        err,\n      );\n    }\n  }\n\n  // Don't await the body — we just want the request to leave the box.\n  // A short timeout avoids tying up the retry loop on a hung processor.\n  const controller = new AbortController();\n  const timer = setTimeout(() => controller.abort(), 5_000);\n\n  try {\n    await fetch(url, {\n      method: \"POST\",\n      headers,\n      body: JSON.stringify({ taskId }),\n      signal: controller.signal,\n    });\n  } finally {\n    clearTimeout(timer);\n  }\n}\n\n/**\n * Start the periodic retry loop. Safe to call multiple times — second call\n * is a no-op.\n */\nexport function startPendingTasksRetryJob(options?: {\n  webhookBaseUrl?: string;\n}): void {\n  if (retryInterval) return;\n  activeWebhookBaseUrl = options?.webhookBaseUrl;\n\n  // Stagger the first run a bit so we don't hammer the DB immediately on boot.\n  initialTimer = setTimeout(() => {\n    void retryStuckPendingTasks().catch((err) => {\n      console.error(\"[integrations] Pending-tasks retry job error:\", err);\n    });\n  }, 10_000);\n  unrefTimer(initialTimer);\n\n  retryInterval = setInterval(() => {\n    void retryStuckPendingTasks().catch((err) => {\n      console.error(\"[integrations] Pending-tasks retry job error:\", err);\n    });\n  }, RETRY_INTERVAL_MS);\n  unrefTimer(retryInterval);\n\n  if (process.env.DEBUG) {\n    console.log(\n      `[integrations] Pending-tasks retry job started (every ${\n        RETRY_INTERVAL_MS / 1000\n      }s)`,\n    );\n  }\n}\n\n/** Stop the retry loop. */\nexport function stopPendingTasksRetryJob(): void {\n  if (initialTimer) {\n    clearTimeout(initialTimer);\n    initialTimer = null;\n  }\n  if (retryInterval) {\n    clearInterval(retryInterval);\n    retryInterval = null;\n  }\n  activeWebhookBaseUrl = undefined;\n}\n\nfunction unrefTimer(timer: ReturnType<typeof setInterval>): void {\n  (timer as unknown as { unref?: () => void }).unref?.();\n}\n"]}

package/dist/integrations/pending-tasks-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pending-tasks-store.d.ts","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAgFA,qDAAqD;AACrD,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,YAAY,GACZ,WAAW,GACX,QAAQ,CAAC;AAEb,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAoBD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBhB;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAU3D;AAED,kCAAkC;AAClC,wBAAsB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAU5E;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAkC7B;AAED,gCAAgC;AAChC,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUjE;AAED,wDAAwD;AACxD,wBAAsB,cAAc,CAClC,EAAE,EAAE,MAAM,EACV,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC,CAUf"}
+{"version":3,"file":"pending-tasks-store.d.ts","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAoFA,qDAAqD;AACrD,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,YAAY,GACZ,WAAW,GACX,QAAQ,CAAC;AAEb,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAoBD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBhB;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAU3D;AAED,kCAAkC;AAClC,wBAAsB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAU5E;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,MAAM,GACT,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAkC7B;AAED,gCAAgC;AAChC,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAUjE;AAED,wDAAwD;AACxD,wBAAsB,cAAc,CAClC,EAAE,EAAE,MAAM,EACV,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC,CAUf"}

package/dist/integrations/pending-tasks-store.js

@@ -41,7 +41,11 @@ async function ensureTable() {
             // catch as "already-enqueued".
             await ensureExternalEventKey(client);
             await client.execute(`CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`);
-        })();
+        })().catch((err) => {
+            // Retry init on the next call after a failed startup.
+            _initPromise = undefined;
+            throw err;
+        });
     }
     return _initPromise;
 }

package/dist/integrations/pending-tasks-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"pending-tasks-store.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAEjE,IAAI,YAAuC,CAAC;AAE5C,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;;;;;;;;;qBASN,OAAO,EAAE;;uBAEP,OAAO,EAAE;uBACT,OAAO,EAAE;yBACP,OAAO,EAAE;;OAE3B,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAClB,8GAA8G,CAC/G,CAAC;YACF,oEAAoE;YACpE,oEAAoE;YACpE,+DAA+D;YAC/D,kEAAkE;YAClE,iEAAiE;YACjE,+BAA+B;YAC/B,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,MAAM,CAAC,OAAO,CAClB,0HAA0H,CAC3H,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAoC;IAEpC,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,CAAC,OAAO,CAClB,wFAAwF,CACzF,CAAC;QACF,OAAO;IACT,CAAC;IACD,wEAAwE;IACxE,uDAAuD;IACvD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAClB,0EAA0E,CAC3E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IACE,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,GAAG,CAAC;aACzB,WAAW,EAAE;aACb,QAAQ,CAAC,WAAW,CAAC,EACxB,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAwBD,SAAS,SAAS,CAAC,GAA4B;IAC7C,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAY;QACpB,QAAQ,EAAE,GAAG,CAAC,QAAkB;QAChC,gBAAgB,EAAE,GAAG,CAAC,kBAA4B;QAClD,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,UAAU,EAAE,GAAG,CAAC,WAAqB;QACrC,KAAK,EAAG,GAAG,CAAC,MAAwB,IAAI,IAAI;QAC5C,MAAM,EAAE,GAAG,CAAC,MAA2B;QACvC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnC,YAAY,EAAG,GAAG,CAAC,aAA+B,IAAI,IAAI;QAC1D,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,WAAW,EACT,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAsB,CAAC;KACvE,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAQvC;IACC,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;+CAEsC;QAC3C,IAAI,EAAE;YACJ,KAAK,CAAC,EAAE;YACR,KAAK,CAAC,QAAQ;YACd,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,OAAO;YACb,KAAK,CAAC,UAAU;YAChB,KAAK,CAAC,KAAK,IAAI,IAAI;YACnB,SAAS;YACT,CAAC;YACD,GAAG;YACH,GAAG;YACH,KAAK,CAAC,gBAAgB,IAAI,IAAI;SAC/B;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,MAAM,CAAC,GAAG,GAAiD,CAAC;IAC5D,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACrB,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,CAAC,4BAA4B;IACjE,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,OAAO,CACL,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACtB,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/B,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAC9B,CAAC;AACJ,CAAC;AAED,kCAAkC;AAClC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAU;IAC7C,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;8DACqD;QAC1D,IAAI,EAAE,CAAC,EAAE,CAAC;KACX,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAU;IAEV,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE,UAAU,EAAE;YACf,CAAC,CAAC;;;yJAGiJ;YACnJ,CAAC,CAAC;;6CAEqC;QACzC,IAAI,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;KAC9B,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;IAE/B,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;IACvD,CAAC;IAED,yEAAyE;IACzE,kDAAkD;IAClD,MAAM,QAAQ,GACX,MAAuD,CAAC,YAAY;QACpE,MAAuD,CAAC,QAAQ,CAAC;IACpE,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IAC7D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,gCAAgC;AAChC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAU;IAChD,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;KAClC,CAAC,CAAC;AACL,CAAC;AAED,wDAAwD;AACxD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,IAAI,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;KACvD,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL-backed pending task queue for integration webhooks.\n *\n * Why this exists: serverless platforms (Netlify Lambda, Vercel, Cloudflare\n * Workers) freeze the function execution as soon as the HTTP response is\n * returned. Fire-and-forget background `Promise`s get killed mid-flight,\n * meaning agent loops triggered from a Slack/Telegram webhook never finish.\n *\n * Solution: persist the inbound message to SQL inside the webhook handler,\n * then dispatch a fresh HTTP POST to a separate processor endpoint. Each\n * invocation gets its own fresh function timeout budget.\n */\nimport { getDbExec, isPostgres, intType } from \"../db/client.js\";\n\nlet _initPromise: Promise<void> | undefined;\n\nasync function ensureTable(): Promise<void> {\n  if (!_initPromise) {\n    _initPromise = (async () => {\n      const client = getDbExec();\n      await client.execute(`\n        CREATE TABLE IF NOT EXISTS integration_pending_tasks (\n          id TEXT PRIMARY KEY,\n          platform TEXT NOT NULL,\n          external_thread_id TEXT NOT NULL,\n          payload TEXT NOT NULL,\n          owner_email TEXT NOT NULL,\n          org_id TEXT,\n          status TEXT NOT NULL,\n          attempts ${intType()} NOT NULL DEFAULT 0,\n          error_message TEXT,\n          created_at ${intType()} NOT NULL,\n          updated_at ${intType()} NOT NULL,\n          completed_at ${intType()}\n        )\n      `);\n      await client.execute(\n        `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,\n      );\n      // Additive migration: add a stable per-event dedup key so duplicate\n      // webhook deliveries from the same platform get rejected at the SQL\n      // layer instead of via an in-memory Map (which doesn't survive\n      // serverless cold starts — H3 in the webhook security audit). The\n      // unique index ensures a duplicate INSERT raises an error we can\n      // catch as \"already-enqueued\".\n      await ensureExternalEventKey(client);\n      await client.execute(\n        `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,\n      );\n    })();\n  }\n  return _initPromise;\n}\n\nasync function ensureExternalEventKey(\n  client: ReturnType<typeof getDbExec>,\n): Promise<void> {\n  if (isPostgres()) {\n    await client.execute(\n      `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,\n    );\n    return;\n  }\n  // SQLite doesn't support `ADD COLUMN IF NOT EXISTS` until 3.35; swallow\n  // the duplicate-column error so reruns are idempotent.\n  try {\n    await client.execute(\n      `ALTER TABLE integration_pending_tasks ADD COLUMN external_event_key TEXT`,\n    );\n  } catch (err: any) {\n    if (\n      !String(err?.message ?? err)\n        .toLowerCase()\n        .includes(\"duplicate\")\n    ) {\n      throw err;\n    }\n  }\n}\n\n/** Status values for an integration pending task. */\nexport type PendingTaskStatus =\n  | \"pending\"\n  | \"processing\"\n  | \"completed\"\n  | \"failed\";\n\nexport interface PendingTask {\n  id: string;\n  platform: string;\n  externalThreadId: string;\n  payload: string;\n  ownerEmail: string;\n  orgId: string | null;\n  status: PendingTaskStatus;\n  attempts: number;\n  errorMessage: string | null;\n  createdAt: number;\n  updatedAt: number;\n  completedAt: number | null;\n}\n\nfunction rowToTask(row: Record<string, unknown>): PendingTask {\n  return {\n    id: row.id as string,\n    platform: row.platform as string,\n    externalThreadId: row.external_thread_id as string,\n    payload: row.payload as string,\n    ownerEmail: row.owner_email as string,\n    orgId: (row.org_id as string | null) ?? null,\n    status: row.status as PendingTaskStatus,\n    attempts: Number(row.attempts ?? 0),\n    errorMessage: (row.error_message as string | null) ?? null,\n    createdAt: Number(row.created_at ?? 0),\n    updatedAt: Number(row.updated_at ?? 0),\n    completedAt:\n      row.completed_at == null ? null : Number(row.completed_at as number),\n  };\n}\n\n/**\n * Insert a new pending task. Returns the generated task id.\n *\n * If `externalEventKey` is supplied, the unique index on\n * `(platform, external_event_key)` will reject duplicates — callers should\n * catch the resulting constraint-violation error and treat it as\n * \"already enqueued\" instead of a hard failure (H3 in the webhook security\n * audit). This is the SQL-backed replacement for the in-memory dedup map.\n */\nexport async function insertPendingTask(input: {\n  id: string;\n  platform: string;\n  externalThreadId: string;\n  payload: string;\n  ownerEmail: string;\n  orgId?: string | null;\n  externalEventKey?: string | null;\n}): Promise<void> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `INSERT INTO integration_pending_tasks\n      (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key)\n      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n    args: [\n      input.id,\n      input.platform,\n      input.externalThreadId,\n      input.payload,\n      input.ownerEmail,\n      input.orgId ?? null,\n      \"pending\",\n      0,\n      now,\n      now,\n      input.externalEventKey ?? null,\n    ],\n  });\n}\n\n/**\n * Returns whether a duplicate-event error from `insertPendingTask` looks\n * like a unique-constraint violation on `(platform, external_event_key)`.\n *\n * Postgres surfaces these as `error.code === \"23505\"`, while SQLite uses\n * a substring match on the error text. Used by the webhook handler to\n * distinguish \"already enqueued\" (silently OK) from genuine insert failures.\n */\nexport function isDuplicateEventError(err: unknown): boolean {\n  const e = err as { code?: string; message?: string } | null;\n  if (!e) return false;\n  if (e.code === \"23505\") return true; // Postgres unique-violation\n  const msg = String(e.message ?? \"\").toLowerCase();\n  return (\n    msg.includes(\"unique\") ||\n    msg.includes(\"duplicate entry\") ||\n    msg.includes(\"duplicate key\")\n  );\n}\n\n/** Fetch a pending task by id. */\nexport async function getPendingTask(id: string): Promise<PendingTask | null> {\n  await ensureTable();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at\n          FROM integration_pending_tasks WHERE id = ? LIMIT 1`,\n    args: [id],\n  });\n  if (rows.length === 0) return null;\n  return rowToTask(rows[0] as Record<string, unknown>);\n}\n\n/**\n * Atomically claim a task: transition pending → processing and increment\n * attempts. Returns the updated task if the transition succeeded, otherwise\n * null (e.g. the task was already claimed by a concurrent worker).\n */\nexport async function claimPendingTask(\n  id: string,\n): Promise<PendingTask | null> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n\n  // Conditional update: only flip if currently pending. Failed tasks are\n  // terminal unless an explicit retry path resets them to pending first.\n  const result = await client.execute({\n    sql: isPostgres()\n      ? `UPDATE integration_pending_tasks\n         SET status = ?, attempts = attempts + 1, updated_at = ?\n         WHERE id = ? AND status = 'pending'\n         RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at`\n      : `UPDATE integration_pending_tasks\n         SET status = ?, attempts = attempts + 1, updated_at = ?\n         WHERE id = ? AND status = 'pending'`,\n    args: [\"processing\", now, id],\n  });\n  const rows = result.rows ?? [];\n\n  if (isPostgres()) {\n    if (rows.length === 0) return null;\n    return rowToTask(rows[0] as Record<string, unknown>);\n  }\n\n  // SQLite: no RETURNING, so re-read after the update. Confirm we actually\n  // moved it into 'processing' (vs. lost the race).\n  const affected =\n    (result as { rowsAffected?: number; rowCount?: number }).rowsAffected ??\n    (result as { rowsAffected?: number; rowCount?: number }).rowCount;\n  if (affected === 0) return null;\n  const fetched = await getPendingTask(id);\n  if (!fetched || fetched.status !== \"processing\") return null;\n  return fetched;\n}\n\n/** Mark a task as completed. */\nexport async function markTaskCompleted(id: string): Promise<void> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `UPDATE integration_pending_tasks\n          SET status = ?, updated_at = ?, completed_at = ?\n          WHERE id = ?`,\n    args: [\"completed\", now, now, id],\n  });\n}\n\n/** Mark a task as failed and stash an error message. */\nexport async function markTaskFailed(\n  id: string,\n  errorMessage: string,\n): Promise<void> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `UPDATE integration_pending_tasks\n          SET status = ?, updated_at = ?, error_message = ?\n          WHERE id = ?`,\n    args: [\"failed\", now, errorMessage.slice(0, 2000), id],\n  });\n}\n"]}
+{"version":3,"file":"pending-tasks-store.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAEjE,IAAI,YAAuC,CAAC;AAE5C,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,MAAM,CAAC,OAAO,CAAC;;;;;;;;;qBASN,OAAO,EAAE;;uBAEP,OAAO,EAAE;uBACT,OAAO,EAAE;yBACP,OAAO,EAAE;;OAE3B,CAAC,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAClB,8GAA8G,CAC/G,CAAC;YACF,oEAAoE;YACpE,oEAAoE;YACpE,+DAA+D;YAC/D,kEAAkE;YAClE,iEAAiE;YACjE,+BAA+B;YAC/B,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,MAAM,CAAC,OAAO,CAClB,0HAA0H,CAC3H,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACjB,sDAAsD;YACtD,YAAY,GAAG,SAAS,CAAC;YACzB,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAoC;IAEpC,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,CAAC,OAAO,CAClB,wFAAwF,CACzF,CAAC;QACF,OAAO;IACT,CAAC;IACD,wEAAwE;IACxE,uDAAuD;IACvD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAClB,0EAA0E,CAC3E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IACE,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,GAAG,CAAC;aACzB,WAAW,EAAE;aACb,QAAQ,CAAC,WAAW,CAAC,EACxB,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAwBD,SAAS,SAAS,CAAC,GAA4B;IAC7C,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAY;QACpB,QAAQ,EAAE,GAAG,CAAC,QAAkB;QAChC,gBAAgB,EAAE,GAAG,CAAC,kBAA4B;QAClD,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,UAAU,EAAE,GAAG,CAAC,WAAqB;QACrC,KAAK,EAAG,GAAG,CAAC,MAAwB,IAAI,IAAI;QAC5C,MAAM,EAAE,GAAG,CAAC,MAA2B;QACvC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnC,YAAY,EAAG,GAAG,CAAC,aAA+B,IAAI,IAAI;QAC1D,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,WAAW,EACT,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAsB,CAAC;KACvE,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAQvC;IACC,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;+CAEsC;QAC3C,IAAI,EAAE;YACJ,KAAK,CAAC,EAAE;YACR,KAAK,CAAC,QAAQ;YACd,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,OAAO;YACb,KAAK,CAAC,UAAU;YAChB,KAAK,CAAC,KAAK,IAAI,IAAI;YACnB,SAAS;YACT,CAAC;YACD,GAAG;YACH,GAAG;YACH,KAAK,CAAC,gBAAgB,IAAI,IAAI;SAC/B;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,MAAM,CAAC,GAAG,GAAiD,CAAC;IAC5D,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACrB,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,CAAC,4BAA4B;IACjE,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,OAAO,CACL,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACtB,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/B,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAC9B,CAAC;AACJ,CAAC;AAED,kCAAkC;AAClC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAU;IAC7C,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;8DACqD;QAC1D,IAAI,EAAE,CAAC,EAAE,CAAC;KACX,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;AACvD,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAU;IAEV,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE,UAAU,EAAE;YACf,CAAC,CAAC;;;yJAGiJ;YACnJ,CAAC,CAAC;;6CAEqC;QACzC,IAAI,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;KAC9B,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;IAE/B,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;IACvD,CAAC;IAED,yEAAyE;IACzE,kDAAkD;IAClD,MAAM,QAAQ,GACX,MAAuD,CAAC,YAAY;QACpE,MAAuD,CAAC,QAAQ,CAAC;IACpE,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IAC7D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,gCAAgC;AAChC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAU;IAChD,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC;KAClC,CAAC,CAAC;AACL,CAAC;AAED,wDAAwD;AACxD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,IAAI,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;KACvD,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL-backed pending task queue for integration webhooks.\n *\n * Why this exists: serverless platforms (Netlify Lambda, Vercel, Cloudflare\n * Workers) freeze the function execution as soon as the HTTP response is\n * returned. Fire-and-forget background `Promise`s get killed mid-flight,\n * meaning agent loops triggered from a Slack/Telegram webhook never finish.\n *\n * Solution: persist the inbound message to SQL inside the webhook handler,\n * then dispatch a fresh HTTP POST to a separate processor endpoint. Each\n * invocation gets its own fresh function timeout budget.\n */\nimport { getDbExec, isPostgres, intType } from \"../db/client.js\";\n\nlet _initPromise: Promise<void> | undefined;\n\nasync function ensureTable(): Promise<void> {\n  if (!_initPromise) {\n    _initPromise = (async () => {\n      const client = getDbExec();\n      await client.execute(`\n        CREATE TABLE IF NOT EXISTS integration_pending_tasks (\n          id TEXT PRIMARY KEY,\n          platform TEXT NOT NULL,\n          external_thread_id TEXT NOT NULL,\n          payload TEXT NOT NULL,\n          owner_email TEXT NOT NULL,\n          org_id TEXT,\n          status TEXT NOT NULL,\n          attempts ${intType()} NOT NULL DEFAULT 0,\n          error_message TEXT,\n          created_at ${intType()} NOT NULL,\n          updated_at ${intType()} NOT NULL,\n          completed_at ${intType()}\n        )\n      `);\n      await client.execute(\n        `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,\n      );\n      // Additive migration: add a stable per-event dedup key so duplicate\n      // webhook deliveries from the same platform get rejected at the SQL\n      // layer instead of via an in-memory Map (which doesn't survive\n      // serverless cold starts — H3 in the webhook security audit). The\n      // unique index ensures a duplicate INSERT raises an error we can\n      // catch as \"already-enqueued\".\n      await ensureExternalEventKey(client);\n      await client.execute(\n        `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,\n      );\n    })().catch((err) => {\n      // Retry init on the next call after a failed startup.\n      _initPromise = undefined;\n      throw err;\n    });\n  }\n  return _initPromise;\n}\n\nasync function ensureExternalEventKey(\n  client: ReturnType<typeof getDbExec>,\n): Promise<void> {\n  if (isPostgres()) {\n    await client.execute(\n      `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,\n    );\n    return;\n  }\n  // SQLite doesn't support `ADD COLUMN IF NOT EXISTS` until 3.35; swallow\n  // the duplicate-column error so reruns are idempotent.\n  try {\n    await client.execute(\n      `ALTER TABLE integration_pending_tasks ADD COLUMN external_event_key TEXT`,\n    );\n  } catch (err: any) {\n    if (\n      !String(err?.message ?? err)\n        .toLowerCase()\n        .includes(\"duplicate\")\n    ) {\n      throw err;\n    }\n  }\n}\n\n/** Status values for an integration pending task. */\nexport type PendingTaskStatus =\n  | \"pending\"\n  | \"processing\"\n  | \"completed\"\n  | \"failed\";\n\nexport interface PendingTask {\n  id: string;\n  platform: string;\n  externalThreadId: string;\n  payload: string;\n  ownerEmail: string;\n  orgId: string | null;\n  status: PendingTaskStatus;\n  attempts: number;\n  errorMessage: string | null;\n  createdAt: number;\n  updatedAt: number;\n  completedAt: number | null;\n}\n\nfunction rowToTask(row: Record<string, unknown>): PendingTask {\n  return {\n    id: row.id as string,\n    platform: row.platform as string,\n    externalThreadId: row.external_thread_id as string,\n    payload: row.payload as string,\n    ownerEmail: row.owner_email as string,\n    orgId: (row.org_id as string | null) ?? null,\n    status: row.status as PendingTaskStatus,\n    attempts: Number(row.attempts ?? 0),\n    errorMessage: (row.error_message as string | null) ?? null,\n    createdAt: Number(row.created_at ?? 0),\n    updatedAt: Number(row.updated_at ?? 0),\n    completedAt:\n      row.completed_at == null ? null : Number(row.completed_at as number),\n  };\n}\n\n/**\n * Insert a new pending task. Returns the generated task id.\n *\n * If `externalEventKey` is supplied, the unique index on\n * `(platform, external_event_key)` will reject duplicates — callers should\n * catch the resulting constraint-violation error and treat it as\n * \"already enqueued\" instead of a hard failure (H3 in the webhook security\n * audit). This is the SQL-backed replacement for the in-memory dedup map.\n */\nexport async function insertPendingTask(input: {\n  id: string;\n  platform: string;\n  externalThreadId: string;\n  payload: string;\n  ownerEmail: string;\n  orgId?: string | null;\n  externalEventKey?: string | null;\n}): Promise<void> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `INSERT INTO integration_pending_tasks\n      (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key)\n      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n    args: [\n      input.id,\n      input.platform,\n      input.externalThreadId,\n      input.payload,\n      input.ownerEmail,\n      input.orgId ?? null,\n      \"pending\",\n      0,\n      now,\n      now,\n      input.externalEventKey ?? null,\n    ],\n  });\n}\n\n/**\n * Returns whether a duplicate-event error from `insertPendingTask` looks\n * like a unique-constraint violation on `(platform, external_event_key)`.\n *\n * Postgres surfaces these as `error.code === \"23505\"`, while SQLite uses\n * a substring match on the error text. Used by the webhook handler to\n * distinguish \"already enqueued\" (silently OK) from genuine insert failures.\n */\nexport function isDuplicateEventError(err: unknown): boolean {\n  const e = err as { code?: string; message?: string } | null;\n  if (!e) return false;\n  if (e.code === \"23505\") return true; // Postgres unique-violation\n  const msg = String(e.message ?? \"\").toLowerCase();\n  return (\n    msg.includes(\"unique\") ||\n    msg.includes(\"duplicate entry\") ||\n    msg.includes(\"duplicate key\")\n  );\n}\n\n/** Fetch a pending task by id. */\nexport async function getPendingTask(id: string): Promise<PendingTask | null> {\n  await ensureTable();\n  const client = getDbExec();\n  const { rows } = await client.execute({\n    sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at\n          FROM integration_pending_tasks WHERE id = ? LIMIT 1`,\n    args: [id],\n  });\n  if (rows.length === 0) return null;\n  return rowToTask(rows[0] as Record<string, unknown>);\n}\n\n/**\n * Atomically claim a task: transition pending → processing and increment\n * attempts. Returns the updated task if the transition succeeded, otherwise\n * null (e.g. the task was already claimed by a concurrent worker).\n */\nexport async function claimPendingTask(\n  id: string,\n): Promise<PendingTask | null> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n\n  // Conditional update: only flip if currently pending. Failed tasks are\n  // terminal unless an explicit retry path resets them to pending first.\n  const result = await client.execute({\n    sql: isPostgres()\n      ? `UPDATE integration_pending_tasks\n         SET status = ?, attempts = attempts + 1, updated_at = ?\n         WHERE id = ? AND status = 'pending'\n         RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at`\n      : `UPDATE integration_pending_tasks\n         SET status = ?, attempts = attempts + 1, updated_at = ?\n         WHERE id = ? AND status = 'pending'`,\n    args: [\"processing\", now, id],\n  });\n  const rows = result.rows ?? [];\n\n  if (isPostgres()) {\n    if (rows.length === 0) return null;\n    return rowToTask(rows[0] as Record<string, unknown>);\n  }\n\n  // SQLite: no RETURNING, so re-read after the update. Confirm we actually\n  // moved it into 'processing' (vs. lost the race).\n  const affected =\n    (result as { rowsAffected?: number; rowCount?: number }).rowsAffected ??\n    (result as { rowsAffected?: number; rowCount?: number }).rowCount;\n  if (affected === 0) return null;\n  const fetched = await getPendingTask(id);\n  if (!fetched || fetched.status !== \"processing\") return null;\n  return fetched;\n}\n\n/** Mark a task as completed. */\nexport async function markTaskCompleted(id: string): Promise<void> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `UPDATE integration_pending_tasks\n          SET status = ?, updated_at = ?, completed_at = ?\n          WHERE id = ?`,\n    args: [\"completed\", now, now, id],\n  });\n}\n\n/** Mark a task as failed and stash an error message. */\nexport async function markTaskFailed(\n  id: string,\n  errorMessage: string,\n): Promise<void> {\n  await ensureTable();\n  const client = getDbExec();\n  const now = Date.now();\n  await client.execute({\n    sql: `UPDATE integration_pending_tasks\n          SET status = ?, updated_at = ?, error_message = ?\n          WHERE id = ?`,\n    args: [\"failed\", now, errorMessage.slice(0, 2000), id],\n  });\n}\n"]}

package/dist/integrations/plugin.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../../src/integrations/plugin.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAEV,yBAAyB,EAE1B,MAAM,YAAY,CAAC;AAkEpB,KAAK,cAAc,GAAG,CAAC,QAAQ,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AA0F9D,KAAK,yBAAyB,GAAG;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAIF,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,yBAAyB,GAClC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAsFlC;AA6ND;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,CAAC,EAAE,yBAAyB,GAClC,cAAc,CAwmChB;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,gBAA6B,CAAC"}
+{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../../src/integrations/plugin.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAEV,yBAAyB,EAE1B,MAAM,YAAY,CAAC;AAmEpB,KAAK,cAAc,GAAG,CAAC,QAAQ,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AA0F9D,KAAK,yBAAyB,GAAG;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAIF,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,yBAAyB,GAClC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAsFlC;AA6ND;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,CAAC,EAAE,yBAAyB,GAClC,cAAc,CAunChB;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,gBAA6B,CAAC"}

package/dist/integrations/plugin.js

@@ -16,6 +16,7 @@ import { emailAdapter } from "./adapters/email.js";
 import { startGoogleDocsPoller, handlePushNotification, } from "./google-docs-poller.js";
 import { startPendingTasksRetryJob } from "./pending-tasks-retry-job.js";
 import { processA2AContinuationById, processDueA2AContinuations, } from "./a2a-continuation-processor.js";
+import { failA2AContinuation } from "./a2a-continuations-store.js";
 import { loadResourcesForPrompt } from "../server/agent-chat-plugin.js";
 import { getTaskQueueStats } from "./task-queue-stats.js";
 import { getSession } from "../server/auth.js";
@@ -1110,9 +1111,19 @@ export function createIntegrationsPlugin(options) {
                     return { error: "Invalid or expired internal token" };
                 }
             }
-            await processA2AContinuationById(continuationId, {
-                adapters: adapterMap,
-            });
+            try {
+                await processA2AContinuationById(continuationId, {
+                    adapters: adapterMap,
+                });
+            }
+            catch (err) {
+                // Mark the continuation failed so it isn't left dangling, and surface
+                // a 500 to the caller instead of leaking an unhandled rejection.
+                await failA2AContinuation(continuationId, err?.message?.slice(0, 500) || "continuation processing failed").catch(() => { });
+                console.error("[integrations] process-a2a-continuation failure:", err);
+                setResponseStatus(event, 500);
+                return { error: "Failed to process A2A continuation" };
+            }
             return { ok: true, continuationId };
         }));
         // ─── Per-platform catch-all ───────────────────────────────────