@agent-native/core 0.119.5 → 0.119.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +6 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/durable-background.ts +1 -0
- package/corpus/core/src/deploy/build.ts +144 -2
- package/corpus/core/src/deploy/workspace-deploy.ts +123 -1
- package/corpus/core/src/integrations/integration-durable-dispatch-config.ts +22 -0
- package/corpus/core/src/integrations/integration-durable-dispatch.ts +190 -0
- package/corpus/core/src/integrations/pending-tasks-retry-job.ts +175 -87
- package/corpus/core/src/integrations/pending-tasks-store.ts +114 -11
- package/corpus/core/src/integrations/plugin.ts +113 -60
- package/corpus/core/src/integrations/task-queue-stats.ts +68 -15
- package/corpus/core/src/integrations/webhook-handler.ts +21 -46
- package/corpus/core/src/server/auth.ts +6 -0
- package/corpus/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/corpus/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/corpus/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/corpus/templates/dispatch/changelog/2026-07-21-recover-stalled-slack-agent-work.md +6 -0
- package/dist/agent/durable-background.d.ts +1 -0
- package/dist/agent/durable-background.d.ts.map +1 -1
- package/dist/agent/durable-background.js +1 -0
- package/dist/agent/durable-background.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/deploy/build.d.ts +3 -0
- package/dist/deploy/build.d.ts.map +1 -1
- package/dist/deploy/build.js +106 -3
- package/dist/deploy/build.js.map +1 -1
- package/dist/deploy/workspace-deploy.d.ts.map +1 -1
- package/dist/deploy/workspace-deploy.js +103 -2
- package/dist/deploy/workspace-deploy.js.map +1 -1
- package/dist/file-upload/actions/upload-image.d.ts +1 -1
- package/dist/integrations/integration-durable-dispatch-config.d.ts +7 -0
- package/dist/integrations/integration-durable-dispatch-config.d.ts.map +1 -0
- package/dist/integrations/integration-durable-dispatch-config.js +16 -0
- package/dist/integrations/integration-durable-dispatch-config.js.map +1 -0
- package/dist/integrations/integration-durable-dispatch.d.ts +23 -0
- package/dist/integrations/integration-durable-dispatch.d.ts.map +1 -0
- package/dist/integrations/integration-durable-dispatch.js +115 -0
- package/dist/integrations/integration-durable-dispatch.js.map +1 -0
- package/dist/integrations/pending-tasks-retry-job.d.ts +13 -1
- package/dist/integrations/pending-tasks-retry-job.d.ts.map +1 -1
- package/dist/integrations/pending-tasks-retry-job.js +133 -74
- package/dist/integrations/pending-tasks-retry-job.js.map +1 -1
- package/dist/integrations/pending-tasks-store.d.ts +14 -2
- package/dist/integrations/pending-tasks-store.d.ts.map +1 -1
- package/dist/integrations/pending-tasks-store.js +75 -11
- package/dist/integrations/pending-tasks-store.js.map +1 -1
- package/dist/integrations/plugin.d.ts.map +1 -1
- package/dist/integrations/plugin.js +88 -55
- package/dist/integrations/plugin.js.map +1 -1
- package/dist/integrations/task-queue-stats.d.ts +15 -3
- package/dist/integrations/task-queue-stats.d.ts.map +1 -1
- package/dist/integrations/task-queue-stats.js +45 -15
- package/dist/integrations/task-queue-stats.js.map +1 -1
- package/dist/integrations/webhook-handler.d.ts +1 -1
- package/dist/integrations/webhook-handler.d.ts.map +1 -1
- package/dist/integrations/webhook-handler.js +18 -42
- package/dist/integrations/webhook-handler.js.map +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/routes.d.ts +5 -5
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/actions/custom-provider-registration.d.ts +5 -5
- package/dist/provider-api/actions/provider-api.d.ts +4 -4
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +3 -3
- package/dist/server/auth.d.ts.map +1 -1
- package/dist/server/auth.js +5 -0
- package/dist/server/auth.js.map +1 -1
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/dist/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/dist/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/dist/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/package.json +1 -1
- package/src/agent/durable-background.ts +1 -0
- package/src/deploy/build.ts +144 -2
- package/src/deploy/workspace-deploy.ts +123 -1
- package/src/integrations/integration-durable-dispatch-config.ts +22 -0
- package/src/integrations/integration-durable-dispatch.ts +190 -0
- package/src/integrations/pending-tasks-retry-job.ts +175 -87
- package/src/integrations/pending-tasks-store.ts +114 -11
- package/src/integrations/plugin.ts +113 -60
- package/src/integrations/task-queue-stats.ts +68 -15
- package/src/integrations/webhook-handler.ts +21 -46
- package/src/server/auth.ts +6 -0
- package/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pending-tasks-retry-job.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-retry-job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAE,yBAAyB,EAAE,MAAM,4BAA4B,CAAC;AACvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,yBAAyB,EAAE,MAAM,0BAA0B,CAAC;AAErE;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,8EAA8E;AAC9E,MAAM,sBAAsB,GAAG,MAAM,CAAC;AACtC,4EAA4E;AAC5E,MAAM,iCAAiC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACxD,MAAM,oCAAoC,GAAG,MAAM,CAAC;AACpD,mEAAmE;AACnE,MAAM,cAAc,GAAG,GAAG,sBAAsB,4BAA4B,CAAC;AAE7E,IAAI,aAAa,GAA0C,IAAI,CAAC;AAChE,IAAI,YAAY,GAAyC,IAAI,CAAC;AAC9D,IAAI,oBAAwC,CAAC;AAC7C;;;GAGG;AACH,IAAI,WAAW,GAAmB,IAAI,CAAC;AAQvC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,cAAuB;IAEvB,MAAM,OAAO,GAAG,cAAc,IAAI,oBAAoB,CAAC;IACvD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,aAAa,GAAG,GAAG,GAAG,sBAAsB,CAAC;IACnD,MAAM,gBAAgB,GAAG,GAAG,GAAG,yBAAyB,EAAE,CAAC;IAE3D,IAAI,SAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YACpC,GAAG,EAAE;;;;;OAKJ;YACD,wEAAwE;YACxE,qEAAqE;YACrE,wEAAwE;YACxE,wEAAwE;YACxE,IAAI,EAAE,CAAC,aAAa,EAAE,aAAa,EAAE,gBAAgB,CAAC;SACvD,CAAC,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,EAAE,EAAE,CAAC,CAAC,EAAY;YAClB,MAAM,EAAE,CAAC,CAAC,MAAgB;YAC1B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;SAClC,CAAC,CAAC,CAAC;QACJ,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,yEAAyE;QACzE,sEAAsE;QACtE,iCAAiC;QACjC,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,WAAW,GAAG,KAAK,CAAC;YACpB,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CACT,yEAAyE,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEnC,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,kEAAkE;YAClE,0CAA0C;YAC1C,IAAI,GAAG,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;gBAC9C,MAAM,MAAM,CAAC,OAAO,CAAC;oBACnB,GAAG,EAAE;;;;;;;;;WASJ;oBACD,IAAI,EAAE;wBACJ,IAAI,CAAC,GAAG,EAAE;wBACV,uBAAuB,yBAAyB,WAAW;wBAC3D,GAAG,CAAC,EAAE;wBACN,GAAG,CAAC,MAAM;qBACX;iBACF,CAAC,CAAC;gBACH,OAAO,CAAC,IAAI,CACV,+BAA+B,GAAG,CAAC,EAAE,aAAa,yBAAyB,4BAA4B,CACxG,CAAC;gBACF,SAAS;YACX,CAAC;YAED,qEAAqE;YACrE,+DAA+D;YAC/D,oDAAoD;YACpD,yEAAyE;YACzE,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;YACvE,MAAM,MAAM,CAAC,OAAO,CAAC;gBACnB,GAAG,EAAE;;;;;SAKJ;gBACD,IAAI,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,CAAC;aAClD,CAAC,CAAC;YAEH,MAAM,eAAe,CAAC,GAAG,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,CAAC,KAAK,CACX,+CAA+C,GAAG,CAAC,EAAE,GAAG,EACxD,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB;IAChC,IACE,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,UAAU,IAAI,UAAU,EACxB,CAAC;QACD,OAAO,oCAAoC,CAAC;IAC9C,CAAC;IACD,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,MAAc,EACd,cAAkC;IAElC,MAAM,OAAO,GACX,cAAc;QACd,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAC5B,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,GAAG;QACf,OAAO,CAAC,GAAG,CAAC,UAAU;QACtB,oBAAoB,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;IAEjD,MAAM,GAAG,GAAG,GAAG,yBAAyB,CAAC,OAAO,CAAC,GAAG,cAAc,EAAE,CAAC;IAErE,2EAA2E;IAC3E,0EAA0E;IAC1E,yEAAyE;IACzE,wEAAwE;IACxE,sBAAsB;IACtB,MAAM,OAAO,GAA2B;QACtC,cAAc,EAAE,kBAAkB;KACnC,CAAC;IACF,IAAI,CAAC;QACH,OAAO,CAAC,eAAe,CAAC,GAAG,UAAU,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC;IACnE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC1C,OAAO,CAAC,KAAK,CACX,4CAA4C,MAAM,gCAAgC;gBAChF,mDAAmD,CACtD,CAAC;YACF,OAAO;QACT,CAAC;QACD,sEAAsE;QACtE,6EAA6E;QAC7E,IAAI,GAAG,YAAY,KAAK,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7D,OAAO,CAAC,KAAK,CACX,4DAA4D,MAAM,GAAG,EACrE,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IAED,oEAAoE;IACpE,sEAAsE;IACtE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,KAAK,CAAC,CAAC;IAE1D,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE;YACf,MAAM,EAAE,MAAM;YACd,OAAO;YACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;YAChC,MAAM,EAAE,UAAU,CAAC,MAAM;SAC1B,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAEzC;IACC,IAAI,aAAa;QAAE,OAAO;IAC1B,oBAAoB,GAAG,OAAO,EAAE,cAAc,CAAC;IAE/C,6EAA6E;IAC7E,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QAC7B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACX,UAAU,CAAC,YAAY,CAAC,CAAC;IAEzB,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QAC/B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACtB,UAAU,CAAC,aAAa,CAAC,CAAC;IAE1B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,yDACE,iBAAiB,GAAG,IACtB,IAAI,CACL,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,wBAAwB;IACtC,IAAI,YAAY,EAAE,CAAC;QACjB,YAAY,CAAC,YAAY,CAAC,CAAC;QAC3B,YAAY,GAAG,IAAI,CAAC;IACtB,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,aAAa,CAAC,aAAa,CAAC,CAAC;QAC7B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,oBAAoB,GAAG,SAAS,CAAC;AACnC,CAAC;AAED,SAAS,UAAU,CAAC,KAAqC;IACtD,KAA2C,CAAC,KAAK,EAAE,EAAE,CAAC;AACzD,CAAC","sourcesContent":["import { getDbExec } from \"../db/client.js\";\nimport { withConfiguredAppBasePath } from \"../server/app-base-path.js\";\nimport { FRAMEWORK_ROUTE_PREFIX } from \"../server/core-routes-plugin.js\";\nimport { signInternalToken } from \"./internal-token.js\";\nimport { MAX_PENDING_TASK_ATTEMPTS } from \"./pending-tasks-store.js\";\n\n/**\n * Retries stuck integration webhook tasks.\n *\n * The integration webhook flow enqueues work into `integration_pending_tasks`\n * (see `pending-tasks-store.ts`) and then fires a self-webhook to the\n * `/_agent-native/integrations/process-task` endpoint to drain the queue.\n * If that fire-and-forget dispatch fails (e.g. transient network blip), the\n * row stays in `pending` forever. Likewise, if the processor is killed mid-\n * processing (function timeout, container shutdown), a row can remain in\n * `processing` forever.\n *\n * This job runs every 60s and re-fires the processor endpoint for tasks that\n * look stuck:\n * - status='pending' AND created_at older than 90s (initial dispatch lost)\n * - status='processing' AND updated_at older than the host-specific\n * function budget (75s on serverless, 5min elsewhere)\n *\n * Retries are capped at MAX_ATTEMPTS attempts; after that the row is marked\n * `failed` permanently so it stops being retried.\n *\n * If the `integration_pending_tasks` table does not yet exist (e.g. older\n * deploy that hasn't run the new webhook flow), this job no-ops silently\n * rather than spamming logs.\n */\n\nconst RETRY_INTERVAL_MS = 60_000;\n/** Tasks pending longer than this are considered stuck on initial dispatch */\nconst PENDING_STUCK_AFTER_MS = 90_000;\n/** Tasks \"processing\" longer than this are considered killed mid-flight. */\nconst DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000;\nconst SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000;\n/** After this many attempts we give up and mark the task failed */\nconst PROCESSOR_PATH = `${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`;\n\nlet retryInterval: ReturnType<typeof setInterval> | null = null;\nlet initialTimer: ReturnType<typeof setTimeout> | null = null;\nlet activeWebhookBaseUrl: string | undefined;\n/**\n * Whether the table exists. Cached after first probe so we don't log every\n * minute when the queue isn't in use yet on a given deployment.\n */\nlet tableExists: boolean | null = null;\n\ninterface StuckTaskRow {\n id: string;\n status: string;\n attempts: number;\n}\n\n/**\n * One pass: find stuck tasks and re-fire the processor for each.\n * Exported for tests and for manual triggers.\n */\nexport async function retryStuckPendingTasks(\n webhookBaseUrl?: string,\n): Promise<void> {\n const baseUrl = webhookBaseUrl ?? activeWebhookBaseUrl;\n const client = getDbExec();\n const now = Date.now();\n const pendingCutoff = now - PENDING_STUCK_AFTER_MS;\n const processingCutoff = now - getProcessingStuckAfterMs();\n\n let stuckRows: StuckTaskRow[];\n try {\n const { rows } = await client.execute({\n sql: `\n SELECT id, status, attempts\n FROM integration_pending_tasks\n WHERE (status = 'pending' AND created_at <= ? AND updated_at <= ?)\n OR (status = 'processing' AND updated_at <= ?)\n `,\n // `updated_at` is initialized to `created_at` on insert, so a genuinely\n // stuck pending row still matches on the first sweep. The retry path\n // below touches `updated_at`, which (with this predicate) keeps the row\n // from being re-selected — and re-firing the processor — on every tick.\n args: [pendingCutoff, pendingCutoff, processingCutoff],\n });\n stuckRows = rows.map((r) => ({\n id: r.id as string,\n status: r.status as string,\n attempts: Number(r.attempts ?? 0),\n }));\n tableExists = true;\n } catch (err) {\n // Most common case: the table hasn't been created yet because no inbound\n // integration webhook has been processed on this deployment. Silently\n // no-op until the table appears.\n if (tableExists !== false) {\n tableExists = false;\n if (process.env.DEBUG) {\n console.log(\n \"[integrations] pending-tasks retry job: table not present yet, skipping\",\n );\n }\n }\n return;\n }\n\n if (stuckRows.length === 0) return;\n\n for (const row of stuckRows) {\n try {\n // Cap retries — mark failed and move on so the row stops bouncing\n // between pending and processing forever.\n if (row.attempts >= MAX_PENDING_TASK_ATTEMPTS) {\n await client.execute({\n sql: `\n UPDATE integration_pending_tasks\n SET status = 'failed',\n updated_at = ?,\n error_message = COALESCE(error_message, ?),\n payload = '{}',\n external_event_key = NULL\n WHERE id = ?\n AND status = ?\n `,\n args: [\n Date.now(),\n `Retry job: exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts`,\n row.id,\n row.status,\n ],\n });\n console.warn(\n `[integrations] Pending task ${row.id} exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts — marking failed`,\n );\n continue;\n }\n\n // Reset stuck `processing` rows back to `pending` so the processor's\n // atomic claim (which only matches pending) can re-acquire it.\n // Without this, processing rows stay stuck forever.\n // For pending rows, just touch updated_at to avoid re-firing every tick.\n const newStatus = row.status === \"processing\" ? \"pending\" : row.status;\n await client.execute({\n sql: `\n UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?\n WHERE id = ?\n AND status = ?\n `,\n args: [newStatus, Date.now(), row.id, row.status],\n });\n\n await refireProcessor(row.id, baseUrl);\n } catch (err) {\n console.error(\n `[integrations] Failed to retry pending task ${row.id}:`,\n err,\n );\n }\n }\n}\n\nfunction getProcessingStuckAfterMs(): number {\n if (\n process.env.NETLIFY ||\n process.env.AWS_LAMBDA_FUNCTION_NAME ||\n process.env.VERCEL ||\n \"__cf_env\" in globalThis\n ) {\n return SERVERLESS_PROCESSING_STUCK_AFTER_MS;\n }\n return DEFAULT_PROCESSING_STUCK_AFTER_MS;\n}\n\n/**\n * Fire-and-forget POST to the processor endpoint for a single task id.\n * Mirrors the original dispatch from the webhook handler, including the\n * short-lived HMAC bearer token bound to this taskId.\n */\nasync function refireProcessor(\n taskId: string,\n webhookBaseUrl: string | undefined,\n): Promise<void> {\n const baseUrl =\n webhookBaseUrl ||\n process.env.WEBHOOK_BASE_URL ||\n process.env.APP_URL ||\n process.env.URL ||\n process.env.DEPLOY_URL ||\n `http://localhost:${process.env.PORT || 3000}`;\n\n const url = `${withConfiguredAppBasePath(baseUrl)}${PROCESSOR_PATH}`;\n\n // Sign with HMAC if A2A_SECRET is configured. In production we MUST sign —\n // an unsigned dispatch in production lets attackers re-trigger any queued\n // task with a guessable id (C3 in the webhook security audit). In dev we\n // fall back to unsigned so contributors can iterate without configuring\n // A2A_SECRET locally.\n const headers: Record<string, string> = {\n \"Content-Type\": \"application/json\",\n };\n try {\n headers[\"Authorization\"] = `Bearer ${signInternalToken(taskId)}`;\n } catch (err) {\n if (process.env.NODE_ENV === \"production\") {\n console.error(\n `[integrations] Refusing to dispatch task ${taskId} — A2A_SECRET not configured. ` +\n \"Set A2A_SECRET to enable signed retry dispatches.\",\n );\n return;\n }\n // Dev: proceed unsigned. Log the underlying error path so a malformed\n // secret (different from \"not set\") doesn't fail silently (L5 in the audit).\n if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) {\n console.error(\n `[integrations] signInternalToken failed unexpectedly for ${taskId}:`,\n err,\n );\n }\n }\n\n // Don't await the body — we just want the request to leave the box.\n // A short timeout avoids tying up the retry loop on a hung processor.\n const controller = new AbortController();\n const timer = setTimeout(() => controller.abort(), 5_000);\n\n try {\n await fetch(url, {\n method: \"POST\",\n headers,\n body: JSON.stringify({ taskId }),\n signal: controller.signal,\n });\n } finally {\n clearTimeout(timer);\n }\n}\n\n/**\n * Start the periodic retry loop. Safe to call multiple times — second call\n * is a no-op.\n */\nexport function startPendingTasksRetryJob(options?: {\n webhookBaseUrl?: string;\n}): void {\n if (retryInterval) return;\n activeWebhookBaseUrl = options?.webhookBaseUrl;\n\n // Stagger the first run a bit so we don't hammer the DB immediately on boot.\n initialTimer = setTimeout(() => {\n void retryStuckPendingTasks().catch((err) => {\n console.error(\"[integrations] Pending-tasks retry job error:\", err);\n });\n }, 10_000);\n unrefTimer(initialTimer);\n\n retryInterval = setInterval(() => {\n void retryStuckPendingTasks().catch((err) => {\n console.error(\"[integrations] Pending-tasks retry job error:\", err);\n });\n }, RETRY_INTERVAL_MS);\n unrefTimer(retryInterval);\n\n if (process.env.DEBUG) {\n console.log(\n `[integrations] Pending-tasks retry job started (every ${\n RETRY_INTERVAL_MS / 1000\n }s)`,\n );\n }\n}\n\n/** Stop the retry loop. */\nexport function stopPendingTasksRetryJob(): void {\n if (initialTimer) {\n clearTimeout(initialTimer);\n initialTimer = null;\n }\n if (retryInterval) {\n clearInterval(retryInterval);\n retryInterval = null;\n }\n activeWebhookBaseUrl = undefined;\n}\n\nfunction unrefTimer(timer: ReturnType<typeof setInterval>): void {\n (timer as unknown as { unref?: () => void }).unref?.();\n}\n"]}
|
|
1
|
+
{"version":3,"file":"pending-tasks-retry-job.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-retry-job.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EACL,0CAA0C,EAC1C,8BAA8B,EAC9B,0CAA0C,GAC3C,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EACL,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,0BAA0B,CAAC;AAElC;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,MAAM,iBAAiB,GAAG,MAAM,CAAC;AACjC,8EAA8E;AAC9E,MAAM,sBAAsB,GAAG,MAAM,CAAC;AACtC,4EAA4E;AAC5E,MAAM,iCAAiC,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACxD,MAAM,oCAAoC,GAAG,MAAM,CAAC;AACpD,MAAM,4CAA4C,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AACpE,mEAAmE;AACnE,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,IAAI,aAAa,GAA0C,IAAI,CAAC;AAChE,IAAI,YAAY,GAAyC,IAAI,CAAC;AAC9D,IAAI,oBAAwC,CAAC;AAC7C;;;GAGG;AACH,IAAI,WAAW,GAAmB,IAAI,CAAC;AA0BvC,SAAS,YAAY,CAAC,MAAe;IACnC,OAAO,MAAM,CACV,MAAoC,CAAC,YAAY;QAC/C,MAAgC,CAAC,QAAQ;QAC1C,CAAC,CACJ,CAAC;AACJ,CAAC;AAED,SAAS,+BAA+B,CAAC,KAAc;IACrD,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IAC7E,OAAO,2DAA2D,CAAC,IAAI,CACrE,OAAO,CACR,CAAC;AACJ,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,MAAM,GAAG,0CAA0C,EAAE,CAAC;IAC5D,IAAI,MAAM,KAAK,IAAI;QAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IACrD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;IAEnE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,KAAK,KAAK,GAAG,EAAE,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,IAAI,KAAK,CAAC,QAAQ,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;YACrE,OAAO,CAAC,IAAI,CACV,gGAAgG,CACjG,CAAC;YACF,IAAI,CAAC,IAAI,CACP,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,KAAK,EACX,OAAO,KAAK,CAAC,KAAK,IAAI,CACvB,CAAC;YACF,SAAS;QACX,CAAC;QACD,OAAO,CAAC,IAAI,CACV,mEAAmE,CACpE,CAAC;QACF,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACtD,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,SAAS,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;AAC5D,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAyC;IAEzC,MAAM,OAAO,GACX,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,OAAO,CAAC,cAAc,IAAI,oBAAoB,CAAC;IAC/D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,mBAAmB,CAAC,CAAC,CAChE,CAAC;IACF,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW;QAClC,CAAC,CAAC,eAAe,EAAE;QACnB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAc,EAAE,CAAC;IACzC,MAAM,MAAM,GAA4B;QACtC,QAAQ,EAAE,CAAC;QACX,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,CAAC;QACf,OAAO,EAAE,CAAC;QACV,cAAc,EAAE,CAAC;KAClB,CAAC;IACF,MAAM,uBAAuB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,aAAa,GAAG,GAAG,GAAG,sBAAsB,CAAC;IACnD,MAAM,gBAAgB,GAAG,GAAG,GAAG,yBAAyB,EAAE,CAAC;IAC3D,MAAM,uBAAuB,GAC3B,GAAG,GAAG,4CAA4C,CAAC;IAErD,IAAI,SAAyB,CAAC;IAC9B,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;YACpC,GAAG,EAAE;;;;;;;;;;;WAWA,QAAQ,CAAC,MAAM;;;OAGnB;YACD,wEAAwE;YACxE,qEAAqE;YACrE,wEAAwE;YACxE,wEAAwE;YACxE,IAAI,EAAE;gBACJ,aAAa;gBACb,aAAa;gBACb,uBAAuB;gBACvB,gBAAgB;gBAChB,GAAG,QAAQ,CAAC,IAAI;gBAChB,KAAK;aACN;SACF,CAAC,CAAC;QACH,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC3B,EAAE,EAAE,CAAC,CAAC,EAAY;YAClB,QAAQ,EAAE,CAAC,CAAC,QAAkB;YAC9B,gBAAgB,EAAE,CAAC,CAAC,kBAA4B;YAChD,MAAM,EAAE,CAAC,CAAC,MAAgB;YAC1B,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC;YACjC,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;YACpC,aAAa,EAAG,CAAC,CAAC,cAAgC,IAAI,IAAI;SAC3D,CAAC,CAAC,CAAC;QACJ,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC;YAAE,MAAM,GAAG,CAAC;QACrD,IAAI,WAAW,KAAK,KAAK,EAAE,CAAC;YAC1B,WAAW,GAAG,KAAK,CAAC;YACpB,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;gBACtB,OAAO,CAAC,GAAG,CACT,yEAAyE,CAC1E,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE;QACnC,MAAM,OAAO,GAAG,0CAA0C,CAAC;YACzD,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;YACtC,eAAe,EAAE,GAAG,CAAC,aAAa;gBAChC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,aAAa,EAAE;gBAClC,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QACH,IAAI,OAAO,CAAC,WAAW,IAAI,CAAC,OAAO;YAAE,OAAO,KAAK,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IACH,MAAM,CAAC,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC;IACnC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAE1C,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,kEAAkE;YAClE,0CAA0C;YAC1C,IAAI,GAAG,CAAC,QAAQ,IAAI,yBAAyB,EAAE,CAAC;gBAC9C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;oBAClC,GAAG,EAAE;;;;;;;;;;WAUJ;oBACD,IAAI,EAAE;wBACJ,IAAI,CAAC,GAAG,EAAE;wBACV,uBAAuB,yBAAyB,WAAW;wBAC3D,GAAG,CAAC,EAAE;wBACN,GAAG,CAAC,MAAM;wBACV,GAAG,CAAC,SAAS;qBACd;iBACF,CAAC,CAAC;gBACH,IAAI,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC/B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;oBACpB,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,YAAY,IAAI,CAAC,CAAC;gBACzB,OAAO,CAAC,IAAI,CACV,+BAA+B,GAAG,CAAC,EAAE,aAAa,yBAAyB,4BAA4B,CACxG,CAAC;gBACF,SAAS;YACX,CAAC;YAED,qEAAqE;YACrE,+DAA+D;YAC/D,oDAAoD;YACpD,yEAAyE;YACzE,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,KAAK,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC;YACvE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;gBAClC,GAAG,EAAE;;;;;;SAMJ;gBACD,IAAI,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,CAAC;aACjE,CAAC,CAAC;YACH,IAAI,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;gBACpB,SAAS;YACX,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,8BAA8B,CAAC;gBACnD,MAAM,EAAE,GAAG,CAAC,EAAE;gBACd,IAAI,EAAE;oBACJ,QAAQ,EAAE,GAAG,CAAC,QAAQ;oBACtB,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;oBACtC,eAAe,EAAE,GAAG,CAAC,aAAa;wBAChC,CAAC,CAAC,EAAE,SAAS,EAAE,GAAG,CAAC,aAAa,EAAE;wBAClC,CAAC,CAAC,SAAS;iBACd;gBACD,OAAO;gBACP,gBAAgB,EAAE,KAAK;aACxB,CAAC,CAAC;YACH,IAAI,OAAO,KAAK,QAAQ;gBAAE,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;;gBAChD,MAAM,CAAC,UAAU,IAAI,CAAC,CAAC;QAC9B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC;YAC3B,OAAO,CAAC,KAAK,CACX,+CAA+C,GAAG,CAAC,EAAE,GAAG,EACxD,GAAG,CACJ,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,yBAAyB;IAChC,IACE,OAAO,CAAC,GAAG,CAAC,OAAO;QACnB,OAAO,CAAC,GAAG,CAAC,wBAAwB;QACpC,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,UAAU,IAAI,UAAU,EACxB,CAAC;QACD,OAAO,oCAAoC,CAAC;IAC9C,CAAC;IACD,OAAO,iCAAiC,CAAC;AAC3C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,yBAAyB,CAAC,OAEzC;IACC,IAAI,aAAa;QAAE,OAAO;IAC1B,oBAAoB,GAAG,OAAO,EAAE,cAAc,CAAC;IAE/C,6EAA6E;IAC7E,YAAY,GAAG,UAAU,CAAC,GAAG,EAAE;QAC7B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,MAAM,CAAC,CAAC;IACX,UAAU,CAAC,YAAY,CAAC,CAAC;IAEzB,aAAa,GAAG,WAAW,CAAC,GAAG,EAAE;QAC/B,KAAK,sBAAsB,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC1C,OAAO,CAAC,KAAK,CAAC,+CAA+C,EAAE,GAAG,CAAC,CAAC;QACtE,CAAC,CAAC,CAAC;IACL,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACtB,UAAU,CAAC,aAAa,CAAC,CAAC;IAE1B,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,yDACE,iBAAiB,GAAG,IACtB,IAAI,CACL,CAAC;IACJ,CAAC;AACH,CAAC;AAED,2BAA2B;AAC3B,MAAM,UAAU,wBAAwB;IACtC,IAAI,YAAY,EAAE,CAAC;QACjB,YAAY,CAAC,YAAY,CAAC,CAAC;QAC3B,YAAY,GAAG,IAAI,CAAC;IACtB,CAAC;IACD,IAAI,aAAa,EAAE,CAAC;QAClB,aAAa,CAAC,aAAa,CAAC,CAAC;QAC7B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,oBAAoB,GAAG,SAAS,CAAC;AACnC,CAAC;AAED,SAAS,UAAU,CAAC,KAAqC;IACtD,KAA2C,CAAC,KAAK,EAAE,EAAE,CAAC;AACzD,CAAC","sourcesContent":["import { getDbExec } from \"../db/client.js\";\nimport {\n configuredIntegrationDurableDispatchScopes,\n dispatchPendingIntegrationTask,\n isIntegrationDurableDispatchEnabledForTask,\n} from \"./integration-durable-dispatch.js\";\nimport {\n ensurePendingTasksTable,\n MAX_PENDING_TASK_ATTEMPTS,\n} from \"./pending-tasks-store.js\";\n\n/**\n * Retries stuck integration webhook tasks.\n *\n * The integration webhook flow enqueues work into `integration_pending_tasks`\n * (see `pending-tasks-store.ts`) and then fires a self-webhook to the\n * `/_agent-native/integrations/process-task` endpoint to drain the queue.\n * If that initial dispatch is lost (e.g. transient network blip), the\n * row stays in `pending` forever. Likewise, if the processor is killed mid-\n * processing (function timeout, container shutdown), a row can remain in\n * `processing` forever.\n *\n * The in-process fallback runs every 60s. Durable deployments also invoke the\n * same bounded sweep from an external scheduler so recovery does not depend on\n * a serverless process remaining alive.\n *\n * Each sweep re-fires the processor endpoint for tasks that\n * look stuck:\n * - status='pending' AND created_at older than 90s (initial dispatch lost)\n * - status='processing' AND updated_at older than the host-specific\n * function budget (75s on serverless, 5min elsewhere)\n *\n * Retries are capped at MAX_ATTEMPTS attempts; after that the row is marked\n * `failed` permanently so it stops being retried.\n *\n * If the `integration_pending_tasks` table does not yet exist (e.g. older\n * deploy that hasn't run the new webhook flow), this job no-ops silently\n * rather than spamming logs.\n */\n\nconst RETRY_INTERVAL_MS = 60_000;\n/** Tasks pending longer than this are considered stuck on initial dispatch */\nconst PENDING_STUCK_AFTER_MS = 90_000;\n/** Tasks \"processing\" longer than this are considered killed mid-flight. */\nconst DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000;\nconst SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000;\nconst DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS = 16 * 60 * 1000;\n/** After this many attempts we give up and mark the task failed */\nconst DEFAULT_SWEEP_LIMIT = 100;\n\nlet retryInterval: ReturnType<typeof setInterval> | null = null;\nlet initialTimer: ReturnType<typeof setTimeout> | null = null;\nlet activeWebhookBaseUrl: string | undefined;\n/**\n * Whether the table exists. Cached after first probe so we don't log every\n * minute when the queue isn't in use yet on a given deployment.\n */\nlet tableExists: boolean | null = null;\n\ninterface StuckTaskRow {\n id: string;\n platform: string;\n externalThreadId: string;\n status: string;\n attempts: number;\n updatedAt: number;\n dispatchScope: string | null;\n}\n\nexport interface PendingTasksSweepResult {\n selected: number;\n dispatched: number;\n markedFailed: number;\n skipped: number;\n dispatchFailed: number;\n}\n\nexport interface PendingTasksSweepOptions {\n webhookBaseUrl?: string;\n limit?: number;\n durableOnly?: boolean;\n}\n\nfunction affectedRows(result: unknown): number {\n return Number(\n (result as { rowsAffected?: number }).rowsAffected ??\n (result as { rowCount?: number }).rowCount ??\n 0,\n );\n}\n\nfunction isMissingPendingTasksTableError(error: unknown): boolean {\n const message = error instanceof Error ? error.message : String(error ?? \"\");\n return /no such table|relation .* does not exist|undefined_table/i.test(\n message,\n );\n}\n\nfunction durableScopeSql(): { clause: string; args: string[] } {\n const scopes = configuredIntegrationDurableDispatchScopes();\n if (scopes === null) return { clause: \"\", args: [] };\n if (scopes.length === 0) return { clause: \" AND 1 = 0\", args: [] };\n\n const clauses: string[] = [];\n const args: string[] = [];\n for (const scope of scopes) {\n if (scope.value === \"*\") {\n clauses.push(\"platform = ?\");\n args.push(scope.platform);\n continue;\n }\n if (scope.platform === \"slack\" && /^[A-Za-z0-9]+$/.test(scope.value)) {\n clauses.push(\n \"(platform = ? AND (external_thread_id = ? OR dispatch_scope = ? OR external_thread_id LIKE ?))\",\n );\n args.push(\n scope.platform,\n scope.value,\n scope.value,\n `%:%:${scope.value}:%`,\n );\n continue;\n }\n clauses.push(\n \"(platform = ? AND (external_thread_id = ? OR dispatch_scope = ?))\",\n );\n args.push(scope.platform, scope.value, scope.value);\n }\n return { clause: ` AND (${clauses.join(\" OR \")})`, args };\n}\n\n/**\n * One pass: find stuck tasks and re-fire the processor for each.\n * Exported for tests and for manual triggers.\n */\nexport async function retryStuckPendingTasks(\n input?: string | PendingTasksSweepOptions,\n): Promise<PendingTasksSweepResult> {\n const options: PendingTasksSweepOptions =\n typeof input === \"string\" ? { webhookBaseUrl: input } : (input ?? {});\n const baseUrl = options.webhookBaseUrl ?? activeWebhookBaseUrl;\n const limit = Math.max(\n 1,\n Math.min(500, Math.floor(options.limit ?? DEFAULT_SWEEP_LIMIT)),\n );\n const scopeSql = options.durableOnly\n ? durableScopeSql()\n : { clause: \"\", args: [] as string[] };\n const result: PendingTasksSweepResult = {\n selected: 0,\n dispatched: 0,\n markedFailed: 0,\n skipped: 0,\n dispatchFailed: 0,\n };\n await ensurePendingTasksTable();\n const client = getDbExec();\n const now = Date.now();\n const pendingCutoff = now - PENDING_STUCK_AFTER_MS;\n const processingCutoff = now - getProcessingStuckAfterMs();\n const durableProcessingCutoff =\n now - DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS;\n\n let stuckRows: StuckTaskRow[];\n try {\n const { rows } = await client.execute({\n sql: `\n SELECT id, platform, external_thread_id, dispatch_scope, status, attempts, updated_at\n FROM integration_pending_tasks\n WHERE ((status = 'pending' AND created_at <= ? AND updated_at <= ?)\n OR (status = 'processing' AND (\n (last_dispatch_outcome = 'background-acknowledged'\n AND updated_at <= ?)\n OR ((last_dispatch_outcome IS NULL\n OR last_dispatch_outcome <> 'background-acknowledged')\n AND updated_at <= ?)\n )))\n ${scopeSql.clause}\n ORDER BY updated_at ASC\n LIMIT ?\n `,\n // `updated_at` is initialized to `created_at` on insert, so a genuinely\n // stuck pending row still matches on the first sweep. The retry path\n // below touches `updated_at`, which (with this predicate) keeps the row\n // from being re-selected — and re-firing the processor — on every tick.\n args: [\n pendingCutoff,\n pendingCutoff,\n durableProcessingCutoff,\n processingCutoff,\n ...scopeSql.args,\n limit,\n ],\n });\n stuckRows = rows.map((r) => ({\n id: r.id as string,\n platform: r.platform as string,\n externalThreadId: r.external_thread_id as string,\n status: r.status as string,\n attempts: Number(r.attempts ?? 0),\n updatedAt: Number(r.updated_at ?? 0),\n dispatchScope: (r.dispatch_scope as string | null) ?? null,\n }));\n tableExists = true;\n } catch (err) {\n if (!isMissingPendingTasksTableError(err)) throw err;\n if (tableExists !== false) {\n tableExists = false;\n if (process.env.DEBUG) {\n console.log(\n \"[integrations] pending-tasks retry job: table not present yet, skipping\",\n );\n }\n }\n return result;\n }\n\n stuckRows = stuckRows.filter((row) => {\n const durable = isIntegrationDurableDispatchEnabledForTask({\n platform: row.platform,\n externalThreadId: row.externalThreadId,\n platformContext: row.dispatchScope\n ? { channelId: row.dispatchScope }\n : undefined,\n });\n if (options.durableOnly && !durable) return false;\n return true;\n });\n result.selected = stuckRows.length;\n if (stuckRows.length === 0) return result;\n\n for (const row of stuckRows) {\n try {\n // Cap retries — mark failed and move on so the row stops bouncing\n // between pending and processing forever.\n if (row.attempts >= MAX_PENDING_TASK_ATTEMPTS) {\n const update = await client.execute({\n sql: `\n UPDATE integration_pending_tasks\n SET status = 'failed',\n updated_at = ?,\n error_message = COALESCE(error_message, ?),\n payload = '{}',\n external_event_key = NULL\n WHERE id = ?\n AND status = ?\n AND updated_at = ?\n `,\n args: [\n Date.now(),\n `Retry job: exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts`,\n row.id,\n row.status,\n row.updatedAt,\n ],\n });\n if (affectedRows(update) === 0) {\n result.skipped += 1;\n continue;\n }\n result.markedFailed += 1;\n console.warn(\n `[integrations] Pending task ${row.id} exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts — marking failed`,\n );\n continue;\n }\n\n // Reset stuck `processing` rows back to `pending` so the processor's\n // atomic claim (which only matches pending) can re-acquire it.\n // Without this, processing rows stay stuck forever.\n // For pending rows, just touch updated_at to avoid re-firing every tick.\n const newStatus = row.status === \"processing\" ? \"pending\" : row.status;\n const update = await client.execute({\n sql: `\n UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?\n WHERE id = ?\n AND status = ?\n AND updated_at = ?\n `,\n args: [newStatus, Date.now(), row.id, row.status, row.updatedAt],\n });\n if (affectedRows(update) === 0) {\n result.skipped += 1;\n continue;\n }\n\n const outcome = await dispatchPendingIntegrationTask({\n taskId: row.id,\n task: {\n platform: row.platform,\n externalThreadId: row.externalThreadId,\n platformContext: row.dispatchScope\n ? { channelId: row.dispatchScope }\n : undefined,\n },\n baseUrl,\n portableSettleMs: 1_000,\n });\n if (outcome === \"failed\") result.dispatchFailed += 1;\n else result.dispatched += 1;\n } catch (err) {\n result.dispatchFailed += 1;\n console.error(\n `[integrations] Failed to retry pending task ${row.id}:`,\n err,\n );\n }\n }\n return result;\n}\n\nfunction getProcessingStuckAfterMs(): number {\n if (\n process.env.NETLIFY ||\n process.env.AWS_LAMBDA_FUNCTION_NAME ||\n process.env.VERCEL ||\n \"__cf_env\" in globalThis\n ) {\n return SERVERLESS_PROCESSING_STUCK_AFTER_MS;\n }\n return DEFAULT_PROCESSING_STUCK_AFTER_MS;\n}\n\n/**\n * Start the periodic retry loop. Safe to call multiple times — second call\n * is a no-op.\n */\nexport function startPendingTasksRetryJob(options?: {\n webhookBaseUrl?: string;\n}): void {\n if (retryInterval) return;\n activeWebhookBaseUrl = options?.webhookBaseUrl;\n\n // Stagger the first run a bit so we don't hammer the DB immediately on boot.\n initialTimer = setTimeout(() => {\n void retryStuckPendingTasks().catch((err) => {\n console.error(\"[integrations] Pending-tasks retry job error:\", err);\n });\n }, 10_000);\n unrefTimer(initialTimer);\n\n retryInterval = setInterval(() => {\n void retryStuckPendingTasks().catch((err) => {\n console.error(\"[integrations] Pending-tasks retry job error:\", err);\n });\n }, RETRY_INTERVAL_MS);\n unrefTimer(retryInterval);\n\n if (process.env.DEBUG) {\n console.log(\n `[integrations] Pending-tasks retry job started (every ${\n RETRY_INTERVAL_MS / 1000\n }s)`,\n );\n }\n}\n\n/** Stop the retry loop. */\nexport function stopPendingTasksRetryJob(): void {\n if (initialTimer) {\n clearTimeout(initialTimer);\n initialTimer = null;\n }\n if (retryInterval) {\n clearInterval(retryInterval);\n retryInterval = null;\n }\n activeWebhookBaseUrl = undefined;\n}\n\nfunction unrefTimer(timer: ReturnType<typeof setInterval>): void {\n (timer as unknown as { unref?: () => void }).unref?.();\n}\n"]}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export declare const MAX_PENDING_TASK_ATTEMPTS = 3;
|
|
2
|
+
export declare function ensurePendingTasksTable(): Promise<void>;
|
|
2
3
|
/** Status values for an integration pending task. */
|
|
3
4
|
export type PendingTaskStatus = "pending" | "processing" | "completed" | "failed";
|
|
4
5
|
export interface PendingTask {
|
|
@@ -10,6 +11,10 @@ export interface PendingTask {
|
|
|
10
11
|
orgId: string | null;
|
|
11
12
|
status: PendingTaskStatus;
|
|
12
13
|
attempts: number;
|
|
14
|
+
dispatchAttempts: number;
|
|
15
|
+
lastDispatchAt: number | null;
|
|
16
|
+
lastDispatchOutcome: string | null;
|
|
17
|
+
dispatchScope: string | null;
|
|
13
18
|
errorMessage: string | null;
|
|
14
19
|
createdAt: number;
|
|
15
20
|
updatedAt: number;
|
|
@@ -40,6 +45,7 @@ export declare function insertPendingTask(input: {
|
|
|
40
45
|
ownerEmail: string;
|
|
41
46
|
orgId?: string | null;
|
|
42
47
|
externalEventKey?: string | null;
|
|
48
|
+
dispatchScope?: string | null;
|
|
43
49
|
}): Promise<void>;
|
|
44
50
|
/**
|
|
45
51
|
* Returns whether a duplicate-event error from `insertPendingTask` looks
|
|
@@ -65,9 +71,15 @@ export declare function resolveIntegrationSourceContext(id: string, ownerEmail:
|
|
|
65
71
|
* attempts. Returns the updated task if the transition succeeded, otherwise
|
|
66
72
|
* null (e.g. the task was already claimed by a concurrent worker).
|
|
67
73
|
*/
|
|
68
|
-
export declare function claimPendingTask(id: string
|
|
74
|
+
export declare function claimPendingTask(id: string, options?: {
|
|
75
|
+
dispatchOutcome?: string;
|
|
76
|
+
}): Promise<PendingTask | null>;
|
|
77
|
+
export declare function recordPendingTaskDispatchAttempt(id: string, outcome: string): Promise<void>;
|
|
69
78
|
/** Next queued turn for a provider thread after its current task completes. */
|
|
70
|
-
export declare function
|
|
79
|
+
export declare function getNextPendingTaskForThread(platform: string, externalThreadId: string): Promise<{
|
|
80
|
+
id: string;
|
|
81
|
+
dispatchScope: string | null;
|
|
82
|
+
} | null>;
|
|
71
83
|
/** Mark a task as completed. */
|
|
72
84
|
export declare function markTaskCompleted(id: string): Promise<void>;
|
|
73
85
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pending-tasks-store.d.ts","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAoBA,eAAO,MAAM,yBAAyB,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"pending-tasks-store.d.ts","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAoBA,eAAO,MAAM,yBAAyB,IAAI,CAAC;AA+F3C,wBAAsB,uBAAuB,IAAI,OAAO,CAAC,IAAI,CAAC,CAE7D;AAsDD,qDAAqD;AACrD,MAAM,MAAM,iBAAiB,GACzB,SAAS,GACT,YAAY,GACZ,WAAW,GACX,QAAQ,CAAC;AAEb,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,mBAAmB,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC5B;AAED;;;;;;GAMG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC,OAAO,CAAC,CAalB;AA2BD;;;;;;;;GAQG;AACH,wBAAsB,iBAAiB,CAAC,KAAK,EAAE;IAC7C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,GAAG,OAAO,CAAC,IAAI,CAAC,CAuBhB;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAU3D;AAED,kCAAkC;AAClC,wBAAsB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAU5E;AAED,MAAM,WAAW,gCAAgC;IAC/C,QAAQ,EAAE,OAAO,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,KAAK,oBAAoB,GAAG,IAAI,CAC9B,WAAW,EACX,UAAU,GAAG,SAAS,GAAG,YAAY,GAAG,OAAO,CAChD,CAAC;AAEF,wBAAgB,4BAA4B,CAC1C,IAAI,EAAE,oBAAoB,GAAG,IAAI,EACjC,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,GAAG,IAAI,GACnB,gCAAgC,GAAG,IAAI,CAyCzC;AAED,iFAAiF;AACjF,wBAAsB,+BAA+B,CACnD,EAAE,EAAE,MAAM,EACV,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,MAAM,GAAG,IAAI,GACnB,OAAO,CAAC,gCAAgC,GAAG,IAAI,CAAC,CAyBlD;AAED;;;;GAIG;AACH,wBAAsB,gBAAgB,CACpC,EAAE,EAAE,MAAM,EACV,OAAO,CAAC,EAAE;IAAE,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,GACrC,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CA4E7B;AAED,wBAAsB,gCAAgC,CACpD,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAef;AAED,+EAA+E;AAC/E,wBAAsB,2BAA2B,CAC/C,QAAQ,EAAE,MAAM,EAChB,gBAAgB,EAAE,MAAM,GACvB,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,GAAG,IAAI,CAAC,CAc9D;AAED,gCAAgC;AAChC,wBAAsB,iBAAiB,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAajE;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,EAAE,EAAE,MAAM,EACV,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,wBAAwB,CAC5C,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wBAAsB,yBAAyB,CAC7C,EAAE,EAAE,MAAM,EACV,OAAO,EAAE,MAAM,EACf,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,wBAAsB,0BAA0B,CAC9C,EAAE,EAAE,MAAM,EACV,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC,CAkBf;AAED,wDAAwD;AACxD,wBAAsB,cAAc,CAClC,EAAE,EAAE,MAAM,EACV,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,IAAI,CAAC,CAUf"}
|
|
@@ -27,6 +27,10 @@ async function ensureTable() {
|
|
|
27
27
|
org_id TEXT,
|
|
28
28
|
status TEXT NOT NULL,
|
|
29
29
|
attempts ${intType()} NOT NULL DEFAULT 0,
|
|
30
|
+
dispatch_attempts ${intType()} NOT NULL DEFAULT 0,
|
|
31
|
+
last_dispatch_at ${intType()},
|
|
32
|
+
last_dispatch_outcome TEXT,
|
|
33
|
+
dispatch_scope TEXT,
|
|
30
34
|
error_message TEXT,
|
|
31
35
|
created_at ${intType()} NOT NULL,
|
|
32
36
|
updated_at ${intType()} NOT NULL,
|
|
@@ -36,8 +40,13 @@ async function ensureTable() {
|
|
|
36
40
|
// PG guard: probe via information_schema, only issue DDL if missing, bounded lock_timeout
|
|
37
41
|
await ensureTableExists("integration_pending_tasks", createSql);
|
|
38
42
|
await ensureColumnExists("integration_pending_tasks", "external_event_key", `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`);
|
|
43
|
+
await ensureColumnExists("integration_pending_tasks", "dispatch_attempts", `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_attempts ${intType()} NOT NULL DEFAULT 0`);
|
|
44
|
+
await ensureColumnExists("integration_pending_tasks", "last_dispatch_at", `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_at ${intType()}`);
|
|
45
|
+
await ensureColumnExists("integration_pending_tasks", "last_dispatch_outcome", `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_outcome TEXT`);
|
|
46
|
+
await ensureColumnExists("integration_pending_tasks", "dispatch_scope", `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_scope TEXT`);
|
|
39
47
|
await ensureIndexExists("idx_pending_tasks_status_created", `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`);
|
|
40
48
|
await ensureIndexExists("idx_pending_tasks_event_key", `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`);
|
|
49
|
+
await ensureIndexExists("idx_pending_tasks_dispatch_scope", `CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`);
|
|
41
50
|
return;
|
|
42
51
|
}
|
|
43
52
|
await client.execute(createSql);
|
|
@@ -49,6 +58,8 @@ async function ensureTable() {
|
|
|
49
58
|
// unique index ensures a duplicate INSERT raises an error we can
|
|
50
59
|
// catch as "already-enqueued".
|
|
51
60
|
await ensureExternalEventKey(client);
|
|
61
|
+
await ensureDispatchColumns(client);
|
|
62
|
+
await client.execute(`CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`);
|
|
52
63
|
await client.execute(`CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`);
|
|
53
64
|
})().catch((err) => {
|
|
54
65
|
// Retry init on the next call after a failed startup.
|
|
@@ -58,6 +69,9 @@ async function ensureTable() {
|
|
|
58
69
|
}
|
|
59
70
|
return _initPromise;
|
|
60
71
|
}
|
|
72
|
+
export async function ensurePendingTasksTable() {
|
|
73
|
+
await ensureTable();
|
|
74
|
+
}
|
|
61
75
|
async function ensureExternalEventKey(client) {
|
|
62
76
|
if (isPostgres()) {
|
|
63
77
|
await client.execute(`ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`);
|
|
@@ -76,6 +90,26 @@ async function ensureExternalEventKey(client) {
|
|
|
76
90
|
}
|
|
77
91
|
}
|
|
78
92
|
}
|
|
93
|
+
async function ensureDispatchColumns(client) {
|
|
94
|
+
const columns = [
|
|
95
|
+
`dispatch_attempts ${intType()} NOT NULL DEFAULT 0`,
|
|
96
|
+
`last_dispatch_at ${intType()}`,
|
|
97
|
+
"last_dispatch_outcome TEXT",
|
|
98
|
+
"dispatch_scope TEXT",
|
|
99
|
+
];
|
|
100
|
+
for (const column of columns) {
|
|
101
|
+
try {
|
|
102
|
+
await client.execute(`ALTER TABLE integration_pending_tasks ADD COLUMN ${column}`);
|
|
103
|
+
}
|
|
104
|
+
catch (err) {
|
|
105
|
+
if (!String(err?.message ?? err)
|
|
106
|
+
.toLowerCase()
|
|
107
|
+
.includes("duplicate")) {
|
|
108
|
+
throw err;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
}
|
|
79
113
|
/**
|
|
80
114
|
* Whether a provider thread currently has queued or executing work.
|
|
81
115
|
*
|
|
@@ -107,6 +141,12 @@ function rowToTask(row) {
|
|
|
107
141
|
orgId: row.org_id ?? null,
|
|
108
142
|
status: row.status,
|
|
109
143
|
attempts: Number(row.attempts ?? 0),
|
|
144
|
+
dispatchAttempts: Number(row.dispatch_attempts ?? 0),
|
|
145
|
+
lastDispatchAt: row.last_dispatch_at == null
|
|
146
|
+
? null
|
|
147
|
+
: Number(row.last_dispatch_at),
|
|
148
|
+
lastDispatchOutcome: row.last_dispatch_outcome ?? null,
|
|
149
|
+
dispatchScope: row.dispatch_scope ?? null,
|
|
110
150
|
errorMessage: row.error_message ?? null,
|
|
111
151
|
createdAt: Number(row.created_at ?? 0),
|
|
112
152
|
updatedAt: Number(row.updated_at ?? 0),
|
|
@@ -128,8 +168,8 @@ export async function insertPendingTask(input) {
|
|
|
128
168
|
const now = Date.now();
|
|
129
169
|
await client.execute({
|
|
130
170
|
sql: `INSERT INTO integration_pending_tasks
|
|
131
|
-
(id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key)
|
|
132
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
171
|
+
(id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key, dispatch_scope)
|
|
172
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
133
173
|
args: [
|
|
134
174
|
input.id,
|
|
135
175
|
input.platform,
|
|
@@ -142,6 +182,7 @@ export async function insertPendingTask(input) {
|
|
|
142
182
|
now,
|
|
143
183
|
now,
|
|
144
184
|
input.externalEventKey ?? null,
|
|
185
|
+
input.dispatchScope ?? null,
|
|
145
186
|
],
|
|
146
187
|
});
|
|
147
188
|
}
|
|
@@ -169,7 +210,7 @@ export async function getPendingTask(id) {
|
|
|
169
210
|
await ensureTable();
|
|
170
211
|
const client = getDbExec();
|
|
171
212
|
const { rows } = await client.execute({
|
|
172
|
-
sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at
|
|
213
|
+
sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at
|
|
173
214
|
FROM integration_pending_tasks WHERE id = ? LIMIT 1`,
|
|
174
215
|
args: [id],
|
|
175
216
|
});
|
|
@@ -242,7 +283,7 @@ export async function resolveIntegrationSourceContext(id, ownerEmail, orgId) {
|
|
|
242
283
|
* attempts. Returns the updated task if the transition succeeded, otherwise
|
|
243
284
|
* null (e.g. the task was already claimed by a concurrent worker).
|
|
244
285
|
*/
|
|
245
|
-
export async function claimPendingTask(id) {
|
|
286
|
+
export async function claimPendingTask(id, options) {
|
|
246
287
|
await ensureTable();
|
|
247
288
|
const client = getDbExec();
|
|
248
289
|
const now = Date.now();
|
|
@@ -251,7 +292,8 @@ export async function claimPendingTask(id) {
|
|
|
251
292
|
const result = await client.execute({
|
|
252
293
|
sql: isPostgres()
|
|
253
294
|
? `UPDATE integration_pending_tasks
|
|
254
|
-
SET status = ?, attempts = attempts + 1, updated_at =
|
|
295
|
+
SET status = ?, attempts = attempts + 1, updated_at = ?,
|
|
296
|
+
last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)
|
|
255
297
|
WHERE id = ? AND status = 'pending'
|
|
256
298
|
AND NOT EXISTS (
|
|
257
299
|
SELECT 1 FROM integration_pending_tasks active
|
|
@@ -273,9 +315,10 @@ export async function claimPendingTask(id) {
|
|
|
273
315
|
)
|
|
274
316
|
)
|
|
275
317
|
)
|
|
276
|
-
RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at`
|
|
318
|
+
RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at`
|
|
277
319
|
: `UPDATE integration_pending_tasks
|
|
278
|
-
SET status = ?, attempts = attempts + 1, updated_at =
|
|
320
|
+
SET status = ?, attempts = attempts + 1, updated_at = ?,
|
|
321
|
+
last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)
|
|
279
322
|
WHERE id = ? AND status = 'pending'
|
|
280
323
|
AND NOT EXISTS (
|
|
281
324
|
SELECT 1 FROM integration_pending_tasks active
|
|
@@ -297,7 +340,7 @@ export async function claimPendingTask(id) {
|
|
|
297
340
|
)
|
|
298
341
|
)
|
|
299
342
|
)`,
|
|
300
|
-
args: ["processing", now, id],
|
|
343
|
+
args: ["processing", now, options?.dispatchOutcome ?? null, id],
|
|
301
344
|
});
|
|
302
345
|
const rows = result.rows ?? [];
|
|
303
346
|
if (isPostgres()) {
|
|
@@ -316,16 +359,37 @@ export async function claimPendingTask(id) {
|
|
|
316
359
|
return null;
|
|
317
360
|
return fetched;
|
|
318
361
|
}
|
|
362
|
+
export async function recordPendingTaskDispatchAttempt(id, outcome) {
|
|
363
|
+
await ensureTable();
|
|
364
|
+
await getDbExec().execute({
|
|
365
|
+
sql: `UPDATE integration_pending_tasks
|
|
366
|
+
SET dispatch_attempts = dispatch_attempts + 1,
|
|
367
|
+
last_dispatch_at = ?,
|
|
368
|
+
last_dispatch_outcome = CASE
|
|
369
|
+
WHEN status = 'processing'
|
|
370
|
+
AND last_dispatch_outcome = 'background-acknowledged'
|
|
371
|
+
THEN last_dispatch_outcome
|
|
372
|
+
ELSE ?
|
|
373
|
+
END
|
|
374
|
+
WHERE id = ?`,
|
|
375
|
+
args: [Date.now(), outcome.slice(0, 80), id],
|
|
376
|
+
});
|
|
377
|
+
}
|
|
319
378
|
/** Next queued turn for a provider thread after its current task completes. */
|
|
320
|
-
export async function
|
|
379
|
+
export async function getNextPendingTaskForThread(platform, externalThreadId) {
|
|
321
380
|
await ensureTable();
|
|
322
381
|
const { rows } = await getDbExec().execute({
|
|
323
|
-
sql: `SELECT id FROM integration_pending_tasks
|
|
382
|
+
sql: `SELECT id, dispatch_scope FROM integration_pending_tasks
|
|
324
383
|
WHERE platform = ? AND external_thread_id = ? AND status = 'pending'
|
|
325
384
|
ORDER BY created_at ASC, id ASC LIMIT 1`,
|
|
326
385
|
args: [platform, externalThreadId],
|
|
327
386
|
});
|
|
328
|
-
return rows[0]?.id
|
|
387
|
+
return rows[0]?.id
|
|
388
|
+
? {
|
|
389
|
+
id: String(rows[0].id),
|
|
390
|
+
dispatchScope: rows[0].dispatch_scope ?? null,
|
|
391
|
+
}
|
|
392
|
+
: null;
|
|
329
393
|
}
|
|
330
394
|
/** Mark a task as completed. */
|
|
331
395
|
export async function markTaskCompleted(id) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pending-tasks-store.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAE5B,IAAI,YAAuC,CAAC;AAC5C,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG;;;;;;;;aAQX,OAAO,EAAE;;eAEP,OAAO,EAAE;eACT,OAAO,EAAE;iBACP,OAAO,EAAE;EACxB,CAAC;YAEG,IAAI,UAAU,EAAE,EAAE,CAAC;gBACjB,0FAA0F;gBAC1F,MAAM,iBAAiB,CAAC,2BAA2B,EAAE,SAAS,CAAC,CAAC;gBAChE,MAAM,kBAAkB,CACtB,2BAA2B,EAC3B,oBAAoB,EACpB,wFAAwF,CACzF,CAAC;gBACF,MAAM,iBAAiB,CACrB,kCAAkC,EAClC,8GAA8G,CAC/G,CAAC;gBACF,MAAM,iBAAiB,CACrB,6BAA6B,EAC7B,0HAA0H,CAC3H,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAChC,MAAM,MAAM,CAAC,OAAO,CAClB,8GAA8G,CAC/G,CAAC;YACF,oEAAoE;YACpE,oEAAoE;YACpE,+DAA+D;YAC/D,kEAAkE;YAClE,iEAAiE;YACjE,+BAA+B;YAC/B,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,MAAM,CAAC,OAAO,CAClB,0HAA0H,CAC3H,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACjB,sDAAsD;YACtD,YAAY,GAAG,SAAS,CAAC;YACzB,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAoC;IAEpC,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,CAAC,OAAO,CAClB,wFAAwF,CACzF,CAAC;QACF,OAAO;IACT,CAAC;IACD,wEAAwE;IACxE,uDAAuD;IACvD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAClB,0EAA0E,CAC3E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IACE,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,GAAG,CAAC;aACzB,WAAW,EAAE;aACb,QAAQ,CAAC,WAAW,CAAC,EACxB,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAwBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,gBAAwB;IAExB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;;;;;cAKK;QACV,IAAI,EAAE,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KACnC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,SAAS,CAAC,GAA4B;IAC7C,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAY;QACpB,QAAQ,EAAE,GAAG,CAAC,QAAkB;QAChC,gBAAgB,EAAE,GAAG,CAAC,kBAA4B;QAClD,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,UAAU,EAAE,GAAG,CAAC,WAAqB;QACrC,KAAK,EAAG,GAAG,CAAC,MAAwB,IAAI,IAAI;QAC5C,MAAM,EAAE,GAAG,CAAC,MAA2B;QACvC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnC,YAAY,EAAG,GAAG,CAAC,aAA+B,IAAI,IAAI;QAC1D,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,WAAW,EACT,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAsB,CAAC;KACvE,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KAQvC;IACC,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;+CAEsC;QAC3C,IAAI,EAAE;YACJ,KAAK,CAAC,EAAE;YACR,KAAK,CAAC,QAAQ;YACd,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,OAAO;YACb,KAAK,CAAC,UAAU;YAChB,KAAK,CAAC,KAAK,IAAI,IAAI;YACnB,SAAS;YACT,CAAC;YACD,GAAG;YACH,GAAG;YACH,KAAK,CAAC,gBAAgB,IAAI,IAAI;SAC/B;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,MAAM,CAAC,GAAG,GAAiD,CAAC;IAC5D,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACrB,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,CAAC,4BAA4B;IACjE,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,OAAO,CACL,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACtB,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/B,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAC9B,CAAC;AACJ,CAAC;AAED,kCAAkC;AAClC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAU;IAC7C,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;8DACqD;QAC1D,IAAI,EAAE,CAAC,EAAE,CAAC;KACX,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;AACvD,CAAC;AAYD,MAAM,UAAU,4BAA4B,CAC1C,IAAiC,EACjC,UAAkB,EAClB,KAAoB;IAEpB,IACE,CAAC,IAAI;QACL,IAAI,CAAC,UAAU,KAAK,UAAU;QAC9B,IAAI,CAAC,KAAK,KAAK,KAAK;QACpB,IAAI,CAAC,QAAQ,KAAK,OAAO,EACzB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAA4B,CAAC;QACpE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC3D,MAAM,cAAc,GAAG,QAAmC,CAAC;QAC3D,IAAI,cAAc,CAAC,QAAQ,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QACrD,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC;QAC3C,IACE,OAAO,SAAS,KAAK,QAAQ;YAC7B,CAAC,SAAS;YACV,SAAS,KAAK,SAAS,CAAC,IAAI,EAAE,EAC9B,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,WAAW,GACf,MAAM,CAAC,QAAQ,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5E,IACE,MAAM,CAAC,QAAQ,KAAK,QAAQ;YAC5B,CAAC,WAAW;YACZ,MAAM,CAAC,QAAQ;YACf,MAAM,CAAC,QAAQ;YACf,MAAM,CAAC,IAAI,EACX,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,EAAU,EACV,UAAkB,EAClB,KAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;;;;;;kBAMS;QACd,IAAI,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;KACrC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAA4B,CAAC;IAC/C,OAAO,4BAA4B,CACjC;QACE,QAAQ,EAAE,GAAG,CAAC,QAAkB;QAChC,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,UAAU,EAAE,GAAG,CAAC,WAAqB;QACrC,KAAK,EAAG,GAAG,CAAC,MAAwB,IAAI,IAAI;KAC7C,EACD,UAAU,EACV,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAU;IAEV,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE,UAAU,EAAE;YACf,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;yJAuBiJ;YACnJ,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;aAsBK;QACT,IAAI,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,EAAE,CAAC;KAC9B,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;IAE/B,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;IACvD,CAAC;IAED,yEAAyE;IACzE,kDAAkD;IAClD,MAAM,QAAQ,GACX,MAAuD,CAAC,YAAY;QACpE,MAAuD,CAAC,QAAQ,CAAC;IACpE,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IAC7D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,QAAgB,EAChB,gBAAwB;IAExB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,EAAE,CAAC,OAAO,CAAC;QACzC,GAAG,EAAE;;8CAEqC;QAC1C,IAAI,EAAE,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KACnC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjD,CAAC;AAED,gCAAgC;AAChC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAU;IAChD,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,qEAAqE;QACrE,uEAAuE;QACvE,4EAA4E;QAC5E,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;KACxC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;KACxD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,EAAU,EACV,OAAe;IAEf,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;KACzB,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,CACpB,MAAoC,CAAC,YAAY;QAC/C,MAAgC,CAAC,QAAQ;QAC1C,CAAC,CACJ,CAAC;IACF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,EAAU,EACV,OAAe,EACf,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;KACxE,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,CACpB,MAAoC,CAAC,YAAY;QAC/C,MAAgC,CAAC,QAAQ;QAC1C,CAAC,CACJ,CAAC;IACF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC,OAAO,CAAC;QACvC,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;KACpE,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,CACpB,MAAoC,CAAC,YAAY;QAC/C,MAAgC,CAAC,QAAQ;QAC1C,CAAC,CACJ,CAAC;IACF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wDAAwD;AACxD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,IAAI,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;KAC7D,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL-backed pending task queue for integration webhooks.\n *\n * Why this exists: serverless platforms (Netlify Lambda, Vercel, Cloudflare\n * Workers) freeze the function execution as soon as the HTTP response is\n * returned. Fire-and-forget background `Promise`s get killed mid-flight,\n * meaning agent loops triggered from a Slack/Telegram webhook never finish.\n *\n * Solution: persist the inbound message to SQL inside the webhook handler,\n * then dispatch a fresh HTTP POST to a separate processor endpoint. Each\n * invocation gets its own fresh function timeout budget.\n */\nimport { getDbExec, isPostgres, intType } from \"../db/client.js\";\nimport {\n ensureTableExists,\n ensureColumnExists,\n ensureIndexExists,\n} from \"../db/ddl-guard.js\";\n\nlet _initPromise: Promise<void> | undefined;\nexport const MAX_PENDING_TASK_ATTEMPTS = 3;\n\nasync function ensureTable(): Promise<void> {\n if (!_initPromise) {\n _initPromise = (async () => {\n const client = getDbExec();\n const createSql = `CREATE TABLE IF NOT EXISTS integration_pending_tasks (\n id TEXT PRIMARY KEY,\n platform TEXT NOT NULL,\n external_thread_id TEXT NOT NULL,\n payload TEXT NOT NULL,\n owner_email TEXT NOT NULL,\n org_id TEXT,\n status TEXT NOT NULL,\n attempts ${intType()} NOT NULL DEFAULT 0,\n error_message TEXT,\n created_at ${intType()} NOT NULL,\n updated_at ${intType()} NOT NULL,\n completed_at ${intType()}\n)`;\n\n if (isPostgres()) {\n // PG guard: probe via information_schema, only issue DDL if missing, bounded lock_timeout\n await ensureTableExists(\"integration_pending_tasks\", createSql);\n await ensureColumnExists(\n \"integration_pending_tasks\",\n \"external_event_key\",\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,\n );\n await ensureIndexExists(\n \"idx_pending_tasks_status_created\",\n `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,\n );\n await ensureIndexExists(\n \"idx_pending_tasks_event_key\",\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,\n );\n return;\n }\n\n await client.execute(createSql);\n await client.execute(\n `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,\n );\n // Additive migration: add a stable per-event dedup key so duplicate\n // webhook deliveries from the same platform get rejected at the SQL\n // layer instead of via an in-memory Map (which doesn't survive\n // serverless cold starts — H3 in the webhook security audit). The\n // unique index ensures a duplicate INSERT raises an error we can\n // catch as \"already-enqueued\".\n await ensureExternalEventKey(client);\n await client.execute(\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,\n );\n })().catch((err) => {\n // Retry init on the next call after a failed startup.\n _initPromise = undefined;\n throw err;\n });\n }\n return _initPromise;\n}\n\nasync function ensureExternalEventKey(\n client: ReturnType<typeof getDbExec>,\n): Promise<void> {\n if (isPostgres()) {\n await client.execute(\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,\n );\n return;\n }\n // SQLite doesn't support `ADD COLUMN IF NOT EXISTS` until 3.35; swallow\n // the duplicate-column error so reruns are idempotent.\n try {\n await client.execute(\n `ALTER TABLE integration_pending_tasks ADD COLUMN external_event_key TEXT`,\n );\n } catch (err: any) {\n if (\n !String(err?.message ?? err)\n .toLowerCase()\n .includes(\"duplicate\")\n ) {\n throw err;\n }\n }\n}\n\n/** Status values for an integration pending task. */\nexport type PendingTaskStatus =\n | \"pending\"\n | \"processing\"\n | \"completed\"\n | \"failed\";\n\nexport interface PendingTask {\n id: string;\n platform: string;\n externalThreadId: string;\n payload: string;\n ownerEmail: string;\n orgId: string | null;\n status: PendingTaskStatus;\n attempts: number;\n errorMessage: string | null;\n createdAt: number;\n updatedAt: number;\n completedAt: number | null;\n}\n\n/**\n * Whether a provider thread currently has queued or executing work.\n *\n * Messaging adapters use this to accept unmentioned replies only while an\n * agent task is active. This prevents broad message subscriptions from turning\n * every channel message into an agent invocation.\n */\nexport async function hasActivePendingTask(\n platform: string,\n externalThreadId: string,\n): Promise<boolean> {\n await ensureTable();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT 1 AS active\n FROM integration_pending_tasks\n WHERE platform = ?\n AND external_thread_id = ?\n AND status IN ('pending', 'processing')\n LIMIT 1`,\n args: [platform, externalThreadId],\n });\n return rows.length > 0;\n}\n\nfunction rowToTask(row: Record<string, unknown>): PendingTask {\n return {\n id: row.id as string,\n platform: row.platform as string,\n externalThreadId: row.external_thread_id as string,\n payload: row.payload as string,\n ownerEmail: row.owner_email as string,\n orgId: (row.org_id as string | null) ?? null,\n status: row.status as PendingTaskStatus,\n attempts: Number(row.attempts ?? 0),\n errorMessage: (row.error_message as string | null) ?? null,\n createdAt: Number(row.created_at ?? 0),\n updatedAt: Number(row.updated_at ?? 0),\n completedAt:\n row.completed_at == null ? null : Number(row.completed_at as number),\n };\n}\n\n/**\n * Insert a new pending task. Returns the generated task id.\n *\n * If `externalEventKey` is supplied, the unique index on\n * `(platform, external_event_key)` will reject duplicates — callers should\n * catch the resulting constraint-violation error and treat it as\n * \"already enqueued\" instead of a hard failure (H3 in the webhook security\n * audit). This is the SQL-backed replacement for the in-memory dedup map.\n */\nexport async function insertPendingTask(input: {\n id: string;\n platform: string;\n externalThreadId: string;\n payload: string;\n ownerEmail: string;\n orgId?: string | null;\n externalEventKey?: string | null;\n}): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `INSERT INTO integration_pending_tasks\n (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n input.id,\n input.platform,\n input.externalThreadId,\n input.payload,\n input.ownerEmail,\n input.orgId ?? null,\n \"pending\",\n 0,\n now,\n now,\n input.externalEventKey ?? null,\n ],\n });\n}\n\n/**\n * Returns whether a duplicate-event error from `insertPendingTask` looks\n * like a unique-constraint violation on `(platform, external_event_key)`.\n *\n * Postgres surfaces these as `error.code === \"23505\"`, while SQLite uses\n * a substring match on the error text. Used by the webhook handler to\n * distinguish \"already enqueued\" (silently OK) from genuine insert failures.\n */\nexport function isDuplicateEventError(err: unknown): boolean {\n const e = err as { code?: string; message?: string } | null;\n if (!e) return false;\n if (e.code === \"23505\") return true; // Postgres unique-violation\n const msg = String(e.message ?? \"\").toLowerCase();\n return (\n msg.includes(\"unique\") ||\n msg.includes(\"duplicate entry\") ||\n msg.includes(\"duplicate key\")\n );\n}\n\n/** Fetch a pending task by id. */\nexport async function getPendingTask(id: string): Promise<PendingTask | null> {\n await ensureTable();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at\n FROM integration_pending_tasks WHERE id = ? LIMIT 1`,\n args: [id],\n });\n if (rows.length === 0) return null;\n return rowToTask(rows[0] as Record<string, unknown>);\n}\n\nexport interface ResolvedIntegrationSourceContext {\n platform: \"slack\";\n sourceUrl: string;\n}\n\ntype PendingTaskSourceRow = Pick<\n PendingTask,\n \"platform\" | \"payload\" | \"ownerEmail\" | \"orgId\"\n>;\n\nexport function sourceContextFromPendingTask(\n task: PendingTaskSourceRow | null,\n ownerEmail: string,\n orgId: string | null,\n): ResolvedIntegrationSourceContext | null {\n if (\n !task ||\n task.ownerEmail !== ownerEmail ||\n task.orgId !== orgId ||\n task.platform !== \"slack\"\n ) {\n return null;\n }\n\n try {\n const payload = JSON.parse(task.payload) as Record<string, unknown>;\n const incoming = payload.incoming;\n if (!incoming || typeof incoming !== \"object\") return null;\n const incomingRecord = incoming as Record<string, unknown>;\n if (incomingRecord.platform !== \"slack\") return null;\n const sourceUrl = incomingRecord.sourceUrl;\n if (\n typeof sourceUrl !== \"string\" ||\n !sourceUrl ||\n sourceUrl !== sourceUrl.trim()\n ) {\n return null;\n }\n\n const parsed = new URL(sourceUrl);\n const isSlackHost =\n parsed.hostname === \"slack.com\" || parsed.hostname.endsWith(\".slack.com\");\n if (\n parsed.protocol !== \"https:\" ||\n !isSlackHost ||\n parsed.username ||\n parsed.password ||\n parsed.port\n ) {\n return null;\n }\n return { platform: \"slack\", sourceUrl };\n } catch {\n return null;\n }\n}\n\n/** Resolve trusted Slack provenance without exposing the stored task payload. */\nexport async function resolveIntegrationSourceContext(\n id: string,\n ownerEmail: string,\n orgId: string | null,\n): Promise<ResolvedIntegrationSourceContext | null> {\n await ensureTable();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT platform, payload, owner_email, org_id\n FROM integration_pending_tasks\n WHERE id = ?\n AND owner_email = ?\n AND (org_id = ? OR (org_id IS NULL AND CAST(? AS TEXT) IS NULL))\n AND platform = 'slack'\n LIMIT 1`,\n args: [id, ownerEmail, orgId, orgId],\n });\n if (rows.length === 0) return null;\n const row = rows[0] as Record<string, unknown>;\n return sourceContextFromPendingTask(\n {\n platform: row.platform as string,\n payload: row.payload as string,\n ownerEmail: row.owner_email as string,\n orgId: (row.org_id as string | null) ?? null,\n },\n ownerEmail,\n orgId,\n );\n}\n\n/**\n * Atomically claim a task: transition pending → processing and increment\n * attempts. Returns the updated task if the transition succeeded, otherwise\n * null (e.g. the task was already claimed by a concurrent worker).\n */\nexport async function claimPendingTask(\n id: string,\n): Promise<PendingTask | null> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n\n // Conditional update: only flip if currently pending. Failed tasks are\n // terminal unless an explicit retry path resets them to pending first.\n const result = await client.execute({\n sql: isPostgres()\n ? `UPDATE integration_pending_tasks\n SET status = ?, attempts = attempts + 1, updated_at = ?\n WHERE id = ? AND status = 'pending'\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks active\n WHERE active.platform = integration_pending_tasks.platform\n AND active.external_thread_id = integration_pending_tasks.external_thread_id\n AND active.status = 'processing'\n AND active.id <> integration_pending_tasks.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks earlier\n WHERE earlier.platform = integration_pending_tasks.platform\n AND earlier.external_thread_id = integration_pending_tasks.external_thread_id\n AND earlier.status = 'pending'\n AND (\n earlier.created_at < integration_pending_tasks.created_at\n OR (\n earlier.created_at = integration_pending_tasks.created_at\n AND earlier.id < integration_pending_tasks.id\n )\n )\n )\n RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at`\n : `UPDATE integration_pending_tasks\n SET status = ?, attempts = attempts + 1, updated_at = ?\n WHERE id = ? AND status = 'pending'\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks active\n WHERE active.platform = integration_pending_tasks.platform\n AND active.external_thread_id = integration_pending_tasks.external_thread_id\n AND active.status = 'processing'\n AND active.id <> integration_pending_tasks.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks earlier\n WHERE earlier.platform = integration_pending_tasks.platform\n AND earlier.external_thread_id = integration_pending_tasks.external_thread_id\n AND earlier.status = 'pending'\n AND (\n earlier.created_at < integration_pending_tasks.created_at\n OR (\n earlier.created_at = integration_pending_tasks.created_at\n AND earlier.id < integration_pending_tasks.id\n )\n )\n )`,\n args: [\"processing\", now, id],\n });\n const rows = result.rows ?? [];\n\n if (isPostgres()) {\n if (rows.length === 0) return null;\n return rowToTask(rows[0] as Record<string, unknown>);\n }\n\n // SQLite: no RETURNING, so re-read after the update. Confirm we actually\n // moved it into 'processing' (vs. lost the race).\n const affected =\n (result as { rowsAffected?: number; rowCount?: number }).rowsAffected ??\n (result as { rowsAffected?: number; rowCount?: number }).rowCount;\n if (affected === 0) return null;\n const fetched = await getPendingTask(id);\n if (!fetched || fetched.status !== \"processing\") return null;\n return fetched;\n}\n\n/** Next queued turn for a provider thread after its current task completes. */\nexport async function getNextPendingTaskIdForThread(\n platform: string,\n externalThreadId: string,\n): Promise<string | null> {\n await ensureTable();\n const { rows } = await getDbExec().execute({\n sql: `SELECT id FROM integration_pending_tasks\n WHERE platform = ? AND external_thread_id = ? AND status = 'pending'\n ORDER BY created_at ASC, id ASC LIMIT 1`,\n args: [platform, externalThreadId],\n });\n return rows[0]?.id ? String(rows[0].id) : null;\n}\n\n/** Mark a task as completed. */\nexport async function markTaskCompleted(id: string): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, completed_at = ?, payload = ?\n WHERE id = ?`,\n // The payload can contain short-lived provider credentials such as a\n // Discord interaction token. Once terminal, no retry needs the inbound\n // body, so erase it instead of retaining secrets or user text indefinitely.\n args: [\"completed\", now, now, \"{}\", id],\n });\n}\n\n/**\n * Return a transiently failed task to the retryable queue without erasing its\n * payload. The payload may contain the only copy of the inbound message and is\n * scrubbed only when the task reaches a permanent terminal state.\n */\nexport async function markTaskRetryable(\n id: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, error_message = ?\n WHERE id = ? AND status = 'processing'`,\n args: [\"pending\", now, errorMessage.slice(0, 2000), id],\n });\n}\n\nexport async function stageTaskDeliveryPayload(\n id: string,\n payload: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n const result = await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET payload = ?, updated_at = ?\n WHERE id = ? AND status = 'processing'`,\n args: [payload, now, id],\n });\n const affected = Number(\n (result as { rowsAffected?: number }).rowsAffected ??\n (result as { rowCount?: number }).rowCount ??\n 0,\n );\n if (affected === 0) {\n throw new Error(\"Integration task is no longer claimable for delivery\");\n }\n}\n\nexport async function markTaskDeliveryRetryable(\n id: string,\n payload: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const result = await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, payload = ?, updated_at = ?, error_message = ?\n WHERE id = ? AND status = 'processing'`,\n args: [\"pending\", payload, Date.now(), errorMessage.slice(0, 2000), id],\n });\n const affected = Number(\n (result as { rowsAffected?: number }).rowsAffected ??\n (result as { rowCount?: number }).rowCount ??\n 0,\n );\n if (affected === 0) {\n throw new Error(\n \"Integration task is no longer claimable for delivery retry\",\n );\n }\n}\n\nexport async function failTaskDeliveryTransition(\n id: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const result = await getDbExec().execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, error_message = ?, payload = ?\n WHERE id = ? AND status = 'processing'`,\n args: [\"failed\", Date.now(), errorMessage.slice(0, 2000), \"{}\", id],\n });\n const affected = Number(\n (result as { rowsAffected?: number }).rowsAffected ??\n (result as { rowCount?: number }).rowCount ??\n 0,\n );\n if (affected === 0) {\n throw new Error(\n \"Integration task delivery failure transition lost its race\",\n );\n }\n}\n\n/** Mark a task as failed and stash an error message. */\nexport async function markTaskFailed(\n id: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, error_message = ?, payload = ?, external_event_key = NULL\n WHERE id = ?`,\n args: [\"failed\", now, errorMessage.slice(0, 2000), \"{}\", id],\n });\n}\n"]}
|
|
1
|
+
{"version":3,"file":"pending-tasks-store.js","sourceRoot":"","sources":["../../src/integrations/pending-tasks-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAE5B,IAAI,YAAuC,CAAC;AAC5C,MAAM,CAAC,MAAM,yBAAyB,GAAG,CAAC,CAAC;AAE3C,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;YAC3B,MAAM,SAAS,GAAG;;;;;;;;aAQX,OAAO,EAAE;sBACA,OAAO,EAAE;qBACV,OAAO,EAAE;;;;eAIf,OAAO,EAAE;eACT,OAAO,EAAE;iBACP,OAAO,EAAE;EACxB,CAAC;YAEG,IAAI,UAAU,EAAE,EAAE,CAAC;gBACjB,0FAA0F;gBAC1F,MAAM,iBAAiB,CAAC,2BAA2B,EAAE,SAAS,CAAC,CAAC;gBAChE,MAAM,kBAAkB,CACtB,2BAA2B,EAC3B,oBAAoB,EACpB,wFAAwF,CACzF,CAAC;gBACF,MAAM,kBAAkB,CACtB,2BAA2B,EAC3B,mBAAmB,EACnB,oFAAoF,OAAO,EAAE,qBAAqB,CACnH,CAAC;gBACF,MAAM,kBAAkB,CACtB,2BAA2B,EAC3B,kBAAkB,EAClB,mFAAmF,OAAO,EAAE,EAAE,CAC/F,CAAC;gBACF,MAAM,kBAAkB,CACtB,2BAA2B,EAC3B,uBAAuB,EACvB,2FAA2F,CAC5F,CAAC;gBACF,MAAM,kBAAkB,CACtB,2BAA2B,EAC3B,gBAAgB,EAChB,oFAAoF,CACrF,CAAC;gBACF,MAAM,iBAAiB,CACrB,kCAAkC,EAClC,8GAA8G,CAC/G,CAAC;gBACF,MAAM,iBAAiB,CACrB,6BAA6B,EAC7B,0HAA0H,CAC3H,CAAC;gBACF,MAAM,iBAAiB,CACrB,kCAAkC,EAClC,oHAAoH,CACrH,CAAC;gBACF,OAAO;YACT,CAAC;YAED,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YAChC,MAAM,MAAM,CAAC,OAAO,CAClB,8GAA8G,CAC/G,CAAC;YACF,oEAAoE;YACpE,oEAAoE;YACpE,+DAA+D;YAC/D,kEAAkE;YAClE,iEAAiE;YACjE,+BAA+B;YAC/B,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,qBAAqB,CAAC,MAAM,CAAC,CAAC;YACpC,MAAM,MAAM,CAAC,OAAO,CAClB,oHAAoH,CACrH,CAAC;YACF,MAAM,MAAM,CAAC,OAAO,CAClB,0HAA0H,CAC3H,CAAC;QACJ,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACjB,sDAAsD;YACtD,YAAY,GAAG,SAAS,CAAC;YACzB,MAAM,GAAG,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB;IAC3C,MAAM,WAAW,EAAE,CAAC;AACtB,CAAC;AAED,KAAK,UAAU,sBAAsB,CACnC,MAAoC;IAEpC,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,MAAM,MAAM,CAAC,OAAO,CAClB,wFAAwF,CACzF,CAAC;QACF,OAAO;IACT,CAAC;IACD,wEAAwE;IACxE,uDAAuD;IACvD,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,OAAO,CAClB,0EAA0E,CAC3E,CAAC;IACJ,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,IACE,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,GAAG,CAAC;aACzB,WAAW,EAAE;aACb,QAAQ,CAAC,WAAW,CAAC,EACxB,CAAC;YACD,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,MAAoC;IAEpC,MAAM,OAAO,GAAG;QACd,qBAAqB,OAAO,EAAE,qBAAqB;QACnD,oBAAoB,OAAO,EAAE,EAAE;QAC/B,4BAA4B;QAC5B,qBAAqB;KACtB,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,OAAO,CAClB,oDAAoD,MAAM,EAAE,CAC7D,CAAC;QACJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IACE,CAAC,MAAM,CAAC,GAAG,EAAE,OAAO,IAAI,GAAG,CAAC;iBACzB,WAAW,EAAE;iBACb,QAAQ,CAAC,WAAW,CAAC,EACxB,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AA4BD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,QAAgB,EAChB,gBAAwB;IAExB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;;;;;cAKK;QACV,IAAI,EAAE,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KACnC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,SAAS,SAAS,CAAC,GAA4B;IAC7C,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAY;QACpB,QAAQ,EAAE,GAAG,CAAC,QAAkB;QAChC,gBAAgB,EAAE,GAAG,CAAC,kBAA4B;QAClD,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,UAAU,EAAE,GAAG,CAAC,WAAqB;QACrC,KAAK,EAAG,GAAG,CAAC,MAAwB,IAAI,IAAI;QAC5C,MAAM,EAAE,GAAG,CAAC,MAA2B;QACvC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,CAAC;QACnC,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,iBAAiB,IAAI,CAAC,CAAC;QACpD,cAAc,EACZ,GAAG,CAAC,gBAAgB,IAAI,IAAI;YAC1B,CAAC,CAAC,IAAI;YACN,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,gBAA0B,CAAC;QAC5C,mBAAmB,EAAG,GAAG,CAAC,qBAAuC,IAAI,IAAI;QACzE,aAAa,EAAG,GAAG,CAAC,cAAgC,IAAI,IAAI;QAC5D,YAAY,EAAG,GAAG,CAAC,aAA+B,IAAI,IAAI;QAC1D,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,IAAI,CAAC,CAAC;QACtC,WAAW,EACT,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,YAAsB,CAAC;KACvE,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,KASvC;IACC,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;kDAEyC;QAC9C,IAAI,EAAE;YACJ,KAAK,CAAC,EAAE;YACR,KAAK,CAAC,QAAQ;YACd,KAAK,CAAC,gBAAgB;YACtB,KAAK,CAAC,OAAO;YACb,KAAK,CAAC,UAAU;YAChB,KAAK,CAAC,KAAK,IAAI,IAAI;YACnB,SAAS;YACT,CAAC;YACD,GAAG;YACH,GAAG;YACH,KAAK,CAAC,gBAAgB,IAAI,IAAI;YAC9B,KAAK,CAAC,aAAa,IAAI,IAAI;SAC5B;KACF,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,GAAY;IAChD,MAAM,CAAC,GAAG,GAAiD,CAAC;IAC5D,IAAI,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IACrB,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC,CAAC,4BAA4B;IACjE,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAClD,OAAO,CACL,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACtB,GAAG,CAAC,QAAQ,CAAC,iBAAiB,CAAC;QAC/B,GAAG,CAAC,QAAQ,CAAC,eAAe,CAAC,CAC9B,CAAC;AACJ,CAAC;AAED,kCAAkC;AAClC,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAU;IAC7C,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;8DACqD;QAC1D,IAAI,EAAE,CAAC,EAAE,CAAC;KACX,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;AACvD,CAAC;AAYD,MAAM,UAAU,4BAA4B,CAC1C,IAAiC,EACjC,UAAkB,EAClB,KAAoB;IAEpB,IACE,CAAC,IAAI;QACL,IAAI,CAAC,UAAU,KAAK,UAAU;QAC9B,IAAI,CAAC,KAAK,KAAK,KAAK;QACpB,IAAI,CAAC,QAAQ,KAAK,OAAO,EACzB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAA4B,CAAC;QACpE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAClC,IAAI,CAAC,QAAQ,IAAI,OAAO,QAAQ,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC3D,MAAM,cAAc,GAAG,QAAmC,CAAC;QAC3D,IAAI,cAAc,CAAC,QAAQ,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QACrD,MAAM,SAAS,GAAG,cAAc,CAAC,SAAS,CAAC;QAC3C,IACE,OAAO,SAAS,KAAK,QAAQ;YAC7B,CAAC,SAAS;YACV,SAAS,KAAK,SAAS,CAAC,IAAI,EAAE,EAC9B,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;QAClC,MAAM,WAAW,GACf,MAAM,CAAC,QAAQ,KAAK,WAAW,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAC5E,IACE,MAAM,CAAC,QAAQ,KAAK,QAAQ;YAC5B,CAAC,WAAW;YACZ,MAAM,CAAC,QAAQ;YACf,MAAM,CAAC,QAAQ;YACf,MAAM,CAAC,IAAI,EACX,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,MAAM,CAAC,KAAK,UAAU,+BAA+B,CACnD,EAAU,EACV,UAAkB,EAClB,KAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QACpC,GAAG,EAAE;;;;;;kBAMS;QACd,IAAI,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,CAAC;KACrC,CAAC,CAAC;IACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAA4B,CAAC;IAC/C,OAAO,4BAA4B,CACjC;QACE,QAAQ,EAAE,GAAG,CAAC,QAAkB;QAChC,OAAO,EAAE,GAAG,CAAC,OAAiB;QAC9B,UAAU,EAAE,GAAG,CAAC,WAAqB;QACrC,KAAK,EAAG,GAAG,CAAC,MAAwB,IAAI,IAAI;KAC7C,EACD,UAAU,EACV,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,EAAU,EACV,OAAsC;IAEtC,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE,UAAU,EAAE;YACf,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;qOAwB6N;YAC/N,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;aAuBK;QACT,IAAI,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,IAAI,IAAI,EAAE,EAAE,CAAC;KAChE,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;IAE/B,IAAI,UAAU,EAAE,EAAE,CAAC;QACjB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACnC,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAA4B,CAAC,CAAC;IACvD,CAAC;IAED,yEAAyE;IACzE,kDAAkD;IAClD,MAAM,QAAQ,GACX,MAAuD,CAAC,YAAY;QACpE,MAAuD,CAAC,QAAQ,CAAC;IACpE,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,EAAE,CAAC,CAAC;IACzC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY;QAAE,OAAO,IAAI,CAAC;IAC7D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gCAAgC,CACpD,EAAU,EACV,OAAe;IAEf,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,SAAS,EAAE,CAAC,OAAO,CAAC;QACxB,GAAG,EAAE;;;;;;;;;uBASc;QACnB,IAAI,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;KAC7C,CAAC,CAAC;AACL,CAAC;AAED,+EAA+E;AAC/E,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,QAAgB,EAChB,gBAAwB;IAExB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,EAAE,CAAC,OAAO,CAAC;QACzC,GAAG,EAAE;;8CAEqC;QAC1C,IAAI,EAAE,CAAC,QAAQ,EAAE,gBAAgB,CAAC;KACnC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE;QAChB,CAAC,CAAC;YACE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACtB,aAAa,EAAG,IAAI,CAAC,CAAC,CAAC,CAAC,cAAgC,IAAI,IAAI;SACjE;QACH,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,gCAAgC;AAChC,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,EAAU;IAChD,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,qEAAqE;QACrE,uEAAuE;QACvE,4EAA4E;QAC5E,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC;KACxC,CAAC,CAAC;AACL,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;KACxD,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAC5C,EAAU,EACV,OAAe;IAEf,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;KACzB,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,CACpB,MAAoC,CAAC,YAAY;QAC/C,MAAgC,CAAC,QAAQ;QAC1C,CAAC,CACJ,CAAC;IACF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,EAAU,EACV,OAAe,EACf,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC;QAClC,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC;KACxE,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,CACpB,MAAoC,CAAC,YAAY;QAC/C,MAAgC,CAAC,QAAQ;QAC1C,CAAC,CACJ,CAAC;IACF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,MAAM,SAAS,EAAE,CAAC,OAAO,CAAC;QACvC,GAAG,EAAE;;iDAEwC;QAC7C,IAAI,EAAE,CAAC,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;KACpE,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,MAAM,CACpB,MAAoC,CAAC,YAAY;QAC/C,MAAgC,CAAC,QAAQ;QAC1C,CAAC,CACJ,CAAC;IACF,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CACb,4DAA4D,CAC7D,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wDAAwD;AACxD,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,EAAU,EACV,YAAoB;IAEpB,MAAM,WAAW,EAAE,CAAC;IACpB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,CAAC,OAAO,CAAC;QACnB,GAAG,EAAE;;uBAEc;QACnB,IAAI,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;KAC7D,CAAC,CAAC;AACL,CAAC","sourcesContent":["/**\n * SQL-backed pending task queue for integration webhooks.\n *\n * Why this exists: serverless platforms (Netlify Lambda, Vercel, Cloudflare\n * Workers) freeze the function execution as soon as the HTTP response is\n * returned. Fire-and-forget background `Promise`s get killed mid-flight,\n * meaning agent loops triggered from a Slack/Telegram webhook never finish.\n *\n * Solution: persist the inbound message to SQL inside the webhook handler,\n * then dispatch a fresh HTTP POST to a separate processor endpoint. Each\n * invocation gets its own fresh function timeout budget.\n */\nimport { getDbExec, isPostgres, intType } from \"../db/client.js\";\nimport {\n ensureTableExists,\n ensureColumnExists,\n ensureIndexExists,\n} from \"../db/ddl-guard.js\";\n\nlet _initPromise: Promise<void> | undefined;\nexport const MAX_PENDING_TASK_ATTEMPTS = 3;\n\nasync function ensureTable(): Promise<void> {\n if (!_initPromise) {\n _initPromise = (async () => {\n const client = getDbExec();\n const createSql = `CREATE TABLE IF NOT EXISTS integration_pending_tasks (\n id TEXT PRIMARY KEY,\n platform TEXT NOT NULL,\n external_thread_id TEXT NOT NULL,\n payload TEXT NOT NULL,\n owner_email TEXT NOT NULL,\n org_id TEXT,\n status TEXT NOT NULL,\n attempts ${intType()} NOT NULL DEFAULT 0,\n dispatch_attempts ${intType()} NOT NULL DEFAULT 0,\n last_dispatch_at ${intType()},\n last_dispatch_outcome TEXT,\n dispatch_scope TEXT,\n error_message TEXT,\n created_at ${intType()} NOT NULL,\n updated_at ${intType()} NOT NULL,\n completed_at ${intType()}\n)`;\n\n if (isPostgres()) {\n // PG guard: probe via information_schema, only issue DDL if missing, bounded lock_timeout\n await ensureTableExists(\"integration_pending_tasks\", createSql);\n await ensureColumnExists(\n \"integration_pending_tasks\",\n \"external_event_key\",\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,\n );\n await ensureColumnExists(\n \"integration_pending_tasks\",\n \"dispatch_attempts\",\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_attempts ${intType()} NOT NULL DEFAULT 0`,\n );\n await ensureColumnExists(\n \"integration_pending_tasks\",\n \"last_dispatch_at\",\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_at ${intType()}`,\n );\n await ensureColumnExists(\n \"integration_pending_tasks\",\n \"last_dispatch_outcome\",\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_outcome TEXT`,\n );\n await ensureColumnExists(\n \"integration_pending_tasks\",\n \"dispatch_scope\",\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_scope TEXT`,\n );\n await ensureIndexExists(\n \"idx_pending_tasks_status_created\",\n `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,\n );\n await ensureIndexExists(\n \"idx_pending_tasks_event_key\",\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,\n );\n await ensureIndexExists(\n \"idx_pending_tasks_dispatch_scope\",\n `CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`,\n );\n return;\n }\n\n await client.execute(createSql);\n await client.execute(\n `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,\n );\n // Additive migration: add a stable per-event dedup key so duplicate\n // webhook deliveries from the same platform get rejected at the SQL\n // layer instead of via an in-memory Map (which doesn't survive\n // serverless cold starts — H3 in the webhook security audit). The\n // unique index ensures a duplicate INSERT raises an error we can\n // catch as \"already-enqueued\".\n await ensureExternalEventKey(client);\n await ensureDispatchColumns(client);\n await client.execute(\n `CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`,\n );\n await client.execute(\n `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,\n );\n })().catch((err) => {\n // Retry init on the next call after a failed startup.\n _initPromise = undefined;\n throw err;\n });\n }\n return _initPromise;\n}\n\nexport async function ensurePendingTasksTable(): Promise<void> {\n await ensureTable();\n}\n\nasync function ensureExternalEventKey(\n client: ReturnType<typeof getDbExec>,\n): Promise<void> {\n if (isPostgres()) {\n await client.execute(\n `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,\n );\n return;\n }\n // SQLite doesn't support `ADD COLUMN IF NOT EXISTS` until 3.35; swallow\n // the duplicate-column error so reruns are idempotent.\n try {\n await client.execute(\n `ALTER TABLE integration_pending_tasks ADD COLUMN external_event_key TEXT`,\n );\n } catch (err: any) {\n if (\n !String(err?.message ?? err)\n .toLowerCase()\n .includes(\"duplicate\")\n ) {\n throw err;\n }\n }\n}\n\nasync function ensureDispatchColumns(\n client: ReturnType<typeof getDbExec>,\n): Promise<void> {\n const columns = [\n `dispatch_attempts ${intType()} NOT NULL DEFAULT 0`,\n `last_dispatch_at ${intType()}`,\n \"last_dispatch_outcome TEXT\",\n \"dispatch_scope TEXT\",\n ];\n for (const column of columns) {\n try {\n await client.execute(\n `ALTER TABLE integration_pending_tasks ADD COLUMN ${column}`,\n );\n } catch (err: any) {\n if (\n !String(err?.message ?? err)\n .toLowerCase()\n .includes(\"duplicate\")\n ) {\n throw err;\n }\n }\n }\n}\n\n/** Status values for an integration pending task. */\nexport type PendingTaskStatus =\n | \"pending\"\n | \"processing\"\n | \"completed\"\n | \"failed\";\n\nexport interface PendingTask {\n id: string;\n platform: string;\n externalThreadId: string;\n payload: string;\n ownerEmail: string;\n orgId: string | null;\n status: PendingTaskStatus;\n attempts: number;\n dispatchAttempts: number;\n lastDispatchAt: number | null;\n lastDispatchOutcome: string | null;\n dispatchScope: string | null;\n errorMessage: string | null;\n createdAt: number;\n updatedAt: number;\n completedAt: number | null;\n}\n\n/**\n * Whether a provider thread currently has queued or executing work.\n *\n * Messaging adapters use this to accept unmentioned replies only while an\n * agent task is active. This prevents broad message subscriptions from turning\n * every channel message into an agent invocation.\n */\nexport async function hasActivePendingTask(\n platform: string,\n externalThreadId: string,\n): Promise<boolean> {\n await ensureTable();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT 1 AS active\n FROM integration_pending_tasks\n WHERE platform = ?\n AND external_thread_id = ?\n AND status IN ('pending', 'processing')\n LIMIT 1`,\n args: [platform, externalThreadId],\n });\n return rows.length > 0;\n}\n\nfunction rowToTask(row: Record<string, unknown>): PendingTask {\n return {\n id: row.id as string,\n platform: row.platform as string,\n externalThreadId: row.external_thread_id as string,\n payload: row.payload as string,\n ownerEmail: row.owner_email as string,\n orgId: (row.org_id as string | null) ?? null,\n status: row.status as PendingTaskStatus,\n attempts: Number(row.attempts ?? 0),\n dispatchAttempts: Number(row.dispatch_attempts ?? 0),\n lastDispatchAt:\n row.last_dispatch_at == null\n ? null\n : Number(row.last_dispatch_at as number),\n lastDispatchOutcome: (row.last_dispatch_outcome as string | null) ?? null,\n dispatchScope: (row.dispatch_scope as string | null) ?? null,\n errorMessage: (row.error_message as string | null) ?? null,\n createdAt: Number(row.created_at ?? 0),\n updatedAt: Number(row.updated_at ?? 0),\n completedAt:\n row.completed_at == null ? null : Number(row.completed_at as number),\n };\n}\n\n/**\n * Insert a new pending task. Returns the generated task id.\n *\n * If `externalEventKey` is supplied, the unique index on\n * `(platform, external_event_key)` will reject duplicates — callers should\n * catch the resulting constraint-violation error and treat it as\n * \"already enqueued\" instead of a hard failure (H3 in the webhook security\n * audit). This is the SQL-backed replacement for the in-memory dedup map.\n */\nexport async function insertPendingTask(input: {\n id: string;\n platform: string;\n externalThreadId: string;\n payload: string;\n ownerEmail: string;\n orgId?: string | null;\n externalEventKey?: string | null;\n dispatchScope?: string | null;\n}): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `INSERT INTO integration_pending_tasks\n (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key, dispatch_scope)\n VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,\n args: [\n input.id,\n input.platform,\n input.externalThreadId,\n input.payload,\n input.ownerEmail,\n input.orgId ?? null,\n \"pending\",\n 0,\n now,\n now,\n input.externalEventKey ?? null,\n input.dispatchScope ?? null,\n ],\n });\n}\n\n/**\n * Returns whether a duplicate-event error from `insertPendingTask` looks\n * like a unique-constraint violation on `(platform, external_event_key)`.\n *\n * Postgres surfaces these as `error.code === \"23505\"`, while SQLite uses\n * a substring match on the error text. Used by the webhook handler to\n * distinguish \"already enqueued\" (silently OK) from genuine insert failures.\n */\nexport function isDuplicateEventError(err: unknown): boolean {\n const e = err as { code?: string; message?: string } | null;\n if (!e) return false;\n if (e.code === \"23505\") return true; // Postgres unique-violation\n const msg = String(e.message ?? \"\").toLowerCase();\n return (\n msg.includes(\"unique\") ||\n msg.includes(\"duplicate entry\") ||\n msg.includes(\"duplicate key\")\n );\n}\n\n/** Fetch a pending task by id. */\nexport async function getPendingTask(id: string): Promise<PendingTask | null> {\n await ensureTable();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at\n FROM integration_pending_tasks WHERE id = ? LIMIT 1`,\n args: [id],\n });\n if (rows.length === 0) return null;\n return rowToTask(rows[0] as Record<string, unknown>);\n}\n\nexport interface ResolvedIntegrationSourceContext {\n platform: \"slack\";\n sourceUrl: string;\n}\n\ntype PendingTaskSourceRow = Pick<\n PendingTask,\n \"platform\" | \"payload\" | \"ownerEmail\" | \"orgId\"\n>;\n\nexport function sourceContextFromPendingTask(\n task: PendingTaskSourceRow | null,\n ownerEmail: string,\n orgId: string | null,\n): ResolvedIntegrationSourceContext | null {\n if (\n !task ||\n task.ownerEmail !== ownerEmail ||\n task.orgId !== orgId ||\n task.platform !== \"slack\"\n ) {\n return null;\n }\n\n try {\n const payload = JSON.parse(task.payload) as Record<string, unknown>;\n const incoming = payload.incoming;\n if (!incoming || typeof incoming !== \"object\") return null;\n const incomingRecord = incoming as Record<string, unknown>;\n if (incomingRecord.platform !== \"slack\") return null;\n const sourceUrl = incomingRecord.sourceUrl;\n if (\n typeof sourceUrl !== \"string\" ||\n !sourceUrl ||\n sourceUrl !== sourceUrl.trim()\n ) {\n return null;\n }\n\n const parsed = new URL(sourceUrl);\n const isSlackHost =\n parsed.hostname === \"slack.com\" || parsed.hostname.endsWith(\".slack.com\");\n if (\n parsed.protocol !== \"https:\" ||\n !isSlackHost ||\n parsed.username ||\n parsed.password ||\n parsed.port\n ) {\n return null;\n }\n return { platform: \"slack\", sourceUrl };\n } catch {\n return null;\n }\n}\n\n/** Resolve trusted Slack provenance without exposing the stored task payload. */\nexport async function resolveIntegrationSourceContext(\n id: string,\n ownerEmail: string,\n orgId: string | null,\n): Promise<ResolvedIntegrationSourceContext | null> {\n await ensureTable();\n const client = getDbExec();\n const { rows } = await client.execute({\n sql: `SELECT platform, payload, owner_email, org_id\n FROM integration_pending_tasks\n WHERE id = ?\n AND owner_email = ?\n AND (org_id = ? OR (org_id IS NULL AND CAST(? AS TEXT) IS NULL))\n AND platform = 'slack'\n LIMIT 1`,\n args: [id, ownerEmail, orgId, orgId],\n });\n if (rows.length === 0) return null;\n const row = rows[0] as Record<string, unknown>;\n return sourceContextFromPendingTask(\n {\n platform: row.platform as string,\n payload: row.payload as string,\n ownerEmail: row.owner_email as string,\n orgId: (row.org_id as string | null) ?? null,\n },\n ownerEmail,\n orgId,\n );\n}\n\n/**\n * Atomically claim a task: transition pending → processing and increment\n * attempts. Returns the updated task if the transition succeeded, otherwise\n * null (e.g. the task was already claimed by a concurrent worker).\n */\nexport async function claimPendingTask(\n id: string,\n options?: { dispatchOutcome?: string },\n): Promise<PendingTask | null> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n\n // Conditional update: only flip if currently pending. Failed tasks are\n // terminal unless an explicit retry path resets them to pending first.\n const result = await client.execute({\n sql: isPostgres()\n ? `UPDATE integration_pending_tasks\n SET status = ?, attempts = attempts + 1, updated_at = ?,\n last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)\n WHERE id = ? AND status = 'pending'\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks active\n WHERE active.platform = integration_pending_tasks.platform\n AND active.external_thread_id = integration_pending_tasks.external_thread_id\n AND active.status = 'processing'\n AND active.id <> integration_pending_tasks.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks earlier\n WHERE earlier.platform = integration_pending_tasks.platform\n AND earlier.external_thread_id = integration_pending_tasks.external_thread_id\n AND earlier.status = 'pending'\n AND (\n earlier.created_at < integration_pending_tasks.created_at\n OR (\n earlier.created_at = integration_pending_tasks.created_at\n AND earlier.id < integration_pending_tasks.id\n )\n )\n )\n RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at`\n : `UPDATE integration_pending_tasks\n SET status = ?, attempts = attempts + 1, updated_at = ?,\n last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)\n WHERE id = ? AND status = 'pending'\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks active\n WHERE active.platform = integration_pending_tasks.platform\n AND active.external_thread_id = integration_pending_tasks.external_thread_id\n AND active.status = 'processing'\n AND active.id <> integration_pending_tasks.id\n )\n AND NOT EXISTS (\n SELECT 1 FROM integration_pending_tasks earlier\n WHERE earlier.platform = integration_pending_tasks.platform\n AND earlier.external_thread_id = integration_pending_tasks.external_thread_id\n AND earlier.status = 'pending'\n AND (\n earlier.created_at < integration_pending_tasks.created_at\n OR (\n earlier.created_at = integration_pending_tasks.created_at\n AND earlier.id < integration_pending_tasks.id\n )\n )\n )`,\n args: [\"processing\", now, options?.dispatchOutcome ?? null, id],\n });\n const rows = result.rows ?? [];\n\n if (isPostgres()) {\n if (rows.length === 0) return null;\n return rowToTask(rows[0] as Record<string, unknown>);\n }\n\n // SQLite: no RETURNING, so re-read after the update. Confirm we actually\n // moved it into 'processing' (vs. lost the race).\n const affected =\n (result as { rowsAffected?: number; rowCount?: number }).rowsAffected ??\n (result as { rowsAffected?: number; rowCount?: number }).rowCount;\n if (affected === 0) return null;\n const fetched = await getPendingTask(id);\n if (!fetched || fetched.status !== \"processing\") return null;\n return fetched;\n}\n\nexport async function recordPendingTaskDispatchAttempt(\n id: string,\n outcome: string,\n): Promise<void> {\n await ensureTable();\n await getDbExec().execute({\n sql: `UPDATE integration_pending_tasks\n SET dispatch_attempts = dispatch_attempts + 1,\n last_dispatch_at = ?,\n last_dispatch_outcome = CASE\n WHEN status = 'processing'\n AND last_dispatch_outcome = 'background-acknowledged'\n THEN last_dispatch_outcome\n ELSE ?\n END\n WHERE id = ?`,\n args: [Date.now(), outcome.slice(0, 80), id],\n });\n}\n\n/** Next queued turn for a provider thread after its current task completes. */\nexport async function getNextPendingTaskForThread(\n platform: string,\n externalThreadId: string,\n): Promise<{ id: string; dispatchScope: string | null } | null> {\n await ensureTable();\n const { rows } = await getDbExec().execute({\n sql: `SELECT id, dispatch_scope FROM integration_pending_tasks\n WHERE platform = ? AND external_thread_id = ? AND status = 'pending'\n ORDER BY created_at ASC, id ASC LIMIT 1`,\n args: [platform, externalThreadId],\n });\n return rows[0]?.id\n ? {\n id: String(rows[0].id),\n dispatchScope: (rows[0].dispatch_scope as string | null) ?? null,\n }\n : null;\n}\n\n/** Mark a task as completed. */\nexport async function markTaskCompleted(id: string): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, completed_at = ?, payload = ?\n WHERE id = ?`,\n // The payload can contain short-lived provider credentials such as a\n // Discord interaction token. Once terminal, no retry needs the inbound\n // body, so erase it instead of retaining secrets or user text indefinitely.\n args: [\"completed\", now, now, \"{}\", id],\n });\n}\n\n/**\n * Return a transiently failed task to the retryable queue without erasing its\n * payload. The payload may contain the only copy of the inbound message and is\n * scrubbed only when the task reaches a permanent terminal state.\n */\nexport async function markTaskRetryable(\n id: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, error_message = ?\n WHERE id = ? AND status = 'processing'`,\n args: [\"pending\", now, errorMessage.slice(0, 2000), id],\n });\n}\n\nexport async function stageTaskDeliveryPayload(\n id: string,\n payload: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n const result = await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET payload = ?, updated_at = ?\n WHERE id = ? AND status = 'processing'`,\n args: [payload, now, id],\n });\n const affected = Number(\n (result as { rowsAffected?: number }).rowsAffected ??\n (result as { rowCount?: number }).rowCount ??\n 0,\n );\n if (affected === 0) {\n throw new Error(\"Integration task is no longer claimable for delivery\");\n }\n}\n\nexport async function markTaskDeliveryRetryable(\n id: string,\n payload: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const result = await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, payload = ?, updated_at = ?, error_message = ?\n WHERE id = ? AND status = 'processing'`,\n args: [\"pending\", payload, Date.now(), errorMessage.slice(0, 2000), id],\n });\n const affected = Number(\n (result as { rowsAffected?: number }).rowsAffected ??\n (result as { rowCount?: number }).rowCount ??\n 0,\n );\n if (affected === 0) {\n throw new Error(\n \"Integration task is no longer claimable for delivery retry\",\n );\n }\n}\n\nexport async function failTaskDeliveryTransition(\n id: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const result = await getDbExec().execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, error_message = ?, payload = ?\n WHERE id = ? AND status = 'processing'`,\n args: [\"failed\", Date.now(), errorMessage.slice(0, 2000), \"{}\", id],\n });\n const affected = Number(\n (result as { rowsAffected?: number }).rowsAffected ??\n (result as { rowCount?: number }).rowCount ??\n 0,\n );\n if (affected === 0) {\n throw new Error(\n \"Integration task delivery failure transition lost its race\",\n );\n }\n}\n\n/** Mark a task as failed and stash an error message. */\nexport async function markTaskFailed(\n id: string,\n errorMessage: string,\n): Promise<void> {\n await ensureTable();\n const client = getDbExec();\n const now = Date.now();\n await client.execute({\n sql: `UPDATE integration_pending_tasks\n SET status = ?, updated_at = ?, error_message = ?, payload = ?, external_event_key = NULL\n WHERE id = ?`,\n args: [\"failed\", now, errorMessage.slice(0, 2000), \"{}\", id],\n });\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../../src/integrations/plugin.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"plugin.d.ts","sourceRoot":"","sources":["../../src/integrations/plugin.ts"],"names":[],"mappings":"AAyCA,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAyGzD,OAAO,KAAK,EACV,eAAe,EACf,yBAAyB,EAK1B,MAAM,YAAY,CAAC;AAYpB,KAAK,cAAc,GAAG,CAAC,QAAQ,EAAE,GAAG,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAuE9D,eAAO,MAAM,sCAAsC;uBACrC,OAAO;;;uBACP,UAAU;;;uBACV,UAAU;;;uBACV,iBAAiB;;;uBACjB,SAAS;;;uBACT,aAAa;;;uBACb,OAAO;;EAIlB,CAAC;AAEJ,eAAO,MAAM,gCAAgC,0GAE5C,CAAC;AAEF,wBAAgB,gCAAgC,IAAI,eAAe,EAAE,CAEpE;AAUD,KAAK,yBAAyB,GAAG;IAC/B,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAgCF,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,yBAAyB,GAClC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAsFlC;AAmOD;;;;;;;;;;GAUG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,CAAC,EAAE,yBAAyB,GAClC,cAAc,CAu0EhB;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,gBAA6B,CAAC"}
|