@agent-native/core 0.119.5 → 0.119.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/corpus/README.md +2 -2
  2. package/corpus/core/CHANGELOG.md +6 -0
  3. package/corpus/core/package.json +1 -1
  4. package/corpus/core/src/agent/durable-background.ts +1 -0
  5. package/corpus/core/src/deploy/build.ts +144 -2
  6. package/corpus/core/src/deploy/workspace-deploy.ts +123 -1
  7. package/corpus/core/src/integrations/integration-durable-dispatch-config.ts +22 -0
  8. package/corpus/core/src/integrations/integration-durable-dispatch.ts +190 -0
  9. package/corpus/core/src/integrations/pending-tasks-retry-job.ts +175 -87
  10. package/corpus/core/src/integrations/pending-tasks-store.ts +114 -11
  11. package/corpus/core/src/integrations/plugin.ts +113 -60
  12. package/corpus/core/src/integrations/task-queue-stats.ts +68 -15
  13. package/corpus/core/src/integrations/webhook-handler.ts +21 -46
  14. package/corpus/core/src/server/auth.ts +6 -0
  15. package/corpus/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
  16. package/corpus/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
  17. package/corpus/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
  18. package/corpus/templates/dispatch/changelog/2026-07-21-recover-stalled-slack-agent-work.md +6 -0
  19. package/dist/agent/durable-background.d.ts +1 -0
  20. package/dist/agent/durable-background.d.ts.map +1 -1
  21. package/dist/agent/durable-background.js +1 -0
  22. package/dist/agent/durable-background.js.map +1 -1
  23. package/dist/collab/routes.d.ts +1 -1
  24. package/dist/collab/struct-routes.d.ts +1 -1
  25. package/dist/deploy/build.d.ts +3 -0
  26. package/dist/deploy/build.d.ts.map +1 -1
  27. package/dist/deploy/build.js +106 -3
  28. package/dist/deploy/build.js.map +1 -1
  29. package/dist/deploy/workspace-deploy.d.ts.map +1 -1
  30. package/dist/deploy/workspace-deploy.js +103 -2
  31. package/dist/deploy/workspace-deploy.js.map +1 -1
  32. package/dist/file-upload/actions/upload-image.d.ts +1 -1
  33. package/dist/integrations/integration-durable-dispatch-config.d.ts +7 -0
  34. package/dist/integrations/integration-durable-dispatch-config.d.ts.map +1 -0
  35. package/dist/integrations/integration-durable-dispatch-config.js +16 -0
  36. package/dist/integrations/integration-durable-dispatch-config.js.map +1 -0
  37. package/dist/integrations/integration-durable-dispatch.d.ts +23 -0
  38. package/dist/integrations/integration-durable-dispatch.d.ts.map +1 -0
  39. package/dist/integrations/integration-durable-dispatch.js +115 -0
  40. package/dist/integrations/integration-durable-dispatch.js.map +1 -0
  41. package/dist/integrations/pending-tasks-retry-job.d.ts +13 -1
  42. package/dist/integrations/pending-tasks-retry-job.d.ts.map +1 -1
  43. package/dist/integrations/pending-tasks-retry-job.js +133 -74
  44. package/dist/integrations/pending-tasks-retry-job.js.map +1 -1
  45. package/dist/integrations/pending-tasks-store.d.ts +14 -2
  46. package/dist/integrations/pending-tasks-store.d.ts.map +1 -1
  47. package/dist/integrations/pending-tasks-store.js +75 -11
  48. package/dist/integrations/pending-tasks-store.js.map +1 -1
  49. package/dist/integrations/plugin.d.ts.map +1 -1
  50. package/dist/integrations/plugin.js +88 -55
  51. package/dist/integrations/plugin.js.map +1 -1
  52. package/dist/integrations/task-queue-stats.d.ts +15 -3
  53. package/dist/integrations/task-queue-stats.d.ts.map +1 -1
  54. package/dist/integrations/task-queue-stats.js +45 -15
  55. package/dist/integrations/task-queue-stats.js.map +1 -1
  56. package/dist/integrations/webhook-handler.d.ts +1 -1
  57. package/dist/integrations/webhook-handler.d.ts.map +1 -1
  58. package/dist/integrations/webhook-handler.js +18 -42
  59. package/dist/integrations/webhook-handler.js.map +1 -1
  60. package/dist/notifications/routes.d.ts +3 -3
  61. package/dist/observability/routes.d.ts +5 -5
  62. package/dist/progress/routes.d.ts +1 -1
  63. package/dist/provider-api/actions/custom-provider-registration.d.ts +5 -5
  64. package/dist/provider-api/actions/provider-api.d.ts +4 -4
  65. package/dist/resources/handlers.d.ts +1 -1
  66. package/dist/secrets/routes.d.ts +3 -3
  67. package/dist/server/auth.d.ts.map +1 -1
  68. package/dist/server/auth.js +5 -0
  69. package/dist/server/auth.js.map +1 -1
  70. package/dist/server/realtime-token.d.ts +1 -1
  71. package/dist/server/transcribe-voice.d.ts +1 -1
  72. package/dist/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
  73. package/dist/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
  74. package/dist/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
  75. package/package.json +1 -1
  76. package/src/agent/durable-background.ts +1 -0
  77. package/src/deploy/build.ts +144 -2
  78. package/src/deploy/workspace-deploy.ts +123 -1
  79. package/src/integrations/integration-durable-dispatch-config.ts +22 -0
  80. package/src/integrations/integration-durable-dispatch.ts +190 -0
  81. package/src/integrations/pending-tasks-retry-job.ts +175 -87
  82. package/src/integrations/pending-tasks-store.ts +114 -11
  83. package/src/integrations/plugin.ts +113 -60
  84. package/src/integrations/task-queue-stats.ts +68 -15
  85. package/src/integrations/webhook-handler.ts +21 -46
  86. package/src/server/auth.ts +6 -0
  87. package/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
  88. package/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
  89. package/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
@@ -1,8 +1,13 @@
1
1
  import { getDbExec } from "../db/client.js";
2
- import { withConfiguredAppBasePath } from "../server/app-base-path.js";
3
- import { FRAMEWORK_ROUTE_PREFIX } from "../server/core-routes-plugin.js";
4
- import { signInternalToken } from "./internal-token.js";
5
- import { MAX_PENDING_TASK_ATTEMPTS } from "./pending-tasks-store.js";
2
+ import {
3
+ configuredIntegrationDurableDispatchScopes,
4
+ dispatchPendingIntegrationTask,
5
+ isIntegrationDurableDispatchEnabledForTask,
6
+ } from "./integration-durable-dispatch.js";
7
+ import {
8
+ ensurePendingTasksTable,
9
+ MAX_PENDING_TASK_ATTEMPTS,
10
+ } from "./pending-tasks-store.js";
6
11
 
7
12
  /**
8
13
  * Retries stuck integration webhook tasks.
@@ -10,12 +15,16 @@ import { MAX_PENDING_TASK_ATTEMPTS } from "./pending-tasks-store.js";
10
15
  * The integration webhook flow enqueues work into `integration_pending_tasks`
11
16
  * (see `pending-tasks-store.ts`) and then fires a self-webhook to the
12
17
  * `/_agent-native/integrations/process-task` endpoint to drain the queue.
13
- * If that fire-and-forget dispatch fails (e.g. transient network blip), the
18
+ * If that initial dispatch is lost (e.g. transient network blip), the
14
19
  * row stays in `pending` forever. Likewise, if the processor is killed mid-
15
20
  * processing (function timeout, container shutdown), a row can remain in
16
21
  * `processing` forever.
17
22
  *
18
- * This job runs every 60s and re-fires the processor endpoint for tasks that
23
+ * The in-process fallback runs every 60s. Durable deployments also invoke the
24
+ * same bounded sweep from an external scheduler so recovery does not depend on
25
+ * a serverless process remaining alive.
26
+ *
27
+ * Each sweep re-fires the processor endpoint for tasks that
19
28
  * look stuck:
20
29
  * - status='pending' AND created_at older than 90s (initial dispatch lost)
21
30
  * - status='processing' AND updated_at older than the host-specific
@@ -35,8 +44,9 @@ const PENDING_STUCK_AFTER_MS = 90_000;
35
44
  /** Tasks "processing" longer than this are considered killed mid-flight. */
36
45
  const DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000;
37
46
  const SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000;
47
+ const DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS = 16 * 60 * 1000;
38
48
  /** After this many attempts we give up and mark the task failed */
39
- const PROCESSOR_PATH = `${FRAMEWORK_ROUTE_PREFIX}/integrations/process-task`;
49
+ const DEFAULT_SWEEP_LIMIT = 100;
40
50
 
41
51
  let retryInterval: ReturnType<typeof setInterval> | null = null;
42
52
  let initialTimer: ReturnType<typeof setTimeout> | null = null;
@@ -49,8 +59,74 @@ let tableExists: boolean | null = null;
49
59
 
50
60
  interface StuckTaskRow {
51
61
  id: string;
62
+ platform: string;
63
+ externalThreadId: string;
52
64
  status: string;
53
65
  attempts: number;
66
+ updatedAt: number;
67
+ dispatchScope: string | null;
68
+ }
69
+
70
+ export interface PendingTasksSweepResult {
71
+ selected: number;
72
+ dispatched: number;
73
+ markedFailed: number;
74
+ skipped: number;
75
+ dispatchFailed: number;
76
+ }
77
+
78
+ export interface PendingTasksSweepOptions {
79
+ webhookBaseUrl?: string;
80
+ limit?: number;
81
+ durableOnly?: boolean;
82
+ }
83
+
84
+ function affectedRows(result: unknown): number {
85
+ return Number(
86
+ (result as { rowsAffected?: number }).rowsAffected ??
87
+ (result as { rowCount?: number }).rowCount ??
88
+ 0,
89
+ );
90
+ }
91
+
92
+ function isMissingPendingTasksTableError(error: unknown): boolean {
93
+ const message = error instanceof Error ? error.message : String(error ?? "");
94
+ return /no such table|relation .* does not exist|undefined_table/i.test(
95
+ message,
96
+ );
97
+ }
98
+
99
+ function durableScopeSql(): { clause: string; args: string[] } {
100
+ const scopes = configuredIntegrationDurableDispatchScopes();
101
+ if (scopes === null) return { clause: "", args: [] };
102
+ if (scopes.length === 0) return { clause: " AND 1 = 0", args: [] };
103
+
104
+ const clauses: string[] = [];
105
+ const args: string[] = [];
106
+ for (const scope of scopes) {
107
+ if (scope.value === "*") {
108
+ clauses.push("platform = ?");
109
+ args.push(scope.platform);
110
+ continue;
111
+ }
112
+ if (scope.platform === "slack" && /^[A-Za-z0-9]+$/.test(scope.value)) {
113
+ clauses.push(
114
+ "(platform = ? AND (external_thread_id = ? OR dispatch_scope = ? OR external_thread_id LIKE ?))",
115
+ );
116
+ args.push(
117
+ scope.platform,
118
+ scope.value,
119
+ scope.value,
120
+ `%:%:${scope.value}:%`,
121
+ );
122
+ continue;
123
+ }
124
+ clauses.push(
125
+ "(platform = ? AND (external_thread_id = ? OR dispatch_scope = ?))",
126
+ );
127
+ args.push(scope.platform, scope.value, scope.value);
128
+ }
129
+ return { clause: ` AND (${clauses.join(" OR ")})`, args };
54
130
  }
55
131
 
56
132
  /**
@@ -58,39 +134,76 @@ interface StuckTaskRow {
58
134
  * Exported for tests and for manual triggers.
59
135
  */
60
136
  export async function retryStuckPendingTasks(
61
- webhookBaseUrl?: string,
62
- ): Promise<void> {
63
- const baseUrl = webhookBaseUrl ?? activeWebhookBaseUrl;
137
+ input?: string | PendingTasksSweepOptions,
138
+ ): Promise<PendingTasksSweepResult> {
139
+ const options: PendingTasksSweepOptions =
140
+ typeof input === "string" ? { webhookBaseUrl: input } : (input ?? {});
141
+ const baseUrl = options.webhookBaseUrl ?? activeWebhookBaseUrl;
142
+ const limit = Math.max(
143
+ 1,
144
+ Math.min(500, Math.floor(options.limit ?? DEFAULT_SWEEP_LIMIT)),
145
+ );
146
+ const scopeSql = options.durableOnly
147
+ ? durableScopeSql()
148
+ : { clause: "", args: [] as string[] };
149
+ const result: PendingTasksSweepResult = {
150
+ selected: 0,
151
+ dispatched: 0,
152
+ markedFailed: 0,
153
+ skipped: 0,
154
+ dispatchFailed: 0,
155
+ };
156
+ await ensurePendingTasksTable();
64
157
  const client = getDbExec();
65
158
  const now = Date.now();
66
159
  const pendingCutoff = now - PENDING_STUCK_AFTER_MS;
67
160
  const processingCutoff = now - getProcessingStuckAfterMs();
161
+ const durableProcessingCutoff =
162
+ now - DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS;
68
163
 
69
164
  let stuckRows: StuckTaskRow[];
70
165
  try {
71
166
  const { rows } = await client.execute({
72
167
  sql: `
73
- SELECT id, status, attempts
168
+ SELECT id, platform, external_thread_id, dispatch_scope, status, attempts, updated_at
74
169
  FROM integration_pending_tasks
75
- WHERE (status = 'pending' AND created_at <= ? AND updated_at <= ?)
76
- OR (status = 'processing' AND updated_at <= ?)
170
+ WHERE ((status = 'pending' AND created_at <= ? AND updated_at <= ?)
171
+ OR (status = 'processing' AND (
172
+ (last_dispatch_outcome = 'background-acknowledged'
173
+ AND updated_at <= ?)
174
+ OR ((last_dispatch_outcome IS NULL
175
+ OR last_dispatch_outcome <> 'background-acknowledged')
176
+ AND updated_at <= ?)
177
+ )))
178
+ ${scopeSql.clause}
179
+ ORDER BY updated_at ASC
180
+ LIMIT ?
77
181
  `,
78
182
  // `updated_at` is initialized to `created_at` on insert, so a genuinely
79
183
  // stuck pending row still matches on the first sweep. The retry path
80
184
  // below touches `updated_at`, which (with this predicate) keeps the row
81
185
  // from being re-selected — and re-firing the processor — on every tick.
82
- args: [pendingCutoff, pendingCutoff, processingCutoff],
186
+ args: [
187
+ pendingCutoff,
188
+ pendingCutoff,
189
+ durableProcessingCutoff,
190
+ processingCutoff,
191
+ ...scopeSql.args,
192
+ limit,
193
+ ],
83
194
  });
84
195
  stuckRows = rows.map((r) => ({
85
196
  id: r.id as string,
197
+ platform: r.platform as string,
198
+ externalThreadId: r.external_thread_id as string,
86
199
  status: r.status as string,
87
200
  attempts: Number(r.attempts ?? 0),
201
+ updatedAt: Number(r.updated_at ?? 0),
202
+ dispatchScope: (r.dispatch_scope as string | null) ?? null,
88
203
  }));
89
204
  tableExists = true;
90
205
  } catch (err) {
91
- // Most common case: the table hasn't been created yet because no inbound
92
- // integration webhook has been processed on this deployment. Silently
93
- // no-op until the table appears.
206
+ if (!isMissingPendingTasksTableError(err)) throw err;
94
207
  if (tableExists !== false) {
95
208
  tableExists = false;
96
209
  if (process.env.DEBUG) {
@@ -99,17 +212,29 @@ export async function retryStuckPendingTasks(
99
212
  );
100
213
  }
101
214
  }
102
- return;
215
+ return result;
103
216
  }
104
217
 
105
- if (stuckRows.length === 0) return;
218
+ stuckRows = stuckRows.filter((row) => {
219
+ const durable = isIntegrationDurableDispatchEnabledForTask({
220
+ platform: row.platform,
221
+ externalThreadId: row.externalThreadId,
222
+ platformContext: row.dispatchScope
223
+ ? { channelId: row.dispatchScope }
224
+ : undefined,
225
+ });
226
+ if (options.durableOnly && !durable) return false;
227
+ return true;
228
+ });
229
+ result.selected = stuckRows.length;
230
+ if (stuckRows.length === 0) return result;
106
231
 
107
232
  for (const row of stuckRows) {
108
233
  try {
109
234
  // Cap retries — mark failed and move on so the row stops bouncing
110
235
  // between pending and processing forever.
111
236
  if (row.attempts >= MAX_PENDING_TASK_ATTEMPTS) {
112
- await client.execute({
237
+ const update = await client.execute({
113
238
  sql: `
114
239
  UPDATE integration_pending_tasks
115
240
  SET status = 'failed',
@@ -119,14 +244,21 @@ export async function retryStuckPendingTasks(
119
244
  external_event_key = NULL
120
245
  WHERE id = ?
121
246
  AND status = ?
247
+ AND updated_at = ?
122
248
  `,
123
249
  args: [
124
250
  Date.now(),
125
251
  `Retry job: exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts`,
126
252
  row.id,
127
253
  row.status,
254
+ row.updatedAt,
128
255
  ],
129
256
  });
257
+ if (affectedRows(update) === 0) {
258
+ result.skipped += 1;
259
+ continue;
260
+ }
261
+ result.markedFailed += 1;
130
262
  console.warn(
131
263
  `[integrations] Pending task ${row.id} exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts — marking failed`,
132
264
  );
@@ -138,24 +270,44 @@ export async function retryStuckPendingTasks(
138
270
  // Without this, processing rows stay stuck forever.
139
271
  // For pending rows, just touch updated_at to avoid re-firing every tick.
140
272
  const newStatus = row.status === "processing" ? "pending" : row.status;
141
- await client.execute({
273
+ const update = await client.execute({
142
274
  sql: `
143
275
  UPDATE integration_pending_tasks
144
276
  SET status = ?, updated_at = ?
145
277
  WHERE id = ?
146
278
  AND status = ?
279
+ AND updated_at = ?
147
280
  `,
148
- args: [newStatus, Date.now(), row.id, row.status],
281
+ args: [newStatus, Date.now(), row.id, row.status, row.updatedAt],
149
282
  });
283
+ if (affectedRows(update) === 0) {
284
+ result.skipped += 1;
285
+ continue;
286
+ }
150
287
 
151
- await refireProcessor(row.id, baseUrl);
288
+ const outcome = await dispatchPendingIntegrationTask({
289
+ taskId: row.id,
290
+ task: {
291
+ platform: row.platform,
292
+ externalThreadId: row.externalThreadId,
293
+ platformContext: row.dispatchScope
294
+ ? { channelId: row.dispatchScope }
295
+ : undefined,
296
+ },
297
+ baseUrl,
298
+ portableSettleMs: 1_000,
299
+ });
300
+ if (outcome === "failed") result.dispatchFailed += 1;
301
+ else result.dispatched += 1;
152
302
  } catch (err) {
303
+ result.dispatchFailed += 1;
153
304
  console.error(
154
305
  `[integrations] Failed to retry pending task ${row.id}:`,
155
306
  err,
156
307
  );
157
308
  }
158
309
  }
310
+ return result;
159
311
  }
160
312
 
161
313
  function getProcessingStuckAfterMs(): number {
@@ -170,70 +322,6 @@ function getProcessingStuckAfterMs(): number {
170
322
  return DEFAULT_PROCESSING_STUCK_AFTER_MS;
171
323
  }
172
324
 
173
- /**
174
- * Fire-and-forget POST to the processor endpoint for a single task id.
175
- * Mirrors the original dispatch from the webhook handler, including the
176
- * short-lived HMAC bearer token bound to this taskId.
177
- */
178
- async function refireProcessor(
179
- taskId: string,
180
- webhookBaseUrl: string | undefined,
181
- ): Promise<void> {
182
- const baseUrl =
183
- webhookBaseUrl ||
184
- process.env.WEBHOOK_BASE_URL ||
185
- process.env.APP_URL ||
186
- process.env.URL ||
187
- process.env.DEPLOY_URL ||
188
- `http://localhost:${process.env.PORT || 3000}`;
189
-
190
- const url = `${withConfiguredAppBasePath(baseUrl)}${PROCESSOR_PATH}`;
191
-
192
- // Sign with HMAC if A2A_SECRET is configured. In production we MUST sign —
193
- // an unsigned dispatch in production lets attackers re-trigger any queued
194
- // task with a guessable id (C3 in the webhook security audit). In dev we
195
- // fall back to unsigned so contributors can iterate without configuring
196
- // A2A_SECRET locally.
197
- const headers: Record<string, string> = {
198
- "Content-Type": "application/json",
199
- };
200
- try {
201
- headers["Authorization"] = `Bearer ${signInternalToken(taskId)}`;
202
- } catch (err) {
203
- if (process.env.NODE_ENV === "production") {
204
- console.error(
205
- `[integrations] Refusing to dispatch task ${taskId} — A2A_SECRET not configured. ` +
206
- "Set A2A_SECRET to enable signed retry dispatches.",
207
- );
208
- return;
209
- }
210
- // Dev: proceed unsigned. Log the underlying error path so a malformed
211
- // secret (different from "not set") doesn't fail silently (L5 in the audit).
212
- if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) {
213
- console.error(
214
- `[integrations] signInternalToken failed unexpectedly for ${taskId}:`,
215
- err,
216
- );
217
- }
218
- }
219
-
220
- // Don't await the body — we just want the request to leave the box.
221
- // A short timeout avoids tying up the retry loop on a hung processor.
222
- const controller = new AbortController();
223
- const timer = setTimeout(() => controller.abort(), 5_000);
224
-
225
- try {
226
- await fetch(url, {
227
- method: "POST",
228
- headers,
229
- body: JSON.stringify({ taskId }),
230
- signal: controller.signal,
231
- });
232
- } finally {
233
- clearTimeout(timer);
234
- }
235
- }
236
-
237
325
  /**
238
326
  * Start the periodic retry loop. Safe to call multiple times — second call
239
327
  * is a no-op.
@@ -33,6 +33,10 @@ async function ensureTable(): Promise<void> {
33
33
  org_id TEXT,
34
34
  status TEXT NOT NULL,
35
35
  attempts ${intType()} NOT NULL DEFAULT 0,
36
+ dispatch_attempts ${intType()} NOT NULL DEFAULT 0,
37
+ last_dispatch_at ${intType()},
38
+ last_dispatch_outcome TEXT,
39
+ dispatch_scope TEXT,
36
40
  error_message TEXT,
37
41
  created_at ${intType()} NOT NULL,
38
42
  updated_at ${intType()} NOT NULL,
@@ -47,6 +51,26 @@ async function ensureTable(): Promise<void> {
47
51
  "external_event_key",
48
52
  `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,
49
53
  );
54
+ await ensureColumnExists(
55
+ "integration_pending_tasks",
56
+ "dispatch_attempts",
57
+ `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_attempts ${intType()} NOT NULL DEFAULT 0`,
58
+ );
59
+ await ensureColumnExists(
60
+ "integration_pending_tasks",
61
+ "last_dispatch_at",
62
+ `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_at ${intType()}`,
63
+ );
64
+ await ensureColumnExists(
65
+ "integration_pending_tasks",
66
+ "last_dispatch_outcome",
67
+ `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_outcome TEXT`,
68
+ );
69
+ await ensureColumnExists(
70
+ "integration_pending_tasks",
71
+ "dispatch_scope",
72
+ `ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_scope TEXT`,
73
+ );
50
74
  await ensureIndexExists(
51
75
  "idx_pending_tasks_status_created",
52
76
  `CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,
@@ -55,6 +79,10 @@ async function ensureTable(): Promise<void> {
55
79
  "idx_pending_tasks_event_key",
56
80
  `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,
57
81
  );
82
+ await ensureIndexExists(
83
+ "idx_pending_tasks_dispatch_scope",
84
+ `CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`,
85
+ );
58
86
  return;
59
87
  }
60
88
 
@@ -69,6 +97,10 @@ async function ensureTable(): Promise<void> {
69
97
  // unique index ensures a duplicate INSERT raises an error we can
70
98
  // catch as "already-enqueued".
71
99
  await ensureExternalEventKey(client);
100
+ await ensureDispatchColumns(client);
101
+ await client.execute(
102
+ `CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`,
103
+ );
72
104
  await client.execute(
73
105
  `CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,
74
106
  );
@@ -81,6 +113,10 @@ async function ensureTable(): Promise<void> {
81
113
  return _initPromise;
82
114
  }
83
115
 
116
+ export async function ensurePendingTasksTable(): Promise<void> {
117
+ await ensureTable();
118
+ }
119
+
84
120
  async function ensureExternalEventKey(
85
121
  client: ReturnType<typeof getDbExec>,
86
122
  ): Promise<void> {
@@ -107,6 +143,32 @@ async function ensureExternalEventKey(
107
143
  }
108
144
  }
109
145
 
146
+ async function ensureDispatchColumns(
147
+ client: ReturnType<typeof getDbExec>,
148
+ ): Promise<void> {
149
+ const columns = [
150
+ `dispatch_attempts ${intType()} NOT NULL DEFAULT 0`,
151
+ `last_dispatch_at ${intType()}`,
152
+ "last_dispatch_outcome TEXT",
153
+ "dispatch_scope TEXT",
154
+ ];
155
+ for (const column of columns) {
156
+ try {
157
+ await client.execute(
158
+ `ALTER TABLE integration_pending_tasks ADD COLUMN ${column}`,
159
+ );
160
+ } catch (err: any) {
161
+ if (
162
+ !String(err?.message ?? err)
163
+ .toLowerCase()
164
+ .includes("duplicate")
165
+ ) {
166
+ throw err;
167
+ }
168
+ }
169
+ }
170
+ }
171
+
110
172
  /** Status values for an integration pending task. */
111
173
  export type PendingTaskStatus =
112
174
  | "pending"
@@ -123,6 +185,10 @@ export interface PendingTask {
123
185
  orgId: string | null;
124
186
  status: PendingTaskStatus;
125
187
  attempts: number;
188
+ dispatchAttempts: number;
189
+ lastDispatchAt: number | null;
190
+ lastDispatchOutcome: string | null;
191
+ dispatchScope: string | null;
126
192
  errorMessage: string | null;
127
193
  createdAt: number;
128
194
  updatedAt: number;
@@ -164,6 +230,13 @@ function rowToTask(row: Record<string, unknown>): PendingTask {
164
230
  orgId: (row.org_id as string | null) ?? null,
165
231
  status: row.status as PendingTaskStatus,
166
232
  attempts: Number(row.attempts ?? 0),
233
+ dispatchAttempts: Number(row.dispatch_attempts ?? 0),
234
+ lastDispatchAt:
235
+ row.last_dispatch_at == null
236
+ ? null
237
+ : Number(row.last_dispatch_at as number),
238
+ lastDispatchOutcome: (row.last_dispatch_outcome as string | null) ?? null,
239
+ dispatchScope: (row.dispatch_scope as string | null) ?? null,
167
240
  errorMessage: (row.error_message as string | null) ?? null,
168
241
  createdAt: Number(row.created_at ?? 0),
169
242
  updatedAt: Number(row.updated_at ?? 0),
@@ -189,14 +262,15 @@ export async function insertPendingTask(input: {
189
262
  ownerEmail: string;
190
263
  orgId?: string | null;
191
264
  externalEventKey?: string | null;
265
+ dispatchScope?: string | null;
192
266
  }): Promise<void> {
193
267
  await ensureTable();
194
268
  const client = getDbExec();
195
269
  const now = Date.now();
196
270
  await client.execute({
197
271
  sql: `INSERT INTO integration_pending_tasks
198
- (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key)
199
- VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
272
+ (id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key, dispatch_scope)
273
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
200
274
  args: [
201
275
  input.id,
202
276
  input.platform,
@@ -209,6 +283,7 @@ export async function insertPendingTask(input: {
209
283
  now,
210
284
  now,
211
285
  input.externalEventKey ?? null,
286
+ input.dispatchScope ?? null,
212
287
  ],
213
288
  });
214
289
  }
@@ -238,7 +313,7 @@ export async function getPendingTask(id: string): Promise<PendingTask | null> {
238
313
  await ensureTable();
239
314
  const client = getDbExec();
240
315
  const { rows } = await client.execute({
241
- sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at
316
+ sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at
242
317
  FROM integration_pending_tasks WHERE id = ? LIMIT 1`,
243
318
  args: [id],
244
319
  });
@@ -342,6 +417,7 @@ export async function resolveIntegrationSourceContext(
342
417
  */
343
418
  export async function claimPendingTask(
344
419
  id: string,
420
+ options?: { dispatchOutcome?: string },
345
421
  ): Promise<PendingTask | null> {
346
422
  await ensureTable();
347
423
  const client = getDbExec();
@@ -352,7 +428,8 @@ export async function claimPendingTask(
352
428
  const result = await client.execute({
353
429
  sql: isPostgres()
354
430
  ? `UPDATE integration_pending_tasks
355
- SET status = ?, attempts = attempts + 1, updated_at = ?
431
+ SET status = ?, attempts = attempts + 1, updated_at = ?,
432
+ last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)
356
433
  WHERE id = ? AND status = 'pending'
357
434
  AND NOT EXISTS (
358
435
  SELECT 1 FROM integration_pending_tasks active
@@ -374,9 +451,10 @@ export async function claimPendingTask(
374
451
  )
375
452
  )
376
453
  )
377
- RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at`
454
+ RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at`
378
455
  : `UPDATE integration_pending_tasks
379
- SET status = ?, attempts = attempts + 1, updated_at = ?
456
+ SET status = ?, attempts = attempts + 1, updated_at = ?,
457
+ last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)
380
458
  WHERE id = ? AND status = 'pending'
381
459
  AND NOT EXISTS (
382
460
  SELECT 1 FROM integration_pending_tasks active
@@ -398,7 +476,7 @@ export async function claimPendingTask(
398
476
  )
399
477
  )
400
478
  )`,
401
- args: ["processing", now, id],
479
+ args: ["processing", now, options?.dispatchOutcome ?? null, id],
402
480
  });
403
481
  const rows = result.rows ?? [];
404
482
 
@@ -418,19 +496,44 @@ export async function claimPendingTask(
418
496
  return fetched;
419
497
  }
420
498
 
499
+ export async function recordPendingTaskDispatchAttempt(
500
+ id: string,
501
+ outcome: string,
502
+ ): Promise<void> {
503
+ await ensureTable();
504
+ await getDbExec().execute({
505
+ sql: `UPDATE integration_pending_tasks
506
+ SET dispatch_attempts = dispatch_attempts + 1,
507
+ last_dispatch_at = ?,
508
+ last_dispatch_outcome = CASE
509
+ WHEN status = 'processing'
510
+ AND last_dispatch_outcome = 'background-acknowledged'
511
+ THEN last_dispatch_outcome
512
+ ELSE ?
513
+ END
514
+ WHERE id = ?`,
515
+ args: [Date.now(), outcome.slice(0, 80), id],
516
+ });
517
+ }
518
+
421
519
  /** Next queued turn for a provider thread after its current task completes. */
422
- export async function getNextPendingTaskIdForThread(
520
+ export async function getNextPendingTaskForThread(
423
521
  platform: string,
424
522
  externalThreadId: string,
425
- ): Promise<string | null> {
523
+ ): Promise<{ id: string; dispatchScope: string | null } | null> {
426
524
  await ensureTable();
427
525
  const { rows } = await getDbExec().execute({
428
- sql: `SELECT id FROM integration_pending_tasks
526
+ sql: `SELECT id, dispatch_scope FROM integration_pending_tasks
429
527
  WHERE platform = ? AND external_thread_id = ? AND status = 'pending'
430
528
  ORDER BY created_at ASC, id ASC LIMIT 1`,
431
529
  args: [platform, externalThreadId],
432
530
  });
433
- return rows[0]?.id ? String(rows[0].id) : null;
531
+ return rows[0]?.id
532
+ ? {
533
+ id: String(rows[0].id),
534
+ dispatchScope: (rows[0].dispatch_scope as string | null) ?? null,
535
+ }
536
+ : null;
434
537
  }
435
538
 
436
539
  /** Mark a task as completed. */