@agent-native/core 0.119.5 → 0.119.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +6 -0
- package/corpus/core/package.json +1 -1
- package/corpus/core/src/agent/durable-background.ts +1 -0
- package/corpus/core/src/deploy/build.ts +144 -2
- package/corpus/core/src/deploy/workspace-deploy.ts +123 -1
- package/corpus/core/src/integrations/integration-durable-dispatch-config.ts +22 -0
- package/corpus/core/src/integrations/integration-durable-dispatch.ts +190 -0
- package/corpus/core/src/integrations/pending-tasks-retry-job.ts +175 -87
- package/corpus/core/src/integrations/pending-tasks-store.ts +114 -11
- package/corpus/core/src/integrations/plugin.ts +113 -60
- package/corpus/core/src/integrations/task-queue-stats.ts +68 -15
- package/corpus/core/src/integrations/webhook-handler.ts +21 -46
- package/corpus/core/src/server/auth.ts +6 -0
- package/corpus/core/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/corpus/core/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/corpus/core/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/corpus/templates/dispatch/changelog/2026-07-21-recover-stalled-slack-agent-work.md +6 -0
- package/dist/agent/durable-background.d.ts +1 -0
- package/dist/agent/durable-background.d.ts.map +1 -1
- package/dist/agent/durable-background.js +1 -0
- package/dist/agent/durable-background.js.map +1 -1
- package/dist/collab/routes.d.ts +1 -1
- package/dist/collab/struct-routes.d.ts +1 -1
- package/dist/deploy/build.d.ts +3 -0
- package/dist/deploy/build.d.ts.map +1 -1
- package/dist/deploy/build.js +106 -3
- package/dist/deploy/build.js.map +1 -1
- package/dist/deploy/workspace-deploy.d.ts.map +1 -1
- package/dist/deploy/workspace-deploy.js +103 -2
- package/dist/deploy/workspace-deploy.js.map +1 -1
- package/dist/file-upload/actions/upload-image.d.ts +1 -1
- package/dist/integrations/integration-durable-dispatch-config.d.ts +7 -0
- package/dist/integrations/integration-durable-dispatch-config.d.ts.map +1 -0
- package/dist/integrations/integration-durable-dispatch-config.js +16 -0
- package/dist/integrations/integration-durable-dispatch-config.js.map +1 -0
- package/dist/integrations/integration-durable-dispatch.d.ts +23 -0
- package/dist/integrations/integration-durable-dispatch.d.ts.map +1 -0
- package/dist/integrations/integration-durable-dispatch.js +115 -0
- package/dist/integrations/integration-durable-dispatch.js.map +1 -0
- package/dist/integrations/pending-tasks-retry-job.d.ts +13 -1
- package/dist/integrations/pending-tasks-retry-job.d.ts.map +1 -1
- package/dist/integrations/pending-tasks-retry-job.js +133 -74
- package/dist/integrations/pending-tasks-retry-job.js.map +1 -1
- package/dist/integrations/pending-tasks-store.d.ts +14 -2
- package/dist/integrations/pending-tasks-store.d.ts.map +1 -1
- package/dist/integrations/pending-tasks-store.js +75 -11
- package/dist/integrations/pending-tasks-store.js.map +1 -1
- package/dist/integrations/plugin.d.ts.map +1 -1
- package/dist/integrations/plugin.js +88 -55
- package/dist/integrations/plugin.js.map +1 -1
- package/dist/integrations/task-queue-stats.d.ts +15 -3
- package/dist/integrations/task-queue-stats.d.ts.map +1 -1
- package/dist/integrations/task-queue-stats.js +45 -15
- package/dist/integrations/task-queue-stats.js.map +1 -1
- package/dist/integrations/webhook-handler.d.ts +1 -1
- package/dist/integrations/webhook-handler.d.ts.map +1 -1
- package/dist/integrations/webhook-handler.js +18 -42
- package/dist/integrations/webhook-handler.js.map +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/observability/routes.d.ts +5 -5
- package/dist/progress/routes.d.ts +1 -1
- package/dist/provider-api/actions/custom-provider-registration.d.ts +5 -5
- package/dist/provider-api/actions/provider-api.d.ts +4 -4
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/secrets/routes.d.ts +3 -3
- package/dist/server/auth.d.ts.map +1 -1
- package/dist/server/auth.js +5 -0
- package/dist/server/auth.js.map +1 -1
- package/dist/server/realtime-token.d.ts +1 -1
- package/dist/server/transcribe-voice.d.ts +1 -1
- package/dist/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/dist/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/dist/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/package.json +1 -1
- package/src/agent/durable-background.ts +1 -0
- package/src/deploy/build.ts +144 -2
- package/src/deploy/workspace-deploy.ts +123 -1
- package/src/integrations/integration-durable-dispatch-config.ts +22 -0
- package/src/integrations/integration-durable-dispatch.ts +190 -0
- package/src/integrations/pending-tasks-retry-job.ts +175 -87
- package/src/integrations/pending-tasks-store.ts +114 -11
- package/src/integrations/plugin.ts +113 -60
- package/src/integrations/task-queue-stats.ts +68 -15
- package/src/integrations/webhook-handler.ts +21 -46
- package/src/server/auth.ts +6 -0
- package/src/templates/default/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/src/templates/headless/.agents/skills/integration-webhooks/SKILL.md +63 -22
- package/src/templates/workspace-core/.agents/skills/integration-webhooks/SKILL.md +63 -22
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import { getDbExec } from "../db/client.js";
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
import {
|
|
3
|
+
configuredIntegrationDurableDispatchScopes,
|
|
4
|
+
dispatchPendingIntegrationTask,
|
|
5
|
+
isIntegrationDurableDispatchEnabledForTask,
|
|
6
|
+
} from "./integration-durable-dispatch.js";
|
|
7
|
+
import {
|
|
8
|
+
ensurePendingTasksTable,
|
|
9
|
+
MAX_PENDING_TASK_ATTEMPTS,
|
|
10
|
+
} from "./pending-tasks-store.js";
|
|
6
11
|
|
|
7
12
|
/**
|
|
8
13
|
* Retries stuck integration webhook tasks.
|
|
@@ -10,12 +15,16 @@ import { MAX_PENDING_TASK_ATTEMPTS } from "./pending-tasks-store.js";
|
|
|
10
15
|
* The integration webhook flow enqueues work into `integration_pending_tasks`
|
|
11
16
|
* (see `pending-tasks-store.ts`) and then fires a self-webhook to the
|
|
12
17
|
* `/_agent-native/integrations/process-task` endpoint to drain the queue.
|
|
13
|
-
* If that
|
|
18
|
+
* If that initial dispatch is lost (e.g. transient network blip), the
|
|
14
19
|
* row stays in `pending` forever. Likewise, if the processor is killed mid-
|
|
15
20
|
* processing (function timeout, container shutdown), a row can remain in
|
|
16
21
|
* `processing` forever.
|
|
17
22
|
*
|
|
18
|
-
*
|
|
23
|
+
* The in-process fallback runs every 60s. Durable deployments also invoke the
|
|
24
|
+
* same bounded sweep from an external scheduler so recovery does not depend on
|
|
25
|
+
* a serverless process remaining alive.
|
|
26
|
+
*
|
|
27
|
+
* Each sweep re-fires the processor endpoint for tasks that
|
|
19
28
|
* look stuck:
|
|
20
29
|
* - status='pending' AND created_at older than 90s (initial dispatch lost)
|
|
21
30
|
* - status='processing' AND updated_at older than the host-specific
|
|
@@ -35,8 +44,9 @@ const PENDING_STUCK_AFTER_MS = 90_000;
|
|
|
35
44
|
/** Tasks "processing" longer than this are considered killed mid-flight. */
|
|
36
45
|
const DEFAULT_PROCESSING_STUCK_AFTER_MS = 5 * 60 * 1000;
|
|
37
46
|
const SERVERLESS_PROCESSING_STUCK_AFTER_MS = 75_000;
|
|
47
|
+
const DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS = 16 * 60 * 1000;
|
|
38
48
|
/** After this many attempts we give up and mark the task failed */
|
|
39
|
-
const
|
|
49
|
+
const DEFAULT_SWEEP_LIMIT = 100;
|
|
40
50
|
|
|
41
51
|
let retryInterval: ReturnType<typeof setInterval> | null = null;
|
|
42
52
|
let initialTimer: ReturnType<typeof setTimeout> | null = null;
|
|
@@ -49,8 +59,74 @@ let tableExists: boolean | null = null;
|
|
|
49
59
|
|
|
50
60
|
interface StuckTaskRow {
|
|
51
61
|
id: string;
|
|
62
|
+
platform: string;
|
|
63
|
+
externalThreadId: string;
|
|
52
64
|
status: string;
|
|
53
65
|
attempts: number;
|
|
66
|
+
updatedAt: number;
|
|
67
|
+
dispatchScope: string | null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export interface PendingTasksSweepResult {
|
|
71
|
+
selected: number;
|
|
72
|
+
dispatched: number;
|
|
73
|
+
markedFailed: number;
|
|
74
|
+
skipped: number;
|
|
75
|
+
dispatchFailed: number;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface PendingTasksSweepOptions {
|
|
79
|
+
webhookBaseUrl?: string;
|
|
80
|
+
limit?: number;
|
|
81
|
+
durableOnly?: boolean;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function affectedRows(result: unknown): number {
|
|
85
|
+
return Number(
|
|
86
|
+
(result as { rowsAffected?: number }).rowsAffected ??
|
|
87
|
+
(result as { rowCount?: number }).rowCount ??
|
|
88
|
+
0,
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function isMissingPendingTasksTableError(error: unknown): boolean {
|
|
93
|
+
const message = error instanceof Error ? error.message : String(error ?? "");
|
|
94
|
+
return /no such table|relation .* does not exist|undefined_table/i.test(
|
|
95
|
+
message,
|
|
96
|
+
);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function durableScopeSql(): { clause: string; args: string[] } {
|
|
100
|
+
const scopes = configuredIntegrationDurableDispatchScopes();
|
|
101
|
+
if (scopes === null) return { clause: "", args: [] };
|
|
102
|
+
if (scopes.length === 0) return { clause: " AND 1 = 0", args: [] };
|
|
103
|
+
|
|
104
|
+
const clauses: string[] = [];
|
|
105
|
+
const args: string[] = [];
|
|
106
|
+
for (const scope of scopes) {
|
|
107
|
+
if (scope.value === "*") {
|
|
108
|
+
clauses.push("platform = ?");
|
|
109
|
+
args.push(scope.platform);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
if (scope.platform === "slack" && /^[A-Za-z0-9]+$/.test(scope.value)) {
|
|
113
|
+
clauses.push(
|
|
114
|
+
"(platform = ? AND (external_thread_id = ? OR dispatch_scope = ? OR external_thread_id LIKE ?))",
|
|
115
|
+
);
|
|
116
|
+
args.push(
|
|
117
|
+
scope.platform,
|
|
118
|
+
scope.value,
|
|
119
|
+
scope.value,
|
|
120
|
+
`%:%:${scope.value}:%`,
|
|
121
|
+
);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
clauses.push(
|
|
125
|
+
"(platform = ? AND (external_thread_id = ? OR dispatch_scope = ?))",
|
|
126
|
+
);
|
|
127
|
+
args.push(scope.platform, scope.value, scope.value);
|
|
128
|
+
}
|
|
129
|
+
return { clause: ` AND (${clauses.join(" OR ")})`, args };
|
|
54
130
|
}
|
|
55
131
|
|
|
56
132
|
/**
|
|
@@ -58,39 +134,76 @@ interface StuckTaskRow {
|
|
|
58
134
|
* Exported for tests and for manual triggers.
|
|
59
135
|
*/
|
|
60
136
|
export async function retryStuckPendingTasks(
|
|
61
|
-
|
|
62
|
-
): Promise<
|
|
63
|
-
const
|
|
137
|
+
input?: string | PendingTasksSweepOptions,
|
|
138
|
+
): Promise<PendingTasksSweepResult> {
|
|
139
|
+
const options: PendingTasksSweepOptions =
|
|
140
|
+
typeof input === "string" ? { webhookBaseUrl: input } : (input ?? {});
|
|
141
|
+
const baseUrl = options.webhookBaseUrl ?? activeWebhookBaseUrl;
|
|
142
|
+
const limit = Math.max(
|
|
143
|
+
1,
|
|
144
|
+
Math.min(500, Math.floor(options.limit ?? DEFAULT_SWEEP_LIMIT)),
|
|
145
|
+
);
|
|
146
|
+
const scopeSql = options.durableOnly
|
|
147
|
+
? durableScopeSql()
|
|
148
|
+
: { clause: "", args: [] as string[] };
|
|
149
|
+
const result: PendingTasksSweepResult = {
|
|
150
|
+
selected: 0,
|
|
151
|
+
dispatched: 0,
|
|
152
|
+
markedFailed: 0,
|
|
153
|
+
skipped: 0,
|
|
154
|
+
dispatchFailed: 0,
|
|
155
|
+
};
|
|
156
|
+
await ensurePendingTasksTable();
|
|
64
157
|
const client = getDbExec();
|
|
65
158
|
const now = Date.now();
|
|
66
159
|
const pendingCutoff = now - PENDING_STUCK_AFTER_MS;
|
|
67
160
|
const processingCutoff = now - getProcessingStuckAfterMs();
|
|
161
|
+
const durableProcessingCutoff =
|
|
162
|
+
now - DURABLE_BACKGROUND_PROCESSING_STUCK_AFTER_MS;
|
|
68
163
|
|
|
69
164
|
let stuckRows: StuckTaskRow[];
|
|
70
165
|
try {
|
|
71
166
|
const { rows } = await client.execute({
|
|
72
167
|
sql: `
|
|
73
|
-
SELECT id, status, attempts
|
|
168
|
+
SELECT id, platform, external_thread_id, dispatch_scope, status, attempts, updated_at
|
|
74
169
|
FROM integration_pending_tasks
|
|
75
|
-
WHERE (status = 'pending' AND created_at <= ? AND updated_at <= ?)
|
|
76
|
-
OR (status = 'processing' AND
|
|
170
|
+
WHERE ((status = 'pending' AND created_at <= ? AND updated_at <= ?)
|
|
171
|
+
OR (status = 'processing' AND (
|
|
172
|
+
(last_dispatch_outcome = 'background-acknowledged'
|
|
173
|
+
AND updated_at <= ?)
|
|
174
|
+
OR ((last_dispatch_outcome IS NULL
|
|
175
|
+
OR last_dispatch_outcome <> 'background-acknowledged')
|
|
176
|
+
AND updated_at <= ?)
|
|
177
|
+
)))
|
|
178
|
+
${scopeSql.clause}
|
|
179
|
+
ORDER BY updated_at ASC
|
|
180
|
+
LIMIT ?
|
|
77
181
|
`,
|
|
78
182
|
// `updated_at` is initialized to `created_at` on insert, so a genuinely
|
|
79
183
|
// stuck pending row still matches on the first sweep. The retry path
|
|
80
184
|
// below touches `updated_at`, which (with this predicate) keeps the row
|
|
81
185
|
// from being re-selected — and re-firing the processor — on every tick.
|
|
82
|
-
args: [
|
|
186
|
+
args: [
|
|
187
|
+
pendingCutoff,
|
|
188
|
+
pendingCutoff,
|
|
189
|
+
durableProcessingCutoff,
|
|
190
|
+
processingCutoff,
|
|
191
|
+
...scopeSql.args,
|
|
192
|
+
limit,
|
|
193
|
+
],
|
|
83
194
|
});
|
|
84
195
|
stuckRows = rows.map((r) => ({
|
|
85
196
|
id: r.id as string,
|
|
197
|
+
platform: r.platform as string,
|
|
198
|
+
externalThreadId: r.external_thread_id as string,
|
|
86
199
|
status: r.status as string,
|
|
87
200
|
attempts: Number(r.attempts ?? 0),
|
|
201
|
+
updatedAt: Number(r.updated_at ?? 0),
|
|
202
|
+
dispatchScope: (r.dispatch_scope as string | null) ?? null,
|
|
88
203
|
}));
|
|
89
204
|
tableExists = true;
|
|
90
205
|
} catch (err) {
|
|
91
|
-
|
|
92
|
-
// integration webhook has been processed on this deployment. Silently
|
|
93
|
-
// no-op until the table appears.
|
|
206
|
+
if (!isMissingPendingTasksTableError(err)) throw err;
|
|
94
207
|
if (tableExists !== false) {
|
|
95
208
|
tableExists = false;
|
|
96
209
|
if (process.env.DEBUG) {
|
|
@@ -99,17 +212,29 @@ export async function retryStuckPendingTasks(
|
|
|
99
212
|
);
|
|
100
213
|
}
|
|
101
214
|
}
|
|
102
|
-
return;
|
|
215
|
+
return result;
|
|
103
216
|
}
|
|
104
217
|
|
|
105
|
-
|
|
218
|
+
stuckRows = stuckRows.filter((row) => {
|
|
219
|
+
const durable = isIntegrationDurableDispatchEnabledForTask({
|
|
220
|
+
platform: row.platform,
|
|
221
|
+
externalThreadId: row.externalThreadId,
|
|
222
|
+
platformContext: row.dispatchScope
|
|
223
|
+
? { channelId: row.dispatchScope }
|
|
224
|
+
: undefined,
|
|
225
|
+
});
|
|
226
|
+
if (options.durableOnly && !durable) return false;
|
|
227
|
+
return true;
|
|
228
|
+
});
|
|
229
|
+
result.selected = stuckRows.length;
|
|
230
|
+
if (stuckRows.length === 0) return result;
|
|
106
231
|
|
|
107
232
|
for (const row of stuckRows) {
|
|
108
233
|
try {
|
|
109
234
|
// Cap retries — mark failed and move on so the row stops bouncing
|
|
110
235
|
// between pending and processing forever.
|
|
111
236
|
if (row.attempts >= MAX_PENDING_TASK_ATTEMPTS) {
|
|
112
|
-
await client.execute({
|
|
237
|
+
const update = await client.execute({
|
|
113
238
|
sql: `
|
|
114
239
|
UPDATE integration_pending_tasks
|
|
115
240
|
SET status = 'failed',
|
|
@@ -119,14 +244,21 @@ export async function retryStuckPendingTasks(
|
|
|
119
244
|
external_event_key = NULL
|
|
120
245
|
WHERE id = ?
|
|
121
246
|
AND status = ?
|
|
247
|
+
AND updated_at = ?
|
|
122
248
|
`,
|
|
123
249
|
args: [
|
|
124
250
|
Date.now(),
|
|
125
251
|
`Retry job: exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts`,
|
|
126
252
|
row.id,
|
|
127
253
|
row.status,
|
|
254
|
+
row.updatedAt,
|
|
128
255
|
],
|
|
129
256
|
});
|
|
257
|
+
if (affectedRows(update) === 0) {
|
|
258
|
+
result.skipped += 1;
|
|
259
|
+
continue;
|
|
260
|
+
}
|
|
261
|
+
result.markedFailed += 1;
|
|
130
262
|
console.warn(
|
|
131
263
|
`[integrations] Pending task ${row.id} exceeded ${MAX_PENDING_TASK_ATTEMPTS} attempts — marking failed`,
|
|
132
264
|
);
|
|
@@ -138,24 +270,44 @@ export async function retryStuckPendingTasks(
|
|
|
138
270
|
// Without this, processing rows stay stuck forever.
|
|
139
271
|
// For pending rows, just touch updated_at to avoid re-firing every tick.
|
|
140
272
|
const newStatus = row.status === "processing" ? "pending" : row.status;
|
|
141
|
-
await client.execute({
|
|
273
|
+
const update = await client.execute({
|
|
142
274
|
sql: `
|
|
143
275
|
UPDATE integration_pending_tasks
|
|
144
276
|
SET status = ?, updated_at = ?
|
|
145
277
|
WHERE id = ?
|
|
146
278
|
AND status = ?
|
|
279
|
+
AND updated_at = ?
|
|
147
280
|
`,
|
|
148
|
-
args: [newStatus, Date.now(), row.id, row.status],
|
|
281
|
+
args: [newStatus, Date.now(), row.id, row.status, row.updatedAt],
|
|
149
282
|
});
|
|
283
|
+
if (affectedRows(update) === 0) {
|
|
284
|
+
result.skipped += 1;
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
150
287
|
|
|
151
|
-
await
|
|
288
|
+
const outcome = await dispatchPendingIntegrationTask({
|
|
289
|
+
taskId: row.id,
|
|
290
|
+
task: {
|
|
291
|
+
platform: row.platform,
|
|
292
|
+
externalThreadId: row.externalThreadId,
|
|
293
|
+
platformContext: row.dispatchScope
|
|
294
|
+
? { channelId: row.dispatchScope }
|
|
295
|
+
: undefined,
|
|
296
|
+
},
|
|
297
|
+
baseUrl,
|
|
298
|
+
portableSettleMs: 1_000,
|
|
299
|
+
});
|
|
300
|
+
if (outcome === "failed") result.dispatchFailed += 1;
|
|
301
|
+
else result.dispatched += 1;
|
|
152
302
|
} catch (err) {
|
|
303
|
+
result.dispatchFailed += 1;
|
|
153
304
|
console.error(
|
|
154
305
|
`[integrations] Failed to retry pending task ${row.id}:`,
|
|
155
306
|
err,
|
|
156
307
|
);
|
|
157
308
|
}
|
|
158
309
|
}
|
|
310
|
+
return result;
|
|
159
311
|
}
|
|
160
312
|
|
|
161
313
|
function getProcessingStuckAfterMs(): number {
|
|
@@ -170,70 +322,6 @@ function getProcessingStuckAfterMs(): number {
|
|
|
170
322
|
return DEFAULT_PROCESSING_STUCK_AFTER_MS;
|
|
171
323
|
}
|
|
172
324
|
|
|
173
|
-
/**
|
|
174
|
-
* Fire-and-forget POST to the processor endpoint for a single task id.
|
|
175
|
-
* Mirrors the original dispatch from the webhook handler, including the
|
|
176
|
-
* short-lived HMAC bearer token bound to this taskId.
|
|
177
|
-
*/
|
|
178
|
-
async function refireProcessor(
|
|
179
|
-
taskId: string,
|
|
180
|
-
webhookBaseUrl: string | undefined,
|
|
181
|
-
): Promise<void> {
|
|
182
|
-
const baseUrl =
|
|
183
|
-
webhookBaseUrl ||
|
|
184
|
-
process.env.WEBHOOK_BASE_URL ||
|
|
185
|
-
process.env.APP_URL ||
|
|
186
|
-
process.env.URL ||
|
|
187
|
-
process.env.DEPLOY_URL ||
|
|
188
|
-
`http://localhost:${process.env.PORT || 3000}`;
|
|
189
|
-
|
|
190
|
-
const url = `${withConfiguredAppBasePath(baseUrl)}${PROCESSOR_PATH}`;
|
|
191
|
-
|
|
192
|
-
// Sign with HMAC if A2A_SECRET is configured. In production we MUST sign —
|
|
193
|
-
// an unsigned dispatch in production lets attackers re-trigger any queued
|
|
194
|
-
// task with a guessable id (C3 in the webhook security audit). In dev we
|
|
195
|
-
// fall back to unsigned so contributors can iterate without configuring
|
|
196
|
-
// A2A_SECRET locally.
|
|
197
|
-
const headers: Record<string, string> = {
|
|
198
|
-
"Content-Type": "application/json",
|
|
199
|
-
};
|
|
200
|
-
try {
|
|
201
|
-
headers["Authorization"] = `Bearer ${signInternalToken(taskId)}`;
|
|
202
|
-
} catch (err) {
|
|
203
|
-
if (process.env.NODE_ENV === "production") {
|
|
204
|
-
console.error(
|
|
205
|
-
`[integrations] Refusing to dispatch task ${taskId} — A2A_SECRET not configured. ` +
|
|
206
|
-
"Set A2A_SECRET to enable signed retry dispatches.",
|
|
207
|
-
);
|
|
208
|
-
return;
|
|
209
|
-
}
|
|
210
|
-
// Dev: proceed unsigned. Log the underlying error path so a malformed
|
|
211
|
-
// secret (different from "not set") doesn't fail silently (L5 in the audit).
|
|
212
|
-
if (err instanceof Error && !/A2A_SECRET/i.test(err.message)) {
|
|
213
|
-
console.error(
|
|
214
|
-
`[integrations] signInternalToken failed unexpectedly for ${taskId}:`,
|
|
215
|
-
err,
|
|
216
|
-
);
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
// Don't await the body — we just want the request to leave the box.
|
|
221
|
-
// A short timeout avoids tying up the retry loop on a hung processor.
|
|
222
|
-
const controller = new AbortController();
|
|
223
|
-
const timer = setTimeout(() => controller.abort(), 5_000);
|
|
224
|
-
|
|
225
|
-
try {
|
|
226
|
-
await fetch(url, {
|
|
227
|
-
method: "POST",
|
|
228
|
-
headers,
|
|
229
|
-
body: JSON.stringify({ taskId }),
|
|
230
|
-
signal: controller.signal,
|
|
231
|
-
});
|
|
232
|
-
} finally {
|
|
233
|
-
clearTimeout(timer);
|
|
234
|
-
}
|
|
235
|
-
}
|
|
236
|
-
|
|
237
325
|
/**
|
|
238
326
|
* Start the periodic retry loop. Safe to call multiple times — second call
|
|
239
327
|
* is a no-op.
|
|
@@ -33,6 +33,10 @@ async function ensureTable(): Promise<void> {
|
|
|
33
33
|
org_id TEXT,
|
|
34
34
|
status TEXT NOT NULL,
|
|
35
35
|
attempts ${intType()} NOT NULL DEFAULT 0,
|
|
36
|
+
dispatch_attempts ${intType()} NOT NULL DEFAULT 0,
|
|
37
|
+
last_dispatch_at ${intType()},
|
|
38
|
+
last_dispatch_outcome TEXT,
|
|
39
|
+
dispatch_scope TEXT,
|
|
36
40
|
error_message TEXT,
|
|
37
41
|
created_at ${intType()} NOT NULL,
|
|
38
42
|
updated_at ${intType()} NOT NULL,
|
|
@@ -47,6 +51,26 @@ async function ensureTable(): Promise<void> {
|
|
|
47
51
|
"external_event_key",
|
|
48
52
|
`ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS external_event_key TEXT`,
|
|
49
53
|
);
|
|
54
|
+
await ensureColumnExists(
|
|
55
|
+
"integration_pending_tasks",
|
|
56
|
+
"dispatch_attempts",
|
|
57
|
+
`ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_attempts ${intType()} NOT NULL DEFAULT 0`,
|
|
58
|
+
);
|
|
59
|
+
await ensureColumnExists(
|
|
60
|
+
"integration_pending_tasks",
|
|
61
|
+
"last_dispatch_at",
|
|
62
|
+
`ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_at ${intType()}`,
|
|
63
|
+
);
|
|
64
|
+
await ensureColumnExists(
|
|
65
|
+
"integration_pending_tasks",
|
|
66
|
+
"last_dispatch_outcome",
|
|
67
|
+
`ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS last_dispatch_outcome TEXT`,
|
|
68
|
+
);
|
|
69
|
+
await ensureColumnExists(
|
|
70
|
+
"integration_pending_tasks",
|
|
71
|
+
"dispatch_scope",
|
|
72
|
+
`ALTER TABLE integration_pending_tasks ADD COLUMN IF NOT EXISTS dispatch_scope TEXT`,
|
|
73
|
+
);
|
|
50
74
|
await ensureIndexExists(
|
|
51
75
|
"idx_pending_tasks_status_created",
|
|
52
76
|
`CREATE INDEX IF NOT EXISTS idx_pending_tasks_status_created ON integration_pending_tasks(status, created_at)`,
|
|
@@ -55,6 +79,10 @@ async function ensureTable(): Promise<void> {
|
|
|
55
79
|
"idx_pending_tasks_event_key",
|
|
56
80
|
`CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,
|
|
57
81
|
);
|
|
82
|
+
await ensureIndexExists(
|
|
83
|
+
"idx_pending_tasks_dispatch_scope",
|
|
84
|
+
`CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`,
|
|
85
|
+
);
|
|
58
86
|
return;
|
|
59
87
|
}
|
|
60
88
|
|
|
@@ -69,6 +97,10 @@ async function ensureTable(): Promise<void> {
|
|
|
69
97
|
// unique index ensures a duplicate INSERT raises an error we can
|
|
70
98
|
// catch as "already-enqueued".
|
|
71
99
|
await ensureExternalEventKey(client);
|
|
100
|
+
await ensureDispatchColumns(client);
|
|
101
|
+
await client.execute(
|
|
102
|
+
`CREATE INDEX IF NOT EXISTS idx_pending_tasks_dispatch_scope ON integration_pending_tasks(platform, dispatch_scope)`,
|
|
103
|
+
);
|
|
72
104
|
await client.execute(
|
|
73
105
|
`CREATE UNIQUE INDEX IF NOT EXISTS idx_pending_tasks_event_key ON integration_pending_tasks(platform, external_event_key)`,
|
|
74
106
|
);
|
|
@@ -81,6 +113,10 @@ async function ensureTable(): Promise<void> {
|
|
|
81
113
|
return _initPromise;
|
|
82
114
|
}
|
|
83
115
|
|
|
116
|
+
export async function ensurePendingTasksTable(): Promise<void> {
|
|
117
|
+
await ensureTable();
|
|
118
|
+
}
|
|
119
|
+
|
|
84
120
|
async function ensureExternalEventKey(
|
|
85
121
|
client: ReturnType<typeof getDbExec>,
|
|
86
122
|
): Promise<void> {
|
|
@@ -107,6 +143,32 @@ async function ensureExternalEventKey(
|
|
|
107
143
|
}
|
|
108
144
|
}
|
|
109
145
|
|
|
146
|
+
async function ensureDispatchColumns(
|
|
147
|
+
client: ReturnType<typeof getDbExec>,
|
|
148
|
+
): Promise<void> {
|
|
149
|
+
const columns = [
|
|
150
|
+
`dispatch_attempts ${intType()} NOT NULL DEFAULT 0`,
|
|
151
|
+
`last_dispatch_at ${intType()}`,
|
|
152
|
+
"last_dispatch_outcome TEXT",
|
|
153
|
+
"dispatch_scope TEXT",
|
|
154
|
+
];
|
|
155
|
+
for (const column of columns) {
|
|
156
|
+
try {
|
|
157
|
+
await client.execute(
|
|
158
|
+
`ALTER TABLE integration_pending_tasks ADD COLUMN ${column}`,
|
|
159
|
+
);
|
|
160
|
+
} catch (err: any) {
|
|
161
|
+
if (
|
|
162
|
+
!String(err?.message ?? err)
|
|
163
|
+
.toLowerCase()
|
|
164
|
+
.includes("duplicate")
|
|
165
|
+
) {
|
|
166
|
+
throw err;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
110
172
|
/** Status values for an integration pending task. */
|
|
111
173
|
export type PendingTaskStatus =
|
|
112
174
|
| "pending"
|
|
@@ -123,6 +185,10 @@ export interface PendingTask {
|
|
|
123
185
|
orgId: string | null;
|
|
124
186
|
status: PendingTaskStatus;
|
|
125
187
|
attempts: number;
|
|
188
|
+
dispatchAttempts: number;
|
|
189
|
+
lastDispatchAt: number | null;
|
|
190
|
+
lastDispatchOutcome: string | null;
|
|
191
|
+
dispatchScope: string | null;
|
|
126
192
|
errorMessage: string | null;
|
|
127
193
|
createdAt: number;
|
|
128
194
|
updatedAt: number;
|
|
@@ -164,6 +230,13 @@ function rowToTask(row: Record<string, unknown>): PendingTask {
|
|
|
164
230
|
orgId: (row.org_id as string | null) ?? null,
|
|
165
231
|
status: row.status as PendingTaskStatus,
|
|
166
232
|
attempts: Number(row.attempts ?? 0),
|
|
233
|
+
dispatchAttempts: Number(row.dispatch_attempts ?? 0),
|
|
234
|
+
lastDispatchAt:
|
|
235
|
+
row.last_dispatch_at == null
|
|
236
|
+
? null
|
|
237
|
+
: Number(row.last_dispatch_at as number),
|
|
238
|
+
lastDispatchOutcome: (row.last_dispatch_outcome as string | null) ?? null,
|
|
239
|
+
dispatchScope: (row.dispatch_scope as string | null) ?? null,
|
|
167
240
|
errorMessage: (row.error_message as string | null) ?? null,
|
|
168
241
|
createdAt: Number(row.created_at ?? 0),
|
|
169
242
|
updatedAt: Number(row.updated_at ?? 0),
|
|
@@ -189,14 +262,15 @@ export async function insertPendingTask(input: {
|
|
|
189
262
|
ownerEmail: string;
|
|
190
263
|
orgId?: string | null;
|
|
191
264
|
externalEventKey?: string | null;
|
|
265
|
+
dispatchScope?: string | null;
|
|
192
266
|
}): Promise<void> {
|
|
193
267
|
await ensureTable();
|
|
194
268
|
const client = getDbExec();
|
|
195
269
|
const now = Date.now();
|
|
196
270
|
await client.execute({
|
|
197
271
|
sql: `INSERT INTO integration_pending_tasks
|
|
198
|
-
(id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key)
|
|
199
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
272
|
+
(id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, created_at, updated_at, external_event_key, dispatch_scope)
|
|
273
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
200
274
|
args: [
|
|
201
275
|
input.id,
|
|
202
276
|
input.platform,
|
|
@@ -209,6 +283,7 @@ export async function insertPendingTask(input: {
|
|
|
209
283
|
now,
|
|
210
284
|
now,
|
|
211
285
|
input.externalEventKey ?? null,
|
|
286
|
+
input.dispatchScope ?? null,
|
|
212
287
|
],
|
|
213
288
|
});
|
|
214
289
|
}
|
|
@@ -238,7 +313,7 @@ export async function getPendingTask(id: string): Promise<PendingTask | null> {
|
|
|
238
313
|
await ensureTable();
|
|
239
314
|
const client = getDbExec();
|
|
240
315
|
const { rows } = await client.execute({
|
|
241
|
-
sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at
|
|
316
|
+
sql: `SELECT id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at
|
|
242
317
|
FROM integration_pending_tasks WHERE id = ? LIMIT 1`,
|
|
243
318
|
args: [id],
|
|
244
319
|
});
|
|
@@ -342,6 +417,7 @@ export async function resolveIntegrationSourceContext(
|
|
|
342
417
|
*/
|
|
343
418
|
export async function claimPendingTask(
|
|
344
419
|
id: string,
|
|
420
|
+
options?: { dispatchOutcome?: string },
|
|
345
421
|
): Promise<PendingTask | null> {
|
|
346
422
|
await ensureTable();
|
|
347
423
|
const client = getDbExec();
|
|
@@ -352,7 +428,8 @@ export async function claimPendingTask(
|
|
|
352
428
|
const result = await client.execute({
|
|
353
429
|
sql: isPostgres()
|
|
354
430
|
? `UPDATE integration_pending_tasks
|
|
355
|
-
SET status = ?, attempts = attempts + 1, updated_at =
|
|
431
|
+
SET status = ?, attempts = attempts + 1, updated_at = ?,
|
|
432
|
+
last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)
|
|
356
433
|
WHERE id = ? AND status = 'pending'
|
|
357
434
|
AND NOT EXISTS (
|
|
358
435
|
SELECT 1 FROM integration_pending_tasks active
|
|
@@ -374,9 +451,10 @@ export async function claimPendingTask(
|
|
|
374
451
|
)
|
|
375
452
|
)
|
|
376
453
|
)
|
|
377
|
-
RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, error_message, created_at, updated_at, completed_at`
|
|
454
|
+
RETURNING id, platform, external_thread_id, payload, owner_email, org_id, status, attempts, dispatch_attempts, last_dispatch_at, last_dispatch_outcome, dispatch_scope, error_message, created_at, updated_at, completed_at`
|
|
378
455
|
: `UPDATE integration_pending_tasks
|
|
379
|
-
SET status = ?, attempts = attempts + 1, updated_at =
|
|
456
|
+
SET status = ?, attempts = attempts + 1, updated_at = ?,
|
|
457
|
+
last_dispatch_outcome = COALESCE(?, last_dispatch_outcome)
|
|
380
458
|
WHERE id = ? AND status = 'pending'
|
|
381
459
|
AND NOT EXISTS (
|
|
382
460
|
SELECT 1 FROM integration_pending_tasks active
|
|
@@ -398,7 +476,7 @@ export async function claimPendingTask(
|
|
|
398
476
|
)
|
|
399
477
|
)
|
|
400
478
|
)`,
|
|
401
|
-
args: ["processing", now, id],
|
|
479
|
+
args: ["processing", now, options?.dispatchOutcome ?? null, id],
|
|
402
480
|
});
|
|
403
481
|
const rows = result.rows ?? [];
|
|
404
482
|
|
|
@@ -418,19 +496,44 @@ export async function claimPendingTask(
|
|
|
418
496
|
return fetched;
|
|
419
497
|
}
|
|
420
498
|
|
|
499
|
+
export async function recordPendingTaskDispatchAttempt(
|
|
500
|
+
id: string,
|
|
501
|
+
outcome: string,
|
|
502
|
+
): Promise<void> {
|
|
503
|
+
await ensureTable();
|
|
504
|
+
await getDbExec().execute({
|
|
505
|
+
sql: `UPDATE integration_pending_tasks
|
|
506
|
+
SET dispatch_attempts = dispatch_attempts + 1,
|
|
507
|
+
last_dispatch_at = ?,
|
|
508
|
+
last_dispatch_outcome = CASE
|
|
509
|
+
WHEN status = 'processing'
|
|
510
|
+
AND last_dispatch_outcome = 'background-acknowledged'
|
|
511
|
+
THEN last_dispatch_outcome
|
|
512
|
+
ELSE ?
|
|
513
|
+
END
|
|
514
|
+
WHERE id = ?`,
|
|
515
|
+
args: [Date.now(), outcome.slice(0, 80), id],
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
|
|
421
519
|
/** Next queued turn for a provider thread after its current task completes. */
|
|
422
|
-
export async function
|
|
520
|
+
export async function getNextPendingTaskForThread(
|
|
423
521
|
platform: string,
|
|
424
522
|
externalThreadId: string,
|
|
425
|
-
): Promise<string | null> {
|
|
523
|
+
): Promise<{ id: string; dispatchScope: string | null } | null> {
|
|
426
524
|
await ensureTable();
|
|
427
525
|
const { rows } = await getDbExec().execute({
|
|
428
|
-
sql: `SELECT id FROM integration_pending_tasks
|
|
526
|
+
sql: `SELECT id, dispatch_scope FROM integration_pending_tasks
|
|
429
527
|
WHERE platform = ? AND external_thread_id = ? AND status = 'pending'
|
|
430
528
|
ORDER BY created_at ASC, id ASC LIMIT 1`,
|
|
431
529
|
args: [platform, externalThreadId],
|
|
432
530
|
});
|
|
433
|
-
return rows[0]?.id
|
|
531
|
+
return rows[0]?.id
|
|
532
|
+
? {
|
|
533
|
+
id: String(rows[0].id),
|
|
534
|
+
dispatchScope: (rows[0].dispatch_scope as string | null) ?? null,
|
|
535
|
+
}
|
|
536
|
+
: null;
|
|
434
537
|
}
|
|
435
538
|
|
|
436
539
|
/** Mark a task as completed. */
|