@nicnocquee/dataqueue 1.31.0 → 1.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.cjs +593 -3
- package/dist/cli.cjs.map +1 -1
- package/dist/cli.d.cts +16 -1
- package/dist/cli.d.ts +16 -1
- package/dist/cli.js +593 -3
- package/dist/cli.js.map +1 -1
- package/dist/index.cjs +2418 -1936
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +151 -16
- package/dist/index.d.ts +151 -16
- package/dist/index.js +2418 -1936
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/backend.ts +70 -4
- package/src/backends/postgres.ts +345 -29
- package/src/backends/redis-scripts.ts +197 -22
- package/src/backends/redis.test.ts +621 -0
- package/src/backends/redis.ts +400 -21
- package/src/cli.test.ts +17 -6
- package/src/cli.ts +28 -2
- package/src/index.ts +12 -29
- package/src/init-command.test.ts +449 -0
- package/src/init-command.ts +709 -0
- package/src/processor.ts +14 -93
- package/src/queue.test.ts +29 -0
- package/src/queue.ts +19 -251
- package/src/types.ts +28 -10
package/dist/index.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
|
-
import { AsyncLocalStorage } from 'async_hooks';
|
|
2
|
-
import { randomUUID } from 'crypto';
|
|
3
1
|
import { Worker } from 'worker_threads';
|
|
2
|
+
import { AsyncLocalStorage } from 'async_hooks';
|
|
4
3
|
import { Pool } from 'pg';
|
|
5
4
|
import { parse } from 'pg-connection-string';
|
|
6
5
|
import fs from 'fs';
|
|
6
|
+
import { randomUUID } from 'crypto';
|
|
7
7
|
import { createRequire } from 'module';
|
|
8
8
|
import { Cron } from 'croner';
|
|
9
9
|
|
|
10
|
+
// src/processor.ts
|
|
11
|
+
|
|
10
12
|
// src/types.ts
|
|
11
13
|
var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
|
|
12
14
|
JobEventType2["Added"] = "added";
|
|
@@ -20,11 +22,11 @@ var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
|
|
|
20
22
|
JobEventType2["Waiting"] = "waiting";
|
|
21
23
|
return JobEventType2;
|
|
22
24
|
})(JobEventType || {});
|
|
23
|
-
var FailureReason = /* @__PURE__ */ ((
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
return
|
|
25
|
+
var FailureReason = /* @__PURE__ */ ((FailureReason4) => {
|
|
26
|
+
FailureReason4["Timeout"] = "timeout";
|
|
27
|
+
FailureReason4["HandlerError"] = "handler_error";
|
|
28
|
+
FailureReason4["NoHandler"] = "no_handler";
|
|
29
|
+
return FailureReason4;
|
|
28
30
|
})(FailureReason || {});
|
|
29
31
|
var WaitSignal = class extends Error {
|
|
30
32
|
constructor(type, waitUntil, tokenId, stepData) {
|
|
@@ -51,804 +53,942 @@ var log = (message) => {
|
|
|
51
53
|
}
|
|
52
54
|
};
|
|
53
55
|
|
|
54
|
-
// src/
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
return this.pool;
|
|
62
|
-
}
|
|
63
|
-
// ── Events ──────────────────────────────────────────────────────────
|
|
64
|
-
async recordJobEvent(jobId, eventType, metadata) {
|
|
65
|
-
const client = await this.pool.connect();
|
|
66
|
-
try {
|
|
67
|
-
await client.query(
|
|
68
|
-
`INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
|
|
69
|
-
[jobId, eventType, metadata ? JSON.stringify(metadata) : null]
|
|
70
|
-
);
|
|
71
|
-
} catch (error) {
|
|
72
|
-
log(`Error recording job event for job ${jobId}: ${error}`);
|
|
73
|
-
} finally {
|
|
74
|
-
client.release();
|
|
75
|
-
}
|
|
76
|
-
}
|
|
77
|
-
async getJobEvents(jobId) {
|
|
78
|
-
const client = await this.pool.connect();
|
|
79
|
-
try {
|
|
80
|
-
const res = await client.query(
|
|
81
|
-
`SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
|
|
82
|
-
[jobId]
|
|
83
|
-
);
|
|
84
|
-
return res.rows;
|
|
85
|
-
} finally {
|
|
86
|
-
client.release();
|
|
87
|
-
}
|
|
88
|
-
}
|
|
89
|
-
// ── Job CRUD ──────────────────────────────────────────────────────────
|
|
90
|
-
async addJob({
|
|
91
|
-
jobType,
|
|
92
|
-
payload,
|
|
93
|
-
maxAttempts = 3,
|
|
94
|
-
priority = 0,
|
|
95
|
-
runAt = null,
|
|
96
|
-
timeoutMs = void 0,
|
|
97
|
-
forceKillOnTimeout = false,
|
|
98
|
-
tags = void 0,
|
|
99
|
-
idempotencyKey = void 0
|
|
100
|
-
}) {
|
|
101
|
-
const client = await this.pool.connect();
|
|
102
|
-
try {
|
|
103
|
-
let result;
|
|
104
|
-
const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
|
|
105
|
-
if (runAt) {
|
|
106
|
-
result = await client.query(
|
|
107
|
-
`INSERT INTO job_queue
|
|
108
|
-
(job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
109
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
110
|
-
${onConflict}
|
|
111
|
-
RETURNING id`,
|
|
112
|
-
[
|
|
113
|
-
jobType,
|
|
114
|
-
payload,
|
|
115
|
-
maxAttempts,
|
|
116
|
-
priority,
|
|
117
|
-
runAt,
|
|
118
|
-
timeoutMs ?? null,
|
|
119
|
-
forceKillOnTimeout ?? false,
|
|
120
|
-
tags ?? null,
|
|
121
|
-
idempotencyKey ?? null
|
|
122
|
-
]
|
|
123
|
-
);
|
|
124
|
-
} else {
|
|
125
|
-
result = await client.query(
|
|
126
|
-
`INSERT INTO job_queue
|
|
127
|
-
(job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
128
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
129
|
-
${onConflict}
|
|
130
|
-
RETURNING id`,
|
|
131
|
-
[
|
|
132
|
-
jobType,
|
|
133
|
-
payload,
|
|
134
|
-
maxAttempts,
|
|
135
|
-
priority,
|
|
136
|
-
timeoutMs ?? null,
|
|
137
|
-
forceKillOnTimeout ?? false,
|
|
138
|
-
tags ?? null,
|
|
139
|
-
idempotencyKey ?? null
|
|
140
|
-
]
|
|
141
|
-
);
|
|
142
|
-
}
|
|
143
|
-
if (result.rows.length === 0 && idempotencyKey) {
|
|
144
|
-
const existing = await client.query(
|
|
145
|
-
`SELECT id FROM job_queue WHERE idempotency_key = $1`,
|
|
146
|
-
[idempotencyKey]
|
|
147
|
-
);
|
|
148
|
-
if (existing.rows.length > 0) {
|
|
149
|
-
log(
|
|
150
|
-
`Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
|
|
151
|
-
);
|
|
152
|
-
return existing.rows[0].id;
|
|
153
|
-
}
|
|
154
|
-
throw new Error(
|
|
155
|
-
`Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
|
|
156
|
-
);
|
|
157
|
-
}
|
|
158
|
-
const jobId = result.rows[0].id;
|
|
159
|
-
log(
|
|
160
|
-
`Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
|
|
56
|
+
// src/processor.ts
|
|
57
|
+
function validateHandlerSerializable(handler, jobType) {
|
|
58
|
+
try {
|
|
59
|
+
const handlerString = handler.toString();
|
|
60
|
+
if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
|
|
61
|
+
throw new Error(
|
|
62
|
+
`Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
|
|
161
63
|
);
|
|
162
|
-
await this.recordJobEvent(jobId, "added" /* Added */, {
|
|
163
|
-
jobType,
|
|
164
|
-
payload,
|
|
165
|
-
tags,
|
|
166
|
-
idempotencyKey
|
|
167
|
-
});
|
|
168
|
-
return jobId;
|
|
169
|
-
} catch (error) {
|
|
170
|
-
log(`Error adding job: ${error}`);
|
|
171
|
-
throw error;
|
|
172
|
-
} finally {
|
|
173
|
-
client.release();
|
|
174
64
|
}
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
try {
|
|
179
|
-
const result = await client.query(
|
|
180
|
-
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
|
|
181
|
-
[id]
|
|
65
|
+
if (handlerString.includes("[native code]")) {
|
|
66
|
+
throw new Error(
|
|
67
|
+
`Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
|
|
182
68
|
);
|
|
183
|
-
if (result.rows.length === 0) {
|
|
184
|
-
log(`Job ${id} not found`);
|
|
185
|
-
return null;
|
|
186
|
-
}
|
|
187
|
-
log(`Found job ${id}`);
|
|
188
|
-
const job = result.rows[0];
|
|
189
|
-
return {
|
|
190
|
-
...job,
|
|
191
|
-
payload: job.payload,
|
|
192
|
-
timeoutMs: job.timeoutMs,
|
|
193
|
-
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
194
|
-
failureReason: job.failureReason
|
|
195
|
-
};
|
|
196
|
-
} catch (error) {
|
|
197
|
-
log(`Error getting job ${id}: ${error}`);
|
|
198
|
-
throw error;
|
|
199
|
-
} finally {
|
|
200
|
-
client.release();
|
|
201
69
|
}
|
|
202
|
-
}
|
|
203
|
-
async getJobsByStatus(status, limit = 100, offset = 0) {
|
|
204
|
-
const client = await this.pool.connect();
|
|
205
70
|
try {
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
71
|
+
new Function("return " + handlerString);
|
|
72
|
+
} catch (parseError) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`Handler for job type "${jobType}" cannot be serialized: ${parseError instanceof Error ? parseError.message : String(parseError)}. When using forceKillOnTimeout, handlers must be serializable functions without closures over external variables.`
|
|
209
75
|
);
|
|
210
|
-
log(`Found ${result.rows.length} jobs by status ${status}`);
|
|
211
|
-
return result.rows.map((job) => ({
|
|
212
|
-
...job,
|
|
213
|
-
payload: job.payload,
|
|
214
|
-
timeoutMs: job.timeoutMs,
|
|
215
|
-
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
216
|
-
failureReason: job.failureReason
|
|
217
|
-
}));
|
|
218
|
-
} catch (error) {
|
|
219
|
-
log(`Error getting jobs by status ${status}: ${error}`);
|
|
220
|
-
throw error;
|
|
221
|
-
} finally {
|
|
222
|
-
client.release();
|
|
223
76
|
}
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
const client = await this.pool.connect();
|
|
227
|
-
try {
|
|
228
|
-
const result = await client.query(
|
|
229
|
-
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
|
|
230
|
-
[limit, offset]
|
|
231
|
-
);
|
|
232
|
-
log(`Found ${result.rows.length} jobs (all)`);
|
|
233
|
-
return result.rows.map((job) => ({
|
|
234
|
-
...job,
|
|
235
|
-
payload: job.payload,
|
|
236
|
-
timeoutMs: job.timeoutMs,
|
|
237
|
-
forceKillOnTimeout: job.forceKillOnTimeout
|
|
238
|
-
}));
|
|
239
|
-
} catch (error) {
|
|
240
|
-
log(`Error getting all jobs: ${error}`);
|
|
77
|
+
} catch (error) {
|
|
78
|
+
if (error instanceof Error) {
|
|
241
79
|
throw error;
|
|
242
|
-
} finally {
|
|
243
|
-
client.release();
|
|
244
80
|
}
|
|
81
|
+
throw new Error(
|
|
82
|
+
`Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
|
|
83
|
+
);
|
|
245
84
|
}
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
|
|
291
|
-
const mode = filters.tags.mode || "all";
|
|
292
|
-
const tagValues = filters.tags.values;
|
|
293
|
-
switch (mode) {
|
|
294
|
-
case "exact":
|
|
295
|
-
where.push(`tags = $${paramIdx++}`);
|
|
296
|
-
params.push(tagValues);
|
|
297
|
-
break;
|
|
298
|
-
case "all":
|
|
299
|
-
where.push(`tags @> $${paramIdx++}`);
|
|
300
|
-
params.push(tagValues);
|
|
301
|
-
break;
|
|
302
|
-
case "any":
|
|
303
|
-
where.push(`tags && $${paramIdx++}`);
|
|
304
|
-
params.push(tagValues);
|
|
305
|
-
break;
|
|
306
|
-
case "none":
|
|
307
|
-
where.push(`NOT (tags && $${paramIdx++})`);
|
|
308
|
-
params.push(tagValues);
|
|
309
|
-
break;
|
|
310
|
-
default:
|
|
311
|
-
where.push(`tags @> $${paramIdx++}`);
|
|
312
|
-
params.push(tagValues);
|
|
85
|
+
}
|
|
86
|
+
async function runHandlerInWorker(handler, payload, timeoutMs, jobType) {
|
|
87
|
+
validateHandlerSerializable(handler, jobType);
|
|
88
|
+
return new Promise((resolve, reject) => {
|
|
89
|
+
const workerCode = `
|
|
90
|
+
(function() {
|
|
91
|
+
const { parentPort, workerData } = require('worker_threads');
|
|
92
|
+
const { handlerCode, payload, timeoutMs } = workerData;
|
|
93
|
+
|
|
94
|
+
// Create an AbortController for the handler
|
|
95
|
+
const controller = new AbortController();
|
|
96
|
+
const signal = controller.signal;
|
|
97
|
+
|
|
98
|
+
// Set up timeout
|
|
99
|
+
const timeoutId = setTimeout(() => {
|
|
100
|
+
controller.abort();
|
|
101
|
+
parentPort.postMessage({ type: 'timeout' });
|
|
102
|
+
}, timeoutMs);
|
|
103
|
+
|
|
104
|
+
try {
|
|
105
|
+
// Execute the handler
|
|
106
|
+
// Note: This uses Function constructor which requires the handler to be serializable.
|
|
107
|
+
// The handler should be validated before reaching this point.
|
|
108
|
+
let handlerFn;
|
|
109
|
+
try {
|
|
110
|
+
// Wrap handlerCode in parentheses to ensure it's treated as an expression
|
|
111
|
+
// This handles both arrow functions and regular functions
|
|
112
|
+
const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
|
|
113
|
+
? handlerCode
|
|
114
|
+
: '(' + handlerCode + ')';
|
|
115
|
+
handlerFn = new Function('return ' + wrappedCode)();
|
|
116
|
+
} catch (parseError) {
|
|
117
|
+
clearTimeout(timeoutId);
|
|
118
|
+
parentPort.postMessage({
|
|
119
|
+
type: 'error',
|
|
120
|
+
error: {
|
|
121
|
+
message: 'Handler cannot be deserialized in worker thread. ' +
|
|
122
|
+
'Ensure your handler is a standalone function without closures over external variables. ' +
|
|
123
|
+
'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
|
|
124
|
+
stack: parseError instanceof Error ? parseError.stack : undefined,
|
|
125
|
+
name: 'SerializationError',
|
|
126
|
+
},
|
|
127
|
+
});
|
|
128
|
+
return;
|
|
313
129
|
}
|
|
130
|
+
|
|
131
|
+
// Ensure handlerFn is actually a function
|
|
132
|
+
if (typeof handlerFn !== 'function') {
|
|
133
|
+
clearTimeout(timeoutId);
|
|
134
|
+
parentPort.postMessage({
|
|
135
|
+
type: 'error',
|
|
136
|
+
error: {
|
|
137
|
+
message: 'Handler deserialization did not produce a function. ' +
|
|
138
|
+
'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
|
|
139
|
+
name: 'SerializationError',
|
|
140
|
+
},
|
|
141
|
+
});
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
handlerFn(payload, signal)
|
|
146
|
+
.then(() => {
|
|
147
|
+
clearTimeout(timeoutId);
|
|
148
|
+
parentPort.postMessage({ type: 'success' });
|
|
149
|
+
})
|
|
150
|
+
.catch((error) => {
|
|
151
|
+
clearTimeout(timeoutId);
|
|
152
|
+
parentPort.postMessage({
|
|
153
|
+
type: 'error',
|
|
154
|
+
error: {
|
|
155
|
+
message: error.message,
|
|
156
|
+
stack: error.stack,
|
|
157
|
+
name: error.name,
|
|
158
|
+
},
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
} catch (error) {
|
|
162
|
+
clearTimeout(timeoutId);
|
|
163
|
+
parentPort.postMessage({
|
|
164
|
+
type: 'error',
|
|
165
|
+
error: {
|
|
166
|
+
message: error.message,
|
|
167
|
+
stack: error.stack,
|
|
168
|
+
name: error.name,
|
|
169
|
+
},
|
|
170
|
+
});
|
|
314
171
|
}
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
172
|
+
})();
|
|
173
|
+
`;
|
|
174
|
+
const worker = new Worker(workerCode, {
|
|
175
|
+
eval: true,
|
|
176
|
+
workerData: {
|
|
177
|
+
handlerCode: handler.toString(),
|
|
178
|
+
payload,
|
|
179
|
+
timeoutMs
|
|
319
180
|
}
|
|
320
|
-
|
|
321
|
-
|
|
181
|
+
});
|
|
182
|
+
let resolved = false;
|
|
183
|
+
worker.on("message", (message) => {
|
|
184
|
+
if (resolved) return;
|
|
185
|
+
resolved = true;
|
|
186
|
+
if (message.type === "success") {
|
|
187
|
+
resolve();
|
|
188
|
+
} else if (message.type === "timeout") {
|
|
189
|
+
const timeoutError = new Error(
|
|
190
|
+
`Job timed out after ${timeoutMs} ms and was forcefully terminated`
|
|
191
|
+
);
|
|
192
|
+
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
193
|
+
reject(timeoutError);
|
|
194
|
+
} else if (message.type === "error") {
|
|
195
|
+
const error = new Error(message.error.message);
|
|
196
|
+
error.stack = message.error.stack;
|
|
197
|
+
error.name = message.error.name;
|
|
198
|
+
reject(error);
|
|
322
199
|
}
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
if (
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
200
|
+
});
|
|
201
|
+
worker.on("error", (error) => {
|
|
202
|
+
if (resolved) return;
|
|
203
|
+
resolved = true;
|
|
204
|
+
reject(error);
|
|
205
|
+
});
|
|
206
|
+
worker.on("exit", (code) => {
|
|
207
|
+
if (resolved) return;
|
|
208
|
+
if (code !== 0) {
|
|
209
|
+
resolved = true;
|
|
210
|
+
reject(new Error(`Worker stopped with exit code ${code}`));
|
|
330
211
|
}
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
}
|
|
212
|
+
});
|
|
213
|
+
setTimeout(() => {
|
|
214
|
+
if (!resolved) {
|
|
215
|
+
resolved = true;
|
|
216
|
+
worker.terminate().then(() => {
|
|
217
|
+
const timeoutError = new Error(
|
|
218
|
+
`Job timed out after ${timeoutMs} ms and was forcefully terminated`
|
|
219
|
+
);
|
|
220
|
+
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
221
|
+
reject(timeoutError);
|
|
222
|
+
}).catch((err) => {
|
|
223
|
+
reject(err);
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
}, timeoutMs + 100);
|
|
227
|
+
});
|
|
228
|
+
}
|
|
229
|
+
function calculateWaitUntil(duration) {
|
|
230
|
+
const now = Date.now();
|
|
231
|
+
let ms = 0;
|
|
232
|
+
if (duration.seconds) ms += duration.seconds * 1e3;
|
|
233
|
+
if (duration.minutes) ms += duration.minutes * 60 * 1e3;
|
|
234
|
+
if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
|
|
235
|
+
if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
|
|
236
|
+
if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
|
|
237
|
+
if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
|
|
238
|
+
if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
|
|
239
|
+
if (ms <= 0) {
|
|
240
|
+
throw new Error(
|
|
241
|
+
"waitFor duration must be positive. Provide at least one positive duration field."
|
|
242
|
+
);
|
|
346
243
|
}
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
default:
|
|
371
|
-
query += " WHERE tags @> $1";
|
|
372
|
-
params = [tags];
|
|
244
|
+
return new Date(now + ms);
|
|
245
|
+
}
|
|
246
|
+
async function resolveCompletedWaits(backend, stepData) {
|
|
247
|
+
for (const key of Object.keys(stepData)) {
|
|
248
|
+
if (!key.startsWith("__wait_")) continue;
|
|
249
|
+
const entry = stepData[key];
|
|
250
|
+
if (!entry || typeof entry !== "object" || entry.completed) continue;
|
|
251
|
+
if (entry.type === "duration" || entry.type === "date") {
|
|
252
|
+
stepData[key] = { ...entry, completed: true };
|
|
253
|
+
} else if (entry.type === "token" && entry.tokenId) {
|
|
254
|
+
const wp = await backend.getWaitpoint(entry.tokenId);
|
|
255
|
+
if (wp && wp.status === "completed") {
|
|
256
|
+
stepData[key] = {
|
|
257
|
+
...entry,
|
|
258
|
+
completed: true,
|
|
259
|
+
result: { ok: true, output: wp.output }
|
|
260
|
+
};
|
|
261
|
+
} else if (wp && wp.status === "timed_out") {
|
|
262
|
+
stepData[key] = {
|
|
263
|
+
...entry,
|
|
264
|
+
completed: true,
|
|
265
|
+
result: { ok: false, error: "Token timed out" }
|
|
266
|
+
};
|
|
373
267
|
}
|
|
374
|
-
query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
|
|
375
|
-
params.push(limit, offset);
|
|
376
|
-
const result = await client.query(query, params);
|
|
377
|
-
log(
|
|
378
|
-
`Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
|
|
379
|
-
);
|
|
380
|
-
return result.rows.map((job) => ({
|
|
381
|
-
...job,
|
|
382
|
-
payload: job.payload,
|
|
383
|
-
timeoutMs: job.timeoutMs,
|
|
384
|
-
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
385
|
-
failureReason: job.failureReason
|
|
386
|
-
}));
|
|
387
|
-
} catch (error) {
|
|
388
|
-
log(
|
|
389
|
-
`Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
|
|
390
|
-
);
|
|
391
|
-
throw error;
|
|
392
|
-
} finally {
|
|
393
|
-
client.release();
|
|
394
268
|
}
|
|
395
269
|
}
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
} else {
|
|
408
|
-
jobTypeFilter = ` AND job_type = $3`;
|
|
409
|
-
params.push(jobType);
|
|
410
|
-
}
|
|
270
|
+
}
|
|
271
|
+
function buildWaitContext(backend, jobId, stepData, baseCtx) {
|
|
272
|
+
let waitCounter = 0;
|
|
273
|
+
const ctx = {
|
|
274
|
+
prolong: baseCtx.prolong,
|
|
275
|
+
onTimeout: baseCtx.onTimeout,
|
|
276
|
+
run: async (stepName, fn) => {
|
|
277
|
+
const cached = stepData[stepName];
|
|
278
|
+
if (cached && typeof cached === "object" && cached.__completed) {
|
|
279
|
+
log(`Step "${stepName}" replayed from cache for job ${jobId}`);
|
|
280
|
+
return cached.result;
|
|
411
281
|
}
|
|
412
|
-
const result = await
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
wait_until = NULL
|
|
424
|
-
WHERE id IN (
|
|
425
|
-
SELECT id FROM job_queue
|
|
426
|
-
WHERE (
|
|
427
|
-
(
|
|
428
|
-
(status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
|
|
429
|
-
AND (attempts < max_attempts)
|
|
430
|
-
AND run_at <= NOW()
|
|
431
|
-
)
|
|
432
|
-
OR (
|
|
433
|
-
status = 'waiting'
|
|
434
|
-
AND wait_until IS NOT NULL
|
|
435
|
-
AND wait_until <= NOW()
|
|
436
|
-
AND wait_token_id IS NULL
|
|
437
|
-
)
|
|
438
|
-
)
|
|
439
|
-
${jobTypeFilter}
|
|
440
|
-
ORDER BY priority DESC, created_at ASC
|
|
441
|
-
LIMIT $2
|
|
442
|
-
FOR UPDATE SKIP LOCKED
|
|
443
|
-
)
|
|
444
|
-
RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
|
|
445
|
-
`,
|
|
446
|
-
params
|
|
447
|
-
);
|
|
448
|
-
log(`Found ${result.rows.length} jobs to process`);
|
|
449
|
-
await client.query("COMMIT");
|
|
450
|
-
if (result.rows.length > 0) {
|
|
451
|
-
await this.recordJobEventsBatch(
|
|
452
|
-
result.rows.map((row) => ({
|
|
453
|
-
jobId: row.id,
|
|
454
|
-
eventType: "processing" /* Processing */
|
|
455
|
-
}))
|
|
456
|
-
);
|
|
282
|
+
const result = await fn();
|
|
283
|
+
stepData[stepName] = { __completed: true, result };
|
|
284
|
+
await backend.updateStepData(jobId, stepData);
|
|
285
|
+
return result;
|
|
286
|
+
},
|
|
287
|
+
waitFor: async (duration) => {
|
|
288
|
+
const waitKey = `__wait_${waitCounter++}`;
|
|
289
|
+
const cached = stepData[waitKey];
|
|
290
|
+
if (cached && typeof cached === "object" && cached.completed) {
|
|
291
|
+
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
292
|
+
return;
|
|
457
293
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
WHERE id = $1 AND status = 'processing'
|
|
481
|
-
`,
|
|
482
|
-
[jobId]
|
|
483
|
-
);
|
|
484
|
-
if (result.rowCount === 0) {
|
|
294
|
+
const waitUntilDate = calculateWaitUntil(duration);
|
|
295
|
+
stepData[waitKey] = { type: "duration", completed: false };
|
|
296
|
+
throw new WaitSignal("duration", waitUntilDate, void 0, stepData);
|
|
297
|
+
},
|
|
298
|
+
waitUntil: async (date) => {
|
|
299
|
+
const waitKey = `__wait_${waitCounter++}`;
|
|
300
|
+
const cached = stepData[waitKey];
|
|
301
|
+
if (cached && typeof cached === "object" && cached.completed) {
|
|
302
|
+
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
303
|
+
return;
|
|
304
|
+
}
|
|
305
|
+
stepData[waitKey] = { type: "date", completed: false };
|
|
306
|
+
throw new WaitSignal("date", date, void 0, stepData);
|
|
307
|
+
},
|
|
308
|
+
createToken: async (options) => {
|
|
309
|
+
const token = await backend.createWaitpoint(jobId, options);
|
|
310
|
+
return token;
|
|
311
|
+
},
|
|
312
|
+
waitForToken: async (tokenId) => {
|
|
313
|
+
const waitKey = `__wait_${waitCounter++}`;
|
|
314
|
+
const cached = stepData[waitKey];
|
|
315
|
+
if (cached && typeof cached === "object" && cached.completed) {
|
|
485
316
|
log(
|
|
486
|
-
`
|
|
317
|
+
`Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
|
|
487
318
|
);
|
|
319
|
+
return cached.result;
|
|
488
320
|
}
|
|
489
|
-
await
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
321
|
+
const wp = await backend.getWaitpoint(tokenId);
|
|
322
|
+
if (wp && wp.status === "completed") {
|
|
323
|
+
const result = {
|
|
324
|
+
ok: true,
|
|
325
|
+
output: wp.output
|
|
326
|
+
};
|
|
327
|
+
stepData[waitKey] = {
|
|
328
|
+
type: "token",
|
|
329
|
+
tokenId,
|
|
330
|
+
completed: true,
|
|
331
|
+
result
|
|
332
|
+
};
|
|
333
|
+
await backend.updateStepData(jobId, stepData);
|
|
334
|
+
return result;
|
|
335
|
+
}
|
|
336
|
+
if (wp && wp.status === "timed_out") {
|
|
337
|
+
const result = {
|
|
338
|
+
ok: false,
|
|
339
|
+
error: "Token timed out"
|
|
340
|
+
};
|
|
341
|
+
stepData[waitKey] = {
|
|
342
|
+
type: "token",
|
|
343
|
+
tokenId,
|
|
344
|
+
completed: true,
|
|
345
|
+
result
|
|
346
|
+
};
|
|
347
|
+
await backend.updateStepData(jobId, stepData);
|
|
348
|
+
return result;
|
|
349
|
+
}
|
|
350
|
+
stepData[waitKey] = { type: "token", tokenId, completed: false };
|
|
351
|
+
throw new WaitSignal("token", void 0, tokenId, stepData);
|
|
352
|
+
},
|
|
353
|
+
setProgress: async (percent) => {
|
|
354
|
+
if (percent < 0 || percent > 100)
|
|
355
|
+
throw new Error("Progress must be between 0 and 100");
|
|
356
|
+
await backend.updateProgress(jobId, Math.round(percent));
|
|
496
357
|
}
|
|
358
|
+
};
|
|
359
|
+
return ctx;
|
|
360
|
+
}
|
|
361
|
+
async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
362
|
+
const handler = jobHandlers[job.jobType];
|
|
363
|
+
if (!handler) {
|
|
364
|
+
await backend.setPendingReasonForUnpickedJobs(
|
|
365
|
+
`No handler registered for job type: ${job.jobType}`,
|
|
366
|
+
job.jobType
|
|
367
|
+
);
|
|
368
|
+
await backend.failJob(
|
|
369
|
+
job.id,
|
|
370
|
+
new Error(`No handler registered for job type: ${job.jobType}`),
|
|
371
|
+
"no_handler" /* NoHandler */
|
|
372
|
+
);
|
|
373
|
+
return;
|
|
497
374
|
}
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
375
|
+
const stepData = { ...job.stepData || {} };
|
|
376
|
+
const hasStepHistory = Object.keys(stepData).some(
|
|
377
|
+
(k) => k.startsWith("__wait_")
|
|
378
|
+
);
|
|
379
|
+
if (hasStepHistory) {
|
|
380
|
+
await resolveCompletedWaits(backend, stepData);
|
|
381
|
+
await backend.updateStepData(job.id, stepData);
|
|
382
|
+
}
|
|
383
|
+
const timeoutMs = job.timeoutMs ?? void 0;
|
|
384
|
+
const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
|
|
385
|
+
let timeoutId;
|
|
386
|
+
const controller = new AbortController();
|
|
387
|
+
try {
|
|
388
|
+
if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
|
|
389
|
+
await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
|
|
390
|
+
} else {
|
|
391
|
+
let onTimeoutCallback;
|
|
392
|
+
let timeoutReject;
|
|
393
|
+
const armTimeout = (ms) => {
|
|
394
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
395
|
+
timeoutId = setTimeout(() => {
|
|
396
|
+
if (onTimeoutCallback) {
|
|
397
|
+
try {
|
|
398
|
+
const extension = onTimeoutCallback();
|
|
399
|
+
if (typeof extension === "number" && extension > 0) {
|
|
400
|
+
backend.prolongJob(job.id).catch(() => {
|
|
401
|
+
});
|
|
402
|
+
armTimeout(extension);
|
|
403
|
+
return;
|
|
404
|
+
}
|
|
405
|
+
} catch (callbackError) {
|
|
406
|
+
log(
|
|
407
|
+
`onTimeout callback threw for job ${job.id}: ${callbackError}`
|
|
408
|
+
);
|
|
521
409
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
410
|
+
}
|
|
411
|
+
controller.abort();
|
|
412
|
+
const timeoutError = new Error(`Job timed out after ${ms} ms`);
|
|
413
|
+
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
414
|
+
if (timeoutReject) {
|
|
415
|
+
timeoutReject(timeoutError);
|
|
416
|
+
}
|
|
417
|
+
}, ms);
|
|
418
|
+
};
|
|
419
|
+
const hasTimeout = timeoutMs != null && timeoutMs > 0;
|
|
420
|
+
const baseCtx = hasTimeout ? {
|
|
421
|
+
prolong: (ms) => {
|
|
422
|
+
const duration = ms ?? timeoutMs;
|
|
423
|
+
if (duration != null && duration > 0) {
|
|
424
|
+
armTimeout(duration);
|
|
425
|
+
backend.prolongJob(job.id).catch(() => {
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
},
|
|
429
|
+
onTimeout: (callback) => {
|
|
430
|
+
onTimeoutCallback = callback;
|
|
431
|
+
}
|
|
432
|
+
} : {
|
|
433
|
+
prolong: () => {
|
|
434
|
+
log("prolong() called but ignored: job has no timeout set");
|
|
435
|
+
},
|
|
436
|
+
onTimeout: () => {
|
|
437
|
+
log("onTimeout() called but ignored: job has no timeout set");
|
|
438
|
+
}
|
|
439
|
+
};
|
|
440
|
+
const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
|
|
441
|
+
if (forceKillOnTimeout && !hasTimeout) {
|
|
527
442
|
log(
|
|
528
|
-
`
|
|
443
|
+
`forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
|
|
529
444
|
);
|
|
530
445
|
}
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
446
|
+
const jobPromise = handler(job.payload, controller.signal, ctx);
|
|
447
|
+
if (hasTimeout) {
|
|
448
|
+
await Promise.race([
|
|
449
|
+
jobPromise,
|
|
450
|
+
new Promise((_, reject) => {
|
|
451
|
+
timeoutReject = reject;
|
|
452
|
+
armTimeout(timeoutMs);
|
|
453
|
+
})
|
|
454
|
+
]);
|
|
455
|
+
} else {
|
|
456
|
+
await jobPromise;
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
460
|
+
await backend.completeJob(job.id);
|
|
461
|
+
} catch (error) {
|
|
462
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
463
|
+
if (error instanceof WaitSignal) {
|
|
464
|
+
log(
|
|
465
|
+
`Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? "none"}, tokenId=${error.tokenId ?? "none"}`
|
|
466
|
+
);
|
|
467
|
+
await backend.waitJob(job.id, {
|
|
468
|
+
waitUntil: error.waitUntil,
|
|
469
|
+
waitTokenId: error.tokenId,
|
|
470
|
+
stepData: error.stepData
|
|
534
471
|
});
|
|
535
|
-
|
|
536
|
-
}
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
472
|
+
return;
|
|
473
|
+
}
|
|
474
|
+
console.error(`Error processing job ${job.id}:`, error);
|
|
475
|
+
let failureReason = "handler_error" /* HandlerError */;
|
|
476
|
+
if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
|
|
477
|
+
failureReason = "timeout" /* Timeout */;
|
|
478
|
+
}
|
|
479
|
+
await backend.failJob(
|
|
480
|
+
job.id,
|
|
481
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
482
|
+
failureReason
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError) {
|
|
487
|
+
const jobs = await backend.getNextBatch(
|
|
488
|
+
workerId,
|
|
489
|
+
batchSize,
|
|
490
|
+
jobType
|
|
491
|
+
);
|
|
492
|
+
if (!concurrency || concurrency >= jobs.length) {
|
|
493
|
+
await Promise.all(
|
|
494
|
+
jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers))
|
|
495
|
+
);
|
|
496
|
+
return jobs.length;
|
|
497
|
+
}
|
|
498
|
+
let idx = 0;
|
|
499
|
+
let running = 0;
|
|
500
|
+
let finished = 0;
|
|
501
|
+
return new Promise((resolve, reject) => {
|
|
502
|
+
const next = () => {
|
|
503
|
+
if (finished === jobs.length) return resolve(jobs.length);
|
|
504
|
+
while (running < concurrency && idx < jobs.length) {
|
|
505
|
+
const job = jobs[idx++];
|
|
506
|
+
running++;
|
|
507
|
+
processJobWithHandlers(backend, job, jobHandlers).then(() => {
|
|
508
|
+
running--;
|
|
509
|
+
finished++;
|
|
510
|
+
next();
|
|
511
|
+
}).catch((err) => {
|
|
512
|
+
running--;
|
|
513
|
+
finished++;
|
|
514
|
+
if (onError) {
|
|
515
|
+
onError(err instanceof Error ? err : new Error(String(err)));
|
|
516
|
+
}
|
|
517
|
+
next();
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
};
|
|
521
|
+
next();
|
|
522
|
+
});
|
|
523
|
+
}
|
|
524
|
+
var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
525
|
+
const {
|
|
526
|
+
workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
|
|
527
|
+
batchSize = 10,
|
|
528
|
+
pollInterval = 5e3,
|
|
529
|
+
onError = (error) => console.error("Job processor error:", error),
|
|
530
|
+
jobType,
|
|
531
|
+
concurrency = 3
|
|
532
|
+
} = options;
|
|
533
|
+
let running = false;
|
|
534
|
+
let intervalId = null;
|
|
535
|
+
let currentBatchPromise = null;
|
|
536
|
+
setLogContext(options.verbose ?? false);
|
|
537
|
+
const processJobs = async () => {
|
|
538
|
+
if (!running) return 0;
|
|
539
|
+
if (onBeforeBatch) {
|
|
540
|
+
try {
|
|
541
|
+
await onBeforeBatch();
|
|
542
|
+
} catch (hookError) {
|
|
543
|
+
log(`onBeforeBatch hook error: ${hookError}`);
|
|
544
|
+
if (onError) {
|
|
545
|
+
onError(
|
|
546
|
+
hookError instanceof Error ? hookError : new Error(String(hookError))
|
|
547
|
+
);
|
|
548
|
+
}
|
|
549
|
+
}
|
|
541
550
|
}
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
551
|
+
log(
|
|
552
|
+
`Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(",") : jobType}` : ""}`
|
|
553
|
+
);
|
|
545
554
|
try {
|
|
546
|
-
await
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
555
|
+
const processed = await processBatchWithHandlers(
|
|
556
|
+
backend,
|
|
557
|
+
workerId,
|
|
558
|
+
batchSize,
|
|
559
|
+
jobType,
|
|
560
|
+
handlers,
|
|
561
|
+
concurrency,
|
|
562
|
+
onError
|
|
553
563
|
);
|
|
554
|
-
|
|
555
|
-
log(`Prolonged job ${jobId}`);
|
|
564
|
+
return processed;
|
|
556
565
|
} catch (error) {
|
|
557
|
-
|
|
558
|
-
} finally {
|
|
559
|
-
client.release();
|
|
566
|
+
onError(error instanceof Error ? error : new Error(String(error)));
|
|
560
567
|
}
|
|
568
|
+
return 0;
|
|
569
|
+
};
|
|
570
|
+
return {
|
|
571
|
+
/**
|
|
572
|
+
* Start the job processor in the background.
|
|
573
|
+
* - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
|
|
574
|
+
* - You have to call the stop method to stop the processor.
|
|
575
|
+
*/
|
|
576
|
+
startInBackground: () => {
|
|
577
|
+
if (running) return;
|
|
578
|
+
log(`Starting job processor with workerId: ${workerId}`);
|
|
579
|
+
running = true;
|
|
580
|
+
const scheduleNext = (immediate) => {
|
|
581
|
+
if (!running) return;
|
|
582
|
+
if (immediate) {
|
|
583
|
+
intervalId = setTimeout(loop, 0);
|
|
584
|
+
} else {
|
|
585
|
+
intervalId = setTimeout(loop, pollInterval);
|
|
586
|
+
}
|
|
587
|
+
};
|
|
588
|
+
const loop = async () => {
|
|
589
|
+
if (!running) return;
|
|
590
|
+
currentBatchPromise = processJobs();
|
|
591
|
+
const processed = await currentBatchPromise;
|
|
592
|
+
currentBatchPromise = null;
|
|
593
|
+
scheduleNext(processed === batchSize);
|
|
594
|
+
};
|
|
595
|
+
loop();
|
|
596
|
+
},
|
|
597
|
+
/**
|
|
598
|
+
* Stop the job processor that runs in the background.
|
|
599
|
+
* Does not wait for in-flight jobs.
|
|
600
|
+
*/
|
|
601
|
+
stop: () => {
|
|
602
|
+
log(`Stopping job processor with workerId: ${workerId}`);
|
|
603
|
+
running = false;
|
|
604
|
+
if (intervalId) {
|
|
605
|
+
clearTimeout(intervalId);
|
|
606
|
+
intervalId = null;
|
|
607
|
+
}
|
|
608
|
+
},
|
|
609
|
+
/**
|
|
610
|
+
* Stop the job processor and wait for all in-flight jobs to complete.
|
|
611
|
+
* Useful for graceful shutdown (e.g., SIGTERM handling).
|
|
612
|
+
*/
|
|
613
|
+
stopAndDrain: async (drainTimeoutMs = 3e4) => {
|
|
614
|
+
log(`Stopping and draining job processor with workerId: ${workerId}`);
|
|
615
|
+
running = false;
|
|
616
|
+
if (intervalId) {
|
|
617
|
+
clearTimeout(intervalId);
|
|
618
|
+
intervalId = null;
|
|
619
|
+
}
|
|
620
|
+
if (currentBatchPromise) {
|
|
621
|
+
await Promise.race([
|
|
622
|
+
currentBatchPromise.catch(() => {
|
|
623
|
+
}),
|
|
624
|
+
new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
|
|
625
|
+
]);
|
|
626
|
+
currentBatchPromise = null;
|
|
627
|
+
}
|
|
628
|
+
log(`Job processor ${workerId} drained`);
|
|
629
|
+
},
|
|
630
|
+
/**
|
|
631
|
+
* Start the job processor synchronously.
|
|
632
|
+
* - This will process all jobs immediately and then stop.
|
|
633
|
+
* - The pollInterval is ignored.
|
|
634
|
+
*/
|
|
635
|
+
start: async () => {
|
|
636
|
+
log(`Starting job processor with workerId: ${workerId}`);
|
|
637
|
+
running = true;
|
|
638
|
+
const processed = await processJobs();
|
|
639
|
+
running = false;
|
|
640
|
+
return processed;
|
|
641
|
+
},
|
|
642
|
+
isRunning: () => running
|
|
643
|
+
};
|
|
644
|
+
};
|
|
645
|
+
function loadPemOrFile(value) {
|
|
646
|
+
if (!value) return void 0;
|
|
647
|
+
if (value.startsWith("file://")) {
|
|
648
|
+
const filePath = value.slice(7);
|
|
649
|
+
return fs.readFileSync(filePath, "utf8");
|
|
561
650
|
}
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
651
|
+
return value;
|
|
652
|
+
}
|
|
653
|
+
var createPool = (config) => {
|
|
654
|
+
let searchPath;
|
|
655
|
+
let ssl = void 0;
|
|
656
|
+
let customCA;
|
|
657
|
+
let sslmode;
|
|
658
|
+
if (config.connectionString) {
|
|
565
659
|
try {
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
)
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
660
|
+
const url = new URL(config.connectionString);
|
|
661
|
+
searchPath = url.searchParams.get("search_path") || void 0;
|
|
662
|
+
sslmode = url.searchParams.get("sslmode") || void 0;
|
|
663
|
+
if (sslmode === "no-verify") {
|
|
664
|
+
ssl = { rejectUnauthorized: false };
|
|
665
|
+
}
|
|
666
|
+
} catch (e) {
|
|
667
|
+
const parsed = parse(config.connectionString);
|
|
668
|
+
if (parsed.options) {
|
|
669
|
+
const match = parsed.options.match(/search_path=([^\s]+)/);
|
|
670
|
+
if (match) {
|
|
671
|
+
searchPath = match[1];
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
sslmode = typeof parsed.sslmode === "string" ? parsed.sslmode : void 0;
|
|
675
|
+
if (sslmode === "no-verify") {
|
|
676
|
+
ssl = { rejectUnauthorized: false };
|
|
677
|
+
}
|
|
575
678
|
}
|
|
576
679
|
}
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
680
|
+
if (config.ssl) {
|
|
681
|
+
if (typeof config.ssl.ca === "string") {
|
|
682
|
+
customCA = config.ssl.ca;
|
|
683
|
+
} else if (typeof process.env.PGSSLROOTCERT === "string") {
|
|
684
|
+
customCA = process.env.PGSSLROOTCERT;
|
|
685
|
+
} else {
|
|
686
|
+
customCA = void 0;
|
|
687
|
+
}
|
|
688
|
+
const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
|
|
689
|
+
ssl = {
|
|
690
|
+
...ssl,
|
|
691
|
+
...caValue ? { ca: caValue } : {},
|
|
692
|
+
cert: loadPemOrFile(
|
|
693
|
+
typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
|
|
694
|
+
),
|
|
695
|
+
key: loadPemOrFile(
|
|
696
|
+
typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
|
|
697
|
+
),
|
|
698
|
+
rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
if (sslmode && customCA) {
|
|
702
|
+
const warning = `
|
|
703
|
+
|
|
704
|
+
\x1B[33m**************************************************
|
|
705
|
+
\u26A0\uFE0F WARNING: SSL CONFIGURATION ISSUE
|
|
706
|
+
**************************************************
|
|
707
|
+
Both sslmode ('${sslmode}') is set in the connection string
|
|
708
|
+
and a custom CA is provided (via config.ssl.ca or PGSSLROOTCERT).
|
|
709
|
+
This combination may cause connection failures or unexpected behavior.
|
|
710
|
+
|
|
711
|
+
Recommended: Remove sslmode from the connection string when using a custom CA.
|
|
712
|
+
**************************************************\x1B[0m
|
|
713
|
+
`;
|
|
714
|
+
console.warn(warning);
|
|
715
|
+
}
|
|
716
|
+
const pool = new Pool({
|
|
717
|
+
...config,
|
|
718
|
+
...ssl ? { ssl } : {}
|
|
719
|
+
});
|
|
720
|
+
if (searchPath) {
|
|
721
|
+
pool.on("connect", (client) => {
|
|
722
|
+
client.query(`SET search_path TO ${searchPath}`);
|
|
723
|
+
});
|
|
724
|
+
}
|
|
725
|
+
return pool;
|
|
726
|
+
};
|
|
727
|
+
var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
|
|
728
|
+
function parseTimeoutString(timeout) {
|
|
729
|
+
const match = timeout.match(/^(\d+)(s|m|h|d)$/);
|
|
730
|
+
if (!match) {
|
|
731
|
+
throw new Error(
|
|
732
|
+
`Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
|
|
733
|
+
);
|
|
734
|
+
}
|
|
735
|
+
const value = parseInt(match[1], 10);
|
|
736
|
+
const unit = match[2];
|
|
737
|
+
let ms;
|
|
738
|
+
switch (unit) {
|
|
739
|
+
case "s":
|
|
740
|
+
ms = value * 1e3;
|
|
741
|
+
break;
|
|
742
|
+
case "m":
|
|
743
|
+
ms = value * 60 * 1e3;
|
|
744
|
+
break;
|
|
745
|
+
case "h":
|
|
746
|
+
ms = value * 60 * 60 * 1e3;
|
|
747
|
+
break;
|
|
748
|
+
case "d":
|
|
749
|
+
ms = value * 24 * 60 * 60 * 1e3;
|
|
750
|
+
break;
|
|
751
|
+
default:
|
|
752
|
+
throw new Error(`Unknown timeout unit: "${unit}"`);
|
|
753
|
+
}
|
|
754
|
+
if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
|
|
755
|
+
throw new Error(
|
|
756
|
+
`Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
|
|
757
|
+
);
|
|
758
|
+
}
|
|
759
|
+
return ms;
|
|
760
|
+
}
|
|
761
|
+
var PostgresBackend = class {
|
|
762
|
+
constructor(pool) {
|
|
763
|
+
this.pool = pool;
|
|
764
|
+
}
|
|
765
|
+
/** Expose the raw pool for advanced usage. */
|
|
766
|
+
getPool() {
|
|
767
|
+
return this.pool;
|
|
768
|
+
}
|
|
769
|
+
// ── Events ──────────────────────────────────────────────────────────
|
|
770
|
+
async recordJobEvent(jobId, eventType, metadata) {
|
|
771
|
+
const client = await this.pool.connect();
|
|
772
|
+
try {
|
|
773
|
+
await client.query(
|
|
774
|
+
`INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
|
|
775
|
+
[jobId, eventType, metadata ? JSON.stringify(metadata) : null]
|
|
593
776
|
);
|
|
594
|
-
if (result.rowCount === 0) {
|
|
595
|
-
log(
|
|
596
|
-
`Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
|
|
597
|
-
);
|
|
598
|
-
}
|
|
599
|
-
await this.recordJobEvent(jobId, "retried" /* Retried */);
|
|
600
|
-
log(`Retried job ${jobId}`);
|
|
601
777
|
} catch (error) {
|
|
602
|
-
log(`Error
|
|
603
|
-
throw error;
|
|
778
|
+
log(`Error recording job event for job ${jobId}: ${error}`);
|
|
604
779
|
} finally {
|
|
605
780
|
client.release();
|
|
606
781
|
}
|
|
607
782
|
}
|
|
608
|
-
async
|
|
783
|
+
async getJobEvents(jobId) {
|
|
609
784
|
const client = await this.pool.connect();
|
|
610
785
|
try {
|
|
611
|
-
await client.query(
|
|
612
|
-
`
|
|
613
|
-
UPDATE job_queue
|
|
614
|
-
SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
|
|
615
|
-
wait_until = NULL, wait_token_id = NULL
|
|
616
|
-
WHERE id = $1 AND status IN ('pending', 'waiting')
|
|
617
|
-
`,
|
|
786
|
+
const res = await client.query(
|
|
787
|
+
`SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
|
|
618
788
|
[jobId]
|
|
619
789
|
);
|
|
620
|
-
|
|
621
|
-
log(`Cancelled job ${jobId}`);
|
|
622
|
-
} catch (error) {
|
|
623
|
-
log(`Error cancelling job ${jobId}: ${error}`);
|
|
624
|
-
throw error;
|
|
790
|
+
return res.rows;
|
|
625
791
|
} finally {
|
|
626
792
|
client.release();
|
|
627
793
|
}
|
|
628
794
|
}
|
|
629
|
-
|
|
795
|
+
// ── Job CRUD ──────────────────────────────────────────────────────────
|
|
796
|
+
async addJob({
|
|
797
|
+
jobType,
|
|
798
|
+
payload,
|
|
799
|
+
maxAttempts = 3,
|
|
800
|
+
priority = 0,
|
|
801
|
+
runAt = null,
|
|
802
|
+
timeoutMs = void 0,
|
|
803
|
+
forceKillOnTimeout = false,
|
|
804
|
+
tags = void 0,
|
|
805
|
+
idempotencyKey = void 0
|
|
806
|
+
}) {
|
|
630
807
|
const client = await this.pool.connect();
|
|
631
808
|
try {
|
|
632
|
-
let
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
break;
|
|
683
|
-
case "all":
|
|
684
|
-
query += ` AND tags @> $${paramIdx++}`;
|
|
685
|
-
params.push(tagValues);
|
|
686
|
-
break;
|
|
687
|
-
case "any":
|
|
688
|
-
query += ` AND tags && $${paramIdx++}`;
|
|
689
|
-
params.push(tagValues);
|
|
690
|
-
break;
|
|
691
|
-
case "none":
|
|
692
|
-
query += ` AND NOT (tags && $${paramIdx++})`;
|
|
693
|
-
params.push(tagValues);
|
|
694
|
-
break;
|
|
695
|
-
default:
|
|
696
|
-
query += ` AND tags @> $${paramIdx++}`;
|
|
697
|
-
params.push(tagValues);
|
|
698
|
-
}
|
|
809
|
+
let result;
|
|
810
|
+
const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
|
|
811
|
+
if (runAt) {
|
|
812
|
+
result = await client.query(
|
|
813
|
+
`INSERT INTO job_queue
|
|
814
|
+
(job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
815
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
816
|
+
${onConflict}
|
|
817
|
+
RETURNING id`,
|
|
818
|
+
[
|
|
819
|
+
jobType,
|
|
820
|
+
payload,
|
|
821
|
+
maxAttempts,
|
|
822
|
+
priority,
|
|
823
|
+
runAt,
|
|
824
|
+
timeoutMs ?? null,
|
|
825
|
+
forceKillOnTimeout ?? false,
|
|
826
|
+
tags ?? null,
|
|
827
|
+
idempotencyKey ?? null
|
|
828
|
+
]
|
|
829
|
+
);
|
|
830
|
+
} else {
|
|
831
|
+
result = await client.query(
|
|
832
|
+
`INSERT INTO job_queue
|
|
833
|
+
(job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
834
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
835
|
+
${onConflict}
|
|
836
|
+
RETURNING id`,
|
|
837
|
+
[
|
|
838
|
+
jobType,
|
|
839
|
+
payload,
|
|
840
|
+
maxAttempts,
|
|
841
|
+
priority,
|
|
842
|
+
timeoutMs ?? null,
|
|
843
|
+
forceKillOnTimeout ?? false,
|
|
844
|
+
tags ?? null,
|
|
845
|
+
idempotencyKey ?? null
|
|
846
|
+
]
|
|
847
|
+
);
|
|
848
|
+
}
|
|
849
|
+
if (result.rows.length === 0 && idempotencyKey) {
|
|
850
|
+
const existing = await client.query(
|
|
851
|
+
`SELECT id FROM job_queue WHERE idempotency_key = $1`,
|
|
852
|
+
[idempotencyKey]
|
|
853
|
+
);
|
|
854
|
+
if (existing.rows.length > 0) {
|
|
855
|
+
log(
|
|
856
|
+
`Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
|
|
857
|
+
);
|
|
858
|
+
return existing.rows[0].id;
|
|
699
859
|
}
|
|
860
|
+
throw new Error(
|
|
861
|
+
`Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
|
|
862
|
+
);
|
|
700
863
|
}
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
864
|
+
const jobId = result.rows[0].id;
|
|
865
|
+
log(
|
|
866
|
+
`Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
|
|
867
|
+
);
|
|
868
|
+
await this.recordJobEvent(jobId, "added" /* Added */, {
|
|
869
|
+
jobType,
|
|
870
|
+
payload,
|
|
871
|
+
tags,
|
|
872
|
+
idempotencyKey
|
|
873
|
+
});
|
|
874
|
+
return jobId;
|
|
705
875
|
} catch (error) {
|
|
706
|
-
log(`Error
|
|
876
|
+
log(`Error adding job: ${error}`);
|
|
707
877
|
throw error;
|
|
708
878
|
} finally {
|
|
709
879
|
client.release();
|
|
710
880
|
}
|
|
711
881
|
}
|
|
712
|
-
async
|
|
882
|
+
async getJob(id) {
|
|
713
883
|
const client = await this.pool.connect();
|
|
714
884
|
try {
|
|
715
|
-
const
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
if (updates.maxAttempts !== void 0) {
|
|
723
|
-
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
724
|
-
params.push(updates.maxAttempts);
|
|
725
|
-
}
|
|
726
|
-
if (updates.priority !== void 0) {
|
|
727
|
-
updateFields.push(`priority = $${paramIdx++}`);
|
|
728
|
-
params.push(updates.priority);
|
|
729
|
-
}
|
|
730
|
-
if (updates.runAt !== void 0) {
|
|
731
|
-
if (updates.runAt === null) {
|
|
732
|
-
updateFields.push(`run_at = NOW()`);
|
|
733
|
-
} else {
|
|
734
|
-
updateFields.push(`run_at = $${paramIdx++}`);
|
|
735
|
-
params.push(updates.runAt);
|
|
736
|
-
}
|
|
737
|
-
}
|
|
738
|
-
if (updates.timeoutMs !== void 0) {
|
|
739
|
-
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
740
|
-
params.push(updates.timeoutMs ?? null);
|
|
741
|
-
}
|
|
742
|
-
if (updates.tags !== void 0) {
|
|
743
|
-
updateFields.push(`tags = $${paramIdx++}`);
|
|
744
|
-
params.push(updates.tags ?? null);
|
|
745
|
-
}
|
|
746
|
-
if (updateFields.length === 0) {
|
|
747
|
-
log(`No fields to update for job ${jobId}`);
|
|
748
|
-
return;
|
|
885
|
+
const result = await client.query(
|
|
886
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
|
|
887
|
+
[id]
|
|
888
|
+
);
|
|
889
|
+
if (result.rows.length === 0) {
|
|
890
|
+
log(`Job ${id} not found`);
|
|
891
|
+
return null;
|
|
749
892
|
}
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
893
|
+
log(`Found job ${id}`);
|
|
894
|
+
const job = result.rows[0];
|
|
895
|
+
return {
|
|
896
|
+
...job,
|
|
897
|
+
payload: job.payload,
|
|
898
|
+
timeoutMs: job.timeoutMs,
|
|
899
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
900
|
+
failureReason: job.failureReason
|
|
901
|
+
};
|
|
902
|
+
} catch (error) {
|
|
903
|
+
log(`Error getting job ${id}: ${error}`);
|
|
904
|
+
throw error;
|
|
905
|
+
} finally {
|
|
906
|
+
client.release();
|
|
907
|
+
}
|
|
908
|
+
}
|
|
909
|
+
async getJobsByStatus(status, limit = 100, offset = 0) {
|
|
910
|
+
const client = await this.pool.connect();
|
|
911
|
+
try {
|
|
912
|
+
const result = await client.query(
|
|
913
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
|
|
914
|
+
[status, limit, offset]
|
|
915
|
+
);
|
|
916
|
+
log(`Found ${result.rows.length} jobs by status ${status}`);
|
|
917
|
+
return result.rows.map((job) => ({
|
|
918
|
+
...job,
|
|
919
|
+
payload: job.payload,
|
|
920
|
+
timeoutMs: job.timeoutMs,
|
|
921
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
922
|
+
failureReason: job.failureReason
|
|
923
|
+
}));
|
|
769
924
|
} catch (error) {
|
|
770
|
-
log(`Error
|
|
925
|
+
log(`Error getting jobs by status ${status}: ${error}`);
|
|
771
926
|
throw error;
|
|
772
927
|
} finally {
|
|
773
928
|
client.release();
|
|
774
929
|
}
|
|
775
930
|
}
|
|
776
|
-
async
|
|
931
|
+
async getAllJobs(limit = 100, offset = 0) {
|
|
777
932
|
const client = await this.pool.connect();
|
|
778
933
|
try {
|
|
779
|
-
const
|
|
934
|
+
const result = await client.query(
|
|
935
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
|
|
936
|
+
[limit, offset]
|
|
937
|
+
);
|
|
938
|
+
log(`Found ${result.rows.length} jobs (all)`);
|
|
939
|
+
return result.rows.map((job) => ({
|
|
940
|
+
...job,
|
|
941
|
+
payload: job.payload,
|
|
942
|
+
timeoutMs: job.timeoutMs,
|
|
943
|
+
forceKillOnTimeout: job.forceKillOnTimeout
|
|
944
|
+
}));
|
|
945
|
+
} catch (error) {
|
|
946
|
+
log(`Error getting all jobs: ${error}`);
|
|
947
|
+
throw error;
|
|
948
|
+
} finally {
|
|
949
|
+
client.release();
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
async getJobs(filters, limit = 100, offset = 0) {
|
|
953
|
+
const client = await this.pool.connect();
|
|
954
|
+
try {
|
|
955
|
+
let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue`;
|
|
780
956
|
const params = [];
|
|
957
|
+
const where = [];
|
|
781
958
|
let paramIdx = 1;
|
|
782
|
-
if (updates.payload !== void 0) {
|
|
783
|
-
updateFields.push(`payload = $${paramIdx++}`);
|
|
784
|
-
params.push(updates.payload);
|
|
785
|
-
}
|
|
786
|
-
if (updates.maxAttempts !== void 0) {
|
|
787
|
-
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
788
|
-
params.push(updates.maxAttempts);
|
|
789
|
-
}
|
|
790
|
-
if (updates.priority !== void 0) {
|
|
791
|
-
updateFields.push(`priority = $${paramIdx++}`);
|
|
792
|
-
params.push(updates.priority);
|
|
793
|
-
}
|
|
794
|
-
if (updates.runAt !== void 0) {
|
|
795
|
-
if (updates.runAt === null) {
|
|
796
|
-
updateFields.push(`run_at = NOW()`);
|
|
797
|
-
} else {
|
|
798
|
-
updateFields.push(`run_at = $${paramIdx++}`);
|
|
799
|
-
params.push(updates.runAt);
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
if (updates.timeoutMs !== void 0) {
|
|
803
|
-
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
804
|
-
params.push(updates.timeoutMs ?? null);
|
|
805
|
-
}
|
|
806
|
-
if (updates.tags !== void 0) {
|
|
807
|
-
updateFields.push(`tags = $${paramIdx++}`);
|
|
808
|
-
params.push(updates.tags ?? null);
|
|
809
|
-
}
|
|
810
|
-
if (updateFields.length === 0) {
|
|
811
|
-
log(`No fields to update for batch edit`);
|
|
812
|
-
return 0;
|
|
813
|
-
}
|
|
814
|
-
updateFields.push(`updated_at = NOW()`);
|
|
815
|
-
let query = `
|
|
816
|
-
UPDATE job_queue
|
|
817
|
-
SET ${updateFields.join(", ")}
|
|
818
|
-
WHERE status = 'pending'`;
|
|
819
959
|
if (filters) {
|
|
820
960
|
if (filters.jobType) {
|
|
821
|
-
|
|
961
|
+
where.push(`job_type = $${paramIdx++}`);
|
|
822
962
|
params.push(filters.jobType);
|
|
823
963
|
}
|
|
824
964
|
if (filters.priority !== void 0) {
|
|
825
|
-
|
|
965
|
+
where.push(`priority = $${paramIdx++}`);
|
|
826
966
|
params.push(filters.priority);
|
|
827
967
|
}
|
|
828
968
|
if (filters.runAt) {
|
|
829
969
|
if (filters.runAt instanceof Date) {
|
|
830
|
-
|
|
970
|
+
where.push(`run_at = $${paramIdx++}`);
|
|
831
971
|
params.push(filters.runAt);
|
|
832
|
-
} else if (typeof filters.runAt === "object") {
|
|
972
|
+
} else if (typeof filters.runAt === "object" && (filters.runAt.gt !== void 0 || filters.runAt.gte !== void 0 || filters.runAt.lt !== void 0 || filters.runAt.lte !== void 0 || filters.runAt.eq !== void 0)) {
|
|
833
973
|
const ops = filters.runAt;
|
|
834
974
|
if (ops.gt) {
|
|
835
|
-
|
|
975
|
+
where.push(`run_at > $${paramIdx++}`);
|
|
836
976
|
params.push(ops.gt);
|
|
837
977
|
}
|
|
838
978
|
if (ops.gte) {
|
|
839
|
-
|
|
979
|
+
where.push(`run_at >= $${paramIdx++}`);
|
|
840
980
|
params.push(ops.gte);
|
|
841
981
|
}
|
|
842
982
|
if (ops.lt) {
|
|
843
|
-
|
|
983
|
+
where.push(`run_at < $${paramIdx++}`);
|
|
844
984
|
params.push(ops.lt);
|
|
845
985
|
}
|
|
846
986
|
if (ops.lte) {
|
|
847
|
-
|
|
987
|
+
where.push(`run_at <= $${paramIdx++}`);
|
|
848
988
|
params.push(ops.lte);
|
|
849
989
|
}
|
|
850
990
|
if (ops.eq) {
|
|
851
|
-
|
|
991
|
+
where.push(`run_at = $${paramIdx++}`);
|
|
852
992
|
params.push(ops.eq);
|
|
853
993
|
}
|
|
854
994
|
}
|
|
@@ -858,1370 +998,1264 @@ var PostgresBackend = class {
|
|
|
858
998
|
const tagValues = filters.tags.values;
|
|
859
999
|
switch (mode) {
|
|
860
1000
|
case "exact":
|
|
861
|
-
|
|
1001
|
+
where.push(`tags = $${paramIdx++}`);
|
|
862
1002
|
params.push(tagValues);
|
|
863
1003
|
break;
|
|
864
1004
|
case "all":
|
|
865
|
-
|
|
1005
|
+
where.push(`tags @> $${paramIdx++}`);
|
|
866
1006
|
params.push(tagValues);
|
|
867
1007
|
break;
|
|
868
1008
|
case "any":
|
|
869
|
-
|
|
1009
|
+
where.push(`tags && $${paramIdx++}`);
|
|
870
1010
|
params.push(tagValues);
|
|
871
1011
|
break;
|
|
872
1012
|
case "none":
|
|
873
|
-
|
|
1013
|
+
where.push(`NOT (tags && $${paramIdx++})`);
|
|
874
1014
|
params.push(tagValues);
|
|
875
1015
|
break;
|
|
876
1016
|
default:
|
|
877
|
-
|
|
1017
|
+
where.push(`tags @> $${paramIdx++}`);
|
|
878
1018
|
params.push(tagValues);
|
|
879
1019
|
}
|
|
880
1020
|
}
|
|
1021
|
+
if (filters.cursor !== void 0) {
|
|
1022
|
+
where.push(`id < $${paramIdx++}`);
|
|
1023
|
+
params.push(filters.cursor);
|
|
1024
|
+
}
|
|
1025
|
+
}
|
|
1026
|
+
if (where.length > 0) {
|
|
1027
|
+
query += ` WHERE ${where.join(" AND ")}`;
|
|
1028
|
+
}
|
|
1029
|
+
paramIdx = params.length + 1;
|
|
1030
|
+
query += ` ORDER BY id DESC LIMIT $${paramIdx++}`;
|
|
1031
|
+
if (!filters?.cursor) {
|
|
1032
|
+
query += ` OFFSET $${paramIdx}`;
|
|
1033
|
+
params.push(limit, offset);
|
|
1034
|
+
} else {
|
|
1035
|
+
params.push(limit);
|
|
881
1036
|
}
|
|
882
|
-
query += "\nRETURNING id";
|
|
883
1037
|
const result = await client.query(query, params);
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
1038
|
+
log(`Found ${result.rows.length} jobs`);
|
|
1039
|
+
return result.rows.map((job) => ({
|
|
1040
|
+
...job,
|
|
1041
|
+
payload: job.payload,
|
|
1042
|
+
timeoutMs: job.timeoutMs,
|
|
1043
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
1044
|
+
failureReason: job.failureReason
|
|
1045
|
+
}));
|
|
1046
|
+
} catch (error) {
|
|
1047
|
+
log(`Error getting jobs: ${error}`);
|
|
1048
|
+
throw error;
|
|
1049
|
+
} finally {
|
|
1050
|
+
client.release();
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
|
|
1054
|
+
const client = await this.pool.connect();
|
|
1055
|
+
try {
|
|
1056
|
+
let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
|
|
1057
|
+
FROM job_queue`;
|
|
1058
|
+
let params = [];
|
|
1059
|
+
switch (mode) {
|
|
1060
|
+
case "exact":
|
|
1061
|
+
query += " WHERE tags = $1";
|
|
1062
|
+
params = [tags];
|
|
1063
|
+
break;
|
|
1064
|
+
case "all":
|
|
1065
|
+
query += " WHERE tags @> $1";
|
|
1066
|
+
params = [tags];
|
|
1067
|
+
break;
|
|
1068
|
+
case "any":
|
|
1069
|
+
query += " WHERE tags && $1";
|
|
1070
|
+
params = [tags];
|
|
1071
|
+
break;
|
|
1072
|
+
case "none":
|
|
1073
|
+
query += " WHERE NOT (tags && $1)";
|
|
1074
|
+
params = [tags];
|
|
1075
|
+
break;
|
|
1076
|
+
default:
|
|
1077
|
+
query += " WHERE tags @> $1";
|
|
1078
|
+
params = [tags];
|
|
1079
|
+
}
|
|
1080
|
+
query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
|
|
1081
|
+
params.push(limit, offset);
|
|
1082
|
+
const result = await client.query(query, params);
|
|
1083
|
+
log(
|
|
1084
|
+
`Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
|
|
1085
|
+
);
|
|
1086
|
+
return result.rows.map((job) => ({
|
|
1087
|
+
...job,
|
|
1088
|
+
payload: job.payload,
|
|
1089
|
+
timeoutMs: job.timeoutMs,
|
|
1090
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
1091
|
+
failureReason: job.failureReason
|
|
1092
|
+
}));
|
|
1093
|
+
} catch (error) {
|
|
1094
|
+
log(
|
|
1095
|
+
`Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
|
|
1096
|
+
);
|
|
1097
|
+
throw error;
|
|
1098
|
+
} finally {
|
|
1099
|
+
client.release();
|
|
1100
|
+
}
|
|
1101
|
+
}
|
|
1102
|
+
// ── Processing lifecycle ──────────────────────────────────────────────
|
|
1103
|
+
async getNextBatch(workerId, batchSize = 10, jobType) {
|
|
1104
|
+
const client = await this.pool.connect();
|
|
1105
|
+
try {
|
|
1106
|
+
await client.query("BEGIN");
|
|
1107
|
+
let jobTypeFilter = "";
|
|
1108
|
+
const params = [workerId, batchSize];
|
|
1109
|
+
if (jobType) {
|
|
1110
|
+
if (Array.isArray(jobType)) {
|
|
1111
|
+
jobTypeFilter = ` AND job_type = ANY($3)`;
|
|
1112
|
+
params.push(jobType);
|
|
1113
|
+
} else {
|
|
1114
|
+
jobTypeFilter = ` AND job_type = $3`;
|
|
1115
|
+
params.push(jobType);
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
const result = await client.query(
|
|
1119
|
+
`
|
|
1120
|
+
UPDATE job_queue
|
|
1121
|
+
SET status = 'processing',
|
|
1122
|
+
locked_at = NOW(),
|
|
1123
|
+
locked_by = $1,
|
|
1124
|
+
attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
|
|
1125
|
+
updated_at = NOW(),
|
|
1126
|
+
pending_reason = NULL,
|
|
1127
|
+
started_at = COALESCE(started_at, NOW()),
|
|
1128
|
+
last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
|
|
1129
|
+
wait_until = NULL
|
|
1130
|
+
WHERE id IN (
|
|
1131
|
+
SELECT id FROM job_queue
|
|
1132
|
+
WHERE (
|
|
1133
|
+
(
|
|
1134
|
+
(status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
|
|
1135
|
+
AND (attempts < max_attempts)
|
|
1136
|
+
AND run_at <= NOW()
|
|
1137
|
+
)
|
|
1138
|
+
OR (
|
|
1139
|
+
status = 'waiting'
|
|
1140
|
+
AND wait_until IS NOT NULL
|
|
1141
|
+
AND wait_until <= NOW()
|
|
1142
|
+
AND wait_token_id IS NULL
|
|
1143
|
+
)
|
|
1144
|
+
)
|
|
1145
|
+
${jobTypeFilter}
|
|
1146
|
+
ORDER BY priority DESC, created_at ASC
|
|
1147
|
+
LIMIT $2
|
|
1148
|
+
FOR UPDATE SKIP LOCKED
|
|
1149
|
+
)
|
|
1150
|
+
RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
|
|
1151
|
+
`,
|
|
1152
|
+
params
|
|
1153
|
+
);
|
|
1154
|
+
log(`Found ${result.rows.length} jobs to process`);
|
|
1155
|
+
await client.query("COMMIT");
|
|
1156
|
+
if (result.rows.length > 0) {
|
|
1157
|
+
await this.recordJobEventsBatch(
|
|
1158
|
+
result.rows.map((row) => ({
|
|
1159
|
+
jobId: row.id,
|
|
1160
|
+
eventType: "processing" /* Processing */
|
|
1161
|
+
}))
|
|
1162
|
+
);
|
|
896
1163
|
}
|
|
897
|
-
|
|
898
|
-
|
|
1164
|
+
return result.rows.map((job) => ({
|
|
1165
|
+
...job,
|
|
1166
|
+
payload: job.payload,
|
|
1167
|
+
timeoutMs: job.timeoutMs,
|
|
1168
|
+
forceKillOnTimeout: job.forceKillOnTimeout
|
|
1169
|
+
}));
|
|
899
1170
|
} catch (error) {
|
|
900
|
-
log(`Error
|
|
1171
|
+
log(`Error getting next batch: ${error}`);
|
|
1172
|
+
await client.query("ROLLBACK");
|
|
901
1173
|
throw error;
|
|
902
1174
|
} finally {
|
|
903
1175
|
client.release();
|
|
904
1176
|
}
|
|
905
1177
|
}
|
|
906
|
-
async
|
|
1178
|
+
async completeJob(jobId) {
|
|
907
1179
|
const client = await this.pool.connect();
|
|
908
1180
|
try {
|
|
909
1181
|
const result = await client.query(
|
|
910
1182
|
`
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
1183
|
+
UPDATE job_queue
|
|
1184
|
+
SET status = 'completed', updated_at = NOW(), completed_at = NOW(),
|
|
1185
|
+
step_data = NULL, wait_until = NULL, wait_token_id = NULL
|
|
1186
|
+
WHERE id = $1 AND status = 'processing'
|
|
915
1187
|
`,
|
|
916
|
-
[
|
|
1188
|
+
[jobId]
|
|
917
1189
|
);
|
|
918
|
-
|
|
919
|
-
|
|
1190
|
+
if (result.rowCount === 0) {
|
|
1191
|
+
log(
|
|
1192
|
+
`Job ${jobId} could not be completed (not in processing state or does not exist)`
|
|
1193
|
+
);
|
|
1194
|
+
}
|
|
1195
|
+
await this.recordJobEvent(jobId, "completed" /* Completed */);
|
|
1196
|
+
log(`Completed job ${jobId}`);
|
|
920
1197
|
} catch (error) {
|
|
921
|
-
log(`Error
|
|
1198
|
+
log(`Error completing job ${jobId}: ${error}`);
|
|
922
1199
|
throw error;
|
|
923
1200
|
} finally {
|
|
924
1201
|
client.release();
|
|
925
1202
|
}
|
|
926
1203
|
}
|
|
927
|
-
async
|
|
1204
|
+
async failJob(jobId, error, failureReason) {
|
|
928
1205
|
const client = await this.pool.connect();
|
|
929
1206
|
try {
|
|
930
1207
|
const result = await client.query(
|
|
931
1208
|
`
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
1209
|
+
UPDATE job_queue
|
|
1210
|
+
SET status = 'failed',
|
|
1211
|
+
updated_at = NOW(),
|
|
1212
|
+
next_attempt_at = CASE
|
|
1213
|
+
WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
|
|
1214
|
+
ELSE NULL
|
|
1215
|
+
END,
|
|
1216
|
+
error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
|
|
1217
|
+
failure_reason = $3,
|
|
1218
|
+
last_failed_at = NOW()
|
|
1219
|
+
WHERE id = $1 AND status IN ('processing', 'pending')
|
|
935
1220
|
`,
|
|
936
|
-
[
|
|
1221
|
+
[
|
|
1222
|
+
jobId,
|
|
1223
|
+
JSON.stringify([
|
|
1224
|
+
{
|
|
1225
|
+
message: error.message || String(error),
|
|
1226
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1227
|
+
}
|
|
1228
|
+
]),
|
|
1229
|
+
failureReason ?? null
|
|
1230
|
+
]
|
|
937
1231
|
);
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
1232
|
+
if (result.rowCount === 0) {
|
|
1233
|
+
log(
|
|
1234
|
+
`Job ${jobId} could not be failed (not in processing/pending state or does not exist)`
|
|
1235
|
+
);
|
|
1236
|
+
}
|
|
1237
|
+
await this.recordJobEvent(jobId, "failed" /* Failed */, {
|
|
1238
|
+
message: error.message || String(error),
|
|
1239
|
+
failureReason
|
|
1240
|
+
});
|
|
1241
|
+
log(`Failed job ${jobId}`);
|
|
1242
|
+
} catch (err) {
|
|
1243
|
+
log(`Error failing job ${jobId}: ${err}`);
|
|
1244
|
+
throw err;
|
|
943
1245
|
} finally {
|
|
944
1246
|
client.release();
|
|
945
1247
|
}
|
|
946
1248
|
}
|
|
947
|
-
async
|
|
1249
|
+
async prolongJob(jobId) {
|
|
948
1250
|
const client = await this.pool.connect();
|
|
949
1251
|
try {
|
|
950
|
-
|
|
1252
|
+
await client.query(
|
|
951
1253
|
`
|
|
952
1254
|
UPDATE job_queue
|
|
953
|
-
SET
|
|
954
|
-
WHERE status = 'processing'
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
|
|
958
|
-
)
|
|
959
|
-
RETURNING id
|
|
960
|
-
`,
|
|
961
|
-
[maxProcessingTimeMinutes]
|
|
1255
|
+
SET locked_at = NOW(), updated_at = NOW()
|
|
1256
|
+
WHERE id = $1 AND status = 'processing'
|
|
1257
|
+
`,
|
|
1258
|
+
[jobId]
|
|
962
1259
|
);
|
|
963
|
-
|
|
964
|
-
|
|
1260
|
+
await this.recordJobEvent(jobId, "prolonged" /* Prolonged */);
|
|
1261
|
+
log(`Prolonged job ${jobId}`);
|
|
965
1262
|
} catch (error) {
|
|
966
|
-
log(`Error
|
|
967
|
-
throw error;
|
|
1263
|
+
log(`Error prolonging job ${jobId}: ${error}`);
|
|
968
1264
|
} finally {
|
|
969
1265
|
client.release();
|
|
970
1266
|
}
|
|
971
1267
|
}
|
|
972
|
-
// ──
|
|
973
|
-
|
|
974
|
-
* Batch-insert multiple job events in a single query.
|
|
975
|
-
* More efficient than individual recordJobEvent calls.
|
|
976
|
-
*/
|
|
977
|
-
async recordJobEventsBatch(events) {
|
|
978
|
-
if (events.length === 0) return;
|
|
1268
|
+
// ── Progress ──────────────────────────────────────────────────────────
|
|
1269
|
+
async updateProgress(jobId, progress) {
|
|
979
1270
|
const client = await this.pool.connect();
|
|
980
1271
|
try {
|
|
981
|
-
const values = [];
|
|
982
|
-
const params = [];
|
|
983
|
-
let paramIdx = 1;
|
|
984
|
-
for (const event of events) {
|
|
985
|
-
values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
|
|
986
|
-
params.push(
|
|
987
|
-
event.jobId,
|
|
988
|
-
event.eventType,
|
|
989
|
-
event.metadata ? JSON.stringify(event.metadata) : null
|
|
990
|
-
);
|
|
991
|
-
}
|
|
992
1272
|
await client.query(
|
|
993
|
-
`
|
|
994
|
-
|
|
1273
|
+
`UPDATE job_queue SET progress = $2, updated_at = NOW() WHERE id = $1`,
|
|
1274
|
+
[jobId, progress]
|
|
995
1275
|
);
|
|
1276
|
+
log(`Updated progress for job ${jobId}: ${progress}%`);
|
|
996
1277
|
} catch (error) {
|
|
997
|
-
log(`Error
|
|
1278
|
+
log(`Error updating progress for job ${jobId}: ${error}`);
|
|
998
1279
|
} finally {
|
|
999
1280
|
client.release();
|
|
1000
1281
|
}
|
|
1001
1282
|
}
|
|
1002
|
-
// ──
|
|
1003
|
-
|
|
1004
|
-
async addCronSchedule(input) {
|
|
1283
|
+
// ── Job management ────────────────────────────────────────────────────
|
|
1284
|
+
async retryJob(jobId) {
|
|
1005
1285
|
const client = await this.pool.connect();
|
|
1006
1286
|
try {
|
|
1007
1287
|
const result = await client.query(
|
|
1008
|
-
`
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
input.maxAttempts,
|
|
1020
|
-
input.priority,
|
|
1021
|
-
input.timeoutMs,
|
|
1022
|
-
input.forceKillOnTimeout,
|
|
1023
|
-
input.tags ?? null,
|
|
1024
|
-
input.timezone,
|
|
1025
|
-
input.allowOverlap,
|
|
1026
|
-
input.nextRunAt
|
|
1027
|
-
]
|
|
1288
|
+
`
|
|
1289
|
+
UPDATE job_queue
|
|
1290
|
+
SET status = 'pending',
|
|
1291
|
+
updated_at = NOW(),
|
|
1292
|
+
locked_at = NULL,
|
|
1293
|
+
locked_by = NULL,
|
|
1294
|
+
next_attempt_at = NOW(),
|
|
1295
|
+
last_retried_at = NOW()
|
|
1296
|
+
WHERE id = $1 AND status IN ('failed', 'processing')
|
|
1297
|
+
`,
|
|
1298
|
+
[jobId]
|
|
1028
1299
|
);
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
} catch (error) {
|
|
1033
|
-
if (error?.code === "23505") {
|
|
1034
|
-
throw new Error(
|
|
1035
|
-
`Cron schedule with name "${input.scheduleName}" already exists`
|
|
1300
|
+
if (result.rowCount === 0) {
|
|
1301
|
+
log(
|
|
1302
|
+
`Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
|
|
1036
1303
|
);
|
|
1037
1304
|
}
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
} finally {
|
|
1041
|
-
client.release();
|
|
1042
|
-
}
|
|
1043
|
-
}
|
|
1044
|
-
/** Get a cron schedule by ID. */
|
|
1045
|
-
async getCronSchedule(id) {
|
|
1046
|
-
const client = await this.pool.connect();
|
|
1047
|
-
try {
|
|
1048
|
-
const result = await client.query(
|
|
1049
|
-
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1050
|
-
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1051
|
-
priority, timeout_ms AS "timeoutMs",
|
|
1052
|
-
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1053
|
-
timezone, allow_overlap AS "allowOverlap", status,
|
|
1054
|
-
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1055
|
-
next_run_at AS "nextRunAt",
|
|
1056
|
-
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1057
|
-
FROM cron_schedules WHERE id = $1`,
|
|
1058
|
-
[id]
|
|
1059
|
-
);
|
|
1060
|
-
if (result.rows.length === 0) return null;
|
|
1061
|
-
return result.rows[0];
|
|
1305
|
+
await this.recordJobEvent(jobId, "retried" /* Retried */);
|
|
1306
|
+
log(`Retried job ${jobId}`);
|
|
1062
1307
|
} catch (error) {
|
|
1063
|
-
log(`Error
|
|
1308
|
+
log(`Error retrying job ${jobId}: ${error}`);
|
|
1064
1309
|
throw error;
|
|
1065
1310
|
} finally {
|
|
1066
1311
|
client.release();
|
|
1067
1312
|
}
|
|
1068
1313
|
}
|
|
1069
|
-
|
|
1070
|
-
async getCronScheduleByName(name) {
|
|
1314
|
+
async cancelJob(jobId) {
|
|
1071
1315
|
const client = await this.pool.connect();
|
|
1072
1316
|
try {
|
|
1073
|
-
|
|
1074
|
-
`
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1082
|
-
FROM cron_schedules WHERE schedule_name = $1`,
|
|
1083
|
-
[name]
|
|
1317
|
+
await client.query(
|
|
1318
|
+
`
|
|
1319
|
+
UPDATE job_queue
|
|
1320
|
+
SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
|
|
1321
|
+
wait_until = NULL, wait_token_id = NULL
|
|
1322
|
+
WHERE id = $1 AND status IN ('pending', 'waiting')
|
|
1323
|
+
`,
|
|
1324
|
+
[jobId]
|
|
1084
1325
|
);
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
} catch (error) {
|
|
1088
|
-
log(`Error getting cron schedule by name "${name}": ${error}`);
|
|
1089
|
-
throw error;
|
|
1090
|
-
} finally {
|
|
1091
|
-
client.release();
|
|
1092
|
-
}
|
|
1093
|
-
}
|
|
1094
|
-
/** List cron schedules, optionally filtered by status. */
|
|
1095
|
-
async listCronSchedules(status) {
|
|
1096
|
-
const client = await this.pool.connect();
|
|
1097
|
-
try {
|
|
1098
|
-
let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1099
|
-
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1100
|
-
priority, timeout_ms AS "timeoutMs",
|
|
1101
|
-
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1102
|
-
timezone, allow_overlap AS "allowOverlap", status,
|
|
1103
|
-
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1104
|
-
next_run_at AS "nextRunAt",
|
|
1105
|
-
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1106
|
-
FROM cron_schedules`;
|
|
1107
|
-
const params = [];
|
|
1108
|
-
if (status) {
|
|
1109
|
-
query += ` WHERE status = $1`;
|
|
1110
|
-
params.push(status);
|
|
1111
|
-
}
|
|
1112
|
-
query += ` ORDER BY created_at ASC`;
|
|
1113
|
-
const result = await client.query(query, params);
|
|
1114
|
-
return result.rows;
|
|
1326
|
+
await this.recordJobEvent(jobId, "cancelled" /* Cancelled */);
|
|
1327
|
+
log(`Cancelled job ${jobId}`);
|
|
1115
1328
|
} catch (error) {
|
|
1116
|
-
log(`Error
|
|
1329
|
+
log(`Error cancelling job ${jobId}: ${error}`);
|
|
1117
1330
|
throw error;
|
|
1118
1331
|
} finally {
|
|
1119
1332
|
client.release();
|
|
1120
1333
|
}
|
|
1121
1334
|
}
|
|
1122
|
-
|
|
1123
|
-
async removeCronSchedule(id) {
|
|
1335
|
+
async cancelAllUpcomingJobs(filters) {
|
|
1124
1336
|
const client = await this.pool.connect();
|
|
1125
1337
|
try {
|
|
1126
|
-
|
|
1127
|
-
|
|
1338
|
+
let query = `
|
|
1339
|
+
UPDATE job_queue
|
|
1340
|
+
SET status = 'cancelled', updated_at = NOW()
|
|
1341
|
+
WHERE status = 'pending'`;
|
|
1342
|
+
const params = [];
|
|
1343
|
+
let paramIdx = 1;
|
|
1344
|
+
if (filters) {
|
|
1345
|
+
if (filters.jobType) {
|
|
1346
|
+
query += ` AND job_type = $${paramIdx++}`;
|
|
1347
|
+
params.push(filters.jobType);
|
|
1348
|
+
}
|
|
1349
|
+
if (filters.priority !== void 0) {
|
|
1350
|
+
query += ` AND priority = $${paramIdx++}`;
|
|
1351
|
+
params.push(filters.priority);
|
|
1352
|
+
}
|
|
1353
|
+
if (filters.runAt) {
|
|
1354
|
+
if (filters.runAt instanceof Date) {
|
|
1355
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1356
|
+
params.push(filters.runAt);
|
|
1357
|
+
} else if (typeof filters.runAt === "object") {
|
|
1358
|
+
const ops = filters.runAt;
|
|
1359
|
+
if (ops.gt) {
|
|
1360
|
+
query += ` AND run_at > $${paramIdx++}`;
|
|
1361
|
+
params.push(ops.gt);
|
|
1362
|
+
}
|
|
1363
|
+
if (ops.gte) {
|
|
1364
|
+
query += ` AND run_at >= $${paramIdx++}`;
|
|
1365
|
+
params.push(ops.gte);
|
|
1366
|
+
}
|
|
1367
|
+
if (ops.lt) {
|
|
1368
|
+
query += ` AND run_at < $${paramIdx++}`;
|
|
1369
|
+
params.push(ops.lt);
|
|
1370
|
+
}
|
|
1371
|
+
if (ops.lte) {
|
|
1372
|
+
query += ` AND run_at <= $${paramIdx++}`;
|
|
1373
|
+
params.push(ops.lte);
|
|
1374
|
+
}
|
|
1375
|
+
if (ops.eq) {
|
|
1376
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1377
|
+
params.push(ops.eq);
|
|
1378
|
+
}
|
|
1379
|
+
}
|
|
1380
|
+
}
|
|
1381
|
+
if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
|
|
1382
|
+
const mode = filters.tags.mode || "all";
|
|
1383
|
+
const tagValues = filters.tags.values;
|
|
1384
|
+
switch (mode) {
|
|
1385
|
+
case "exact":
|
|
1386
|
+
query += ` AND tags = $${paramIdx++}`;
|
|
1387
|
+
params.push(tagValues);
|
|
1388
|
+
break;
|
|
1389
|
+
case "all":
|
|
1390
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1391
|
+
params.push(tagValues);
|
|
1392
|
+
break;
|
|
1393
|
+
case "any":
|
|
1394
|
+
query += ` AND tags && $${paramIdx++}`;
|
|
1395
|
+
params.push(tagValues);
|
|
1396
|
+
break;
|
|
1397
|
+
case "none":
|
|
1398
|
+
query += ` AND NOT (tags && $${paramIdx++})`;
|
|
1399
|
+
params.push(tagValues);
|
|
1400
|
+
break;
|
|
1401
|
+
default:
|
|
1402
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1403
|
+
params.push(tagValues);
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
query += "\nRETURNING id";
|
|
1408
|
+
const result = await client.query(query, params);
|
|
1409
|
+
log(`Cancelled ${result.rowCount} jobs`);
|
|
1410
|
+
return result.rowCount || 0;
|
|
1128
1411
|
} catch (error) {
|
|
1129
|
-
log(`Error
|
|
1412
|
+
log(`Error cancelling upcoming jobs: ${error}`);
|
|
1130
1413
|
throw error;
|
|
1131
1414
|
} finally {
|
|
1132
1415
|
client.release();
|
|
1133
1416
|
}
|
|
1134
1417
|
}
|
|
1135
|
-
|
|
1136
|
-
async pauseCronSchedule(id) {
|
|
1418
|
+
async editJob(jobId, updates) {
|
|
1137
1419
|
const client = await this.pool.connect();
|
|
1138
1420
|
try {
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
)
|
|
1143
|
-
|
|
1421
|
+
const updateFields = [];
|
|
1422
|
+
const params = [];
|
|
1423
|
+
let paramIdx = 1;
|
|
1424
|
+
if (updates.payload !== void 0) {
|
|
1425
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
1426
|
+
params.push(updates.payload);
|
|
1427
|
+
}
|
|
1428
|
+
if (updates.maxAttempts !== void 0) {
|
|
1429
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
1430
|
+
params.push(updates.maxAttempts);
|
|
1431
|
+
}
|
|
1432
|
+
if (updates.priority !== void 0) {
|
|
1433
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
1434
|
+
params.push(updates.priority);
|
|
1435
|
+
}
|
|
1436
|
+
if (updates.runAt !== void 0) {
|
|
1437
|
+
if (updates.runAt === null) {
|
|
1438
|
+
updateFields.push(`run_at = NOW()`);
|
|
1439
|
+
} else {
|
|
1440
|
+
updateFields.push(`run_at = $${paramIdx++}`);
|
|
1441
|
+
params.push(updates.runAt);
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
if (updates.timeoutMs !== void 0) {
|
|
1445
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
1446
|
+
params.push(updates.timeoutMs ?? null);
|
|
1447
|
+
}
|
|
1448
|
+
if (updates.tags !== void 0) {
|
|
1449
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
1450
|
+
params.push(updates.tags ?? null);
|
|
1451
|
+
}
|
|
1452
|
+
if (updateFields.length === 0) {
|
|
1453
|
+
log(`No fields to update for job ${jobId}`);
|
|
1454
|
+
return;
|
|
1455
|
+
}
|
|
1456
|
+
updateFields.push(`updated_at = NOW()`);
|
|
1457
|
+
params.push(jobId);
|
|
1458
|
+
const query = `
|
|
1459
|
+
UPDATE job_queue
|
|
1460
|
+
SET ${updateFields.join(", ")}
|
|
1461
|
+
WHERE id = $${paramIdx} AND status = 'pending'
|
|
1462
|
+
`;
|
|
1463
|
+
await client.query(query, params);
|
|
1464
|
+
const metadata = {};
|
|
1465
|
+
if (updates.payload !== void 0) metadata.payload = updates.payload;
|
|
1466
|
+
if (updates.maxAttempts !== void 0)
|
|
1467
|
+
metadata.maxAttempts = updates.maxAttempts;
|
|
1468
|
+
if (updates.priority !== void 0) metadata.priority = updates.priority;
|
|
1469
|
+
if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
|
|
1470
|
+
if (updates.timeoutMs !== void 0)
|
|
1471
|
+
metadata.timeoutMs = updates.timeoutMs;
|
|
1472
|
+
if (updates.tags !== void 0) metadata.tags = updates.tags;
|
|
1473
|
+
await this.recordJobEvent(jobId, "edited" /* Edited */, metadata);
|
|
1474
|
+
log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
|
|
1144
1475
|
} catch (error) {
|
|
1145
|
-
log(`Error
|
|
1476
|
+
log(`Error editing job ${jobId}: ${error}`);
|
|
1146
1477
|
throw error;
|
|
1147
1478
|
} finally {
|
|
1148
1479
|
client.release();
|
|
1149
1480
|
}
|
|
1150
1481
|
}
|
|
1151
|
-
|
|
1152
|
-
async resumeCronSchedule(id) {
|
|
1482
|
+
async editAllPendingJobs(filters = void 0, updates) {
|
|
1153
1483
|
const client = await this.pool.connect();
|
|
1154
1484
|
try {
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
)
|
|
1159
|
-
|
|
1485
|
+
const updateFields = [];
|
|
1486
|
+
const params = [];
|
|
1487
|
+
let paramIdx = 1;
|
|
1488
|
+
if (updates.payload !== void 0) {
|
|
1489
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
1490
|
+
params.push(updates.payload);
|
|
1491
|
+
}
|
|
1492
|
+
if (updates.maxAttempts !== void 0) {
|
|
1493
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
1494
|
+
params.push(updates.maxAttempts);
|
|
1495
|
+
}
|
|
1496
|
+
if (updates.priority !== void 0) {
|
|
1497
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
1498
|
+
params.push(updates.priority);
|
|
1499
|
+
}
|
|
1500
|
+
if (updates.runAt !== void 0) {
|
|
1501
|
+
if (updates.runAt === null) {
|
|
1502
|
+
updateFields.push(`run_at = NOW()`);
|
|
1503
|
+
} else {
|
|
1504
|
+
updateFields.push(`run_at = $${paramIdx++}`);
|
|
1505
|
+
params.push(updates.runAt);
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
if (updates.timeoutMs !== void 0) {
|
|
1509
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
1510
|
+
params.push(updates.timeoutMs ?? null);
|
|
1511
|
+
}
|
|
1512
|
+
if (updates.tags !== void 0) {
|
|
1513
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
1514
|
+
params.push(updates.tags ?? null);
|
|
1515
|
+
}
|
|
1516
|
+
if (updateFields.length === 0) {
|
|
1517
|
+
log(`No fields to update for batch edit`);
|
|
1518
|
+
return 0;
|
|
1519
|
+
}
|
|
1520
|
+
updateFields.push(`updated_at = NOW()`);
|
|
1521
|
+
let query = `
|
|
1522
|
+
UPDATE job_queue
|
|
1523
|
+
SET ${updateFields.join(", ")}
|
|
1524
|
+
WHERE status = 'pending'`;
|
|
1525
|
+
if (filters) {
|
|
1526
|
+
if (filters.jobType) {
|
|
1527
|
+
query += ` AND job_type = $${paramIdx++}`;
|
|
1528
|
+
params.push(filters.jobType);
|
|
1529
|
+
}
|
|
1530
|
+
if (filters.priority !== void 0) {
|
|
1531
|
+
query += ` AND priority = $${paramIdx++}`;
|
|
1532
|
+
params.push(filters.priority);
|
|
1533
|
+
}
|
|
1534
|
+
if (filters.runAt) {
|
|
1535
|
+
if (filters.runAt instanceof Date) {
|
|
1536
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1537
|
+
params.push(filters.runAt);
|
|
1538
|
+
} else if (typeof filters.runAt === "object") {
|
|
1539
|
+
const ops = filters.runAt;
|
|
1540
|
+
if (ops.gt) {
|
|
1541
|
+
query += ` AND run_at > $${paramIdx++}`;
|
|
1542
|
+
params.push(ops.gt);
|
|
1543
|
+
}
|
|
1544
|
+
if (ops.gte) {
|
|
1545
|
+
query += ` AND run_at >= $${paramIdx++}`;
|
|
1546
|
+
params.push(ops.gte);
|
|
1547
|
+
}
|
|
1548
|
+
if (ops.lt) {
|
|
1549
|
+
query += ` AND run_at < $${paramIdx++}`;
|
|
1550
|
+
params.push(ops.lt);
|
|
1551
|
+
}
|
|
1552
|
+
if (ops.lte) {
|
|
1553
|
+
query += ` AND run_at <= $${paramIdx++}`;
|
|
1554
|
+
params.push(ops.lte);
|
|
1555
|
+
}
|
|
1556
|
+
if (ops.eq) {
|
|
1557
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1558
|
+
params.push(ops.eq);
|
|
1559
|
+
}
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
|
|
1563
|
+
const mode = filters.tags.mode || "all";
|
|
1564
|
+
const tagValues = filters.tags.values;
|
|
1565
|
+
switch (mode) {
|
|
1566
|
+
case "exact":
|
|
1567
|
+
query += ` AND tags = $${paramIdx++}`;
|
|
1568
|
+
params.push(tagValues);
|
|
1569
|
+
break;
|
|
1570
|
+
case "all":
|
|
1571
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1572
|
+
params.push(tagValues);
|
|
1573
|
+
break;
|
|
1574
|
+
case "any":
|
|
1575
|
+
query += ` AND tags && $${paramIdx++}`;
|
|
1576
|
+
params.push(tagValues);
|
|
1577
|
+
break;
|
|
1578
|
+
case "none":
|
|
1579
|
+
query += ` AND NOT (tags && $${paramIdx++})`;
|
|
1580
|
+
params.push(tagValues);
|
|
1581
|
+
break;
|
|
1582
|
+
default:
|
|
1583
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1584
|
+
params.push(tagValues);
|
|
1585
|
+
}
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
query += "\nRETURNING id";
|
|
1589
|
+
const result = await client.query(query, params);
|
|
1590
|
+
const editedCount = result.rowCount || 0;
|
|
1591
|
+
const metadata = {};
|
|
1592
|
+
if (updates.payload !== void 0) metadata.payload = updates.payload;
|
|
1593
|
+
if (updates.maxAttempts !== void 0)
|
|
1594
|
+
metadata.maxAttempts = updates.maxAttempts;
|
|
1595
|
+
if (updates.priority !== void 0) metadata.priority = updates.priority;
|
|
1596
|
+
if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
|
|
1597
|
+
if (updates.timeoutMs !== void 0)
|
|
1598
|
+
metadata.timeoutMs = updates.timeoutMs;
|
|
1599
|
+
if (updates.tags !== void 0) metadata.tags = updates.tags;
|
|
1600
|
+
for (const row of result.rows) {
|
|
1601
|
+
await this.recordJobEvent(row.id, "edited" /* Edited */, metadata);
|
|
1602
|
+
}
|
|
1603
|
+
log(`Edited ${editedCount} pending jobs: ${JSON.stringify(metadata)}`);
|
|
1604
|
+
return editedCount;
|
|
1160
1605
|
} catch (error) {
|
|
1161
|
-
log(`Error
|
|
1606
|
+
log(`Error editing pending jobs: ${error}`);
|
|
1162
1607
|
throw error;
|
|
1163
1608
|
} finally {
|
|
1164
1609
|
client.release();
|
|
1165
1610
|
}
|
|
1166
1611
|
}
|
|
1167
|
-
/**
|
|
1168
|
-
|
|
1169
|
-
|
|
1612
|
+
/**
|
|
1613
|
+
* Delete completed jobs older than the given number of days.
|
|
1614
|
+
* Deletes in batches of 1000 to avoid long-running transactions
|
|
1615
|
+
* and excessive WAL bloat at scale.
|
|
1616
|
+
*
|
|
1617
|
+
* @param daysToKeep - Number of days to retain completed jobs (default 30).
|
|
1618
|
+
* @param batchSize - Number of rows to delete per batch (default 1000).
|
|
1619
|
+
* @returns Total number of deleted jobs.
|
|
1620
|
+
*/
|
|
1621
|
+
async cleanupOldJobs(daysToKeep = 30, batchSize = 1e3) {
|
|
1622
|
+
let totalDeleted = 0;
|
|
1170
1623
|
try {
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
|
|
1196
|
-
params.push(updates.forceKillOnTimeout);
|
|
1197
|
-
}
|
|
1198
|
-
if (updates.tags !== void 0) {
|
|
1199
|
-
updateFields.push(`tags = $${paramIdx++}`);
|
|
1200
|
-
params.push(updates.tags);
|
|
1201
|
-
}
|
|
1202
|
-
if (updates.timezone !== void 0) {
|
|
1203
|
-
updateFields.push(`timezone = $${paramIdx++}`);
|
|
1204
|
-
params.push(updates.timezone);
|
|
1205
|
-
}
|
|
1206
|
-
if (updates.allowOverlap !== void 0) {
|
|
1207
|
-
updateFields.push(`allow_overlap = $${paramIdx++}`);
|
|
1208
|
-
params.push(updates.allowOverlap);
|
|
1209
|
-
}
|
|
1210
|
-
if (nextRunAt !== void 0) {
|
|
1211
|
-
updateFields.push(`next_run_at = $${paramIdx++}`);
|
|
1212
|
-
params.push(nextRunAt);
|
|
1213
|
-
}
|
|
1214
|
-
if (updateFields.length === 0) {
|
|
1215
|
-
log(`No fields to update for cron schedule ${id}`);
|
|
1216
|
-
return;
|
|
1217
|
-
}
|
|
1218
|
-
updateFields.push(`updated_at = NOW()`);
|
|
1219
|
-
params.push(id);
|
|
1220
|
-
const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
|
|
1221
|
-
await client.query(query, params);
|
|
1222
|
-
log(`Edited cron schedule ${id}`);
|
|
1624
|
+
let deletedInBatch;
|
|
1625
|
+
do {
|
|
1626
|
+
const client = await this.pool.connect();
|
|
1627
|
+
try {
|
|
1628
|
+
const result = await client.query(
|
|
1629
|
+
`
|
|
1630
|
+
DELETE FROM job_queue
|
|
1631
|
+
WHERE id IN (
|
|
1632
|
+
SELECT id FROM job_queue
|
|
1633
|
+
WHERE status = 'completed'
|
|
1634
|
+
AND updated_at < NOW() - INTERVAL '1 day' * $1::int
|
|
1635
|
+
LIMIT $2
|
|
1636
|
+
)
|
|
1637
|
+
`,
|
|
1638
|
+
[daysToKeep, batchSize]
|
|
1639
|
+
);
|
|
1640
|
+
deletedInBatch = result.rowCount || 0;
|
|
1641
|
+
totalDeleted += deletedInBatch;
|
|
1642
|
+
} finally {
|
|
1643
|
+
client.release();
|
|
1644
|
+
}
|
|
1645
|
+
} while (deletedInBatch === batchSize);
|
|
1646
|
+
log(`Deleted ${totalDeleted} old jobs`);
|
|
1647
|
+
return totalDeleted;
|
|
1223
1648
|
} catch (error) {
|
|
1224
|
-
log(`Error
|
|
1649
|
+
log(`Error cleaning up old jobs: ${error}`);
|
|
1225
1650
|
throw error;
|
|
1226
|
-
} finally {
|
|
1227
|
-
client.release();
|
|
1228
1651
|
}
|
|
1229
1652
|
}
|
|
1230
1653
|
/**
|
|
1231
|
-
*
|
|
1232
|
-
*
|
|
1654
|
+
* Delete job events older than the given number of days.
|
|
1655
|
+
* Deletes in batches of 1000 to avoid long-running transactions
|
|
1656
|
+
* and excessive WAL bloat at scale.
|
|
1657
|
+
*
|
|
1658
|
+
* @param daysToKeep - Number of days to retain events (default 30).
|
|
1659
|
+
* @param batchSize - Number of rows to delete per batch (default 1000).
|
|
1660
|
+
* @returns Total number of deleted events.
|
|
1233
1661
|
*/
|
|
1234
|
-
async
|
|
1235
|
-
|
|
1662
|
+
async cleanupOldJobEvents(daysToKeep = 30, batchSize = 1e3) {
|
|
1663
|
+
let totalDeleted = 0;
|
|
1236
1664
|
try {
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1665
|
+
let deletedInBatch;
|
|
1666
|
+
do {
|
|
1667
|
+
const client = await this.pool.connect();
|
|
1668
|
+
try {
|
|
1669
|
+
const result = await client.query(
|
|
1670
|
+
`
|
|
1671
|
+
DELETE FROM job_events
|
|
1672
|
+
WHERE id IN (
|
|
1673
|
+
SELECT id FROM job_events
|
|
1674
|
+
WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
|
|
1675
|
+
LIMIT $2
|
|
1676
|
+
)
|
|
1677
|
+
`,
|
|
1678
|
+
[daysToKeep, batchSize]
|
|
1679
|
+
);
|
|
1680
|
+
deletedInBatch = result.rowCount || 0;
|
|
1681
|
+
totalDeleted += deletedInBatch;
|
|
1682
|
+
} finally {
|
|
1683
|
+
client.release();
|
|
1684
|
+
}
|
|
1685
|
+
} while (deletedInBatch === batchSize);
|
|
1686
|
+
log(`Deleted ${totalDeleted} old job events`);
|
|
1687
|
+
return totalDeleted;
|
|
1255
1688
|
} catch (error) {
|
|
1256
|
-
|
|
1257
|
-
log("cron_schedules table does not exist, skipping cron enqueue");
|
|
1258
|
-
return [];
|
|
1259
|
-
}
|
|
1260
|
-
log(`Error getting due cron schedules: ${error}`);
|
|
1689
|
+
log(`Error cleaning up old job events: ${error}`);
|
|
1261
1690
|
throw error;
|
|
1262
|
-
} finally {
|
|
1263
|
-
client.release();
|
|
1264
1691
|
}
|
|
1265
1692
|
}
|
|
1266
|
-
|
|
1267
|
-
* Update a cron schedule after a job has been enqueued.
|
|
1268
|
-
* Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
|
|
1269
|
-
*/
|
|
1270
|
-
async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
|
|
1693
|
+
async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
|
|
1271
1694
|
const client = await this.pool.connect();
|
|
1272
1695
|
try {
|
|
1273
|
-
await client.query(
|
|
1274
|
-
`
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1696
|
+
const result = await client.query(
|
|
1697
|
+
`
|
|
1698
|
+
UPDATE job_queue
|
|
1699
|
+
SET status = 'pending', locked_at = NULL, locked_by = NULL, updated_at = NOW()
|
|
1700
|
+
WHERE status = 'processing'
|
|
1701
|
+
AND locked_at < NOW() - GREATEST(
|
|
1702
|
+
INTERVAL '1 minute' * $1::int,
|
|
1703
|
+
INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
|
|
1704
|
+
)
|
|
1705
|
+
RETURNING id
|
|
1706
|
+
`,
|
|
1707
|
+
[maxProcessingTimeMinutes]
|
|
1284
1708
|
);
|
|
1709
|
+
log(`Reclaimed ${result.rowCount} stuck jobs`);
|
|
1710
|
+
return result.rowCount || 0;
|
|
1285
1711
|
} catch (error) {
|
|
1286
|
-
log(`Error
|
|
1712
|
+
log(`Error reclaiming stuck jobs: ${error}`);
|
|
1287
1713
|
throw error;
|
|
1288
1714
|
} finally {
|
|
1289
1715
|
client.release();
|
|
1290
1716
|
}
|
|
1291
1717
|
}
|
|
1292
1718
|
// ── Internal helpers ──────────────────────────────────────────────────
|
|
1293
|
-
|
|
1719
|
+
/**
|
|
1720
|
+
* Batch-insert multiple job events in a single query.
|
|
1721
|
+
* More efficient than individual recordJobEvent calls.
|
|
1722
|
+
*/
|
|
1723
|
+
async recordJobEventsBatch(events) {
|
|
1724
|
+
if (events.length === 0) return;
|
|
1294
1725
|
const client = await this.pool.connect();
|
|
1295
1726
|
try {
|
|
1296
|
-
|
|
1297
|
-
const params = [
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
}
|
|
1307
|
-
await client.query(
|
|
1308
|
-
`UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
|
|
1309
|
-
params
|
|
1310
|
-
);
|
|
1311
|
-
} finally {
|
|
1312
|
-
client.release();
|
|
1313
|
-
}
|
|
1314
|
-
}
|
|
1315
|
-
};
|
|
1316
|
-
var recordJobEvent = async (pool, jobId, eventType, metadata) => new PostgresBackend(pool).recordJobEvent(jobId, eventType, metadata);
|
|
1317
|
-
var waitJob = async (pool, jobId, options) => {
|
|
1318
|
-
const client = await pool.connect();
|
|
1319
|
-
try {
|
|
1320
|
-
const result = await client.query(
|
|
1321
|
-
`
|
|
1322
|
-
UPDATE job_queue
|
|
1323
|
-
SET status = 'waiting',
|
|
1324
|
-
wait_until = $2,
|
|
1325
|
-
wait_token_id = $3,
|
|
1326
|
-
step_data = $4,
|
|
1327
|
-
locked_at = NULL,
|
|
1328
|
-
locked_by = NULL,
|
|
1329
|
-
updated_at = NOW()
|
|
1330
|
-
WHERE id = $1 AND status = 'processing'
|
|
1331
|
-
`,
|
|
1332
|
-
[
|
|
1333
|
-
jobId,
|
|
1334
|
-
options.waitUntil ?? null,
|
|
1335
|
-
options.waitTokenId ?? null,
|
|
1336
|
-
JSON.stringify(options.stepData)
|
|
1337
|
-
]
|
|
1338
|
-
);
|
|
1339
|
-
if (result.rowCount === 0) {
|
|
1340
|
-
log(
|
|
1341
|
-
`Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
|
|
1342
|
-
);
|
|
1343
|
-
return;
|
|
1344
|
-
}
|
|
1345
|
-
await recordJobEvent(pool, jobId, "waiting" /* Waiting */, {
|
|
1346
|
-
waitUntil: options.waitUntil?.toISOString() ?? null,
|
|
1347
|
-
waitTokenId: options.waitTokenId ?? null
|
|
1348
|
-
});
|
|
1349
|
-
log(`Job ${jobId} set to waiting`);
|
|
1350
|
-
} catch (error) {
|
|
1351
|
-
log(`Error setting job ${jobId} to waiting: ${error}`);
|
|
1352
|
-
throw error;
|
|
1353
|
-
} finally {
|
|
1354
|
-
client.release();
|
|
1355
|
-
}
|
|
1356
|
-
};
|
|
1357
|
-
var updateStepData = async (pool, jobId, stepData) => {
|
|
1358
|
-
const client = await pool.connect();
|
|
1359
|
-
try {
|
|
1360
|
-
await client.query(
|
|
1361
|
-
`UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
|
|
1362
|
-
[jobId, JSON.stringify(stepData)]
|
|
1363
|
-
);
|
|
1364
|
-
} catch (error) {
|
|
1365
|
-
log(`Error updating step_data for job ${jobId}: ${error}`);
|
|
1366
|
-
} finally {
|
|
1367
|
-
client.release();
|
|
1368
|
-
}
|
|
1369
|
-
};
|
|
1370
|
-
var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
|
|
1371
|
-
function parseTimeoutString(timeout) {
|
|
1372
|
-
const match = timeout.match(/^(\d+)(s|m|h|d)$/);
|
|
1373
|
-
if (!match) {
|
|
1374
|
-
throw new Error(
|
|
1375
|
-
`Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
|
|
1376
|
-
);
|
|
1377
|
-
}
|
|
1378
|
-
const value = parseInt(match[1], 10);
|
|
1379
|
-
const unit = match[2];
|
|
1380
|
-
let ms;
|
|
1381
|
-
switch (unit) {
|
|
1382
|
-
case "s":
|
|
1383
|
-
ms = value * 1e3;
|
|
1384
|
-
break;
|
|
1385
|
-
case "m":
|
|
1386
|
-
ms = value * 60 * 1e3;
|
|
1387
|
-
break;
|
|
1388
|
-
case "h":
|
|
1389
|
-
ms = value * 60 * 60 * 1e3;
|
|
1390
|
-
break;
|
|
1391
|
-
case "d":
|
|
1392
|
-
ms = value * 24 * 60 * 60 * 1e3;
|
|
1393
|
-
break;
|
|
1394
|
-
default:
|
|
1395
|
-
throw new Error(`Unknown timeout unit: "${unit}"`);
|
|
1396
|
-
}
|
|
1397
|
-
if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
|
|
1398
|
-
throw new Error(
|
|
1399
|
-
`Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
|
|
1400
|
-
);
|
|
1401
|
-
}
|
|
1402
|
-
return ms;
|
|
1403
|
-
}
|
|
1404
|
-
var createWaitpoint = async (pool, jobId, options) => {
|
|
1405
|
-
const client = await pool.connect();
|
|
1406
|
-
try {
|
|
1407
|
-
const id = `wp_${randomUUID()}`;
|
|
1408
|
-
let timeoutAt = null;
|
|
1409
|
-
if (options?.timeout) {
|
|
1410
|
-
const ms = parseTimeoutString(options.timeout);
|
|
1411
|
-
timeoutAt = new Date(Date.now() + ms);
|
|
1412
|
-
}
|
|
1413
|
-
await client.query(
|
|
1414
|
-
`INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
|
|
1415
|
-
[id, jobId, timeoutAt, options?.tags ?? null]
|
|
1416
|
-
);
|
|
1417
|
-
log(`Created waitpoint ${id} for job ${jobId}`);
|
|
1418
|
-
return { id };
|
|
1419
|
-
} catch (error) {
|
|
1420
|
-
log(`Error creating waitpoint: ${error}`);
|
|
1421
|
-
throw error;
|
|
1422
|
-
} finally {
|
|
1423
|
-
client.release();
|
|
1424
|
-
}
|
|
1425
|
-
};
|
|
1426
|
-
var completeWaitpoint = async (pool, tokenId, data) => {
|
|
1427
|
-
const client = await pool.connect();
|
|
1428
|
-
try {
|
|
1429
|
-
await client.query("BEGIN");
|
|
1430
|
-
const wpResult = await client.query(
|
|
1431
|
-
`UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
|
|
1432
|
-
WHERE id = $1 AND status = 'waiting'
|
|
1433
|
-
RETURNING job_id`,
|
|
1434
|
-
[tokenId, data != null ? JSON.stringify(data) : null]
|
|
1435
|
-
);
|
|
1436
|
-
if (wpResult.rows.length === 0) {
|
|
1437
|
-
await client.query("ROLLBACK");
|
|
1438
|
-
log(`Waitpoint ${tokenId} not found or already completed`);
|
|
1439
|
-
return;
|
|
1440
|
-
}
|
|
1441
|
-
const jobId = wpResult.rows[0].job_id;
|
|
1442
|
-
if (jobId != null) {
|
|
1727
|
+
const values = [];
|
|
1728
|
+
const params = [];
|
|
1729
|
+
let paramIdx = 1;
|
|
1730
|
+
for (const event of events) {
|
|
1731
|
+
values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
|
|
1732
|
+
params.push(
|
|
1733
|
+
event.jobId,
|
|
1734
|
+
event.eventType,
|
|
1735
|
+
event.metadata ? JSON.stringify(event.metadata) : null
|
|
1736
|
+
);
|
|
1737
|
+
}
|
|
1443
1738
|
await client.query(
|
|
1444
|
-
`
|
|
1445
|
-
|
|
1446
|
-
WHERE id = $1 AND status = 'waiting'`,
|
|
1447
|
-
[jobId]
|
|
1739
|
+
`INSERT INTO job_events (job_id, event_type, metadata) VALUES ${values.join(", ")}`,
|
|
1740
|
+
params
|
|
1448
1741
|
);
|
|
1742
|
+
} catch (error) {
|
|
1743
|
+
log(`Error recording batch job events: ${error}`);
|
|
1744
|
+
} finally {
|
|
1745
|
+
client.release();
|
|
1449
1746
|
}
|
|
1450
|
-
await client.query("COMMIT");
|
|
1451
|
-
log(`Completed waitpoint ${tokenId} for job ${jobId}`);
|
|
1452
|
-
} catch (error) {
|
|
1453
|
-
await client.query("ROLLBACK");
|
|
1454
|
-
log(`Error completing waitpoint ${tokenId}: ${error}`);
|
|
1455
|
-
throw error;
|
|
1456
|
-
} finally {
|
|
1457
|
-
client.release();
|
|
1458
|
-
}
|
|
1459
|
-
};
|
|
1460
|
-
var getWaitpoint = async (pool, tokenId) => {
|
|
1461
|
-
const client = await pool.connect();
|
|
1462
|
-
try {
|
|
1463
|
-
const result = await client.query(
|
|
1464
|
-
`SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
|
|
1465
|
-
[tokenId]
|
|
1466
|
-
);
|
|
1467
|
-
if (result.rows.length === 0) return null;
|
|
1468
|
-
return result.rows[0];
|
|
1469
|
-
} catch (error) {
|
|
1470
|
-
log(`Error getting waitpoint ${tokenId}: ${error}`);
|
|
1471
|
-
throw error;
|
|
1472
|
-
} finally {
|
|
1473
|
-
client.release();
|
|
1474
1747
|
}
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1748
|
+
// ── Cron schedules ──────────────────────────────────────────────────
|
|
1749
|
+
/** Create a cron schedule and return its ID. */
|
|
1750
|
+
async addCronSchedule(input) {
|
|
1751
|
+
const client = await this.pool.connect();
|
|
1752
|
+
try {
|
|
1753
|
+
const result = await client.query(
|
|
1754
|
+
`INSERT INTO cron_schedules
|
|
1755
|
+
(schedule_name, cron_expression, job_type, payload, max_attempts,
|
|
1756
|
+
priority, timeout_ms, force_kill_on_timeout, tags, timezone,
|
|
1757
|
+
allow_overlap, next_run_at)
|
|
1758
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
1759
|
+
RETURNING id`,
|
|
1760
|
+
[
|
|
1761
|
+
input.scheduleName,
|
|
1762
|
+
input.cronExpression,
|
|
1763
|
+
input.jobType,
|
|
1764
|
+
input.payload,
|
|
1765
|
+
input.maxAttempts,
|
|
1766
|
+
input.priority,
|
|
1767
|
+
input.timeoutMs,
|
|
1768
|
+
input.forceKillOnTimeout,
|
|
1769
|
+
input.tags ?? null,
|
|
1770
|
+
input.timezone,
|
|
1771
|
+
input.allowOverlap,
|
|
1772
|
+
input.nextRunAt
|
|
1773
|
+
]
|
|
1774
|
+
);
|
|
1775
|
+
const id = result.rows[0].id;
|
|
1776
|
+
log(`Added cron schedule ${id}: "${input.scheduleName}"`);
|
|
1777
|
+
return id;
|
|
1778
|
+
} catch (error) {
|
|
1779
|
+
if (error?.code === "23505") {
|
|
1780
|
+
throw new Error(
|
|
1781
|
+
`Cron schedule with name "${input.scheduleName}" already exists`
|
|
1493
1782
|
);
|
|
1494
1783
|
}
|
|
1784
|
+
log(`Error adding cron schedule: ${error}`);
|
|
1785
|
+
throw error;
|
|
1786
|
+
} finally {
|
|
1787
|
+
client.release();
|
|
1495
1788
|
}
|
|
1496
|
-
await client.query("COMMIT");
|
|
1497
|
-
const count = result.rowCount || 0;
|
|
1498
|
-
if (count > 0) {
|
|
1499
|
-
log(`Expired ${count} timed-out waitpoints`);
|
|
1500
|
-
}
|
|
1501
|
-
return count;
|
|
1502
|
-
} catch (error) {
|
|
1503
|
-
await client.query("ROLLBACK");
|
|
1504
|
-
log(`Error expiring timed-out waitpoints: ${error}`);
|
|
1505
|
-
throw error;
|
|
1506
|
-
} finally {
|
|
1507
|
-
client.release();
|
|
1508
|
-
}
|
|
1509
|
-
};
|
|
1510
|
-
function tryExtractPool(backend) {
|
|
1511
|
-
if (backend instanceof PostgresBackend) {
|
|
1512
|
-
return backend.getPool();
|
|
1513
1789
|
}
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
waitUntil: async () => {
|
|
1530
|
-
throw waitError();
|
|
1531
|
-
},
|
|
1532
|
-
createToken: async () => {
|
|
1533
|
-
throw waitError();
|
|
1534
|
-
},
|
|
1535
|
-
waitForToken: async () => {
|
|
1536
|
-
throw waitError();
|
|
1537
|
-
},
|
|
1538
|
-
setProgress: async (percent) => {
|
|
1539
|
-
if (percent < 0 || percent > 100)
|
|
1540
|
-
throw new Error("Progress must be between 0 and 100");
|
|
1541
|
-
await backend.updateProgress(jobId, Math.round(percent));
|
|
1542
|
-
}
|
|
1543
|
-
};
|
|
1544
|
-
}
|
|
1545
|
-
function validateHandlerSerializable(handler, jobType) {
|
|
1546
|
-
try {
|
|
1547
|
-
const handlerString = handler.toString();
|
|
1548
|
-
if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
|
|
1549
|
-
throw new Error(
|
|
1550
|
-
`Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
|
|
1551
|
-
);
|
|
1552
|
-
}
|
|
1553
|
-
if (handlerString.includes("[native code]")) {
|
|
1554
|
-
throw new Error(
|
|
1555
|
-
`Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
|
|
1790
|
+
/** Get a cron schedule by ID. */
|
|
1791
|
+
async getCronSchedule(id) {
|
|
1792
|
+
const client = await this.pool.connect();
|
|
1793
|
+
try {
|
|
1794
|
+
const result = await client.query(
|
|
1795
|
+
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1796
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1797
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1798
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1799
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1800
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1801
|
+
next_run_at AS "nextRunAt",
|
|
1802
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1803
|
+
FROM cron_schedules WHERE id = $1`,
|
|
1804
|
+
[id]
|
|
1556
1805
|
);
|
|
1806
|
+
if (result.rows.length === 0) return null;
|
|
1807
|
+
return result.rows[0];
|
|
1808
|
+
} catch (error) {
|
|
1809
|
+
log(`Error getting cron schedule ${id}: ${error}`);
|
|
1810
|
+
throw error;
|
|
1811
|
+
} finally {
|
|
1812
|
+
client.release();
|
|
1557
1813
|
}
|
|
1814
|
+
}
|
|
1815
|
+
/** Get a cron schedule by its unique name. */
|
|
1816
|
+
async getCronScheduleByName(name) {
|
|
1817
|
+
const client = await this.pool.connect();
|
|
1558
1818
|
try {
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1819
|
+
const result = await client.query(
|
|
1820
|
+
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1821
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1822
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1823
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1824
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1825
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1826
|
+
next_run_at AS "nextRunAt",
|
|
1827
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1828
|
+
FROM cron_schedules WHERE schedule_name = $1`,
|
|
1829
|
+
[name]
|
|
1563
1830
|
);
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1831
|
+
if (result.rows.length === 0) return null;
|
|
1832
|
+
return result.rows[0];
|
|
1833
|
+
} catch (error) {
|
|
1834
|
+
log(`Error getting cron schedule by name "${name}": ${error}`);
|
|
1567
1835
|
throw error;
|
|
1836
|
+
} finally {
|
|
1837
|
+
client.release();
|
|
1568
1838
|
}
|
|
1569
|
-
throw new Error(
|
|
1570
|
-
`Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
|
|
1571
|
-
);
|
|
1572
1839
|
}
|
|
1573
|
-
|
|
1574
|
-
async
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
}, timeoutMs);
|
|
1591
|
-
|
|
1592
|
-
try {
|
|
1593
|
-
// Execute the handler
|
|
1594
|
-
// Note: This uses Function constructor which requires the handler to be serializable.
|
|
1595
|
-
// The handler should be validated before reaching this point.
|
|
1596
|
-
let handlerFn;
|
|
1597
|
-
try {
|
|
1598
|
-
// Wrap handlerCode in parentheses to ensure it's treated as an expression
|
|
1599
|
-
// This handles both arrow functions and regular functions
|
|
1600
|
-
const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
|
|
1601
|
-
? handlerCode
|
|
1602
|
-
: '(' + handlerCode + ')';
|
|
1603
|
-
handlerFn = new Function('return ' + wrappedCode)();
|
|
1604
|
-
} catch (parseError) {
|
|
1605
|
-
clearTimeout(timeoutId);
|
|
1606
|
-
parentPort.postMessage({
|
|
1607
|
-
type: 'error',
|
|
1608
|
-
error: {
|
|
1609
|
-
message: 'Handler cannot be deserialized in worker thread. ' +
|
|
1610
|
-
'Ensure your handler is a standalone function without closures over external variables. ' +
|
|
1611
|
-
'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
|
|
1612
|
-
stack: parseError instanceof Error ? parseError.stack : undefined,
|
|
1613
|
-
name: 'SerializationError',
|
|
1614
|
-
},
|
|
1615
|
-
});
|
|
1616
|
-
return;
|
|
1617
|
-
}
|
|
1618
|
-
|
|
1619
|
-
// Ensure handlerFn is actually a function
|
|
1620
|
-
if (typeof handlerFn !== 'function') {
|
|
1621
|
-
clearTimeout(timeoutId);
|
|
1622
|
-
parentPort.postMessage({
|
|
1623
|
-
type: 'error',
|
|
1624
|
-
error: {
|
|
1625
|
-
message: 'Handler deserialization did not produce a function. ' +
|
|
1626
|
-
'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
|
|
1627
|
-
name: 'SerializationError',
|
|
1628
|
-
},
|
|
1629
|
-
});
|
|
1630
|
-
return;
|
|
1631
|
-
}
|
|
1632
|
-
|
|
1633
|
-
handlerFn(payload, signal)
|
|
1634
|
-
.then(() => {
|
|
1635
|
-
clearTimeout(timeoutId);
|
|
1636
|
-
parentPort.postMessage({ type: 'success' });
|
|
1637
|
-
})
|
|
1638
|
-
.catch((error) => {
|
|
1639
|
-
clearTimeout(timeoutId);
|
|
1640
|
-
parentPort.postMessage({
|
|
1641
|
-
type: 'error',
|
|
1642
|
-
error: {
|
|
1643
|
-
message: error.message,
|
|
1644
|
-
stack: error.stack,
|
|
1645
|
-
name: error.name,
|
|
1646
|
-
},
|
|
1647
|
-
});
|
|
1648
|
-
});
|
|
1649
|
-
} catch (error) {
|
|
1650
|
-
clearTimeout(timeoutId);
|
|
1651
|
-
parentPort.postMessage({
|
|
1652
|
-
type: 'error',
|
|
1653
|
-
error: {
|
|
1654
|
-
message: error.message,
|
|
1655
|
-
stack: error.stack,
|
|
1656
|
-
name: error.name,
|
|
1657
|
-
},
|
|
1658
|
-
});
|
|
1659
|
-
}
|
|
1660
|
-
})();
|
|
1661
|
-
`;
|
|
1662
|
-
const worker = new Worker(workerCode, {
|
|
1663
|
-
eval: true,
|
|
1664
|
-
workerData: {
|
|
1665
|
-
handlerCode: handler.toString(),
|
|
1666
|
-
payload,
|
|
1667
|
-
timeoutMs
|
|
1840
|
+
/** List cron schedules, optionally filtered by status. */
|
|
1841
|
+
async listCronSchedules(status) {
|
|
1842
|
+
const client = await this.pool.connect();
|
|
1843
|
+
try {
|
|
1844
|
+
let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1845
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1846
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1847
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1848
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1849
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1850
|
+
next_run_at AS "nextRunAt",
|
|
1851
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1852
|
+
FROM cron_schedules`;
|
|
1853
|
+
const params = [];
|
|
1854
|
+
if (status) {
|
|
1855
|
+
query += ` WHERE status = $1`;
|
|
1856
|
+
params.push(status);
|
|
1668
1857
|
}
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1858
|
+
query += ` ORDER BY created_at ASC`;
|
|
1859
|
+
const result = await client.query(query, params);
|
|
1860
|
+
return result.rows;
|
|
1861
|
+
} catch (error) {
|
|
1862
|
+
log(`Error listing cron schedules: ${error}`);
|
|
1863
|
+
throw error;
|
|
1864
|
+
} finally {
|
|
1865
|
+
client.release();
|
|
1866
|
+
}
|
|
1867
|
+
}
|
|
1868
|
+
/** Delete a cron schedule by ID. */
|
|
1869
|
+
async removeCronSchedule(id) {
|
|
1870
|
+
const client = await this.pool.connect();
|
|
1871
|
+
try {
|
|
1872
|
+
await client.query(`DELETE FROM cron_schedules WHERE id = $1`, [id]);
|
|
1873
|
+
log(`Removed cron schedule ${id}`);
|
|
1874
|
+
} catch (error) {
|
|
1875
|
+
log(`Error removing cron schedule ${id}: ${error}`);
|
|
1876
|
+
throw error;
|
|
1877
|
+
} finally {
|
|
1878
|
+
client.release();
|
|
1879
|
+
}
|
|
1880
|
+
}
|
|
1881
|
+
/** Pause a cron schedule. */
|
|
1882
|
+
async pauseCronSchedule(id) {
|
|
1883
|
+
const client = await this.pool.connect();
|
|
1884
|
+
try {
|
|
1885
|
+
await client.query(
|
|
1886
|
+
`UPDATE cron_schedules SET status = 'paused', updated_at = NOW() WHERE id = $1`,
|
|
1887
|
+
[id]
|
|
1888
|
+
);
|
|
1889
|
+
log(`Paused cron schedule ${id}`);
|
|
1890
|
+
} catch (error) {
|
|
1891
|
+
log(`Error pausing cron schedule ${id}: ${error}`);
|
|
1892
|
+
throw error;
|
|
1893
|
+
} finally {
|
|
1894
|
+
client.release();
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
/** Resume a paused cron schedule. */
|
|
1898
|
+
async resumeCronSchedule(id) {
|
|
1899
|
+
const client = await this.pool.connect();
|
|
1900
|
+
try {
|
|
1901
|
+
await client.query(
|
|
1902
|
+
`UPDATE cron_schedules SET status = 'active', updated_at = NOW() WHERE id = $1`,
|
|
1903
|
+
[id]
|
|
1904
|
+
);
|
|
1905
|
+
log(`Resumed cron schedule ${id}`);
|
|
1906
|
+
} catch (error) {
|
|
1907
|
+
log(`Error resuming cron schedule ${id}: ${error}`);
|
|
1908
|
+
throw error;
|
|
1909
|
+
} finally {
|
|
1910
|
+
client.release();
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
/** Edit a cron schedule. */
|
|
1914
|
+
async editCronSchedule(id, updates, nextRunAt) {
|
|
1915
|
+
const client = await this.pool.connect();
|
|
1916
|
+
try {
|
|
1917
|
+
const updateFields = [];
|
|
1918
|
+
const params = [];
|
|
1919
|
+
let paramIdx = 1;
|
|
1920
|
+
if (updates.cronExpression !== void 0) {
|
|
1921
|
+
updateFields.push(`cron_expression = $${paramIdx++}`);
|
|
1922
|
+
params.push(updates.cronExpression);
|
|
1687
1923
|
}
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
resolved = true;
|
|
1692
|
-
reject(error);
|
|
1693
|
-
});
|
|
1694
|
-
worker.on("exit", (code) => {
|
|
1695
|
-
if (resolved) return;
|
|
1696
|
-
if (code !== 0) {
|
|
1697
|
-
resolved = true;
|
|
1698
|
-
reject(new Error(`Worker stopped with exit code ${code}`));
|
|
1924
|
+
if (updates.payload !== void 0) {
|
|
1925
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
1926
|
+
params.push(updates.payload);
|
|
1699
1927
|
}
|
|
1700
|
-
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
resolved = true;
|
|
1704
|
-
worker.terminate().then(() => {
|
|
1705
|
-
const timeoutError = new Error(
|
|
1706
|
-
`Job timed out after ${timeoutMs} ms and was forcefully terminated`
|
|
1707
|
-
);
|
|
1708
|
-
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
1709
|
-
reject(timeoutError);
|
|
1710
|
-
}).catch((err) => {
|
|
1711
|
-
reject(err);
|
|
1712
|
-
});
|
|
1928
|
+
if (updates.maxAttempts !== void 0) {
|
|
1929
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
1930
|
+
params.push(updates.maxAttempts);
|
|
1713
1931
|
}
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
function calculateWaitUntil(duration) {
|
|
1718
|
-
const now = Date.now();
|
|
1719
|
-
let ms = 0;
|
|
1720
|
-
if (duration.seconds) ms += duration.seconds * 1e3;
|
|
1721
|
-
if (duration.minutes) ms += duration.minutes * 60 * 1e3;
|
|
1722
|
-
if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
|
|
1723
|
-
if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
|
|
1724
|
-
if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
|
|
1725
|
-
if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
|
|
1726
|
-
if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
|
|
1727
|
-
if (ms <= 0) {
|
|
1728
|
-
throw new Error(
|
|
1729
|
-
"waitFor duration must be positive. Provide at least one positive duration field."
|
|
1730
|
-
);
|
|
1731
|
-
}
|
|
1732
|
-
return new Date(now + ms);
|
|
1733
|
-
}
|
|
1734
|
-
async function resolveCompletedWaits(pool, stepData) {
|
|
1735
|
-
for (const key of Object.keys(stepData)) {
|
|
1736
|
-
if (!key.startsWith("__wait_")) continue;
|
|
1737
|
-
const entry = stepData[key];
|
|
1738
|
-
if (!entry || typeof entry !== "object" || entry.completed) continue;
|
|
1739
|
-
if (entry.type === "duration" || entry.type === "date") {
|
|
1740
|
-
stepData[key] = { ...entry, completed: true };
|
|
1741
|
-
} else if (entry.type === "token" && entry.tokenId) {
|
|
1742
|
-
const wp = await getWaitpoint(pool, entry.tokenId);
|
|
1743
|
-
if (wp && wp.status === "completed") {
|
|
1744
|
-
stepData[key] = {
|
|
1745
|
-
...entry,
|
|
1746
|
-
completed: true,
|
|
1747
|
-
result: { ok: true, output: wp.output }
|
|
1748
|
-
};
|
|
1749
|
-
} else if (wp && wp.status === "timed_out") {
|
|
1750
|
-
stepData[key] = {
|
|
1751
|
-
...entry,
|
|
1752
|
-
completed: true,
|
|
1753
|
-
result: { ok: false, error: "Token timed out" }
|
|
1754
|
-
};
|
|
1932
|
+
if (updates.priority !== void 0) {
|
|
1933
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
1934
|
+
params.push(updates.priority);
|
|
1755
1935
|
}
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
function buildWaitContext(backend, pool, jobId, stepData, baseCtx) {
|
|
1760
|
-
let waitCounter = 0;
|
|
1761
|
-
const ctx = {
|
|
1762
|
-
prolong: baseCtx.prolong,
|
|
1763
|
-
onTimeout: baseCtx.onTimeout,
|
|
1764
|
-
run: async (stepName, fn) => {
|
|
1765
|
-
const cached = stepData[stepName];
|
|
1766
|
-
if (cached && typeof cached === "object" && cached.__completed) {
|
|
1767
|
-
log(`Step "${stepName}" replayed from cache for job ${jobId}`);
|
|
1768
|
-
return cached.result;
|
|
1936
|
+
if (updates.timeoutMs !== void 0) {
|
|
1937
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
1938
|
+
params.push(updates.timeoutMs);
|
|
1769
1939
|
}
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
return result;
|
|
1774
|
-
},
|
|
1775
|
-
waitFor: async (duration) => {
|
|
1776
|
-
const waitKey = `__wait_${waitCounter++}`;
|
|
1777
|
-
const cached = stepData[waitKey];
|
|
1778
|
-
if (cached && typeof cached === "object" && cached.completed) {
|
|
1779
|
-
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
1780
|
-
return;
|
|
1940
|
+
if (updates.forceKillOnTimeout !== void 0) {
|
|
1941
|
+
updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
|
|
1942
|
+
params.push(updates.forceKillOnTimeout);
|
|
1781
1943
|
}
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
},
|
|
1786
|
-
waitUntil: async (date) => {
|
|
1787
|
-
const waitKey = `__wait_${waitCounter++}`;
|
|
1788
|
-
const cached = stepData[waitKey];
|
|
1789
|
-
if (cached && typeof cached === "object" && cached.completed) {
|
|
1790
|
-
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
1791
|
-
return;
|
|
1944
|
+
if (updates.tags !== void 0) {
|
|
1945
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
1946
|
+
params.push(updates.tags);
|
|
1792
1947
|
}
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
createToken: async (options) => {
|
|
1797
|
-
const token = await createWaitpoint(pool, jobId, options);
|
|
1798
|
-
return token;
|
|
1799
|
-
},
|
|
1800
|
-
waitForToken: async (tokenId) => {
|
|
1801
|
-
const waitKey = `__wait_${waitCounter++}`;
|
|
1802
|
-
const cached = stepData[waitKey];
|
|
1803
|
-
if (cached && typeof cached === "object" && cached.completed) {
|
|
1804
|
-
log(
|
|
1805
|
-
`Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
|
|
1806
|
-
);
|
|
1807
|
-
return cached.result;
|
|
1948
|
+
if (updates.timezone !== void 0) {
|
|
1949
|
+
updateFields.push(`timezone = $${paramIdx++}`);
|
|
1950
|
+
params.push(updates.timezone);
|
|
1808
1951
|
}
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
ok: true,
|
|
1813
|
-
output: wp.output
|
|
1814
|
-
};
|
|
1815
|
-
stepData[waitKey] = {
|
|
1816
|
-
type: "token",
|
|
1817
|
-
tokenId,
|
|
1818
|
-
completed: true,
|
|
1819
|
-
result
|
|
1820
|
-
};
|
|
1821
|
-
await updateStepData(pool, jobId, stepData);
|
|
1822
|
-
return result;
|
|
1952
|
+
if (updates.allowOverlap !== void 0) {
|
|
1953
|
+
updateFields.push(`allow_overlap = $${paramIdx++}`);
|
|
1954
|
+
params.push(updates.allowOverlap);
|
|
1823
1955
|
}
|
|
1824
|
-
if (
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
tokenId,
|
|
1832
|
-
completed: true,
|
|
1833
|
-
result
|
|
1834
|
-
};
|
|
1835
|
-
await updateStepData(pool, jobId, stepData);
|
|
1836
|
-
return result;
|
|
1956
|
+
if (nextRunAt !== void 0) {
|
|
1957
|
+
updateFields.push(`next_run_at = $${paramIdx++}`);
|
|
1958
|
+
params.push(nextRunAt);
|
|
1959
|
+
}
|
|
1960
|
+
if (updateFields.length === 0) {
|
|
1961
|
+
log(`No fields to update for cron schedule ${id}`);
|
|
1962
|
+
return;
|
|
1837
1963
|
}
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1964
|
+
updateFields.push(`updated_at = NOW()`);
|
|
1965
|
+
params.push(id);
|
|
1966
|
+
const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
|
|
1967
|
+
await client.query(query, params);
|
|
1968
|
+
log(`Edited cron schedule ${id}`);
|
|
1969
|
+
} catch (error) {
|
|
1970
|
+
log(`Error editing cron schedule ${id}: ${error}`);
|
|
1971
|
+
throw error;
|
|
1972
|
+
} finally {
|
|
1973
|
+
client.release();
|
|
1845
1974
|
}
|
|
1846
|
-
};
|
|
1847
|
-
return ctx;
|
|
1848
|
-
}
|
|
1849
|
-
async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
1850
|
-
const handler = jobHandlers[job.jobType];
|
|
1851
|
-
if (!handler) {
|
|
1852
|
-
await backend.setPendingReasonForUnpickedJobs(
|
|
1853
|
-
`No handler registered for job type: ${job.jobType}`,
|
|
1854
|
-
job.jobType
|
|
1855
|
-
);
|
|
1856
|
-
await backend.failJob(
|
|
1857
|
-
job.id,
|
|
1858
|
-
new Error(`No handler registered for job type: ${job.jobType}`),
|
|
1859
|
-
"no_handler" /* NoHandler */
|
|
1860
|
-
);
|
|
1861
|
-
return;
|
|
1862
|
-
}
|
|
1863
|
-
const stepData = { ...job.stepData || {} };
|
|
1864
|
-
const pool = tryExtractPool(backend);
|
|
1865
|
-
const hasStepHistory = Object.keys(stepData).some(
|
|
1866
|
-
(k) => k.startsWith("__wait_")
|
|
1867
|
-
);
|
|
1868
|
-
if (hasStepHistory && pool) {
|
|
1869
|
-
await resolveCompletedWaits(pool, stepData);
|
|
1870
|
-
await updateStepData(pool, job.id, stepData);
|
|
1871
1975
|
}
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
const timeoutError = new Error(`Job timed out after ${ms} ms`);
|
|
1902
|
-
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
1903
|
-
if (timeoutReject) {
|
|
1904
|
-
timeoutReject(timeoutError);
|
|
1905
|
-
}
|
|
1906
|
-
}, ms);
|
|
1907
|
-
};
|
|
1908
|
-
const hasTimeout = timeoutMs != null && timeoutMs > 0;
|
|
1909
|
-
const baseCtx = hasTimeout ? {
|
|
1910
|
-
prolong: (ms) => {
|
|
1911
|
-
const duration = ms ?? timeoutMs;
|
|
1912
|
-
if (duration != null && duration > 0) {
|
|
1913
|
-
armTimeout(duration);
|
|
1914
|
-
backend.prolongJob(job.id).catch(() => {
|
|
1915
|
-
});
|
|
1916
|
-
}
|
|
1917
|
-
},
|
|
1918
|
-
onTimeout: (callback) => {
|
|
1919
|
-
onTimeoutCallback = callback;
|
|
1920
|
-
}
|
|
1921
|
-
} : {
|
|
1922
|
-
prolong: () => {
|
|
1923
|
-
log("prolong() called but ignored: job has no timeout set");
|
|
1924
|
-
},
|
|
1925
|
-
onTimeout: () => {
|
|
1926
|
-
log("onTimeout() called but ignored: job has no timeout set");
|
|
1927
|
-
}
|
|
1928
|
-
};
|
|
1929
|
-
const ctx = pool ? buildWaitContext(backend, pool, job.id, stepData, baseCtx) : buildBasicContext(backend, job.id, baseCtx);
|
|
1930
|
-
if (forceKillOnTimeout && !hasTimeout) {
|
|
1931
|
-
log(
|
|
1932
|
-
`forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
|
|
1933
|
-
);
|
|
1934
|
-
}
|
|
1935
|
-
const jobPromise = handler(job.payload, controller.signal, ctx);
|
|
1936
|
-
if (hasTimeout) {
|
|
1937
|
-
await Promise.race([
|
|
1938
|
-
jobPromise,
|
|
1939
|
-
new Promise((_, reject) => {
|
|
1940
|
-
timeoutReject = reject;
|
|
1941
|
-
armTimeout(timeoutMs);
|
|
1942
|
-
})
|
|
1943
|
-
]);
|
|
1944
|
-
} else {
|
|
1945
|
-
await jobPromise;
|
|
1976
|
+
/**
|
|
1977
|
+
* Atomically fetch all active cron schedules whose nextRunAt <= NOW().
|
|
1978
|
+
* Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
|
|
1979
|
+
*/
|
|
1980
|
+
async getDueCronSchedules() {
|
|
1981
|
+
const client = await this.pool.connect();
|
|
1982
|
+
try {
|
|
1983
|
+
const result = await client.query(
|
|
1984
|
+
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1985
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1986
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1987
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1988
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1989
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1990
|
+
next_run_at AS "nextRunAt",
|
|
1991
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1992
|
+
FROM cron_schedules
|
|
1993
|
+
WHERE status = 'active'
|
|
1994
|
+
AND next_run_at IS NOT NULL
|
|
1995
|
+
AND next_run_at <= NOW()
|
|
1996
|
+
ORDER BY next_run_at ASC
|
|
1997
|
+
FOR UPDATE SKIP LOCKED`
|
|
1998
|
+
);
|
|
1999
|
+
log(`Found ${result.rows.length} due cron schedules`);
|
|
2000
|
+
return result.rows;
|
|
2001
|
+
} catch (error) {
|
|
2002
|
+
if (error?.code === "42P01") {
|
|
2003
|
+
log("cron_schedules table does not exist, skipping cron enqueue");
|
|
2004
|
+
return [];
|
|
1946
2005
|
}
|
|
2006
|
+
log(`Error getting due cron schedules: ${error}`);
|
|
2007
|
+
throw error;
|
|
2008
|
+
} finally {
|
|
2009
|
+
client.release();
|
|
1947
2010
|
}
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
1952
|
-
|
|
1953
|
-
|
|
1954
|
-
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
2011
|
+
}
|
|
2012
|
+
/**
|
|
2013
|
+
* Update a cron schedule after a job has been enqueued.
|
|
2014
|
+
* Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
|
|
2015
|
+
*/
|
|
2016
|
+
async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
|
|
2017
|
+
const client = await this.pool.connect();
|
|
2018
|
+
try {
|
|
2019
|
+
await client.query(
|
|
2020
|
+
`UPDATE cron_schedules
|
|
2021
|
+
SET last_enqueued_at = $2,
|
|
2022
|
+
last_job_id = $3,
|
|
2023
|
+
next_run_at = $4,
|
|
2024
|
+
updated_at = NOW()
|
|
2025
|
+
WHERE id = $1`,
|
|
2026
|
+
[id, lastEnqueuedAt, lastJobId, nextRunAt]
|
|
2027
|
+
);
|
|
1963
2028
|
log(
|
|
1964
|
-
`
|
|
2029
|
+
`Updated cron schedule ${id}: lastJobId=${lastJobId}, nextRunAt=${nextRunAt?.toISOString() ?? "null"}`
|
|
1965
2030
|
);
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
1970
|
-
|
|
1971
|
-
return;
|
|
1972
|
-
}
|
|
1973
|
-
console.error(`Error processing job ${job.id}:`, error);
|
|
1974
|
-
let failureReason = "handler_error" /* HandlerError */;
|
|
1975
|
-
if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
|
|
1976
|
-
failureReason = "timeout" /* Timeout */;
|
|
2031
|
+
} catch (error) {
|
|
2032
|
+
log(`Error updating cron schedule ${id} after enqueue: ${error}`);
|
|
2033
|
+
throw error;
|
|
2034
|
+
} finally {
|
|
2035
|
+
client.release();
|
|
1977
2036
|
}
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
onError(err instanceof Error ? err : new Error(String(err)));
|
|
2015
|
-
}
|
|
2016
|
-
next();
|
|
2017
|
-
});
|
|
2018
|
-
}
|
|
2019
|
-
};
|
|
2020
|
-
next();
|
|
2021
|
-
});
|
|
2022
|
-
}
|
|
2023
|
-
var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
2024
|
-
const {
|
|
2025
|
-
workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
|
|
2026
|
-
batchSize = 10,
|
|
2027
|
-
pollInterval = 5e3,
|
|
2028
|
-
onError = (error) => console.error("Job processor error:", error),
|
|
2029
|
-
jobType,
|
|
2030
|
-
concurrency = 3
|
|
2031
|
-
} = options;
|
|
2032
|
-
let running = false;
|
|
2033
|
-
let intervalId = null;
|
|
2034
|
-
let currentBatchPromise = null;
|
|
2035
|
-
setLogContext(options.verbose ?? false);
|
|
2036
|
-
const processJobs = async () => {
|
|
2037
|
-
if (!running) return 0;
|
|
2038
|
-
if (onBeforeBatch) {
|
|
2039
|
-
try {
|
|
2040
|
-
await onBeforeBatch();
|
|
2041
|
-
} catch (hookError) {
|
|
2042
|
-
log(`onBeforeBatch hook error: ${hookError}`);
|
|
2043
|
-
if (onError) {
|
|
2044
|
-
onError(
|
|
2045
|
-
hookError instanceof Error ? hookError : new Error(String(hookError))
|
|
2046
|
-
);
|
|
2047
|
-
}
|
|
2037
|
+
}
|
|
2038
|
+
// ── Wait / step-data support ────────────────────────────────────────
|
|
2039
|
+
/**
|
|
2040
|
+
* Transition a job from 'processing' to 'waiting' status.
|
|
2041
|
+
* Persists step data so the handler can resume from where it left off.
|
|
2042
|
+
*
|
|
2043
|
+
* @param jobId - The job to pause.
|
|
2044
|
+
* @param options - Wait configuration including optional waitUntil date, token ID, and step data.
|
|
2045
|
+
*/
|
|
2046
|
+
async waitJob(jobId, options) {
|
|
2047
|
+
const client = await this.pool.connect();
|
|
2048
|
+
try {
|
|
2049
|
+
const result = await client.query(
|
|
2050
|
+
`
|
|
2051
|
+
UPDATE job_queue
|
|
2052
|
+
SET status = 'waiting',
|
|
2053
|
+
wait_until = $2,
|
|
2054
|
+
wait_token_id = $3,
|
|
2055
|
+
step_data = $4,
|
|
2056
|
+
locked_at = NULL,
|
|
2057
|
+
locked_by = NULL,
|
|
2058
|
+
updated_at = NOW()
|
|
2059
|
+
WHERE id = $1 AND status = 'processing'
|
|
2060
|
+
`,
|
|
2061
|
+
[
|
|
2062
|
+
jobId,
|
|
2063
|
+
options.waitUntil ?? null,
|
|
2064
|
+
options.waitTokenId ?? null,
|
|
2065
|
+
JSON.stringify(options.stepData)
|
|
2066
|
+
]
|
|
2067
|
+
);
|
|
2068
|
+
if (result.rowCount === 0) {
|
|
2069
|
+
log(
|
|
2070
|
+
`Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
|
|
2071
|
+
);
|
|
2072
|
+
return;
|
|
2048
2073
|
}
|
|
2074
|
+
await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
|
|
2075
|
+
waitUntil: options.waitUntil?.toISOString() ?? null,
|
|
2076
|
+
waitTokenId: options.waitTokenId ?? null
|
|
2077
|
+
});
|
|
2078
|
+
log(`Job ${jobId} set to waiting`);
|
|
2079
|
+
} catch (error) {
|
|
2080
|
+
log(`Error setting job ${jobId} to waiting: ${error}`);
|
|
2081
|
+
throw error;
|
|
2082
|
+
} finally {
|
|
2083
|
+
client.release();
|
|
2049
2084
|
}
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2085
|
+
}
|
|
2086
|
+
/**
|
|
2087
|
+
* Persist step data for a job. Called after each ctx.run() step completes.
|
|
2088
|
+
* Best-effort: does not throw to avoid killing the running handler.
|
|
2089
|
+
*
|
|
2090
|
+
* @param jobId - The job to update.
|
|
2091
|
+
* @param stepData - The step data to persist.
|
|
2092
|
+
*/
|
|
2093
|
+
async updateStepData(jobId, stepData) {
|
|
2094
|
+
const client = await this.pool.connect();
|
|
2053
2095
|
try {
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
batchSize,
|
|
2058
|
-
jobType,
|
|
2059
|
-
handlers,
|
|
2060
|
-
concurrency,
|
|
2061
|
-
onError
|
|
2096
|
+
await client.query(
|
|
2097
|
+
`UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
|
|
2098
|
+
[jobId, JSON.stringify(stepData)]
|
|
2062
2099
|
);
|
|
2063
|
-
return processed;
|
|
2064
2100
|
} catch (error) {
|
|
2065
|
-
|
|
2101
|
+
log(`Error updating step_data for job ${jobId}: ${error}`);
|
|
2102
|
+
} finally {
|
|
2103
|
+
client.release();
|
|
2066
2104
|
}
|
|
2067
|
-
return 0;
|
|
2068
|
-
};
|
|
2069
|
-
return {
|
|
2070
|
-
/**
|
|
2071
|
-
* Start the job processor in the background.
|
|
2072
|
-
* - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
|
|
2073
|
-
* - You have to call the stop method to stop the processor.
|
|
2074
|
-
*/
|
|
2075
|
-
startInBackground: () => {
|
|
2076
|
-
if (running) return;
|
|
2077
|
-
log(`Starting job processor with workerId: ${workerId}`);
|
|
2078
|
-
running = true;
|
|
2079
|
-
const scheduleNext = (immediate) => {
|
|
2080
|
-
if (!running) return;
|
|
2081
|
-
if (immediate) {
|
|
2082
|
-
intervalId = setTimeout(loop, 0);
|
|
2083
|
-
} else {
|
|
2084
|
-
intervalId = setTimeout(loop, pollInterval);
|
|
2085
|
-
}
|
|
2086
|
-
};
|
|
2087
|
-
const loop = async () => {
|
|
2088
|
-
if (!running) return;
|
|
2089
|
-
currentBatchPromise = processJobs();
|
|
2090
|
-
const processed = await currentBatchPromise;
|
|
2091
|
-
currentBatchPromise = null;
|
|
2092
|
-
scheduleNext(processed === batchSize);
|
|
2093
|
-
};
|
|
2094
|
-
loop();
|
|
2095
|
-
},
|
|
2096
|
-
/**
|
|
2097
|
-
* Stop the job processor that runs in the background.
|
|
2098
|
-
* Does not wait for in-flight jobs.
|
|
2099
|
-
*/
|
|
2100
|
-
stop: () => {
|
|
2101
|
-
log(`Stopping job processor with workerId: ${workerId}`);
|
|
2102
|
-
running = false;
|
|
2103
|
-
if (intervalId) {
|
|
2104
|
-
clearTimeout(intervalId);
|
|
2105
|
-
intervalId = null;
|
|
2106
|
-
}
|
|
2107
|
-
},
|
|
2108
|
-
/**
|
|
2109
|
-
* Stop the job processor and wait for all in-flight jobs to complete.
|
|
2110
|
-
* Useful for graceful shutdown (e.g., SIGTERM handling).
|
|
2111
|
-
*/
|
|
2112
|
-
stopAndDrain: async (drainTimeoutMs = 3e4) => {
|
|
2113
|
-
log(`Stopping and draining job processor with workerId: ${workerId}`);
|
|
2114
|
-
running = false;
|
|
2115
|
-
if (intervalId) {
|
|
2116
|
-
clearTimeout(intervalId);
|
|
2117
|
-
intervalId = null;
|
|
2118
|
-
}
|
|
2119
|
-
if (currentBatchPromise) {
|
|
2120
|
-
await Promise.race([
|
|
2121
|
-
currentBatchPromise.catch(() => {
|
|
2122
|
-
}),
|
|
2123
|
-
new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
|
|
2124
|
-
]);
|
|
2125
|
-
currentBatchPromise = null;
|
|
2126
|
-
}
|
|
2127
|
-
log(`Job processor ${workerId} drained`);
|
|
2128
|
-
},
|
|
2129
|
-
/**
|
|
2130
|
-
* Start the job processor synchronously.
|
|
2131
|
-
* - This will process all jobs immediately and then stop.
|
|
2132
|
-
* - The pollInterval is ignored.
|
|
2133
|
-
*/
|
|
2134
|
-
start: async () => {
|
|
2135
|
-
log(`Starting job processor with workerId: ${workerId}`);
|
|
2136
|
-
running = true;
|
|
2137
|
-
const processed = await processJobs();
|
|
2138
|
-
running = false;
|
|
2139
|
-
return processed;
|
|
2140
|
-
},
|
|
2141
|
-
isRunning: () => running
|
|
2142
|
-
};
|
|
2143
|
-
};
|
|
2144
|
-
function loadPemOrFile(value) {
|
|
2145
|
-
if (!value) return void 0;
|
|
2146
|
-
if (value.startsWith("file://")) {
|
|
2147
|
-
const filePath = value.slice(7);
|
|
2148
|
-
return fs.readFileSync(filePath, "utf8");
|
|
2149
2105
|
}
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2106
|
+
/**
|
|
2107
|
+
* Create a waitpoint token in the database.
|
|
2108
|
+
*
|
|
2109
|
+
* @param jobId - The job ID to associate with the token (null if created outside a handler).
|
|
2110
|
+
* @param options - Optional timeout string (e.g. '10m', '1h') and tags.
|
|
2111
|
+
* @returns The created waitpoint with its unique ID.
|
|
2112
|
+
*/
|
|
2113
|
+
async createWaitpoint(jobId, options) {
|
|
2114
|
+
const client = await this.pool.connect();
|
|
2158
2115
|
try {
|
|
2159
|
-
const
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2116
|
+
const id = `wp_${randomUUID()}`;
|
|
2117
|
+
let timeoutAt = null;
|
|
2118
|
+
if (options?.timeout) {
|
|
2119
|
+
const ms = parseTimeoutString(options.timeout);
|
|
2120
|
+
timeoutAt = new Date(Date.now() + ms);
|
|
2164
2121
|
}
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2122
|
+
await client.query(
|
|
2123
|
+
`INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
|
|
2124
|
+
[id, jobId, timeoutAt, options?.tags ?? null]
|
|
2125
|
+
);
|
|
2126
|
+
log(`Created waitpoint ${id} for job ${jobId}`);
|
|
2127
|
+
return { id };
|
|
2128
|
+
} catch (error) {
|
|
2129
|
+
log(`Error creating waitpoint: ${error}`);
|
|
2130
|
+
throw error;
|
|
2131
|
+
} finally {
|
|
2132
|
+
client.release();
|
|
2133
|
+
}
|
|
2134
|
+
}
|
|
2135
|
+
/**
|
|
2136
|
+
* Complete a waitpoint token and move the associated job back to 'pending'.
|
|
2137
|
+
*
|
|
2138
|
+
* @param tokenId - The waitpoint token ID to complete.
|
|
2139
|
+
* @param data - Optional data to pass to the waiting handler.
|
|
2140
|
+
*/
|
|
2141
|
+
async completeWaitpoint(tokenId, data) {
|
|
2142
|
+
const client = await this.pool.connect();
|
|
2143
|
+
try {
|
|
2144
|
+
await client.query("BEGIN");
|
|
2145
|
+
const wpResult = await client.query(
|
|
2146
|
+
`UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
|
|
2147
|
+
WHERE id = $1 AND status = 'waiting'
|
|
2148
|
+
RETURNING job_id`,
|
|
2149
|
+
[tokenId, data != null ? JSON.stringify(data) : null]
|
|
2150
|
+
);
|
|
2151
|
+
if (wpResult.rows.length === 0) {
|
|
2152
|
+
await client.query("ROLLBACK");
|
|
2153
|
+
log(`Waitpoint ${tokenId} not found or already completed`);
|
|
2154
|
+
return;
|
|
2172
2155
|
}
|
|
2173
|
-
|
|
2174
|
-
if (
|
|
2175
|
-
|
|
2156
|
+
const jobId = wpResult.rows[0].job_id;
|
|
2157
|
+
if (jobId != null) {
|
|
2158
|
+
await client.query(
|
|
2159
|
+
`UPDATE job_queue
|
|
2160
|
+
SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
|
|
2161
|
+
WHERE id = $1 AND status = 'waiting'`,
|
|
2162
|
+
[jobId]
|
|
2163
|
+
);
|
|
2176
2164
|
}
|
|
2165
|
+
await client.query("COMMIT");
|
|
2166
|
+
log(`Completed waitpoint ${tokenId} for job ${jobId}`);
|
|
2167
|
+
} catch (error) {
|
|
2168
|
+
await client.query("ROLLBACK");
|
|
2169
|
+
log(`Error completing waitpoint ${tokenId}: ${error}`);
|
|
2170
|
+
throw error;
|
|
2171
|
+
} finally {
|
|
2172
|
+
client.release();
|
|
2177
2173
|
}
|
|
2178
2174
|
}
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2175
|
+
/**
|
|
2176
|
+
* Retrieve a waitpoint token by its ID.
|
|
2177
|
+
*
|
|
2178
|
+
* @param tokenId - The waitpoint token ID to look up.
|
|
2179
|
+
* @returns The waitpoint record, or null if not found.
|
|
2180
|
+
*/
|
|
2181
|
+
async getWaitpoint(tokenId) {
|
|
2182
|
+
const client = await this.pool.connect();
|
|
2183
|
+
try {
|
|
2184
|
+
const result = await client.query(
|
|
2185
|
+
`SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
|
|
2186
|
+
[tokenId]
|
|
2187
|
+
);
|
|
2188
|
+
if (result.rows.length === 0) return null;
|
|
2189
|
+
return result.rows[0];
|
|
2190
|
+
} catch (error) {
|
|
2191
|
+
log(`Error getting waitpoint ${tokenId}: ${error}`);
|
|
2192
|
+
throw error;
|
|
2193
|
+
} finally {
|
|
2194
|
+
client.release();
|
|
2186
2195
|
}
|
|
2187
|
-
const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
|
|
2188
|
-
ssl = {
|
|
2189
|
-
...ssl,
|
|
2190
|
-
...caValue ? { ca: caValue } : {},
|
|
2191
|
-
cert: loadPemOrFile(
|
|
2192
|
-
typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
|
|
2193
|
-
),
|
|
2194
|
-
key: loadPemOrFile(
|
|
2195
|
-
typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
|
|
2196
|
-
),
|
|
2197
|
-
rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
|
|
2198
|
-
};
|
|
2199
2196
|
}
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2197
|
+
/**
|
|
2198
|
+
* Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
|
|
2199
|
+
*
|
|
2200
|
+
* @returns The number of tokens that were expired.
|
|
2201
|
+
*/
|
|
2202
|
+
async expireTimedOutWaitpoints() {
|
|
2203
|
+
const client = await this.pool.connect();
|
|
2204
|
+
try {
|
|
2205
|
+
await client.query("BEGIN");
|
|
2206
|
+
const result = await client.query(
|
|
2207
|
+
`UPDATE waitpoints
|
|
2208
|
+
SET status = 'timed_out'
|
|
2209
|
+
WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
|
|
2210
|
+
RETURNING id, job_id`
|
|
2211
|
+
);
|
|
2212
|
+
for (const row of result.rows) {
|
|
2213
|
+
if (row.job_id != null) {
|
|
2214
|
+
await client.query(
|
|
2215
|
+
`UPDATE job_queue
|
|
2216
|
+
SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
|
|
2217
|
+
WHERE id = $1 AND status = 'waiting'`,
|
|
2218
|
+
[row.job_id]
|
|
2219
|
+
);
|
|
2220
|
+
}
|
|
2221
|
+
}
|
|
2222
|
+
await client.query("COMMIT");
|
|
2223
|
+
const count = result.rowCount || 0;
|
|
2224
|
+
if (count > 0) {
|
|
2225
|
+
log(`Expired ${count} timed-out waitpoints`);
|
|
2226
|
+
}
|
|
2227
|
+
return count;
|
|
2228
|
+
} catch (error) {
|
|
2229
|
+
await client.query("ROLLBACK");
|
|
2230
|
+
log(`Error expiring timed-out waitpoints: ${error}`);
|
|
2231
|
+
throw error;
|
|
2232
|
+
} finally {
|
|
2233
|
+
client.release();
|
|
2234
|
+
}
|
|
2214
2235
|
}
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2236
|
+
// ── Internal helpers ──────────────────────────────────────────────────
|
|
2237
|
+
async setPendingReasonForUnpickedJobs(reason, jobType) {
|
|
2238
|
+
const client = await this.pool.connect();
|
|
2239
|
+
try {
|
|
2240
|
+
let jobTypeFilter = "";
|
|
2241
|
+
const params = [reason];
|
|
2242
|
+
if (jobType) {
|
|
2243
|
+
if (Array.isArray(jobType)) {
|
|
2244
|
+
jobTypeFilter = ` AND job_type = ANY($2)`;
|
|
2245
|
+
params.push(jobType);
|
|
2246
|
+
} else {
|
|
2247
|
+
jobTypeFilter = ` AND job_type = $2`;
|
|
2248
|
+
params.push(jobType);
|
|
2249
|
+
}
|
|
2250
|
+
}
|
|
2251
|
+
await client.query(
|
|
2252
|
+
`UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
|
|
2253
|
+
params
|
|
2254
|
+
);
|
|
2255
|
+
} finally {
|
|
2256
|
+
client.release();
|
|
2257
|
+
}
|
|
2223
2258
|
}
|
|
2224
|
-
return pool;
|
|
2225
2259
|
};
|
|
2226
2260
|
|
|
2227
2261
|
// src/backends/redis-scripts.ts
|
|
@@ -2278,7 +2312,10 @@ redis.call('HMSET', jobKey,
|
|
|
2278
2312
|
'lastFailedAt', 'null',
|
|
2279
2313
|
'lastCancelledAt', 'null',
|
|
2280
2314
|
'tags', tagsJson,
|
|
2281
|
-
'idempotencyKey', idempotencyKey
|
|
2315
|
+
'idempotencyKey', idempotencyKey,
|
|
2316
|
+
'waitUntil', 'null',
|
|
2317
|
+
'waitTokenId', 'null',
|
|
2318
|
+
'stepData', 'null'
|
|
2282
2319
|
)
|
|
2283
2320
|
|
|
2284
2321
|
-- Status index
|
|
@@ -2361,7 +2398,25 @@ for _, jobId in ipairs(retries) do
|
|
|
2361
2398
|
redis.call('ZREM', prefix .. 'retry', jobId)
|
|
2362
2399
|
end
|
|
2363
2400
|
|
|
2364
|
-
-- 3.
|
|
2401
|
+
-- 3. Move ready waiting jobs (time-based, no token) into queue
|
|
2402
|
+
local waitingJobs = redis.call('ZRANGEBYSCORE', prefix .. 'waiting', '-inf', nowMs, 'LIMIT', 0, 200)
|
|
2403
|
+
for _, jobId in ipairs(waitingJobs) do
|
|
2404
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2405
|
+
local status = redis.call('HGET', jk, 'status')
|
|
2406
|
+
local waitTokenId = redis.call('HGET', jk, 'waitTokenId')
|
|
2407
|
+
if status == 'waiting' and (waitTokenId == false or waitTokenId == 'null') then
|
|
2408
|
+
local pri = tonumber(redis.call('HGET', jk, 'priority') or '0')
|
|
2409
|
+
local ca = tonumber(redis.call('HGET', jk, 'createdAt'))
|
|
2410
|
+
local score = pri * ${SCORE_RANGE} + (${SCORE_RANGE} - ca)
|
|
2411
|
+
redis.call('ZADD', prefix .. 'queue', score, jobId)
|
|
2412
|
+
redis.call('SREM', prefix .. 'status:waiting', jobId)
|
|
2413
|
+
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2414
|
+
redis.call('HMSET', jk, 'status', 'pending', 'waitUntil', 'null')
|
|
2415
|
+
end
|
|
2416
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2417
|
+
end
|
|
2418
|
+
|
|
2419
|
+
-- 4. Parse job type filter
|
|
2365
2420
|
local filterTypes = nil
|
|
2366
2421
|
if jobTypeFilter ~= "null" then
|
|
2367
2422
|
-- Could be a JSON array or a plain string
|
|
@@ -2374,7 +2429,7 @@ if jobTypeFilter ~= "null" then
|
|
|
2374
2429
|
end
|
|
2375
2430
|
end
|
|
2376
2431
|
|
|
2377
|
-
--
|
|
2432
|
+
-- 5. Pop candidates from queue (highest score first)
|
|
2378
2433
|
-- We pop more than batchSize because some may be filtered out
|
|
2379
2434
|
local popCount = batchSize * 3
|
|
2380
2435
|
local candidates = redis.call('ZPOPMAX', prefix .. 'queue', popCount)
|
|
@@ -2458,7 +2513,10 @@ local jk = prefix .. 'job:' .. jobId
|
|
|
2458
2513
|
redis.call('HMSET', jk,
|
|
2459
2514
|
'status', 'completed',
|
|
2460
2515
|
'updatedAt', nowMs,
|
|
2461
|
-
'completedAt', nowMs
|
|
2516
|
+
'completedAt', nowMs,
|
|
2517
|
+
'stepData', 'null',
|
|
2518
|
+
'waitUntil', 'null',
|
|
2519
|
+
'waitTokenId', 'null'
|
|
2462
2520
|
)
|
|
2463
2521
|
redis.call('SREM', prefix .. 'status:processing', jobId)
|
|
2464
2522
|
redis.call('SADD', prefix .. 'status:completed', jobId)
|
|
@@ -2517,6 +2575,7 @@ local nowMs = tonumber(ARGV[2])
|
|
|
2517
2575
|
local jk = prefix .. 'job:' .. jobId
|
|
2518
2576
|
|
|
2519
2577
|
local oldStatus = redis.call('HGET', jk, 'status')
|
|
2578
|
+
if oldStatus ~= 'failed' and oldStatus ~= 'processing' then return 0 end
|
|
2520
2579
|
|
|
2521
2580
|
redis.call('HMSET', jk,
|
|
2522
2581
|
'status', 'pending',
|
|
@@ -2528,9 +2587,7 @@ redis.call('HMSET', jk,
|
|
|
2528
2587
|
)
|
|
2529
2588
|
|
|
2530
2589
|
-- Remove from old status, add to pending
|
|
2531
|
-
|
|
2532
|
-
redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
|
|
2533
|
-
end
|
|
2590
|
+
redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
|
|
2534
2591
|
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2535
2592
|
|
|
2536
2593
|
-- Remove from retry sorted set if present
|
|
@@ -2551,18 +2608,21 @@ local nowMs = ARGV[2]
|
|
|
2551
2608
|
local jk = prefix .. 'job:' .. jobId
|
|
2552
2609
|
|
|
2553
2610
|
local status = redis.call('HGET', jk, 'status')
|
|
2554
|
-
if status ~= 'pending' then return 0 end
|
|
2611
|
+
if status ~= 'pending' and status ~= 'waiting' then return 0 end
|
|
2555
2612
|
|
|
2556
2613
|
redis.call('HMSET', jk,
|
|
2557
2614
|
'status', 'cancelled',
|
|
2558
2615
|
'updatedAt', nowMs,
|
|
2559
|
-
'lastCancelledAt', nowMs
|
|
2616
|
+
'lastCancelledAt', nowMs,
|
|
2617
|
+
'waitUntil', 'null',
|
|
2618
|
+
'waitTokenId', 'null'
|
|
2560
2619
|
)
|
|
2561
|
-
redis.call('SREM', prefix .. 'status:
|
|
2620
|
+
redis.call('SREM', prefix .. 'status:' .. status, jobId)
|
|
2562
2621
|
redis.call('SADD', prefix .. 'status:cancelled', jobId)
|
|
2563
|
-
-- Remove from queue / delayed
|
|
2622
|
+
-- Remove from queue / delayed / waiting
|
|
2564
2623
|
redis.call('ZREM', prefix .. 'queue', jobId)
|
|
2565
2624
|
redis.call('ZREM', prefix .. 'delayed', jobId)
|
|
2625
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2566
2626
|
|
|
2567
2627
|
return 1
|
|
2568
2628
|
`;
|
|
@@ -2630,18 +2690,16 @@ end
|
|
|
2630
2690
|
|
|
2631
2691
|
return count
|
|
2632
2692
|
`;
|
|
2633
|
-
var
|
|
2693
|
+
var CLEANUP_OLD_JOBS_BATCH_SCRIPT = `
|
|
2634
2694
|
local prefix = KEYS[1]
|
|
2635
2695
|
local cutoffMs = tonumber(ARGV[1])
|
|
2636
|
-
|
|
2637
|
-
local completed = redis.call('SMEMBERS', prefix .. 'status:completed')
|
|
2638
2696
|
local count = 0
|
|
2639
2697
|
|
|
2640
|
-
for
|
|
2698
|
+
for i = 2, #ARGV do
|
|
2699
|
+
local jobId = ARGV[i]
|
|
2641
2700
|
local jk = prefix .. 'job:' .. jobId
|
|
2642
2701
|
local updatedAt = tonumber(redis.call('HGET', jk, 'updatedAt'))
|
|
2643
2702
|
if updatedAt and updatedAt < cutoffMs then
|
|
2644
|
-
-- Remove all indexes
|
|
2645
2703
|
local jobType = redis.call('HGET', jk, 'jobType')
|
|
2646
2704
|
local tagsJson = redis.call('HGET', jk, 'tags')
|
|
2647
2705
|
local idempotencyKey = redis.call('HGET', jk, 'idempotencyKey')
|
|
@@ -2664,7 +2722,6 @@ for _, jobId in ipairs(completed) do
|
|
|
2664
2722
|
if idempotencyKey and idempotencyKey ~= 'null' then
|
|
2665
2723
|
redis.call('DEL', prefix .. 'idempotency:' .. idempotencyKey)
|
|
2666
2724
|
end
|
|
2667
|
-
-- Delete events
|
|
2668
2725
|
redis.call('DEL', prefix .. 'events:' .. jobId)
|
|
2669
2726
|
|
|
2670
2727
|
count = count + 1
|
|
@@ -2673,8 +2730,158 @@ end
|
|
|
2673
2730
|
|
|
2674
2731
|
return count
|
|
2675
2732
|
`;
|
|
2733
|
+
var WAIT_JOB_SCRIPT = `
|
|
2734
|
+
local prefix = KEYS[1]
|
|
2735
|
+
local jobId = ARGV[1]
|
|
2736
|
+
local waitUntilMs = ARGV[2]
|
|
2737
|
+
local waitTokenId = ARGV[3]
|
|
2738
|
+
local stepDataJson = ARGV[4]
|
|
2739
|
+
local nowMs = ARGV[5]
|
|
2740
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2741
|
+
|
|
2742
|
+
local status = redis.call('HGET', jk, 'status')
|
|
2743
|
+
if status ~= 'processing' then return 0 end
|
|
2744
|
+
|
|
2745
|
+
redis.call('HMSET', jk,
|
|
2746
|
+
'status', 'waiting',
|
|
2747
|
+
'waitUntil', waitUntilMs,
|
|
2748
|
+
'waitTokenId', waitTokenId,
|
|
2749
|
+
'stepData', stepDataJson,
|
|
2750
|
+
'lockedAt', 'null',
|
|
2751
|
+
'lockedBy', 'null',
|
|
2752
|
+
'updatedAt', nowMs
|
|
2753
|
+
)
|
|
2754
|
+
redis.call('SREM', prefix .. 'status:processing', jobId)
|
|
2755
|
+
redis.call('SADD', prefix .. 'status:waiting', jobId)
|
|
2756
|
+
|
|
2757
|
+
-- Add to waiting sorted set if time-based wait
|
|
2758
|
+
if waitUntilMs ~= 'null' then
|
|
2759
|
+
redis.call('ZADD', prefix .. 'waiting', tonumber(waitUntilMs), jobId)
|
|
2760
|
+
end
|
|
2761
|
+
|
|
2762
|
+
return 1
|
|
2763
|
+
`;
|
|
2764
|
+
var COMPLETE_WAITPOINT_SCRIPT = `
|
|
2765
|
+
local prefix = KEYS[1]
|
|
2766
|
+
local tokenId = ARGV[1]
|
|
2767
|
+
local outputJson = ARGV[2]
|
|
2768
|
+
local nowMs = ARGV[3]
|
|
2769
|
+
local wpk = prefix .. 'waitpoint:' .. tokenId
|
|
2770
|
+
|
|
2771
|
+
local wpStatus = redis.call('HGET', wpk, 'status')
|
|
2772
|
+
if not wpStatus or wpStatus ~= 'waiting' then return 0 end
|
|
2773
|
+
|
|
2774
|
+
redis.call('HMSET', wpk,
|
|
2775
|
+
'status', 'completed',
|
|
2776
|
+
'output', outputJson,
|
|
2777
|
+
'completedAt', nowMs
|
|
2778
|
+
)
|
|
2779
|
+
|
|
2780
|
+
-- Move associated job back to pending
|
|
2781
|
+
local jobId = redis.call('HGET', wpk, 'jobId')
|
|
2782
|
+
if jobId and jobId ~= 'null' then
|
|
2783
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2784
|
+
local jobStatus = redis.call('HGET', jk, 'status')
|
|
2785
|
+
if jobStatus == 'waiting' then
|
|
2786
|
+
redis.call('HMSET', jk,
|
|
2787
|
+
'status', 'pending',
|
|
2788
|
+
'waitTokenId', 'null',
|
|
2789
|
+
'waitUntil', 'null',
|
|
2790
|
+
'updatedAt', nowMs
|
|
2791
|
+
)
|
|
2792
|
+
redis.call('SREM', prefix .. 'status:waiting', jobId)
|
|
2793
|
+
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2794
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2795
|
+
|
|
2796
|
+
-- Re-add to queue
|
|
2797
|
+
local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
|
|
2798
|
+
local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
|
|
2799
|
+
local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
|
|
2800
|
+
redis.call('ZADD', prefix .. 'queue', score, jobId)
|
|
2801
|
+
end
|
|
2802
|
+
end
|
|
2803
|
+
|
|
2804
|
+
return 1
|
|
2805
|
+
`;
|
|
2806
|
+
var EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT = `
|
|
2807
|
+
local prefix = KEYS[1]
|
|
2808
|
+
local nowMs = tonumber(ARGV[1])
|
|
2809
|
+
|
|
2810
|
+
local expiredIds = redis.call('ZRANGEBYSCORE', prefix .. 'waitpoint_timeout', '-inf', nowMs)
|
|
2811
|
+
local count = 0
|
|
2812
|
+
|
|
2813
|
+
for _, tokenId in ipairs(expiredIds) do
|
|
2814
|
+
local wpk = prefix .. 'waitpoint:' .. tokenId
|
|
2815
|
+
local wpStatus = redis.call('HGET', wpk, 'status')
|
|
2816
|
+
if wpStatus == 'waiting' then
|
|
2817
|
+
redis.call('HMSET', wpk,
|
|
2818
|
+
'status', 'timed_out'
|
|
2819
|
+
)
|
|
2820
|
+
|
|
2821
|
+
-- Move associated job back to pending
|
|
2822
|
+
local jobId = redis.call('HGET', wpk, 'jobId')
|
|
2823
|
+
if jobId and jobId ~= 'null' then
|
|
2824
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2825
|
+
local jobStatus = redis.call('HGET', jk, 'status')
|
|
2826
|
+
if jobStatus == 'waiting' then
|
|
2827
|
+
redis.call('HMSET', jk,
|
|
2828
|
+
'status', 'pending',
|
|
2829
|
+
'waitTokenId', 'null',
|
|
2830
|
+
'waitUntil', 'null',
|
|
2831
|
+
'updatedAt', nowMs
|
|
2832
|
+
)
|
|
2833
|
+
redis.call('SREM', prefix .. 'status:waiting', jobId)
|
|
2834
|
+
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2835
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2836
|
+
|
|
2837
|
+
local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
|
|
2838
|
+
local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
|
|
2839
|
+
local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
|
|
2840
|
+
redis.call('ZADD', prefix .. 'queue', score, jobId)
|
|
2841
|
+
end
|
|
2842
|
+
end
|
|
2843
|
+
|
|
2844
|
+
count = count + 1
|
|
2845
|
+
end
|
|
2846
|
+
redis.call('ZREM', prefix .. 'waitpoint_timeout', tokenId)
|
|
2847
|
+
end
|
|
2676
2848
|
|
|
2677
|
-
|
|
2849
|
+
return count
|
|
2850
|
+
`;
|
|
2851
|
+
var MAX_TIMEOUT_MS2 = 365 * 24 * 60 * 60 * 1e3;
|
|
2852
|
+
function parseTimeoutString2(timeout) {
|
|
2853
|
+
const match = timeout.match(/^(\d+)(s|m|h|d)$/);
|
|
2854
|
+
if (!match) {
|
|
2855
|
+
throw new Error(
|
|
2856
|
+
`Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
|
|
2857
|
+
);
|
|
2858
|
+
}
|
|
2859
|
+
const value = parseInt(match[1], 10);
|
|
2860
|
+
const unit = match[2];
|
|
2861
|
+
let ms;
|
|
2862
|
+
switch (unit) {
|
|
2863
|
+
case "s":
|
|
2864
|
+
ms = value * 1e3;
|
|
2865
|
+
break;
|
|
2866
|
+
case "m":
|
|
2867
|
+
ms = value * 60 * 1e3;
|
|
2868
|
+
break;
|
|
2869
|
+
case "h":
|
|
2870
|
+
ms = value * 60 * 60 * 1e3;
|
|
2871
|
+
break;
|
|
2872
|
+
case "d":
|
|
2873
|
+
ms = value * 24 * 60 * 60 * 1e3;
|
|
2874
|
+
break;
|
|
2875
|
+
default:
|
|
2876
|
+
throw new Error(`Unknown timeout unit: "${unit}"`);
|
|
2877
|
+
}
|
|
2878
|
+
if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS2) {
|
|
2879
|
+
throw new Error(
|
|
2880
|
+
`Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
|
|
2881
|
+
);
|
|
2882
|
+
}
|
|
2883
|
+
return ms;
|
|
2884
|
+
}
|
|
2678
2885
|
function hashToObject(arr) {
|
|
2679
2886
|
const obj = {};
|
|
2680
2887
|
for (let i = 0; i < arr.length; i += 2) {
|
|
@@ -2740,9 +2947,20 @@ function deserializeJob(h) {
|
|
|
2740
2947
|
lastCancelledAt: dateOrNull(h.lastCancelledAt),
|
|
2741
2948
|
tags,
|
|
2742
2949
|
idempotencyKey: nullish(h.idempotencyKey),
|
|
2743
|
-
progress: numOrNull(h.progress)
|
|
2950
|
+
progress: numOrNull(h.progress),
|
|
2951
|
+
waitUntil: dateOrNull(h.waitUntil),
|
|
2952
|
+
waitTokenId: nullish(h.waitTokenId),
|
|
2953
|
+
stepData: parseStepData(h.stepData)
|
|
2744
2954
|
};
|
|
2745
2955
|
}
|
|
2956
|
+
function parseStepData(raw) {
|
|
2957
|
+
if (!raw || raw === "null") return void 0;
|
|
2958
|
+
try {
|
|
2959
|
+
return JSON.parse(raw);
|
|
2960
|
+
} catch {
|
|
2961
|
+
return void 0;
|
|
2962
|
+
}
|
|
2963
|
+
}
|
|
2746
2964
|
var RedisBackend = class {
|
|
2747
2965
|
constructor(redisConfig) {
|
|
2748
2966
|
let IORedis;
|
|
@@ -2898,8 +3116,14 @@ var RedisBackend = class {
|
|
|
2898
3116
|
if (filters.runAt) {
|
|
2899
3117
|
jobs = this.filterByRunAt(jobs, filters.runAt);
|
|
2900
3118
|
}
|
|
3119
|
+
if (filters.cursor !== void 0) {
|
|
3120
|
+
jobs = jobs.filter((j) => j.id < filters.cursor);
|
|
3121
|
+
}
|
|
3122
|
+
}
|
|
3123
|
+
jobs.sort((a, b) => b.id - a.id);
|
|
3124
|
+
if (filters?.cursor !== void 0) {
|
|
3125
|
+
return jobs.slice(0, limit);
|
|
2901
3126
|
}
|
|
2902
|
-
jobs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
|
2903
3127
|
return jobs.slice(offset, offset + limit);
|
|
2904
3128
|
}
|
|
2905
3129
|
async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
|
|
@@ -3131,35 +3355,302 @@ var RedisBackend = class {
|
|
|
3131
3355
|
log(`Edited ${count} pending jobs`);
|
|
3132
3356
|
return count;
|
|
3133
3357
|
}
|
|
3134
|
-
|
|
3358
|
+
/**
|
|
3359
|
+
* Delete completed jobs older than the given number of days.
|
|
3360
|
+
* Uses SSCAN to iterate the completed set in batches, avoiding
|
|
3361
|
+
* loading all IDs into memory and preventing long Redis blocks.
|
|
3362
|
+
*
|
|
3363
|
+
* @param daysToKeep - Number of days to retain completed jobs (default 30).
|
|
3364
|
+
* @param batchSize - Number of IDs to scan per SSCAN iteration (default 200).
|
|
3365
|
+
* @returns Total number of deleted jobs.
|
|
3366
|
+
*/
|
|
3367
|
+
async cleanupOldJobs(daysToKeep = 30, batchSize = 200) {
|
|
3368
|
+
const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
|
|
3369
|
+
const setKey = `${this.prefix}status:completed`;
|
|
3370
|
+
let totalDeleted = 0;
|
|
3371
|
+
let cursor = "0";
|
|
3372
|
+
do {
|
|
3373
|
+
const [nextCursor, ids] = await this.client.sscan(
|
|
3374
|
+
setKey,
|
|
3375
|
+
cursor,
|
|
3376
|
+
"COUNT",
|
|
3377
|
+
batchSize
|
|
3378
|
+
);
|
|
3379
|
+
cursor = nextCursor;
|
|
3380
|
+
if (ids.length > 0) {
|
|
3381
|
+
const result = await this.client.eval(
|
|
3382
|
+
CLEANUP_OLD_JOBS_BATCH_SCRIPT,
|
|
3383
|
+
1,
|
|
3384
|
+
this.prefix,
|
|
3385
|
+
cutoffMs,
|
|
3386
|
+
...ids
|
|
3387
|
+
);
|
|
3388
|
+
totalDeleted += Number(result);
|
|
3389
|
+
}
|
|
3390
|
+
} while (cursor !== "0");
|
|
3391
|
+
log(`Deleted ${totalDeleted} old jobs`);
|
|
3392
|
+
return totalDeleted;
|
|
3393
|
+
}
|
|
3394
|
+
/**
|
|
3395
|
+
* Delete job events older than the given number of days.
|
|
3396
|
+
* Iterates all event lists and removes events whose createdAt is before the cutoff.
|
|
3397
|
+
* Also removes orphaned event lists (where the job no longer exists).
|
|
3398
|
+
*
|
|
3399
|
+
* @param daysToKeep - Number of days to retain events (default 30).
|
|
3400
|
+
* @param batchSize - Number of event keys to scan per SCAN iteration (default 200).
|
|
3401
|
+
* @returns Total number of deleted events.
|
|
3402
|
+
*/
|
|
3403
|
+
async cleanupOldJobEvents(daysToKeep = 30, batchSize = 200) {
|
|
3135
3404
|
const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
|
|
3405
|
+
const pattern = `${this.prefix}events:*`;
|
|
3406
|
+
let totalDeleted = 0;
|
|
3407
|
+
let cursor = "0";
|
|
3408
|
+
do {
|
|
3409
|
+
const [nextCursor, keys] = await this.client.scan(
|
|
3410
|
+
cursor,
|
|
3411
|
+
"MATCH",
|
|
3412
|
+
pattern,
|
|
3413
|
+
"COUNT",
|
|
3414
|
+
batchSize
|
|
3415
|
+
);
|
|
3416
|
+
cursor = nextCursor;
|
|
3417
|
+
for (const key of keys) {
|
|
3418
|
+
const jobIdStr = key.slice(`${this.prefix}events:`.length);
|
|
3419
|
+
const jobExists = await this.client.exists(
|
|
3420
|
+
`${this.prefix}job:${jobIdStr}`
|
|
3421
|
+
);
|
|
3422
|
+
if (!jobExists) {
|
|
3423
|
+
const len = await this.client.llen(key);
|
|
3424
|
+
await this.client.del(key);
|
|
3425
|
+
totalDeleted += len;
|
|
3426
|
+
continue;
|
|
3427
|
+
}
|
|
3428
|
+
const events = await this.client.lrange(key, 0, -1);
|
|
3429
|
+
const kept = [];
|
|
3430
|
+
for (const raw of events) {
|
|
3431
|
+
try {
|
|
3432
|
+
const e = JSON.parse(raw);
|
|
3433
|
+
if (e.createdAt >= cutoffMs) {
|
|
3434
|
+
kept.push(raw);
|
|
3435
|
+
} else {
|
|
3436
|
+
totalDeleted++;
|
|
3437
|
+
}
|
|
3438
|
+
} catch {
|
|
3439
|
+
totalDeleted++;
|
|
3440
|
+
}
|
|
3441
|
+
}
|
|
3442
|
+
if (kept.length === 0) {
|
|
3443
|
+
await this.client.del(key);
|
|
3444
|
+
} else if (kept.length < events.length) {
|
|
3445
|
+
const pipeline = this.client.pipeline();
|
|
3446
|
+
pipeline.del(key);
|
|
3447
|
+
for (const raw of kept) {
|
|
3448
|
+
pipeline.rpush(key, raw);
|
|
3449
|
+
}
|
|
3450
|
+
await pipeline.exec();
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
} while (cursor !== "0");
|
|
3454
|
+
log(`Deleted ${totalDeleted} old job events`);
|
|
3455
|
+
return totalDeleted;
|
|
3456
|
+
}
|
|
3457
|
+
async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
|
|
3458
|
+
const maxAgeMs = maxProcessingTimeMinutes * 60 * 1e3;
|
|
3459
|
+
const now = this.nowMs();
|
|
3136
3460
|
const result = await this.client.eval(
|
|
3137
|
-
|
|
3461
|
+
RECLAIM_STUCK_JOBS_SCRIPT,
|
|
3138
3462
|
1,
|
|
3139
3463
|
this.prefix,
|
|
3140
|
-
|
|
3464
|
+
maxAgeMs,
|
|
3465
|
+
now
|
|
3141
3466
|
);
|
|
3142
|
-
log(`
|
|
3467
|
+
log(`Reclaimed ${result} stuck jobs`);
|
|
3143
3468
|
return Number(result);
|
|
3144
3469
|
}
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3470
|
+
// ── Wait / step-data support ────────────────────────────────────────
|
|
3471
|
+
/**
|
|
3472
|
+
* Transition a job from 'processing' to 'waiting' status.
|
|
3473
|
+
* Persists step data so the handler can resume from where it left off.
|
|
3474
|
+
*
|
|
3475
|
+
* @param jobId - The job to pause.
|
|
3476
|
+
* @param options - Wait configuration including optional waitUntil date, token ID, and step data.
|
|
3477
|
+
*/
|
|
3478
|
+
async waitJob(jobId, options) {
|
|
3479
|
+
const now = this.nowMs();
|
|
3480
|
+
const waitUntilMs = options.waitUntil ? options.waitUntil.getTime().toString() : "null";
|
|
3481
|
+
const waitTokenId = options.waitTokenId ?? "null";
|
|
3482
|
+
const stepDataJson = JSON.stringify(options.stepData);
|
|
3483
|
+
const result = await this.client.eval(
|
|
3484
|
+
WAIT_JOB_SCRIPT,
|
|
3485
|
+
1,
|
|
3486
|
+
this.prefix,
|
|
3487
|
+
jobId,
|
|
3488
|
+
waitUntilMs,
|
|
3489
|
+
waitTokenId,
|
|
3490
|
+
stepDataJson,
|
|
3491
|
+
now
|
|
3148
3492
|
);
|
|
3149
|
-
|
|
3493
|
+
if (Number(result) === 0) {
|
|
3494
|
+
log(
|
|
3495
|
+
`Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
|
|
3496
|
+
);
|
|
3497
|
+
return;
|
|
3498
|
+
}
|
|
3499
|
+
await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
|
|
3500
|
+
waitUntil: options.waitUntil?.toISOString() ?? null,
|
|
3501
|
+
waitTokenId: options.waitTokenId ?? null
|
|
3502
|
+
});
|
|
3503
|
+
log(`Job ${jobId} set to waiting`);
|
|
3150
3504
|
}
|
|
3151
|
-
|
|
3152
|
-
|
|
3505
|
+
/**
|
|
3506
|
+
* Persist step data for a job. Called after each ctx.run() step completes.
|
|
3507
|
+
* Best-effort: does not throw to avoid killing the running handler.
|
|
3508
|
+
*
|
|
3509
|
+
* @param jobId - The job to update.
|
|
3510
|
+
* @param stepData - The step data to persist.
|
|
3511
|
+
*/
|
|
3512
|
+
async updateStepData(jobId, stepData) {
|
|
3513
|
+
try {
|
|
3514
|
+
const now = this.nowMs();
|
|
3515
|
+
await this.client.hset(
|
|
3516
|
+
`${this.prefix}job:${jobId}`,
|
|
3517
|
+
"stepData",
|
|
3518
|
+
JSON.stringify(stepData),
|
|
3519
|
+
"updatedAt",
|
|
3520
|
+
now.toString()
|
|
3521
|
+
);
|
|
3522
|
+
} catch (error) {
|
|
3523
|
+
log(`Error updating stepData for job ${jobId}: ${error}`);
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
/**
|
|
3527
|
+
* Create a waitpoint token.
|
|
3528
|
+
*
|
|
3529
|
+
* @param jobId - The job ID to associate with the token (null if created outside a handler).
|
|
3530
|
+
* @param options - Optional timeout string (e.g. '10m', '1h') and tags.
|
|
3531
|
+
* @returns The created waitpoint with its unique ID.
|
|
3532
|
+
*/
|
|
3533
|
+
async createWaitpoint(jobId, options) {
|
|
3534
|
+
const id = `wp_${randomUUID()}`;
|
|
3535
|
+
const now = this.nowMs();
|
|
3536
|
+
let timeoutAt = null;
|
|
3537
|
+
if (options?.timeout) {
|
|
3538
|
+
const ms = parseTimeoutString2(options.timeout);
|
|
3539
|
+
timeoutAt = now + ms;
|
|
3540
|
+
}
|
|
3541
|
+
const key = `${this.prefix}waitpoint:${id}`;
|
|
3542
|
+
const fields = [
|
|
3543
|
+
"id",
|
|
3544
|
+
id,
|
|
3545
|
+
"jobId",
|
|
3546
|
+
jobId !== null ? jobId.toString() : "null",
|
|
3547
|
+
"status",
|
|
3548
|
+
"waiting",
|
|
3549
|
+
"output",
|
|
3550
|
+
"null",
|
|
3551
|
+
"timeoutAt",
|
|
3552
|
+
timeoutAt !== null ? timeoutAt.toString() : "null",
|
|
3553
|
+
"createdAt",
|
|
3554
|
+
now.toString(),
|
|
3555
|
+
"completedAt",
|
|
3556
|
+
"null",
|
|
3557
|
+
"tags",
|
|
3558
|
+
options?.tags ? JSON.stringify(options.tags) : "null"
|
|
3559
|
+
];
|
|
3560
|
+
await this.client.hmset(key, ...fields);
|
|
3561
|
+
if (timeoutAt !== null) {
|
|
3562
|
+
await this.client.zadd(`${this.prefix}waitpoint_timeout`, timeoutAt, id);
|
|
3563
|
+
}
|
|
3564
|
+
log(`Created waitpoint ${id} for job ${jobId}`);
|
|
3565
|
+
return { id };
|
|
3566
|
+
}
|
|
3567
|
+
/**
|
|
3568
|
+
* Complete a waitpoint token and move the associated job back to 'pending'.
|
|
3569
|
+
*
|
|
3570
|
+
* @param tokenId - The waitpoint token ID to complete.
|
|
3571
|
+
* @param data - Optional data to pass to the waiting handler.
|
|
3572
|
+
*/
|
|
3573
|
+
async completeWaitpoint(tokenId, data) {
|
|
3153
3574
|
const now = this.nowMs();
|
|
3575
|
+
const outputJson = data != null ? JSON.stringify(data) : "null";
|
|
3154
3576
|
const result = await this.client.eval(
|
|
3155
|
-
|
|
3577
|
+
COMPLETE_WAITPOINT_SCRIPT,
|
|
3156
3578
|
1,
|
|
3157
3579
|
this.prefix,
|
|
3158
|
-
|
|
3580
|
+
tokenId,
|
|
3581
|
+
outputJson,
|
|
3159
3582
|
now
|
|
3160
3583
|
);
|
|
3161
|
-
|
|
3162
|
-
|
|
3584
|
+
if (Number(result) === 0) {
|
|
3585
|
+
log(`Waitpoint ${tokenId} not found or already completed`);
|
|
3586
|
+
return;
|
|
3587
|
+
}
|
|
3588
|
+
log(`Completed waitpoint ${tokenId}`);
|
|
3589
|
+
}
|
|
3590
|
+
/**
|
|
3591
|
+
* Retrieve a waitpoint token by its ID.
|
|
3592
|
+
*
|
|
3593
|
+
* @param tokenId - The waitpoint token ID to look up.
|
|
3594
|
+
* @returns The waitpoint record, or null if not found.
|
|
3595
|
+
*/
|
|
3596
|
+
async getWaitpoint(tokenId) {
|
|
3597
|
+
const data = await this.client.hgetall(
|
|
3598
|
+
`${this.prefix}waitpoint:${tokenId}`
|
|
3599
|
+
);
|
|
3600
|
+
if (!data || Object.keys(data).length === 0) return null;
|
|
3601
|
+
const nullish = (v) => v === void 0 || v === "null" || v === "" ? null : v;
|
|
3602
|
+
const numOrNull = (v) => {
|
|
3603
|
+
const n = nullish(v);
|
|
3604
|
+
return n === null ? null : Number(n);
|
|
3605
|
+
};
|
|
3606
|
+
const dateOrNull = (v) => {
|
|
3607
|
+
const n = numOrNull(v);
|
|
3608
|
+
return n === null ? null : new Date(n);
|
|
3609
|
+
};
|
|
3610
|
+
let output = null;
|
|
3611
|
+
if (data.output && data.output !== "null") {
|
|
3612
|
+
try {
|
|
3613
|
+
output = JSON.parse(data.output);
|
|
3614
|
+
} catch {
|
|
3615
|
+
output = data.output;
|
|
3616
|
+
}
|
|
3617
|
+
}
|
|
3618
|
+
let tags = null;
|
|
3619
|
+
if (data.tags && data.tags !== "null") {
|
|
3620
|
+
try {
|
|
3621
|
+
tags = JSON.parse(data.tags);
|
|
3622
|
+
} catch {
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
return {
|
|
3626
|
+
id: data.id,
|
|
3627
|
+
jobId: numOrNull(data.jobId),
|
|
3628
|
+
status: data.status,
|
|
3629
|
+
output,
|
|
3630
|
+
timeoutAt: dateOrNull(data.timeoutAt),
|
|
3631
|
+
createdAt: new Date(Number(data.createdAt)),
|
|
3632
|
+
completedAt: dateOrNull(data.completedAt),
|
|
3633
|
+
tags
|
|
3634
|
+
};
|
|
3635
|
+
}
|
|
3636
|
+
/**
|
|
3637
|
+
* Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
|
|
3638
|
+
*
|
|
3639
|
+
* @returns The number of tokens that were expired.
|
|
3640
|
+
*/
|
|
3641
|
+
async expireTimedOutWaitpoints() {
|
|
3642
|
+
const now = this.nowMs();
|
|
3643
|
+
const result = await this.client.eval(
|
|
3644
|
+
EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT,
|
|
3645
|
+
1,
|
|
3646
|
+
this.prefix,
|
|
3647
|
+
now
|
|
3648
|
+
);
|
|
3649
|
+
const count = Number(result);
|
|
3650
|
+
if (count > 0) {
|
|
3651
|
+
log(`Expired ${count} timed-out waitpoints`);
|
|
3652
|
+
}
|
|
3653
|
+
return count;
|
|
3163
3654
|
}
|
|
3164
3655
|
// ── Internal helpers ──────────────────────────────────────────────────
|
|
3165
3656
|
async setPendingReasonForUnpickedJobs(reason, jobType) {
|
|
@@ -3708,10 +4199,9 @@ var initJobQueue = (config) => {
|
|
|
3708
4199
|
const backendType = config.backend ?? "postgres";
|
|
3709
4200
|
setLogContext(config.verbose ?? false);
|
|
3710
4201
|
let backend;
|
|
3711
|
-
let pool;
|
|
3712
4202
|
if (backendType === "postgres") {
|
|
3713
4203
|
const pgConfig = config;
|
|
3714
|
-
pool = createPool(pgConfig.databaseConfig);
|
|
4204
|
+
const pool = createPool(pgConfig.databaseConfig);
|
|
3715
4205
|
backend = new PostgresBackend(pool);
|
|
3716
4206
|
} else if (backendType === "redis") {
|
|
3717
4207
|
const redisConfig = config.redisConfig;
|
|
@@ -3719,14 +4209,6 @@ var initJobQueue = (config) => {
|
|
|
3719
4209
|
} else {
|
|
3720
4210
|
throw new Error(`Unknown backend: ${backendType}`);
|
|
3721
4211
|
}
|
|
3722
|
-
const requirePool = () => {
|
|
3723
|
-
if (!pool) {
|
|
3724
|
-
throw new Error(
|
|
3725
|
-
'Wait/Token features require the PostgreSQL backend. Configure with backend: "postgres" to use these features.'
|
|
3726
|
-
);
|
|
3727
|
-
}
|
|
3728
|
-
return pool;
|
|
3729
|
-
};
|
|
3730
4212
|
const enqueueDueCronJobsImpl = async () => {
|
|
3731
4213
|
const dueSchedules = await backend.getDueCronSchedules();
|
|
3732
4214
|
let count = 0;
|
|
@@ -3793,8 +4275,8 @@ var initJobQueue = (config) => {
|
|
|
3793
4275
|
config.verbose ?? false
|
|
3794
4276
|
),
|
|
3795
4277
|
retryJob: (jobId) => backend.retryJob(jobId),
|
|
3796
|
-
cleanupOldJobs: (daysToKeep) => backend.cleanupOldJobs(daysToKeep),
|
|
3797
|
-
cleanupOldJobEvents: (daysToKeep) => backend.cleanupOldJobEvents(daysToKeep),
|
|
4278
|
+
cleanupOldJobs: (daysToKeep, batchSize) => backend.cleanupOldJobs(daysToKeep, batchSize),
|
|
4279
|
+
cleanupOldJobEvents: (daysToKeep, batchSize) => backend.cleanupOldJobEvents(daysToKeep, batchSize),
|
|
3798
4280
|
cancelJob: withLogContext(
|
|
3799
4281
|
(jobId) => backend.cancelJob(jobId),
|
|
3800
4282
|
config.verbose ?? false
|
|
@@ -3831,21 +4313,21 @@ var initJobQueue = (config) => {
|
|
|
3831
4313
|
(jobId) => backend.getJobEvents(jobId),
|
|
3832
4314
|
config.verbose ?? false
|
|
3833
4315
|
),
|
|
3834
|
-
// Wait / Token support (
|
|
4316
|
+
// Wait / Token support (works with all backends)
|
|
3835
4317
|
createToken: withLogContext(
|
|
3836
|
-
(options) => createWaitpoint(
|
|
4318
|
+
(options) => backend.createWaitpoint(null, options),
|
|
3837
4319
|
config.verbose ?? false
|
|
3838
4320
|
),
|
|
3839
4321
|
completeToken: withLogContext(
|
|
3840
|
-
(tokenId, data) => completeWaitpoint(
|
|
4322
|
+
(tokenId, data) => backend.completeWaitpoint(tokenId, data),
|
|
3841
4323
|
config.verbose ?? false
|
|
3842
4324
|
),
|
|
3843
4325
|
getToken: withLogContext(
|
|
3844
|
-
(tokenId) => getWaitpoint(
|
|
4326
|
+
(tokenId) => backend.getWaitpoint(tokenId),
|
|
3845
4327
|
config.verbose ?? false
|
|
3846
4328
|
),
|
|
3847
4329
|
expireTimedOutTokens: withLogContext(
|
|
3848
|
-
() => expireTimedOutWaitpoints(
|
|
4330
|
+
() => backend.expireTimedOutWaitpoints(),
|
|
3849
4331
|
config.verbose ?? false
|
|
3850
4332
|
),
|
|
3851
4333
|
// Cron schedule operations
|
|
@@ -3926,7 +4408,7 @@ var initJobQueue = (config) => {
|
|
|
3926
4408
|
),
|
|
3927
4409
|
// Advanced access
|
|
3928
4410
|
getPool: () => {
|
|
3929
|
-
if (
|
|
4411
|
+
if (!(backend instanceof PostgresBackend)) {
|
|
3930
4412
|
throw new Error(
|
|
3931
4413
|
"getPool() is only available with the PostgreSQL backend."
|
|
3932
4414
|
);
|