@nicnocquee/dataqueue 1.31.0 → 1.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +2418 -1936
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +151 -16
- package/dist/index.d.ts +151 -16
- package/dist/index.js +2418 -1936
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/backend.ts +70 -4
- package/src/backends/postgres.ts +345 -29
- package/src/backends/redis-scripts.ts +197 -22
- package/src/backends/redis.test.ts +621 -0
- package/src/backends/redis.ts +400 -21
- package/src/index.ts +12 -29
- package/src/processor.ts +14 -93
- package/src/queue.test.ts +29 -0
- package/src/queue.ts +19 -251
- package/src/types.ts +28 -10
package/dist/index.cjs
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
var async_hooks = require('async_hooks');
|
|
4
|
-
var crypto = require('crypto');
|
|
5
3
|
var worker_threads = require('worker_threads');
|
|
4
|
+
var async_hooks = require('async_hooks');
|
|
6
5
|
var pg = require('pg');
|
|
7
6
|
var pgConnectionString = require('pg-connection-string');
|
|
8
7
|
var fs = require('fs');
|
|
8
|
+
var crypto = require('crypto');
|
|
9
9
|
var module$1 = require('module');
|
|
10
10
|
var croner = require('croner');
|
|
11
11
|
|
|
@@ -14,6 +14,8 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
|
14
14
|
|
|
15
15
|
var fs__default = /*#__PURE__*/_interopDefault(fs);
|
|
16
16
|
|
|
17
|
+
// src/processor.ts
|
|
18
|
+
|
|
17
19
|
// src/types.ts
|
|
18
20
|
var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
|
|
19
21
|
JobEventType2["Added"] = "added";
|
|
@@ -27,11 +29,11 @@ var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
|
|
|
27
29
|
JobEventType2["Waiting"] = "waiting";
|
|
28
30
|
return JobEventType2;
|
|
29
31
|
})(JobEventType || {});
|
|
30
|
-
var FailureReason = /* @__PURE__ */ ((
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
return
|
|
32
|
+
var FailureReason = /* @__PURE__ */ ((FailureReason4) => {
|
|
33
|
+
FailureReason4["Timeout"] = "timeout";
|
|
34
|
+
FailureReason4["HandlerError"] = "handler_error";
|
|
35
|
+
FailureReason4["NoHandler"] = "no_handler";
|
|
36
|
+
return FailureReason4;
|
|
35
37
|
})(FailureReason || {});
|
|
36
38
|
var WaitSignal = class extends Error {
|
|
37
39
|
constructor(type, waitUntil, tokenId, stepData) {
|
|
@@ -58,804 +60,942 @@ var log = (message) => {
|
|
|
58
60
|
}
|
|
59
61
|
};
|
|
60
62
|
|
|
61
|
-
// src/
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
return this.pool;
|
|
69
|
-
}
|
|
70
|
-
// ── Events ──────────────────────────────────────────────────────────
|
|
71
|
-
async recordJobEvent(jobId, eventType, metadata) {
|
|
72
|
-
const client = await this.pool.connect();
|
|
73
|
-
try {
|
|
74
|
-
await client.query(
|
|
75
|
-
`INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
|
|
76
|
-
[jobId, eventType, metadata ? JSON.stringify(metadata) : null]
|
|
77
|
-
);
|
|
78
|
-
} catch (error) {
|
|
79
|
-
log(`Error recording job event for job ${jobId}: ${error}`);
|
|
80
|
-
} finally {
|
|
81
|
-
client.release();
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
async getJobEvents(jobId) {
|
|
85
|
-
const client = await this.pool.connect();
|
|
86
|
-
try {
|
|
87
|
-
const res = await client.query(
|
|
88
|
-
`SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
|
|
89
|
-
[jobId]
|
|
90
|
-
);
|
|
91
|
-
return res.rows;
|
|
92
|
-
} finally {
|
|
93
|
-
client.release();
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
// ── Job CRUD ──────────────────────────────────────────────────────────
|
|
97
|
-
async addJob({
|
|
98
|
-
jobType,
|
|
99
|
-
payload,
|
|
100
|
-
maxAttempts = 3,
|
|
101
|
-
priority = 0,
|
|
102
|
-
runAt = null,
|
|
103
|
-
timeoutMs = void 0,
|
|
104
|
-
forceKillOnTimeout = false,
|
|
105
|
-
tags = void 0,
|
|
106
|
-
idempotencyKey = void 0
|
|
107
|
-
}) {
|
|
108
|
-
const client = await this.pool.connect();
|
|
109
|
-
try {
|
|
110
|
-
let result;
|
|
111
|
-
const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
|
|
112
|
-
if (runAt) {
|
|
113
|
-
result = await client.query(
|
|
114
|
-
`INSERT INTO job_queue
|
|
115
|
-
(job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
116
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
117
|
-
${onConflict}
|
|
118
|
-
RETURNING id`,
|
|
119
|
-
[
|
|
120
|
-
jobType,
|
|
121
|
-
payload,
|
|
122
|
-
maxAttempts,
|
|
123
|
-
priority,
|
|
124
|
-
runAt,
|
|
125
|
-
timeoutMs ?? null,
|
|
126
|
-
forceKillOnTimeout ?? false,
|
|
127
|
-
tags ?? null,
|
|
128
|
-
idempotencyKey ?? null
|
|
129
|
-
]
|
|
130
|
-
);
|
|
131
|
-
} else {
|
|
132
|
-
result = await client.query(
|
|
133
|
-
`INSERT INTO job_queue
|
|
134
|
-
(job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
135
|
-
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
136
|
-
${onConflict}
|
|
137
|
-
RETURNING id`,
|
|
138
|
-
[
|
|
139
|
-
jobType,
|
|
140
|
-
payload,
|
|
141
|
-
maxAttempts,
|
|
142
|
-
priority,
|
|
143
|
-
timeoutMs ?? null,
|
|
144
|
-
forceKillOnTimeout ?? false,
|
|
145
|
-
tags ?? null,
|
|
146
|
-
idempotencyKey ?? null
|
|
147
|
-
]
|
|
148
|
-
);
|
|
149
|
-
}
|
|
150
|
-
if (result.rows.length === 0 && idempotencyKey) {
|
|
151
|
-
const existing = await client.query(
|
|
152
|
-
`SELECT id FROM job_queue WHERE idempotency_key = $1`,
|
|
153
|
-
[idempotencyKey]
|
|
154
|
-
);
|
|
155
|
-
if (existing.rows.length > 0) {
|
|
156
|
-
log(
|
|
157
|
-
`Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
|
|
158
|
-
);
|
|
159
|
-
return existing.rows[0].id;
|
|
160
|
-
}
|
|
161
|
-
throw new Error(
|
|
162
|
-
`Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
|
|
163
|
-
);
|
|
164
|
-
}
|
|
165
|
-
const jobId = result.rows[0].id;
|
|
166
|
-
log(
|
|
167
|
-
`Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
|
|
63
|
+
// src/processor.ts
|
|
64
|
+
function validateHandlerSerializable(handler, jobType) {
|
|
65
|
+
try {
|
|
66
|
+
const handlerString = handler.toString();
|
|
67
|
+
if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
|
|
68
|
+
throw new Error(
|
|
69
|
+
`Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
|
|
168
70
|
);
|
|
169
|
-
await this.recordJobEvent(jobId, "added" /* Added */, {
|
|
170
|
-
jobType,
|
|
171
|
-
payload,
|
|
172
|
-
tags,
|
|
173
|
-
idempotencyKey
|
|
174
|
-
});
|
|
175
|
-
return jobId;
|
|
176
|
-
} catch (error) {
|
|
177
|
-
log(`Error adding job: ${error}`);
|
|
178
|
-
throw error;
|
|
179
|
-
} finally {
|
|
180
|
-
client.release();
|
|
181
71
|
}
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
try {
|
|
186
|
-
const result = await client.query(
|
|
187
|
-
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
|
|
188
|
-
[id]
|
|
72
|
+
if (handlerString.includes("[native code]")) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
`Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
|
|
189
75
|
);
|
|
190
|
-
if (result.rows.length === 0) {
|
|
191
|
-
log(`Job ${id} not found`);
|
|
192
|
-
return null;
|
|
193
|
-
}
|
|
194
|
-
log(`Found job ${id}`);
|
|
195
|
-
const job = result.rows[0];
|
|
196
|
-
return {
|
|
197
|
-
...job,
|
|
198
|
-
payload: job.payload,
|
|
199
|
-
timeoutMs: job.timeoutMs,
|
|
200
|
-
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
201
|
-
failureReason: job.failureReason
|
|
202
|
-
};
|
|
203
|
-
} catch (error) {
|
|
204
|
-
log(`Error getting job ${id}: ${error}`);
|
|
205
|
-
throw error;
|
|
206
|
-
} finally {
|
|
207
|
-
client.release();
|
|
208
76
|
}
|
|
209
|
-
}
|
|
210
|
-
async getJobsByStatus(status, limit = 100, offset = 0) {
|
|
211
|
-
const client = await this.pool.connect();
|
|
212
77
|
try {
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
78
|
+
new Function("return " + handlerString);
|
|
79
|
+
} catch (parseError) {
|
|
80
|
+
throw new Error(
|
|
81
|
+
`Handler for job type "${jobType}" cannot be serialized: ${parseError instanceof Error ? parseError.message : String(parseError)}. When using forceKillOnTimeout, handlers must be serializable functions without closures over external variables.`
|
|
216
82
|
);
|
|
217
|
-
log(`Found ${result.rows.length} jobs by status ${status}`);
|
|
218
|
-
return result.rows.map((job) => ({
|
|
219
|
-
...job,
|
|
220
|
-
payload: job.payload,
|
|
221
|
-
timeoutMs: job.timeoutMs,
|
|
222
|
-
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
223
|
-
failureReason: job.failureReason
|
|
224
|
-
}));
|
|
225
|
-
} catch (error) {
|
|
226
|
-
log(`Error getting jobs by status ${status}: ${error}`);
|
|
227
|
-
throw error;
|
|
228
|
-
} finally {
|
|
229
|
-
client.release();
|
|
230
83
|
}
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
const client = await this.pool.connect();
|
|
234
|
-
try {
|
|
235
|
-
const result = await client.query(
|
|
236
|
-
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
|
|
237
|
-
[limit, offset]
|
|
238
|
-
);
|
|
239
|
-
log(`Found ${result.rows.length} jobs (all)`);
|
|
240
|
-
return result.rows.map((job) => ({
|
|
241
|
-
...job,
|
|
242
|
-
payload: job.payload,
|
|
243
|
-
timeoutMs: job.timeoutMs,
|
|
244
|
-
forceKillOnTimeout: job.forceKillOnTimeout
|
|
245
|
-
}));
|
|
246
|
-
} catch (error) {
|
|
247
|
-
log(`Error getting all jobs: ${error}`);
|
|
84
|
+
} catch (error) {
|
|
85
|
+
if (error instanceof Error) {
|
|
248
86
|
throw error;
|
|
249
|
-
} finally {
|
|
250
|
-
client.release();
|
|
251
87
|
}
|
|
88
|
+
throw new Error(
|
|
89
|
+
`Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
|
|
90
|
+
);
|
|
252
91
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
|
|
298
|
-
const mode = filters.tags.mode || "all";
|
|
299
|
-
const tagValues = filters.tags.values;
|
|
300
|
-
switch (mode) {
|
|
301
|
-
case "exact":
|
|
302
|
-
where.push(`tags = $${paramIdx++}`);
|
|
303
|
-
params.push(tagValues);
|
|
304
|
-
break;
|
|
305
|
-
case "all":
|
|
306
|
-
where.push(`tags @> $${paramIdx++}`);
|
|
307
|
-
params.push(tagValues);
|
|
308
|
-
break;
|
|
309
|
-
case "any":
|
|
310
|
-
where.push(`tags && $${paramIdx++}`);
|
|
311
|
-
params.push(tagValues);
|
|
312
|
-
break;
|
|
313
|
-
case "none":
|
|
314
|
-
where.push(`NOT (tags && $${paramIdx++})`);
|
|
315
|
-
params.push(tagValues);
|
|
316
|
-
break;
|
|
317
|
-
default:
|
|
318
|
-
where.push(`tags @> $${paramIdx++}`);
|
|
319
|
-
params.push(tagValues);
|
|
92
|
+
}
|
|
93
|
+
async function runHandlerInWorker(handler, payload, timeoutMs, jobType) {
|
|
94
|
+
validateHandlerSerializable(handler, jobType);
|
|
95
|
+
return new Promise((resolve, reject) => {
|
|
96
|
+
const workerCode = `
|
|
97
|
+
(function() {
|
|
98
|
+
const { parentPort, workerData } = require('worker_threads');
|
|
99
|
+
const { handlerCode, payload, timeoutMs } = workerData;
|
|
100
|
+
|
|
101
|
+
// Create an AbortController for the handler
|
|
102
|
+
const controller = new AbortController();
|
|
103
|
+
const signal = controller.signal;
|
|
104
|
+
|
|
105
|
+
// Set up timeout
|
|
106
|
+
const timeoutId = setTimeout(() => {
|
|
107
|
+
controller.abort();
|
|
108
|
+
parentPort.postMessage({ type: 'timeout' });
|
|
109
|
+
}, timeoutMs);
|
|
110
|
+
|
|
111
|
+
try {
|
|
112
|
+
// Execute the handler
|
|
113
|
+
// Note: This uses Function constructor which requires the handler to be serializable.
|
|
114
|
+
// The handler should be validated before reaching this point.
|
|
115
|
+
let handlerFn;
|
|
116
|
+
try {
|
|
117
|
+
// Wrap handlerCode in parentheses to ensure it's treated as an expression
|
|
118
|
+
// This handles both arrow functions and regular functions
|
|
119
|
+
const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
|
|
120
|
+
? handlerCode
|
|
121
|
+
: '(' + handlerCode + ')';
|
|
122
|
+
handlerFn = new Function('return ' + wrappedCode)();
|
|
123
|
+
} catch (parseError) {
|
|
124
|
+
clearTimeout(timeoutId);
|
|
125
|
+
parentPort.postMessage({
|
|
126
|
+
type: 'error',
|
|
127
|
+
error: {
|
|
128
|
+
message: 'Handler cannot be deserialized in worker thread. ' +
|
|
129
|
+
'Ensure your handler is a standalone function without closures over external variables. ' +
|
|
130
|
+
'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
|
|
131
|
+
stack: parseError instanceof Error ? parseError.stack : undefined,
|
|
132
|
+
name: 'SerializationError',
|
|
133
|
+
},
|
|
134
|
+
});
|
|
135
|
+
return;
|
|
320
136
|
}
|
|
137
|
+
|
|
138
|
+
// Ensure handlerFn is actually a function
|
|
139
|
+
if (typeof handlerFn !== 'function') {
|
|
140
|
+
clearTimeout(timeoutId);
|
|
141
|
+
parentPort.postMessage({
|
|
142
|
+
type: 'error',
|
|
143
|
+
error: {
|
|
144
|
+
message: 'Handler deserialization did not produce a function. ' +
|
|
145
|
+
'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
|
|
146
|
+
name: 'SerializationError',
|
|
147
|
+
},
|
|
148
|
+
});
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
handlerFn(payload, signal)
|
|
153
|
+
.then(() => {
|
|
154
|
+
clearTimeout(timeoutId);
|
|
155
|
+
parentPort.postMessage({ type: 'success' });
|
|
156
|
+
})
|
|
157
|
+
.catch((error) => {
|
|
158
|
+
clearTimeout(timeoutId);
|
|
159
|
+
parentPort.postMessage({
|
|
160
|
+
type: 'error',
|
|
161
|
+
error: {
|
|
162
|
+
message: error.message,
|
|
163
|
+
stack: error.stack,
|
|
164
|
+
name: error.name,
|
|
165
|
+
},
|
|
166
|
+
});
|
|
167
|
+
});
|
|
168
|
+
} catch (error) {
|
|
169
|
+
clearTimeout(timeoutId);
|
|
170
|
+
parentPort.postMessage({
|
|
171
|
+
type: 'error',
|
|
172
|
+
error: {
|
|
173
|
+
message: error.message,
|
|
174
|
+
stack: error.stack,
|
|
175
|
+
name: error.name,
|
|
176
|
+
},
|
|
177
|
+
});
|
|
321
178
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
179
|
+
})();
|
|
180
|
+
`;
|
|
181
|
+
const worker = new worker_threads.Worker(workerCode, {
|
|
182
|
+
eval: true,
|
|
183
|
+
workerData: {
|
|
184
|
+
handlerCode: handler.toString(),
|
|
185
|
+
payload,
|
|
186
|
+
timeoutMs
|
|
326
187
|
}
|
|
327
|
-
|
|
328
|
-
|
|
188
|
+
});
|
|
189
|
+
let resolved = false;
|
|
190
|
+
worker.on("message", (message) => {
|
|
191
|
+
if (resolved) return;
|
|
192
|
+
resolved = true;
|
|
193
|
+
if (message.type === "success") {
|
|
194
|
+
resolve();
|
|
195
|
+
} else if (message.type === "timeout") {
|
|
196
|
+
const timeoutError = new Error(
|
|
197
|
+
`Job timed out after ${timeoutMs} ms and was forcefully terminated`
|
|
198
|
+
);
|
|
199
|
+
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
200
|
+
reject(timeoutError);
|
|
201
|
+
} else if (message.type === "error") {
|
|
202
|
+
const error = new Error(message.error.message);
|
|
203
|
+
error.stack = message.error.stack;
|
|
204
|
+
error.name = message.error.name;
|
|
205
|
+
reject(error);
|
|
329
206
|
}
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
if (
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
207
|
+
});
|
|
208
|
+
worker.on("error", (error) => {
|
|
209
|
+
if (resolved) return;
|
|
210
|
+
resolved = true;
|
|
211
|
+
reject(error);
|
|
212
|
+
});
|
|
213
|
+
worker.on("exit", (code) => {
|
|
214
|
+
if (resolved) return;
|
|
215
|
+
if (code !== 0) {
|
|
216
|
+
resolved = true;
|
|
217
|
+
reject(new Error(`Worker stopped with exit code ${code}`));
|
|
337
218
|
}
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
}
|
|
219
|
+
});
|
|
220
|
+
setTimeout(() => {
|
|
221
|
+
if (!resolved) {
|
|
222
|
+
resolved = true;
|
|
223
|
+
worker.terminate().then(() => {
|
|
224
|
+
const timeoutError = new Error(
|
|
225
|
+
`Job timed out after ${timeoutMs} ms and was forcefully terminated`
|
|
226
|
+
);
|
|
227
|
+
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
228
|
+
reject(timeoutError);
|
|
229
|
+
}).catch((err) => {
|
|
230
|
+
reject(err);
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
}, timeoutMs + 100);
|
|
234
|
+
});
|
|
235
|
+
}
|
|
236
|
+
function calculateWaitUntil(duration) {
|
|
237
|
+
const now = Date.now();
|
|
238
|
+
let ms = 0;
|
|
239
|
+
if (duration.seconds) ms += duration.seconds * 1e3;
|
|
240
|
+
if (duration.minutes) ms += duration.minutes * 60 * 1e3;
|
|
241
|
+
if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
|
|
242
|
+
if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
|
|
243
|
+
if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
|
|
244
|
+
if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
|
|
245
|
+
if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
|
|
246
|
+
if (ms <= 0) {
|
|
247
|
+
throw new Error(
|
|
248
|
+
"waitFor duration must be positive. Provide at least one positive duration field."
|
|
249
|
+
);
|
|
353
250
|
}
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
default:
|
|
378
|
-
query += " WHERE tags @> $1";
|
|
379
|
-
params = [tags];
|
|
251
|
+
return new Date(now + ms);
|
|
252
|
+
}
|
|
253
|
+
async function resolveCompletedWaits(backend, stepData) {
|
|
254
|
+
for (const key of Object.keys(stepData)) {
|
|
255
|
+
if (!key.startsWith("__wait_")) continue;
|
|
256
|
+
const entry = stepData[key];
|
|
257
|
+
if (!entry || typeof entry !== "object" || entry.completed) continue;
|
|
258
|
+
if (entry.type === "duration" || entry.type === "date") {
|
|
259
|
+
stepData[key] = { ...entry, completed: true };
|
|
260
|
+
} else if (entry.type === "token" && entry.tokenId) {
|
|
261
|
+
const wp = await backend.getWaitpoint(entry.tokenId);
|
|
262
|
+
if (wp && wp.status === "completed") {
|
|
263
|
+
stepData[key] = {
|
|
264
|
+
...entry,
|
|
265
|
+
completed: true,
|
|
266
|
+
result: { ok: true, output: wp.output }
|
|
267
|
+
};
|
|
268
|
+
} else if (wp && wp.status === "timed_out") {
|
|
269
|
+
stepData[key] = {
|
|
270
|
+
...entry,
|
|
271
|
+
completed: true,
|
|
272
|
+
result: { ok: false, error: "Token timed out" }
|
|
273
|
+
};
|
|
380
274
|
}
|
|
381
|
-
query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
|
|
382
|
-
params.push(limit, offset);
|
|
383
|
-
const result = await client.query(query, params);
|
|
384
|
-
log(
|
|
385
|
-
`Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
|
|
386
|
-
);
|
|
387
|
-
return result.rows.map((job) => ({
|
|
388
|
-
...job,
|
|
389
|
-
payload: job.payload,
|
|
390
|
-
timeoutMs: job.timeoutMs,
|
|
391
|
-
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
392
|
-
failureReason: job.failureReason
|
|
393
|
-
}));
|
|
394
|
-
} catch (error) {
|
|
395
|
-
log(
|
|
396
|
-
`Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
|
|
397
|
-
);
|
|
398
|
-
throw error;
|
|
399
|
-
} finally {
|
|
400
|
-
client.release();
|
|
401
275
|
}
|
|
402
276
|
}
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
} else {
|
|
415
|
-
jobTypeFilter = ` AND job_type = $3`;
|
|
416
|
-
params.push(jobType);
|
|
417
|
-
}
|
|
277
|
+
}
|
|
278
|
+
function buildWaitContext(backend, jobId, stepData, baseCtx) {
|
|
279
|
+
let waitCounter = 0;
|
|
280
|
+
const ctx = {
|
|
281
|
+
prolong: baseCtx.prolong,
|
|
282
|
+
onTimeout: baseCtx.onTimeout,
|
|
283
|
+
run: async (stepName, fn) => {
|
|
284
|
+
const cached = stepData[stepName];
|
|
285
|
+
if (cached && typeof cached === "object" && cached.__completed) {
|
|
286
|
+
log(`Step "${stepName}" replayed from cache for job ${jobId}`);
|
|
287
|
+
return cached.result;
|
|
418
288
|
}
|
|
419
|
-
const result = await
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
wait_until = NULL
|
|
431
|
-
WHERE id IN (
|
|
432
|
-
SELECT id FROM job_queue
|
|
433
|
-
WHERE (
|
|
434
|
-
(
|
|
435
|
-
(status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
|
|
436
|
-
AND (attempts < max_attempts)
|
|
437
|
-
AND run_at <= NOW()
|
|
438
|
-
)
|
|
439
|
-
OR (
|
|
440
|
-
status = 'waiting'
|
|
441
|
-
AND wait_until IS NOT NULL
|
|
442
|
-
AND wait_until <= NOW()
|
|
443
|
-
AND wait_token_id IS NULL
|
|
444
|
-
)
|
|
445
|
-
)
|
|
446
|
-
${jobTypeFilter}
|
|
447
|
-
ORDER BY priority DESC, created_at ASC
|
|
448
|
-
LIMIT $2
|
|
449
|
-
FOR UPDATE SKIP LOCKED
|
|
450
|
-
)
|
|
451
|
-
RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
|
|
452
|
-
`,
|
|
453
|
-
params
|
|
454
|
-
);
|
|
455
|
-
log(`Found ${result.rows.length} jobs to process`);
|
|
456
|
-
await client.query("COMMIT");
|
|
457
|
-
if (result.rows.length > 0) {
|
|
458
|
-
await this.recordJobEventsBatch(
|
|
459
|
-
result.rows.map((row) => ({
|
|
460
|
-
jobId: row.id,
|
|
461
|
-
eventType: "processing" /* Processing */
|
|
462
|
-
}))
|
|
463
|
-
);
|
|
289
|
+
const result = await fn();
|
|
290
|
+
stepData[stepName] = { __completed: true, result };
|
|
291
|
+
await backend.updateStepData(jobId, stepData);
|
|
292
|
+
return result;
|
|
293
|
+
},
|
|
294
|
+
waitFor: async (duration) => {
|
|
295
|
+
const waitKey = `__wait_${waitCounter++}`;
|
|
296
|
+
const cached = stepData[waitKey];
|
|
297
|
+
if (cached && typeof cached === "object" && cached.completed) {
|
|
298
|
+
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
299
|
+
return;
|
|
464
300
|
}
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
WHERE id = $1 AND status = 'processing'
|
|
488
|
-
`,
|
|
489
|
-
[jobId]
|
|
490
|
-
);
|
|
491
|
-
if (result.rowCount === 0) {
|
|
301
|
+
const waitUntilDate = calculateWaitUntil(duration);
|
|
302
|
+
stepData[waitKey] = { type: "duration", completed: false };
|
|
303
|
+
throw new WaitSignal("duration", waitUntilDate, void 0, stepData);
|
|
304
|
+
},
|
|
305
|
+
waitUntil: async (date) => {
|
|
306
|
+
const waitKey = `__wait_${waitCounter++}`;
|
|
307
|
+
const cached = stepData[waitKey];
|
|
308
|
+
if (cached && typeof cached === "object" && cached.completed) {
|
|
309
|
+
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
stepData[waitKey] = { type: "date", completed: false };
|
|
313
|
+
throw new WaitSignal("date", date, void 0, stepData);
|
|
314
|
+
},
|
|
315
|
+
createToken: async (options) => {
|
|
316
|
+
const token = await backend.createWaitpoint(jobId, options);
|
|
317
|
+
return token;
|
|
318
|
+
},
|
|
319
|
+
waitForToken: async (tokenId) => {
|
|
320
|
+
const waitKey = `__wait_${waitCounter++}`;
|
|
321
|
+
const cached = stepData[waitKey];
|
|
322
|
+
if (cached && typeof cached === "object" && cached.completed) {
|
|
492
323
|
log(
|
|
493
|
-
`
|
|
324
|
+
`Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
|
|
494
325
|
);
|
|
326
|
+
return cached.result;
|
|
495
327
|
}
|
|
496
|
-
await
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
328
|
+
const wp = await backend.getWaitpoint(tokenId);
|
|
329
|
+
if (wp && wp.status === "completed") {
|
|
330
|
+
const result = {
|
|
331
|
+
ok: true,
|
|
332
|
+
output: wp.output
|
|
333
|
+
};
|
|
334
|
+
stepData[waitKey] = {
|
|
335
|
+
type: "token",
|
|
336
|
+
tokenId,
|
|
337
|
+
completed: true,
|
|
338
|
+
result
|
|
339
|
+
};
|
|
340
|
+
await backend.updateStepData(jobId, stepData);
|
|
341
|
+
return result;
|
|
342
|
+
}
|
|
343
|
+
if (wp && wp.status === "timed_out") {
|
|
344
|
+
const result = {
|
|
345
|
+
ok: false,
|
|
346
|
+
error: "Token timed out"
|
|
347
|
+
};
|
|
348
|
+
stepData[waitKey] = {
|
|
349
|
+
type: "token",
|
|
350
|
+
tokenId,
|
|
351
|
+
completed: true,
|
|
352
|
+
result
|
|
353
|
+
};
|
|
354
|
+
await backend.updateStepData(jobId, stepData);
|
|
355
|
+
return result;
|
|
356
|
+
}
|
|
357
|
+
stepData[waitKey] = { type: "token", tokenId, completed: false };
|
|
358
|
+
throw new WaitSignal("token", void 0, tokenId, stepData);
|
|
359
|
+
},
|
|
360
|
+
setProgress: async (percent) => {
|
|
361
|
+
if (percent < 0 || percent > 100)
|
|
362
|
+
throw new Error("Progress must be between 0 and 100");
|
|
363
|
+
await backend.updateProgress(jobId, Math.round(percent));
|
|
503
364
|
}
|
|
365
|
+
};
|
|
366
|
+
return ctx;
|
|
367
|
+
}
|
|
368
|
+
async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
369
|
+
const handler = jobHandlers[job.jobType];
|
|
370
|
+
if (!handler) {
|
|
371
|
+
await backend.setPendingReasonForUnpickedJobs(
|
|
372
|
+
`No handler registered for job type: ${job.jobType}`,
|
|
373
|
+
job.jobType
|
|
374
|
+
);
|
|
375
|
+
await backend.failJob(
|
|
376
|
+
job.id,
|
|
377
|
+
new Error(`No handler registered for job type: ${job.jobType}`),
|
|
378
|
+
"no_handler" /* NoHandler */
|
|
379
|
+
);
|
|
380
|
+
return;
|
|
504
381
|
}
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
382
|
+
const stepData = { ...job.stepData || {} };
|
|
383
|
+
const hasStepHistory = Object.keys(stepData).some(
|
|
384
|
+
(k) => k.startsWith("__wait_")
|
|
385
|
+
);
|
|
386
|
+
if (hasStepHistory) {
|
|
387
|
+
await resolveCompletedWaits(backend, stepData);
|
|
388
|
+
await backend.updateStepData(job.id, stepData);
|
|
389
|
+
}
|
|
390
|
+
const timeoutMs = job.timeoutMs ?? void 0;
|
|
391
|
+
const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
|
|
392
|
+
let timeoutId;
|
|
393
|
+
const controller = new AbortController();
|
|
394
|
+
try {
|
|
395
|
+
if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
|
|
396
|
+
await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
|
|
397
|
+
} else {
|
|
398
|
+
let onTimeoutCallback;
|
|
399
|
+
let timeoutReject;
|
|
400
|
+
const armTimeout = (ms) => {
|
|
401
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
402
|
+
timeoutId = setTimeout(() => {
|
|
403
|
+
if (onTimeoutCallback) {
|
|
404
|
+
try {
|
|
405
|
+
const extension = onTimeoutCallback();
|
|
406
|
+
if (typeof extension === "number" && extension > 0) {
|
|
407
|
+
backend.prolongJob(job.id).catch(() => {
|
|
408
|
+
});
|
|
409
|
+
armTimeout(extension);
|
|
410
|
+
return;
|
|
411
|
+
}
|
|
412
|
+
} catch (callbackError) {
|
|
413
|
+
log(
|
|
414
|
+
`onTimeout callback threw for job ${job.id}: ${callbackError}`
|
|
415
|
+
);
|
|
528
416
|
}
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
417
|
+
}
|
|
418
|
+
controller.abort();
|
|
419
|
+
const timeoutError = new Error(`Job timed out after ${ms} ms`);
|
|
420
|
+
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
421
|
+
if (timeoutReject) {
|
|
422
|
+
timeoutReject(timeoutError);
|
|
423
|
+
}
|
|
424
|
+
}, ms);
|
|
425
|
+
};
|
|
426
|
+
const hasTimeout = timeoutMs != null && timeoutMs > 0;
|
|
427
|
+
const baseCtx = hasTimeout ? {
|
|
428
|
+
prolong: (ms) => {
|
|
429
|
+
const duration = ms ?? timeoutMs;
|
|
430
|
+
if (duration != null && duration > 0) {
|
|
431
|
+
armTimeout(duration);
|
|
432
|
+
backend.prolongJob(job.id).catch(() => {
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
},
|
|
436
|
+
onTimeout: (callback) => {
|
|
437
|
+
onTimeoutCallback = callback;
|
|
438
|
+
}
|
|
439
|
+
} : {
|
|
440
|
+
prolong: () => {
|
|
441
|
+
log("prolong() called but ignored: job has no timeout set");
|
|
442
|
+
},
|
|
443
|
+
onTimeout: () => {
|
|
444
|
+
log("onTimeout() called but ignored: job has no timeout set");
|
|
445
|
+
}
|
|
446
|
+
};
|
|
447
|
+
const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
|
|
448
|
+
if (forceKillOnTimeout && !hasTimeout) {
|
|
534
449
|
log(
|
|
535
|
-
`
|
|
450
|
+
`forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
|
|
536
451
|
);
|
|
537
452
|
}
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
453
|
+
const jobPromise = handler(job.payload, controller.signal, ctx);
|
|
454
|
+
if (hasTimeout) {
|
|
455
|
+
await Promise.race([
|
|
456
|
+
jobPromise,
|
|
457
|
+
new Promise((_, reject) => {
|
|
458
|
+
timeoutReject = reject;
|
|
459
|
+
armTimeout(timeoutMs);
|
|
460
|
+
})
|
|
461
|
+
]);
|
|
462
|
+
} else {
|
|
463
|
+
await jobPromise;
|
|
464
|
+
}
|
|
465
|
+
}
|
|
466
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
467
|
+
await backend.completeJob(job.id);
|
|
468
|
+
} catch (error) {
|
|
469
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
470
|
+
if (error instanceof WaitSignal) {
|
|
471
|
+
log(
|
|
472
|
+
`Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? "none"}, tokenId=${error.tokenId ?? "none"}`
|
|
473
|
+
);
|
|
474
|
+
await backend.waitJob(job.id, {
|
|
475
|
+
waitUntil: error.waitUntil,
|
|
476
|
+
waitTokenId: error.tokenId,
|
|
477
|
+
stepData: error.stepData
|
|
541
478
|
});
|
|
542
|
-
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
479
|
+
return;
|
|
480
|
+
}
|
|
481
|
+
console.error(`Error processing job ${job.id}:`, error);
|
|
482
|
+
let failureReason = "handler_error" /* HandlerError */;
|
|
483
|
+
if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
|
|
484
|
+
failureReason = "timeout" /* Timeout */;
|
|
485
|
+
}
|
|
486
|
+
await backend.failJob(
|
|
487
|
+
job.id,
|
|
488
|
+
error instanceof Error ? error : new Error(String(error)),
|
|
489
|
+
failureReason
|
|
490
|
+
);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError) {
|
|
494
|
+
const jobs = await backend.getNextBatch(
|
|
495
|
+
workerId,
|
|
496
|
+
batchSize,
|
|
497
|
+
jobType
|
|
498
|
+
);
|
|
499
|
+
if (!concurrency || concurrency >= jobs.length) {
|
|
500
|
+
await Promise.all(
|
|
501
|
+
jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers))
|
|
502
|
+
);
|
|
503
|
+
return jobs.length;
|
|
504
|
+
}
|
|
505
|
+
let idx = 0;
|
|
506
|
+
let running = 0;
|
|
507
|
+
let finished = 0;
|
|
508
|
+
return new Promise((resolve, reject) => {
|
|
509
|
+
const next = () => {
|
|
510
|
+
if (finished === jobs.length) return resolve(jobs.length);
|
|
511
|
+
while (running < concurrency && idx < jobs.length) {
|
|
512
|
+
const job = jobs[idx++];
|
|
513
|
+
running++;
|
|
514
|
+
processJobWithHandlers(backend, job, jobHandlers).then(() => {
|
|
515
|
+
running--;
|
|
516
|
+
finished++;
|
|
517
|
+
next();
|
|
518
|
+
}).catch((err) => {
|
|
519
|
+
running--;
|
|
520
|
+
finished++;
|
|
521
|
+
if (onError) {
|
|
522
|
+
onError(err instanceof Error ? err : new Error(String(err)));
|
|
523
|
+
}
|
|
524
|
+
next();
|
|
525
|
+
});
|
|
526
|
+
}
|
|
527
|
+
};
|
|
528
|
+
next();
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
532
|
+
const {
|
|
533
|
+
workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
|
|
534
|
+
batchSize = 10,
|
|
535
|
+
pollInterval = 5e3,
|
|
536
|
+
onError = (error) => console.error("Job processor error:", error),
|
|
537
|
+
jobType,
|
|
538
|
+
concurrency = 3
|
|
539
|
+
} = options;
|
|
540
|
+
let running = false;
|
|
541
|
+
let intervalId = null;
|
|
542
|
+
let currentBatchPromise = null;
|
|
543
|
+
setLogContext(options.verbose ?? false);
|
|
544
|
+
const processJobs = async () => {
|
|
545
|
+
if (!running) return 0;
|
|
546
|
+
if (onBeforeBatch) {
|
|
547
|
+
try {
|
|
548
|
+
await onBeforeBatch();
|
|
549
|
+
} catch (hookError) {
|
|
550
|
+
log(`onBeforeBatch hook error: ${hookError}`);
|
|
551
|
+
if (onError) {
|
|
552
|
+
onError(
|
|
553
|
+
hookError instanceof Error ? hookError : new Error(String(hookError))
|
|
554
|
+
);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
548
557
|
}
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
558
|
+
log(
|
|
559
|
+
`Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(",") : jobType}` : ""}`
|
|
560
|
+
);
|
|
552
561
|
try {
|
|
553
|
-
await
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
562
|
+
const processed = await processBatchWithHandlers(
|
|
563
|
+
backend,
|
|
564
|
+
workerId,
|
|
565
|
+
batchSize,
|
|
566
|
+
jobType,
|
|
567
|
+
handlers,
|
|
568
|
+
concurrency,
|
|
569
|
+
onError
|
|
560
570
|
);
|
|
561
|
-
|
|
562
|
-
log(`Prolonged job ${jobId}`);
|
|
571
|
+
return processed;
|
|
563
572
|
} catch (error) {
|
|
564
|
-
|
|
565
|
-
} finally {
|
|
566
|
-
client.release();
|
|
573
|
+
onError(error instanceof Error ? error : new Error(String(error)));
|
|
567
574
|
}
|
|
575
|
+
return 0;
|
|
576
|
+
};
|
|
577
|
+
return {
|
|
578
|
+
/**
|
|
579
|
+
* Start the job processor in the background.
|
|
580
|
+
* - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
|
|
581
|
+
* - You have to call the stop method to stop the processor.
|
|
582
|
+
*/
|
|
583
|
+
startInBackground: () => {
|
|
584
|
+
if (running) return;
|
|
585
|
+
log(`Starting job processor with workerId: ${workerId}`);
|
|
586
|
+
running = true;
|
|
587
|
+
const scheduleNext = (immediate) => {
|
|
588
|
+
if (!running) return;
|
|
589
|
+
if (immediate) {
|
|
590
|
+
intervalId = setTimeout(loop, 0);
|
|
591
|
+
} else {
|
|
592
|
+
intervalId = setTimeout(loop, pollInterval);
|
|
593
|
+
}
|
|
594
|
+
};
|
|
595
|
+
const loop = async () => {
|
|
596
|
+
if (!running) return;
|
|
597
|
+
currentBatchPromise = processJobs();
|
|
598
|
+
const processed = await currentBatchPromise;
|
|
599
|
+
currentBatchPromise = null;
|
|
600
|
+
scheduleNext(processed === batchSize);
|
|
601
|
+
};
|
|
602
|
+
loop();
|
|
603
|
+
},
|
|
604
|
+
/**
|
|
605
|
+
* Stop the job processor that runs in the background.
|
|
606
|
+
* Does not wait for in-flight jobs.
|
|
607
|
+
*/
|
|
608
|
+
stop: () => {
|
|
609
|
+
log(`Stopping job processor with workerId: ${workerId}`);
|
|
610
|
+
running = false;
|
|
611
|
+
if (intervalId) {
|
|
612
|
+
clearTimeout(intervalId);
|
|
613
|
+
intervalId = null;
|
|
614
|
+
}
|
|
615
|
+
},
|
|
616
|
+
/**
|
|
617
|
+
* Stop the job processor and wait for all in-flight jobs to complete.
|
|
618
|
+
* Useful for graceful shutdown (e.g., SIGTERM handling).
|
|
619
|
+
*/
|
|
620
|
+
stopAndDrain: async (drainTimeoutMs = 3e4) => {
|
|
621
|
+
log(`Stopping and draining job processor with workerId: ${workerId}`);
|
|
622
|
+
running = false;
|
|
623
|
+
if (intervalId) {
|
|
624
|
+
clearTimeout(intervalId);
|
|
625
|
+
intervalId = null;
|
|
626
|
+
}
|
|
627
|
+
if (currentBatchPromise) {
|
|
628
|
+
await Promise.race([
|
|
629
|
+
currentBatchPromise.catch(() => {
|
|
630
|
+
}),
|
|
631
|
+
new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
|
|
632
|
+
]);
|
|
633
|
+
currentBatchPromise = null;
|
|
634
|
+
}
|
|
635
|
+
log(`Job processor ${workerId} drained`);
|
|
636
|
+
},
|
|
637
|
+
/**
|
|
638
|
+
* Start the job processor synchronously.
|
|
639
|
+
* - This will process all jobs immediately and then stop.
|
|
640
|
+
* - The pollInterval is ignored.
|
|
641
|
+
*/
|
|
642
|
+
start: async () => {
|
|
643
|
+
log(`Starting job processor with workerId: ${workerId}`);
|
|
644
|
+
running = true;
|
|
645
|
+
const processed = await processJobs();
|
|
646
|
+
running = false;
|
|
647
|
+
return processed;
|
|
648
|
+
},
|
|
649
|
+
isRunning: () => running
|
|
650
|
+
};
|
|
651
|
+
};
|
|
652
|
+
function loadPemOrFile(value) {
|
|
653
|
+
if (!value) return void 0;
|
|
654
|
+
if (value.startsWith("file://")) {
|
|
655
|
+
const filePath = value.slice(7);
|
|
656
|
+
return fs__default.default.readFileSync(filePath, "utf8");
|
|
568
657
|
}
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
658
|
+
return value;
|
|
659
|
+
}
|
|
660
|
+
var createPool = (config) => {
|
|
661
|
+
let searchPath;
|
|
662
|
+
let ssl = void 0;
|
|
663
|
+
let customCA;
|
|
664
|
+
let sslmode;
|
|
665
|
+
if (config.connectionString) {
|
|
572
666
|
try {
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
)
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
667
|
+
const url = new URL(config.connectionString);
|
|
668
|
+
searchPath = url.searchParams.get("search_path") || void 0;
|
|
669
|
+
sslmode = url.searchParams.get("sslmode") || void 0;
|
|
670
|
+
if (sslmode === "no-verify") {
|
|
671
|
+
ssl = { rejectUnauthorized: false };
|
|
672
|
+
}
|
|
673
|
+
} catch (e) {
|
|
674
|
+
const parsed = pgConnectionString.parse(config.connectionString);
|
|
675
|
+
if (parsed.options) {
|
|
676
|
+
const match = parsed.options.match(/search_path=([^\s]+)/);
|
|
677
|
+
if (match) {
|
|
678
|
+
searchPath = match[1];
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
sslmode = typeof parsed.sslmode === "string" ? parsed.sslmode : void 0;
|
|
682
|
+
if (sslmode === "no-verify") {
|
|
683
|
+
ssl = { rejectUnauthorized: false };
|
|
684
|
+
}
|
|
582
685
|
}
|
|
583
686
|
}
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
687
|
+
if (config.ssl) {
|
|
688
|
+
if (typeof config.ssl.ca === "string") {
|
|
689
|
+
customCA = config.ssl.ca;
|
|
690
|
+
} else if (typeof process.env.PGSSLROOTCERT === "string") {
|
|
691
|
+
customCA = process.env.PGSSLROOTCERT;
|
|
692
|
+
} else {
|
|
693
|
+
customCA = void 0;
|
|
694
|
+
}
|
|
695
|
+
const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
|
|
696
|
+
ssl = {
|
|
697
|
+
...ssl,
|
|
698
|
+
...caValue ? { ca: caValue } : {},
|
|
699
|
+
cert: loadPemOrFile(
|
|
700
|
+
typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
|
|
701
|
+
),
|
|
702
|
+
key: loadPemOrFile(
|
|
703
|
+
typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
|
|
704
|
+
),
|
|
705
|
+
rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
if (sslmode && customCA) {
|
|
709
|
+
const warning = `
|
|
710
|
+
|
|
711
|
+
\x1B[33m**************************************************
|
|
712
|
+
\u26A0\uFE0F WARNING: SSL CONFIGURATION ISSUE
|
|
713
|
+
**************************************************
|
|
714
|
+
Both sslmode ('${sslmode}') is set in the connection string
|
|
715
|
+
and a custom CA is provided (via config.ssl.ca or PGSSLROOTCERT).
|
|
716
|
+
This combination may cause connection failures or unexpected behavior.
|
|
717
|
+
|
|
718
|
+
Recommended: Remove sslmode from the connection string when using a custom CA.
|
|
719
|
+
**************************************************\x1B[0m
|
|
720
|
+
`;
|
|
721
|
+
console.warn(warning);
|
|
722
|
+
}
|
|
723
|
+
const pool = new pg.Pool({
|
|
724
|
+
...config,
|
|
725
|
+
...ssl ? { ssl } : {}
|
|
726
|
+
});
|
|
727
|
+
if (searchPath) {
|
|
728
|
+
pool.on("connect", (client) => {
|
|
729
|
+
client.query(`SET search_path TO ${searchPath}`);
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
return pool;
|
|
733
|
+
};
|
|
734
|
+
var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
|
|
735
|
+
function parseTimeoutString(timeout) {
|
|
736
|
+
const match = timeout.match(/^(\d+)(s|m|h|d)$/);
|
|
737
|
+
if (!match) {
|
|
738
|
+
throw new Error(
|
|
739
|
+
`Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
|
|
740
|
+
);
|
|
741
|
+
}
|
|
742
|
+
const value = parseInt(match[1], 10);
|
|
743
|
+
const unit = match[2];
|
|
744
|
+
let ms;
|
|
745
|
+
switch (unit) {
|
|
746
|
+
case "s":
|
|
747
|
+
ms = value * 1e3;
|
|
748
|
+
break;
|
|
749
|
+
case "m":
|
|
750
|
+
ms = value * 60 * 1e3;
|
|
751
|
+
break;
|
|
752
|
+
case "h":
|
|
753
|
+
ms = value * 60 * 60 * 1e3;
|
|
754
|
+
break;
|
|
755
|
+
case "d":
|
|
756
|
+
ms = value * 24 * 60 * 60 * 1e3;
|
|
757
|
+
break;
|
|
758
|
+
default:
|
|
759
|
+
throw new Error(`Unknown timeout unit: "${unit}"`);
|
|
760
|
+
}
|
|
761
|
+
if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
|
|
762
|
+
throw new Error(
|
|
763
|
+
`Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
|
|
764
|
+
);
|
|
765
|
+
}
|
|
766
|
+
return ms;
|
|
767
|
+
}
|
|
768
|
+
var PostgresBackend = class {
|
|
769
|
+
constructor(pool) {
|
|
770
|
+
this.pool = pool;
|
|
771
|
+
}
|
|
772
|
+
/** Expose the raw pool for advanced usage. */
|
|
773
|
+
getPool() {
|
|
774
|
+
return this.pool;
|
|
775
|
+
}
|
|
776
|
+
// ── Events ──────────────────────────────────────────────────────────
|
|
777
|
+
async recordJobEvent(jobId, eventType, metadata) {
|
|
778
|
+
const client = await this.pool.connect();
|
|
779
|
+
try {
|
|
780
|
+
await client.query(
|
|
781
|
+
`INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
|
|
782
|
+
[jobId, eventType, metadata ? JSON.stringify(metadata) : null]
|
|
600
783
|
);
|
|
601
|
-
if (result.rowCount === 0) {
|
|
602
|
-
log(
|
|
603
|
-
`Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
|
|
604
|
-
);
|
|
605
|
-
}
|
|
606
|
-
await this.recordJobEvent(jobId, "retried" /* Retried */);
|
|
607
|
-
log(`Retried job ${jobId}`);
|
|
608
784
|
} catch (error) {
|
|
609
|
-
log(`Error
|
|
610
|
-
throw error;
|
|
785
|
+
log(`Error recording job event for job ${jobId}: ${error}`);
|
|
611
786
|
} finally {
|
|
612
787
|
client.release();
|
|
613
788
|
}
|
|
614
789
|
}
|
|
615
|
-
async
|
|
790
|
+
async getJobEvents(jobId) {
|
|
616
791
|
const client = await this.pool.connect();
|
|
617
792
|
try {
|
|
618
|
-
await client.query(
|
|
619
|
-
`
|
|
620
|
-
UPDATE job_queue
|
|
621
|
-
SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
|
|
622
|
-
wait_until = NULL, wait_token_id = NULL
|
|
623
|
-
WHERE id = $1 AND status IN ('pending', 'waiting')
|
|
624
|
-
`,
|
|
793
|
+
const res = await client.query(
|
|
794
|
+
`SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
|
|
625
795
|
[jobId]
|
|
626
796
|
);
|
|
627
|
-
|
|
628
|
-
log(`Cancelled job ${jobId}`);
|
|
629
|
-
} catch (error) {
|
|
630
|
-
log(`Error cancelling job ${jobId}: ${error}`);
|
|
631
|
-
throw error;
|
|
797
|
+
return res.rows;
|
|
632
798
|
} finally {
|
|
633
799
|
client.release();
|
|
634
800
|
}
|
|
635
801
|
}
|
|
636
|
-
|
|
802
|
+
// ── Job CRUD ──────────────────────────────────────────────────────────
|
|
803
|
+
async addJob({
|
|
804
|
+
jobType,
|
|
805
|
+
payload,
|
|
806
|
+
maxAttempts = 3,
|
|
807
|
+
priority = 0,
|
|
808
|
+
runAt = null,
|
|
809
|
+
timeoutMs = void 0,
|
|
810
|
+
forceKillOnTimeout = false,
|
|
811
|
+
tags = void 0,
|
|
812
|
+
idempotencyKey = void 0
|
|
813
|
+
}) {
|
|
637
814
|
const client = await this.pool.connect();
|
|
638
815
|
try {
|
|
639
|
-
let
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
break;
|
|
690
|
-
case "all":
|
|
691
|
-
query += ` AND tags @> $${paramIdx++}`;
|
|
692
|
-
params.push(tagValues);
|
|
693
|
-
break;
|
|
694
|
-
case "any":
|
|
695
|
-
query += ` AND tags && $${paramIdx++}`;
|
|
696
|
-
params.push(tagValues);
|
|
697
|
-
break;
|
|
698
|
-
case "none":
|
|
699
|
-
query += ` AND NOT (tags && $${paramIdx++})`;
|
|
700
|
-
params.push(tagValues);
|
|
701
|
-
break;
|
|
702
|
-
default:
|
|
703
|
-
query += ` AND tags @> $${paramIdx++}`;
|
|
704
|
-
params.push(tagValues);
|
|
705
|
-
}
|
|
816
|
+
let result;
|
|
817
|
+
const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
|
|
818
|
+
if (runAt) {
|
|
819
|
+
result = await client.query(
|
|
820
|
+
`INSERT INTO job_queue
|
|
821
|
+
(job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
822
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
|
|
823
|
+
${onConflict}
|
|
824
|
+
RETURNING id`,
|
|
825
|
+
[
|
|
826
|
+
jobType,
|
|
827
|
+
payload,
|
|
828
|
+
maxAttempts,
|
|
829
|
+
priority,
|
|
830
|
+
runAt,
|
|
831
|
+
timeoutMs ?? null,
|
|
832
|
+
forceKillOnTimeout ?? false,
|
|
833
|
+
tags ?? null,
|
|
834
|
+
idempotencyKey ?? null
|
|
835
|
+
]
|
|
836
|
+
);
|
|
837
|
+
} else {
|
|
838
|
+
result = await client.query(
|
|
839
|
+
`INSERT INTO job_queue
|
|
840
|
+
(job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
|
|
841
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
842
|
+
${onConflict}
|
|
843
|
+
RETURNING id`,
|
|
844
|
+
[
|
|
845
|
+
jobType,
|
|
846
|
+
payload,
|
|
847
|
+
maxAttempts,
|
|
848
|
+
priority,
|
|
849
|
+
timeoutMs ?? null,
|
|
850
|
+
forceKillOnTimeout ?? false,
|
|
851
|
+
tags ?? null,
|
|
852
|
+
idempotencyKey ?? null
|
|
853
|
+
]
|
|
854
|
+
);
|
|
855
|
+
}
|
|
856
|
+
if (result.rows.length === 0 && idempotencyKey) {
|
|
857
|
+
const existing = await client.query(
|
|
858
|
+
`SELECT id FROM job_queue WHERE idempotency_key = $1`,
|
|
859
|
+
[idempotencyKey]
|
|
860
|
+
);
|
|
861
|
+
if (existing.rows.length > 0) {
|
|
862
|
+
log(
|
|
863
|
+
`Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
|
|
864
|
+
);
|
|
865
|
+
return existing.rows[0].id;
|
|
706
866
|
}
|
|
867
|
+
throw new Error(
|
|
868
|
+
`Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
|
|
869
|
+
);
|
|
707
870
|
}
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
871
|
+
const jobId = result.rows[0].id;
|
|
872
|
+
log(
|
|
873
|
+
`Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
|
|
874
|
+
);
|
|
875
|
+
await this.recordJobEvent(jobId, "added" /* Added */, {
|
|
876
|
+
jobType,
|
|
877
|
+
payload,
|
|
878
|
+
tags,
|
|
879
|
+
idempotencyKey
|
|
880
|
+
});
|
|
881
|
+
return jobId;
|
|
712
882
|
} catch (error) {
|
|
713
|
-
log(`Error
|
|
883
|
+
log(`Error adding job: ${error}`);
|
|
714
884
|
throw error;
|
|
715
885
|
} finally {
|
|
716
886
|
client.release();
|
|
717
887
|
}
|
|
718
888
|
}
|
|
719
|
-
async
|
|
889
|
+
async getJob(id) {
|
|
720
890
|
const client = await this.pool.connect();
|
|
721
891
|
try {
|
|
722
|
-
const
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
if (updates.maxAttempts !== void 0) {
|
|
730
|
-
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
731
|
-
params.push(updates.maxAttempts);
|
|
732
|
-
}
|
|
733
|
-
if (updates.priority !== void 0) {
|
|
734
|
-
updateFields.push(`priority = $${paramIdx++}`);
|
|
735
|
-
params.push(updates.priority);
|
|
736
|
-
}
|
|
737
|
-
if (updates.runAt !== void 0) {
|
|
738
|
-
if (updates.runAt === null) {
|
|
739
|
-
updateFields.push(`run_at = NOW()`);
|
|
740
|
-
} else {
|
|
741
|
-
updateFields.push(`run_at = $${paramIdx++}`);
|
|
742
|
-
params.push(updates.runAt);
|
|
743
|
-
}
|
|
744
|
-
}
|
|
745
|
-
if (updates.timeoutMs !== void 0) {
|
|
746
|
-
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
747
|
-
params.push(updates.timeoutMs ?? null);
|
|
748
|
-
}
|
|
749
|
-
if (updates.tags !== void 0) {
|
|
750
|
-
updateFields.push(`tags = $${paramIdx++}`);
|
|
751
|
-
params.push(updates.tags ?? null);
|
|
752
|
-
}
|
|
753
|
-
if (updateFields.length === 0) {
|
|
754
|
-
log(`No fields to update for job ${jobId}`);
|
|
755
|
-
return;
|
|
892
|
+
const result = await client.query(
|
|
893
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
|
|
894
|
+
[id]
|
|
895
|
+
);
|
|
896
|
+
if (result.rows.length === 0) {
|
|
897
|
+
log(`Job ${id} not found`);
|
|
898
|
+
return null;
|
|
756
899
|
}
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
900
|
+
log(`Found job ${id}`);
|
|
901
|
+
const job = result.rows[0];
|
|
902
|
+
return {
|
|
903
|
+
...job,
|
|
904
|
+
payload: job.payload,
|
|
905
|
+
timeoutMs: job.timeoutMs,
|
|
906
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
907
|
+
failureReason: job.failureReason
|
|
908
|
+
};
|
|
909
|
+
} catch (error) {
|
|
910
|
+
log(`Error getting job ${id}: ${error}`);
|
|
911
|
+
throw error;
|
|
912
|
+
} finally {
|
|
913
|
+
client.release();
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
async getJobsByStatus(status, limit = 100, offset = 0) {
|
|
917
|
+
const client = await this.pool.connect();
|
|
918
|
+
try {
|
|
919
|
+
const result = await client.query(
|
|
920
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
|
|
921
|
+
[status, limit, offset]
|
|
922
|
+
);
|
|
923
|
+
log(`Found ${result.rows.length} jobs by status ${status}`);
|
|
924
|
+
return result.rows.map((job) => ({
|
|
925
|
+
...job,
|
|
926
|
+
payload: job.payload,
|
|
927
|
+
timeoutMs: job.timeoutMs,
|
|
928
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
929
|
+
failureReason: job.failureReason
|
|
930
|
+
}));
|
|
776
931
|
} catch (error) {
|
|
777
|
-
log(`Error
|
|
932
|
+
log(`Error getting jobs by status ${status}: ${error}`);
|
|
778
933
|
throw error;
|
|
779
934
|
} finally {
|
|
780
935
|
client.release();
|
|
781
936
|
}
|
|
782
937
|
}
|
|
783
|
-
async
|
|
938
|
+
async getAllJobs(limit = 100, offset = 0) {
|
|
784
939
|
const client = await this.pool.connect();
|
|
785
940
|
try {
|
|
786
|
-
const
|
|
941
|
+
const result = await client.query(
|
|
942
|
+
`SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
|
|
943
|
+
[limit, offset]
|
|
944
|
+
);
|
|
945
|
+
log(`Found ${result.rows.length} jobs (all)`);
|
|
946
|
+
return result.rows.map((job) => ({
|
|
947
|
+
...job,
|
|
948
|
+
payload: job.payload,
|
|
949
|
+
timeoutMs: job.timeoutMs,
|
|
950
|
+
forceKillOnTimeout: job.forceKillOnTimeout
|
|
951
|
+
}));
|
|
952
|
+
} catch (error) {
|
|
953
|
+
log(`Error getting all jobs: ${error}`);
|
|
954
|
+
throw error;
|
|
955
|
+
} finally {
|
|
956
|
+
client.release();
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
async getJobs(filters, limit = 100, offset = 0) {
|
|
960
|
+
const client = await this.pool.connect();
|
|
961
|
+
try {
|
|
962
|
+
let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue`;
|
|
787
963
|
const params = [];
|
|
964
|
+
const where = [];
|
|
788
965
|
let paramIdx = 1;
|
|
789
|
-
if (updates.payload !== void 0) {
|
|
790
|
-
updateFields.push(`payload = $${paramIdx++}`);
|
|
791
|
-
params.push(updates.payload);
|
|
792
|
-
}
|
|
793
|
-
if (updates.maxAttempts !== void 0) {
|
|
794
|
-
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
795
|
-
params.push(updates.maxAttempts);
|
|
796
|
-
}
|
|
797
|
-
if (updates.priority !== void 0) {
|
|
798
|
-
updateFields.push(`priority = $${paramIdx++}`);
|
|
799
|
-
params.push(updates.priority);
|
|
800
|
-
}
|
|
801
|
-
if (updates.runAt !== void 0) {
|
|
802
|
-
if (updates.runAt === null) {
|
|
803
|
-
updateFields.push(`run_at = NOW()`);
|
|
804
|
-
} else {
|
|
805
|
-
updateFields.push(`run_at = $${paramIdx++}`);
|
|
806
|
-
params.push(updates.runAt);
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
if (updates.timeoutMs !== void 0) {
|
|
810
|
-
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
811
|
-
params.push(updates.timeoutMs ?? null);
|
|
812
|
-
}
|
|
813
|
-
if (updates.tags !== void 0) {
|
|
814
|
-
updateFields.push(`tags = $${paramIdx++}`);
|
|
815
|
-
params.push(updates.tags ?? null);
|
|
816
|
-
}
|
|
817
|
-
if (updateFields.length === 0) {
|
|
818
|
-
log(`No fields to update for batch edit`);
|
|
819
|
-
return 0;
|
|
820
|
-
}
|
|
821
|
-
updateFields.push(`updated_at = NOW()`);
|
|
822
|
-
let query = `
|
|
823
|
-
UPDATE job_queue
|
|
824
|
-
SET ${updateFields.join(", ")}
|
|
825
|
-
WHERE status = 'pending'`;
|
|
826
966
|
if (filters) {
|
|
827
967
|
if (filters.jobType) {
|
|
828
|
-
|
|
968
|
+
where.push(`job_type = $${paramIdx++}`);
|
|
829
969
|
params.push(filters.jobType);
|
|
830
970
|
}
|
|
831
971
|
if (filters.priority !== void 0) {
|
|
832
|
-
|
|
972
|
+
where.push(`priority = $${paramIdx++}`);
|
|
833
973
|
params.push(filters.priority);
|
|
834
974
|
}
|
|
835
975
|
if (filters.runAt) {
|
|
836
976
|
if (filters.runAt instanceof Date) {
|
|
837
|
-
|
|
977
|
+
where.push(`run_at = $${paramIdx++}`);
|
|
838
978
|
params.push(filters.runAt);
|
|
839
|
-
} else if (typeof filters.runAt === "object") {
|
|
979
|
+
} else if (typeof filters.runAt === "object" && (filters.runAt.gt !== void 0 || filters.runAt.gte !== void 0 || filters.runAt.lt !== void 0 || filters.runAt.lte !== void 0 || filters.runAt.eq !== void 0)) {
|
|
840
980
|
const ops = filters.runAt;
|
|
841
981
|
if (ops.gt) {
|
|
842
|
-
|
|
982
|
+
where.push(`run_at > $${paramIdx++}`);
|
|
843
983
|
params.push(ops.gt);
|
|
844
984
|
}
|
|
845
985
|
if (ops.gte) {
|
|
846
|
-
|
|
986
|
+
where.push(`run_at >= $${paramIdx++}`);
|
|
847
987
|
params.push(ops.gte);
|
|
848
988
|
}
|
|
849
989
|
if (ops.lt) {
|
|
850
|
-
|
|
990
|
+
where.push(`run_at < $${paramIdx++}`);
|
|
851
991
|
params.push(ops.lt);
|
|
852
992
|
}
|
|
853
993
|
if (ops.lte) {
|
|
854
|
-
|
|
994
|
+
where.push(`run_at <= $${paramIdx++}`);
|
|
855
995
|
params.push(ops.lte);
|
|
856
996
|
}
|
|
857
997
|
if (ops.eq) {
|
|
858
|
-
|
|
998
|
+
where.push(`run_at = $${paramIdx++}`);
|
|
859
999
|
params.push(ops.eq);
|
|
860
1000
|
}
|
|
861
1001
|
}
|
|
@@ -865,1370 +1005,1264 @@ var PostgresBackend = class {
|
|
|
865
1005
|
const tagValues = filters.tags.values;
|
|
866
1006
|
switch (mode) {
|
|
867
1007
|
case "exact":
|
|
868
|
-
|
|
1008
|
+
where.push(`tags = $${paramIdx++}`);
|
|
869
1009
|
params.push(tagValues);
|
|
870
1010
|
break;
|
|
871
1011
|
case "all":
|
|
872
|
-
|
|
1012
|
+
where.push(`tags @> $${paramIdx++}`);
|
|
873
1013
|
params.push(tagValues);
|
|
874
1014
|
break;
|
|
875
1015
|
case "any":
|
|
876
|
-
|
|
1016
|
+
where.push(`tags && $${paramIdx++}`);
|
|
877
1017
|
params.push(tagValues);
|
|
878
1018
|
break;
|
|
879
1019
|
case "none":
|
|
880
|
-
|
|
1020
|
+
where.push(`NOT (tags && $${paramIdx++})`);
|
|
881
1021
|
params.push(tagValues);
|
|
882
1022
|
break;
|
|
883
1023
|
default:
|
|
884
|
-
|
|
1024
|
+
where.push(`tags @> $${paramIdx++}`);
|
|
885
1025
|
params.push(tagValues);
|
|
886
1026
|
}
|
|
887
1027
|
}
|
|
1028
|
+
if (filters.cursor !== void 0) {
|
|
1029
|
+
where.push(`id < $${paramIdx++}`);
|
|
1030
|
+
params.push(filters.cursor);
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
if (where.length > 0) {
|
|
1034
|
+
query += ` WHERE ${where.join(" AND ")}`;
|
|
1035
|
+
}
|
|
1036
|
+
paramIdx = params.length + 1;
|
|
1037
|
+
query += ` ORDER BY id DESC LIMIT $${paramIdx++}`;
|
|
1038
|
+
if (!filters?.cursor) {
|
|
1039
|
+
query += ` OFFSET $${paramIdx}`;
|
|
1040
|
+
params.push(limit, offset);
|
|
1041
|
+
} else {
|
|
1042
|
+
params.push(limit);
|
|
888
1043
|
}
|
|
889
|
-
query += "\nRETURNING id";
|
|
890
1044
|
const result = await client.query(query, params);
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
1045
|
+
log(`Found ${result.rows.length} jobs`);
|
|
1046
|
+
return result.rows.map((job) => ({
|
|
1047
|
+
...job,
|
|
1048
|
+
payload: job.payload,
|
|
1049
|
+
timeoutMs: job.timeoutMs,
|
|
1050
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
1051
|
+
failureReason: job.failureReason
|
|
1052
|
+
}));
|
|
1053
|
+
} catch (error) {
|
|
1054
|
+
log(`Error getting jobs: ${error}`);
|
|
1055
|
+
throw error;
|
|
1056
|
+
} finally {
|
|
1057
|
+
client.release();
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
|
|
1061
|
+
const client = await this.pool.connect();
|
|
1062
|
+
try {
|
|
1063
|
+
let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
|
|
1064
|
+
FROM job_queue`;
|
|
1065
|
+
let params = [];
|
|
1066
|
+
switch (mode) {
|
|
1067
|
+
case "exact":
|
|
1068
|
+
query += " WHERE tags = $1";
|
|
1069
|
+
params = [tags];
|
|
1070
|
+
break;
|
|
1071
|
+
case "all":
|
|
1072
|
+
query += " WHERE tags @> $1";
|
|
1073
|
+
params = [tags];
|
|
1074
|
+
break;
|
|
1075
|
+
case "any":
|
|
1076
|
+
query += " WHERE tags && $1";
|
|
1077
|
+
params = [tags];
|
|
1078
|
+
break;
|
|
1079
|
+
case "none":
|
|
1080
|
+
query += " WHERE NOT (tags && $1)";
|
|
1081
|
+
params = [tags];
|
|
1082
|
+
break;
|
|
1083
|
+
default:
|
|
1084
|
+
query += " WHERE tags @> $1";
|
|
1085
|
+
params = [tags];
|
|
1086
|
+
}
|
|
1087
|
+
query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
|
|
1088
|
+
params.push(limit, offset);
|
|
1089
|
+
const result = await client.query(query, params);
|
|
1090
|
+
log(
|
|
1091
|
+
`Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
|
|
1092
|
+
);
|
|
1093
|
+
return result.rows.map((job) => ({
|
|
1094
|
+
...job,
|
|
1095
|
+
payload: job.payload,
|
|
1096
|
+
timeoutMs: job.timeoutMs,
|
|
1097
|
+
forceKillOnTimeout: job.forceKillOnTimeout,
|
|
1098
|
+
failureReason: job.failureReason
|
|
1099
|
+
}));
|
|
1100
|
+
} catch (error) {
|
|
1101
|
+
log(
|
|
1102
|
+
`Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
|
|
1103
|
+
);
|
|
1104
|
+
throw error;
|
|
1105
|
+
} finally {
|
|
1106
|
+
client.release();
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
// ── Processing lifecycle ──────────────────────────────────────────────
|
|
1110
|
+
async getNextBatch(workerId, batchSize = 10, jobType) {
|
|
1111
|
+
const client = await this.pool.connect();
|
|
1112
|
+
try {
|
|
1113
|
+
await client.query("BEGIN");
|
|
1114
|
+
let jobTypeFilter = "";
|
|
1115
|
+
const params = [workerId, batchSize];
|
|
1116
|
+
if (jobType) {
|
|
1117
|
+
if (Array.isArray(jobType)) {
|
|
1118
|
+
jobTypeFilter = ` AND job_type = ANY($3)`;
|
|
1119
|
+
params.push(jobType);
|
|
1120
|
+
} else {
|
|
1121
|
+
jobTypeFilter = ` AND job_type = $3`;
|
|
1122
|
+
params.push(jobType);
|
|
1123
|
+
}
|
|
1124
|
+
}
|
|
1125
|
+
const result = await client.query(
|
|
1126
|
+
`
|
|
1127
|
+
UPDATE job_queue
|
|
1128
|
+
SET status = 'processing',
|
|
1129
|
+
locked_at = NOW(),
|
|
1130
|
+
locked_by = $1,
|
|
1131
|
+
attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
|
|
1132
|
+
updated_at = NOW(),
|
|
1133
|
+
pending_reason = NULL,
|
|
1134
|
+
started_at = COALESCE(started_at, NOW()),
|
|
1135
|
+
last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
|
|
1136
|
+
wait_until = NULL
|
|
1137
|
+
WHERE id IN (
|
|
1138
|
+
SELECT id FROM job_queue
|
|
1139
|
+
WHERE (
|
|
1140
|
+
(
|
|
1141
|
+
(status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
|
|
1142
|
+
AND (attempts < max_attempts)
|
|
1143
|
+
AND run_at <= NOW()
|
|
1144
|
+
)
|
|
1145
|
+
OR (
|
|
1146
|
+
status = 'waiting'
|
|
1147
|
+
AND wait_until IS NOT NULL
|
|
1148
|
+
AND wait_until <= NOW()
|
|
1149
|
+
AND wait_token_id IS NULL
|
|
1150
|
+
)
|
|
1151
|
+
)
|
|
1152
|
+
${jobTypeFilter}
|
|
1153
|
+
ORDER BY priority DESC, created_at ASC
|
|
1154
|
+
LIMIT $2
|
|
1155
|
+
FOR UPDATE SKIP LOCKED
|
|
1156
|
+
)
|
|
1157
|
+
RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
|
|
1158
|
+
`,
|
|
1159
|
+
params
|
|
1160
|
+
);
|
|
1161
|
+
log(`Found ${result.rows.length} jobs to process`);
|
|
1162
|
+
await client.query("COMMIT");
|
|
1163
|
+
if (result.rows.length > 0) {
|
|
1164
|
+
await this.recordJobEventsBatch(
|
|
1165
|
+
result.rows.map((row) => ({
|
|
1166
|
+
jobId: row.id,
|
|
1167
|
+
eventType: "processing" /* Processing */
|
|
1168
|
+
}))
|
|
1169
|
+
);
|
|
903
1170
|
}
|
|
904
|
-
|
|
905
|
-
|
|
1171
|
+
return result.rows.map((job) => ({
|
|
1172
|
+
...job,
|
|
1173
|
+
payload: job.payload,
|
|
1174
|
+
timeoutMs: job.timeoutMs,
|
|
1175
|
+
forceKillOnTimeout: job.forceKillOnTimeout
|
|
1176
|
+
}));
|
|
906
1177
|
} catch (error) {
|
|
907
|
-
log(`Error
|
|
1178
|
+
log(`Error getting next batch: ${error}`);
|
|
1179
|
+
await client.query("ROLLBACK");
|
|
908
1180
|
throw error;
|
|
909
1181
|
} finally {
|
|
910
1182
|
client.release();
|
|
911
1183
|
}
|
|
912
1184
|
}
|
|
913
|
-
async
|
|
1185
|
+
async completeJob(jobId) {
|
|
914
1186
|
const client = await this.pool.connect();
|
|
915
1187
|
try {
|
|
916
1188
|
const result = await client.query(
|
|
917
1189
|
`
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
1190
|
+
UPDATE job_queue
|
|
1191
|
+
SET status = 'completed', updated_at = NOW(), completed_at = NOW(),
|
|
1192
|
+
step_data = NULL, wait_until = NULL, wait_token_id = NULL
|
|
1193
|
+
WHERE id = $1 AND status = 'processing'
|
|
922
1194
|
`,
|
|
923
|
-
[
|
|
1195
|
+
[jobId]
|
|
924
1196
|
);
|
|
925
|
-
|
|
926
|
-
|
|
1197
|
+
if (result.rowCount === 0) {
|
|
1198
|
+
log(
|
|
1199
|
+
`Job ${jobId} could not be completed (not in processing state or does not exist)`
|
|
1200
|
+
);
|
|
1201
|
+
}
|
|
1202
|
+
await this.recordJobEvent(jobId, "completed" /* Completed */);
|
|
1203
|
+
log(`Completed job ${jobId}`);
|
|
927
1204
|
} catch (error) {
|
|
928
|
-
log(`Error
|
|
1205
|
+
log(`Error completing job ${jobId}: ${error}`);
|
|
929
1206
|
throw error;
|
|
930
1207
|
} finally {
|
|
931
1208
|
client.release();
|
|
932
1209
|
}
|
|
933
1210
|
}
|
|
934
|
-
async
|
|
1211
|
+
async failJob(jobId, error, failureReason) {
|
|
935
1212
|
const client = await this.pool.connect();
|
|
936
1213
|
try {
|
|
937
1214
|
const result = await client.query(
|
|
938
1215
|
`
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
1216
|
+
UPDATE job_queue
|
|
1217
|
+
SET status = 'failed',
|
|
1218
|
+
updated_at = NOW(),
|
|
1219
|
+
next_attempt_at = CASE
|
|
1220
|
+
WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
|
|
1221
|
+
ELSE NULL
|
|
1222
|
+
END,
|
|
1223
|
+
error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
|
|
1224
|
+
failure_reason = $3,
|
|
1225
|
+
last_failed_at = NOW()
|
|
1226
|
+
WHERE id = $1 AND status IN ('processing', 'pending')
|
|
942
1227
|
`,
|
|
943
|
-
[
|
|
1228
|
+
[
|
|
1229
|
+
jobId,
|
|
1230
|
+
JSON.stringify([
|
|
1231
|
+
{
|
|
1232
|
+
message: error.message || String(error),
|
|
1233
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
1234
|
+
}
|
|
1235
|
+
]),
|
|
1236
|
+
failureReason ?? null
|
|
1237
|
+
]
|
|
944
1238
|
);
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
1239
|
+
if (result.rowCount === 0) {
|
|
1240
|
+
log(
|
|
1241
|
+
`Job ${jobId} could not be failed (not in processing/pending state or does not exist)`
|
|
1242
|
+
);
|
|
1243
|
+
}
|
|
1244
|
+
await this.recordJobEvent(jobId, "failed" /* Failed */, {
|
|
1245
|
+
message: error.message || String(error),
|
|
1246
|
+
failureReason
|
|
1247
|
+
});
|
|
1248
|
+
log(`Failed job ${jobId}`);
|
|
1249
|
+
} catch (err) {
|
|
1250
|
+
log(`Error failing job ${jobId}: ${err}`);
|
|
1251
|
+
throw err;
|
|
950
1252
|
} finally {
|
|
951
1253
|
client.release();
|
|
952
1254
|
}
|
|
953
1255
|
}
|
|
954
|
-
async
|
|
1256
|
+
async prolongJob(jobId) {
|
|
955
1257
|
const client = await this.pool.connect();
|
|
956
1258
|
try {
|
|
957
|
-
|
|
1259
|
+
await client.query(
|
|
958
1260
|
`
|
|
959
1261
|
UPDATE job_queue
|
|
960
|
-
SET
|
|
961
|
-
WHERE status = 'processing'
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
|
|
965
|
-
)
|
|
966
|
-
RETURNING id
|
|
967
|
-
`,
|
|
968
|
-
[maxProcessingTimeMinutes]
|
|
1262
|
+
SET locked_at = NOW(), updated_at = NOW()
|
|
1263
|
+
WHERE id = $1 AND status = 'processing'
|
|
1264
|
+
`,
|
|
1265
|
+
[jobId]
|
|
969
1266
|
);
|
|
970
|
-
|
|
971
|
-
|
|
1267
|
+
await this.recordJobEvent(jobId, "prolonged" /* Prolonged */);
|
|
1268
|
+
log(`Prolonged job ${jobId}`);
|
|
972
1269
|
} catch (error) {
|
|
973
|
-
log(`Error
|
|
974
|
-
throw error;
|
|
1270
|
+
log(`Error prolonging job ${jobId}: ${error}`);
|
|
975
1271
|
} finally {
|
|
976
1272
|
client.release();
|
|
977
1273
|
}
|
|
978
1274
|
}
|
|
979
|
-
// ──
|
|
980
|
-
|
|
981
|
-
* Batch-insert multiple job events in a single query.
|
|
982
|
-
* More efficient than individual recordJobEvent calls.
|
|
983
|
-
*/
|
|
984
|
-
async recordJobEventsBatch(events) {
|
|
985
|
-
if (events.length === 0) return;
|
|
1275
|
+
// ── Progress ──────────────────────────────────────────────────────────
|
|
1276
|
+
async updateProgress(jobId, progress) {
|
|
986
1277
|
const client = await this.pool.connect();
|
|
987
1278
|
try {
|
|
988
|
-
const values = [];
|
|
989
|
-
const params = [];
|
|
990
|
-
let paramIdx = 1;
|
|
991
|
-
for (const event of events) {
|
|
992
|
-
values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
|
|
993
|
-
params.push(
|
|
994
|
-
event.jobId,
|
|
995
|
-
event.eventType,
|
|
996
|
-
event.metadata ? JSON.stringify(event.metadata) : null
|
|
997
|
-
);
|
|
998
|
-
}
|
|
999
1279
|
await client.query(
|
|
1000
|
-
`
|
|
1001
|
-
|
|
1280
|
+
`UPDATE job_queue SET progress = $2, updated_at = NOW() WHERE id = $1`,
|
|
1281
|
+
[jobId, progress]
|
|
1002
1282
|
);
|
|
1283
|
+
log(`Updated progress for job ${jobId}: ${progress}%`);
|
|
1003
1284
|
} catch (error) {
|
|
1004
|
-
log(`Error
|
|
1285
|
+
log(`Error updating progress for job ${jobId}: ${error}`);
|
|
1005
1286
|
} finally {
|
|
1006
1287
|
client.release();
|
|
1007
1288
|
}
|
|
1008
1289
|
}
|
|
1009
|
-
// ──
|
|
1010
|
-
|
|
1011
|
-
async addCronSchedule(input) {
|
|
1290
|
+
// ── Job management ────────────────────────────────────────────────────
|
|
1291
|
+
async retryJob(jobId) {
|
|
1012
1292
|
const client = await this.pool.connect();
|
|
1013
1293
|
try {
|
|
1014
1294
|
const result = await client.query(
|
|
1015
|
-
`
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
input.maxAttempts,
|
|
1027
|
-
input.priority,
|
|
1028
|
-
input.timeoutMs,
|
|
1029
|
-
input.forceKillOnTimeout,
|
|
1030
|
-
input.tags ?? null,
|
|
1031
|
-
input.timezone,
|
|
1032
|
-
input.allowOverlap,
|
|
1033
|
-
input.nextRunAt
|
|
1034
|
-
]
|
|
1295
|
+
`
|
|
1296
|
+
UPDATE job_queue
|
|
1297
|
+
SET status = 'pending',
|
|
1298
|
+
updated_at = NOW(),
|
|
1299
|
+
locked_at = NULL,
|
|
1300
|
+
locked_by = NULL,
|
|
1301
|
+
next_attempt_at = NOW(),
|
|
1302
|
+
last_retried_at = NOW()
|
|
1303
|
+
WHERE id = $1 AND status IN ('failed', 'processing')
|
|
1304
|
+
`,
|
|
1305
|
+
[jobId]
|
|
1035
1306
|
);
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
} catch (error) {
|
|
1040
|
-
if (error?.code === "23505") {
|
|
1041
|
-
throw new Error(
|
|
1042
|
-
`Cron schedule with name "${input.scheduleName}" already exists`
|
|
1307
|
+
if (result.rowCount === 0) {
|
|
1308
|
+
log(
|
|
1309
|
+
`Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
|
|
1043
1310
|
);
|
|
1044
1311
|
}
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
} finally {
|
|
1048
|
-
client.release();
|
|
1049
|
-
}
|
|
1050
|
-
}
|
|
1051
|
-
/** Get a cron schedule by ID. */
|
|
1052
|
-
async getCronSchedule(id) {
|
|
1053
|
-
const client = await this.pool.connect();
|
|
1054
|
-
try {
|
|
1055
|
-
const result = await client.query(
|
|
1056
|
-
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1057
|
-
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1058
|
-
priority, timeout_ms AS "timeoutMs",
|
|
1059
|
-
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1060
|
-
timezone, allow_overlap AS "allowOverlap", status,
|
|
1061
|
-
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1062
|
-
next_run_at AS "nextRunAt",
|
|
1063
|
-
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1064
|
-
FROM cron_schedules WHERE id = $1`,
|
|
1065
|
-
[id]
|
|
1066
|
-
);
|
|
1067
|
-
if (result.rows.length === 0) return null;
|
|
1068
|
-
return result.rows[0];
|
|
1312
|
+
await this.recordJobEvent(jobId, "retried" /* Retried */);
|
|
1313
|
+
log(`Retried job ${jobId}`);
|
|
1069
1314
|
} catch (error) {
|
|
1070
|
-
log(`Error
|
|
1315
|
+
log(`Error retrying job ${jobId}: ${error}`);
|
|
1071
1316
|
throw error;
|
|
1072
1317
|
} finally {
|
|
1073
1318
|
client.release();
|
|
1074
1319
|
}
|
|
1075
1320
|
}
|
|
1076
|
-
|
|
1077
|
-
async getCronScheduleByName(name) {
|
|
1321
|
+
async cancelJob(jobId) {
|
|
1078
1322
|
const client = await this.pool.connect();
|
|
1079
1323
|
try {
|
|
1080
|
-
|
|
1081
|
-
`
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1089
|
-
FROM cron_schedules WHERE schedule_name = $1`,
|
|
1090
|
-
[name]
|
|
1324
|
+
await client.query(
|
|
1325
|
+
`
|
|
1326
|
+
UPDATE job_queue
|
|
1327
|
+
SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
|
|
1328
|
+
wait_until = NULL, wait_token_id = NULL
|
|
1329
|
+
WHERE id = $1 AND status IN ('pending', 'waiting')
|
|
1330
|
+
`,
|
|
1331
|
+
[jobId]
|
|
1091
1332
|
);
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
} catch (error) {
|
|
1095
|
-
log(`Error getting cron schedule by name "${name}": ${error}`);
|
|
1096
|
-
throw error;
|
|
1097
|
-
} finally {
|
|
1098
|
-
client.release();
|
|
1099
|
-
}
|
|
1100
|
-
}
|
|
1101
|
-
/** List cron schedules, optionally filtered by status. */
|
|
1102
|
-
async listCronSchedules(status) {
|
|
1103
|
-
const client = await this.pool.connect();
|
|
1104
|
-
try {
|
|
1105
|
-
let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1106
|
-
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1107
|
-
priority, timeout_ms AS "timeoutMs",
|
|
1108
|
-
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1109
|
-
timezone, allow_overlap AS "allowOverlap", status,
|
|
1110
|
-
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1111
|
-
next_run_at AS "nextRunAt",
|
|
1112
|
-
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1113
|
-
FROM cron_schedules`;
|
|
1114
|
-
const params = [];
|
|
1115
|
-
if (status) {
|
|
1116
|
-
query += ` WHERE status = $1`;
|
|
1117
|
-
params.push(status);
|
|
1118
|
-
}
|
|
1119
|
-
query += ` ORDER BY created_at ASC`;
|
|
1120
|
-
const result = await client.query(query, params);
|
|
1121
|
-
return result.rows;
|
|
1333
|
+
await this.recordJobEvent(jobId, "cancelled" /* Cancelled */);
|
|
1334
|
+
log(`Cancelled job ${jobId}`);
|
|
1122
1335
|
} catch (error) {
|
|
1123
|
-
log(`Error
|
|
1336
|
+
log(`Error cancelling job ${jobId}: ${error}`);
|
|
1124
1337
|
throw error;
|
|
1125
1338
|
} finally {
|
|
1126
1339
|
client.release();
|
|
1127
1340
|
}
|
|
1128
1341
|
}
|
|
1129
|
-
|
|
1130
|
-
async removeCronSchedule(id) {
|
|
1342
|
+
async cancelAllUpcomingJobs(filters) {
|
|
1131
1343
|
const client = await this.pool.connect();
|
|
1132
1344
|
try {
|
|
1133
|
-
|
|
1134
|
-
|
|
1345
|
+
let query = `
|
|
1346
|
+
UPDATE job_queue
|
|
1347
|
+
SET status = 'cancelled', updated_at = NOW()
|
|
1348
|
+
WHERE status = 'pending'`;
|
|
1349
|
+
const params = [];
|
|
1350
|
+
let paramIdx = 1;
|
|
1351
|
+
if (filters) {
|
|
1352
|
+
if (filters.jobType) {
|
|
1353
|
+
query += ` AND job_type = $${paramIdx++}`;
|
|
1354
|
+
params.push(filters.jobType);
|
|
1355
|
+
}
|
|
1356
|
+
if (filters.priority !== void 0) {
|
|
1357
|
+
query += ` AND priority = $${paramIdx++}`;
|
|
1358
|
+
params.push(filters.priority);
|
|
1359
|
+
}
|
|
1360
|
+
if (filters.runAt) {
|
|
1361
|
+
if (filters.runAt instanceof Date) {
|
|
1362
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1363
|
+
params.push(filters.runAt);
|
|
1364
|
+
} else if (typeof filters.runAt === "object") {
|
|
1365
|
+
const ops = filters.runAt;
|
|
1366
|
+
if (ops.gt) {
|
|
1367
|
+
query += ` AND run_at > $${paramIdx++}`;
|
|
1368
|
+
params.push(ops.gt);
|
|
1369
|
+
}
|
|
1370
|
+
if (ops.gte) {
|
|
1371
|
+
query += ` AND run_at >= $${paramIdx++}`;
|
|
1372
|
+
params.push(ops.gte);
|
|
1373
|
+
}
|
|
1374
|
+
if (ops.lt) {
|
|
1375
|
+
query += ` AND run_at < $${paramIdx++}`;
|
|
1376
|
+
params.push(ops.lt);
|
|
1377
|
+
}
|
|
1378
|
+
if (ops.lte) {
|
|
1379
|
+
query += ` AND run_at <= $${paramIdx++}`;
|
|
1380
|
+
params.push(ops.lte);
|
|
1381
|
+
}
|
|
1382
|
+
if (ops.eq) {
|
|
1383
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1384
|
+
params.push(ops.eq);
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
}
|
|
1388
|
+
if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
|
|
1389
|
+
const mode = filters.tags.mode || "all";
|
|
1390
|
+
const tagValues = filters.tags.values;
|
|
1391
|
+
switch (mode) {
|
|
1392
|
+
case "exact":
|
|
1393
|
+
query += ` AND tags = $${paramIdx++}`;
|
|
1394
|
+
params.push(tagValues);
|
|
1395
|
+
break;
|
|
1396
|
+
case "all":
|
|
1397
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1398
|
+
params.push(tagValues);
|
|
1399
|
+
break;
|
|
1400
|
+
case "any":
|
|
1401
|
+
query += ` AND tags && $${paramIdx++}`;
|
|
1402
|
+
params.push(tagValues);
|
|
1403
|
+
break;
|
|
1404
|
+
case "none":
|
|
1405
|
+
query += ` AND NOT (tags && $${paramIdx++})`;
|
|
1406
|
+
params.push(tagValues);
|
|
1407
|
+
break;
|
|
1408
|
+
default:
|
|
1409
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1410
|
+
params.push(tagValues);
|
|
1411
|
+
}
|
|
1412
|
+
}
|
|
1413
|
+
}
|
|
1414
|
+
query += "\nRETURNING id";
|
|
1415
|
+
const result = await client.query(query, params);
|
|
1416
|
+
log(`Cancelled ${result.rowCount} jobs`);
|
|
1417
|
+
return result.rowCount || 0;
|
|
1135
1418
|
} catch (error) {
|
|
1136
|
-
log(`Error
|
|
1419
|
+
log(`Error cancelling upcoming jobs: ${error}`);
|
|
1137
1420
|
throw error;
|
|
1138
1421
|
} finally {
|
|
1139
1422
|
client.release();
|
|
1140
1423
|
}
|
|
1141
1424
|
}
|
|
1142
|
-
|
|
1143
|
-
async pauseCronSchedule(id) {
|
|
1425
|
+
async editJob(jobId, updates) {
|
|
1144
1426
|
const client = await this.pool.connect();
|
|
1145
1427
|
try {
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
)
|
|
1150
|
-
|
|
1428
|
+
const updateFields = [];
|
|
1429
|
+
const params = [];
|
|
1430
|
+
let paramIdx = 1;
|
|
1431
|
+
if (updates.payload !== void 0) {
|
|
1432
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
1433
|
+
params.push(updates.payload);
|
|
1434
|
+
}
|
|
1435
|
+
if (updates.maxAttempts !== void 0) {
|
|
1436
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
1437
|
+
params.push(updates.maxAttempts);
|
|
1438
|
+
}
|
|
1439
|
+
if (updates.priority !== void 0) {
|
|
1440
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
1441
|
+
params.push(updates.priority);
|
|
1442
|
+
}
|
|
1443
|
+
if (updates.runAt !== void 0) {
|
|
1444
|
+
if (updates.runAt === null) {
|
|
1445
|
+
updateFields.push(`run_at = NOW()`);
|
|
1446
|
+
} else {
|
|
1447
|
+
updateFields.push(`run_at = $${paramIdx++}`);
|
|
1448
|
+
params.push(updates.runAt);
|
|
1449
|
+
}
|
|
1450
|
+
}
|
|
1451
|
+
if (updates.timeoutMs !== void 0) {
|
|
1452
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
1453
|
+
params.push(updates.timeoutMs ?? null);
|
|
1454
|
+
}
|
|
1455
|
+
if (updates.tags !== void 0) {
|
|
1456
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
1457
|
+
params.push(updates.tags ?? null);
|
|
1458
|
+
}
|
|
1459
|
+
if (updateFields.length === 0) {
|
|
1460
|
+
log(`No fields to update for job ${jobId}`);
|
|
1461
|
+
return;
|
|
1462
|
+
}
|
|
1463
|
+
updateFields.push(`updated_at = NOW()`);
|
|
1464
|
+
params.push(jobId);
|
|
1465
|
+
const query = `
|
|
1466
|
+
UPDATE job_queue
|
|
1467
|
+
SET ${updateFields.join(", ")}
|
|
1468
|
+
WHERE id = $${paramIdx} AND status = 'pending'
|
|
1469
|
+
`;
|
|
1470
|
+
await client.query(query, params);
|
|
1471
|
+
const metadata = {};
|
|
1472
|
+
if (updates.payload !== void 0) metadata.payload = updates.payload;
|
|
1473
|
+
if (updates.maxAttempts !== void 0)
|
|
1474
|
+
metadata.maxAttempts = updates.maxAttempts;
|
|
1475
|
+
if (updates.priority !== void 0) metadata.priority = updates.priority;
|
|
1476
|
+
if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
|
|
1477
|
+
if (updates.timeoutMs !== void 0)
|
|
1478
|
+
metadata.timeoutMs = updates.timeoutMs;
|
|
1479
|
+
if (updates.tags !== void 0) metadata.tags = updates.tags;
|
|
1480
|
+
await this.recordJobEvent(jobId, "edited" /* Edited */, metadata);
|
|
1481
|
+
log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
|
|
1151
1482
|
} catch (error) {
|
|
1152
|
-
log(`Error
|
|
1483
|
+
log(`Error editing job ${jobId}: ${error}`);
|
|
1153
1484
|
throw error;
|
|
1154
1485
|
} finally {
|
|
1155
1486
|
client.release();
|
|
1156
1487
|
}
|
|
1157
1488
|
}
|
|
1158
|
-
|
|
1159
|
-
async resumeCronSchedule(id) {
|
|
1489
|
+
async editAllPendingJobs(filters = void 0, updates) {
|
|
1160
1490
|
const client = await this.pool.connect();
|
|
1161
1491
|
try {
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
)
|
|
1166
|
-
|
|
1492
|
+
const updateFields = [];
|
|
1493
|
+
const params = [];
|
|
1494
|
+
let paramIdx = 1;
|
|
1495
|
+
if (updates.payload !== void 0) {
|
|
1496
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
1497
|
+
params.push(updates.payload);
|
|
1498
|
+
}
|
|
1499
|
+
if (updates.maxAttempts !== void 0) {
|
|
1500
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
1501
|
+
params.push(updates.maxAttempts);
|
|
1502
|
+
}
|
|
1503
|
+
if (updates.priority !== void 0) {
|
|
1504
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
1505
|
+
params.push(updates.priority);
|
|
1506
|
+
}
|
|
1507
|
+
if (updates.runAt !== void 0) {
|
|
1508
|
+
if (updates.runAt === null) {
|
|
1509
|
+
updateFields.push(`run_at = NOW()`);
|
|
1510
|
+
} else {
|
|
1511
|
+
updateFields.push(`run_at = $${paramIdx++}`);
|
|
1512
|
+
params.push(updates.runAt);
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
if (updates.timeoutMs !== void 0) {
|
|
1516
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
1517
|
+
params.push(updates.timeoutMs ?? null);
|
|
1518
|
+
}
|
|
1519
|
+
if (updates.tags !== void 0) {
|
|
1520
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
1521
|
+
params.push(updates.tags ?? null);
|
|
1522
|
+
}
|
|
1523
|
+
if (updateFields.length === 0) {
|
|
1524
|
+
log(`No fields to update for batch edit`);
|
|
1525
|
+
return 0;
|
|
1526
|
+
}
|
|
1527
|
+
updateFields.push(`updated_at = NOW()`);
|
|
1528
|
+
let query = `
|
|
1529
|
+
UPDATE job_queue
|
|
1530
|
+
SET ${updateFields.join(", ")}
|
|
1531
|
+
WHERE status = 'pending'`;
|
|
1532
|
+
if (filters) {
|
|
1533
|
+
if (filters.jobType) {
|
|
1534
|
+
query += ` AND job_type = $${paramIdx++}`;
|
|
1535
|
+
params.push(filters.jobType);
|
|
1536
|
+
}
|
|
1537
|
+
if (filters.priority !== void 0) {
|
|
1538
|
+
query += ` AND priority = $${paramIdx++}`;
|
|
1539
|
+
params.push(filters.priority);
|
|
1540
|
+
}
|
|
1541
|
+
if (filters.runAt) {
|
|
1542
|
+
if (filters.runAt instanceof Date) {
|
|
1543
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1544
|
+
params.push(filters.runAt);
|
|
1545
|
+
} else if (typeof filters.runAt === "object") {
|
|
1546
|
+
const ops = filters.runAt;
|
|
1547
|
+
if (ops.gt) {
|
|
1548
|
+
query += ` AND run_at > $${paramIdx++}`;
|
|
1549
|
+
params.push(ops.gt);
|
|
1550
|
+
}
|
|
1551
|
+
if (ops.gte) {
|
|
1552
|
+
query += ` AND run_at >= $${paramIdx++}`;
|
|
1553
|
+
params.push(ops.gte);
|
|
1554
|
+
}
|
|
1555
|
+
if (ops.lt) {
|
|
1556
|
+
query += ` AND run_at < $${paramIdx++}`;
|
|
1557
|
+
params.push(ops.lt);
|
|
1558
|
+
}
|
|
1559
|
+
if (ops.lte) {
|
|
1560
|
+
query += ` AND run_at <= $${paramIdx++}`;
|
|
1561
|
+
params.push(ops.lte);
|
|
1562
|
+
}
|
|
1563
|
+
if (ops.eq) {
|
|
1564
|
+
query += ` AND run_at = $${paramIdx++}`;
|
|
1565
|
+
params.push(ops.eq);
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
|
|
1570
|
+
const mode = filters.tags.mode || "all";
|
|
1571
|
+
const tagValues = filters.tags.values;
|
|
1572
|
+
switch (mode) {
|
|
1573
|
+
case "exact":
|
|
1574
|
+
query += ` AND tags = $${paramIdx++}`;
|
|
1575
|
+
params.push(tagValues);
|
|
1576
|
+
break;
|
|
1577
|
+
case "all":
|
|
1578
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1579
|
+
params.push(tagValues);
|
|
1580
|
+
break;
|
|
1581
|
+
case "any":
|
|
1582
|
+
query += ` AND tags && $${paramIdx++}`;
|
|
1583
|
+
params.push(tagValues);
|
|
1584
|
+
break;
|
|
1585
|
+
case "none":
|
|
1586
|
+
query += ` AND NOT (tags && $${paramIdx++})`;
|
|
1587
|
+
params.push(tagValues);
|
|
1588
|
+
break;
|
|
1589
|
+
default:
|
|
1590
|
+
query += ` AND tags @> $${paramIdx++}`;
|
|
1591
|
+
params.push(tagValues);
|
|
1592
|
+
}
|
|
1593
|
+
}
|
|
1594
|
+
}
|
|
1595
|
+
query += "\nRETURNING id";
|
|
1596
|
+
const result = await client.query(query, params);
|
|
1597
|
+
const editedCount = result.rowCount || 0;
|
|
1598
|
+
const metadata = {};
|
|
1599
|
+
if (updates.payload !== void 0) metadata.payload = updates.payload;
|
|
1600
|
+
if (updates.maxAttempts !== void 0)
|
|
1601
|
+
metadata.maxAttempts = updates.maxAttempts;
|
|
1602
|
+
if (updates.priority !== void 0) metadata.priority = updates.priority;
|
|
1603
|
+
if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
|
|
1604
|
+
if (updates.timeoutMs !== void 0)
|
|
1605
|
+
metadata.timeoutMs = updates.timeoutMs;
|
|
1606
|
+
if (updates.tags !== void 0) metadata.tags = updates.tags;
|
|
1607
|
+
for (const row of result.rows) {
|
|
1608
|
+
await this.recordJobEvent(row.id, "edited" /* Edited */, metadata);
|
|
1609
|
+
}
|
|
1610
|
+
log(`Edited ${editedCount} pending jobs: ${JSON.stringify(metadata)}`);
|
|
1611
|
+
return editedCount;
|
|
1167
1612
|
} catch (error) {
|
|
1168
|
-
log(`Error
|
|
1613
|
+
log(`Error editing pending jobs: ${error}`);
|
|
1169
1614
|
throw error;
|
|
1170
1615
|
} finally {
|
|
1171
1616
|
client.release();
|
|
1172
1617
|
}
|
|
1173
1618
|
}
|
|
1174
|
-
/**
|
|
1175
|
-
|
|
1176
|
-
|
|
1619
|
+
/**
|
|
1620
|
+
* Delete completed jobs older than the given number of days.
|
|
1621
|
+
* Deletes in batches of 1000 to avoid long-running transactions
|
|
1622
|
+
* and excessive WAL bloat at scale.
|
|
1623
|
+
*
|
|
1624
|
+
* @param daysToKeep - Number of days to retain completed jobs (default 30).
|
|
1625
|
+
* @param batchSize - Number of rows to delete per batch (default 1000).
|
|
1626
|
+
* @returns Total number of deleted jobs.
|
|
1627
|
+
*/
|
|
1628
|
+
async cleanupOldJobs(daysToKeep = 30, batchSize = 1e3) {
|
|
1629
|
+
let totalDeleted = 0;
|
|
1177
1630
|
try {
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
}
|
|
1201
|
-
|
|
1202
|
-
updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
|
|
1203
|
-
params.push(updates.forceKillOnTimeout);
|
|
1204
|
-
}
|
|
1205
|
-
if (updates.tags !== void 0) {
|
|
1206
|
-
updateFields.push(`tags = $${paramIdx++}`);
|
|
1207
|
-
params.push(updates.tags);
|
|
1208
|
-
}
|
|
1209
|
-
if (updates.timezone !== void 0) {
|
|
1210
|
-
updateFields.push(`timezone = $${paramIdx++}`);
|
|
1211
|
-
params.push(updates.timezone);
|
|
1212
|
-
}
|
|
1213
|
-
if (updates.allowOverlap !== void 0) {
|
|
1214
|
-
updateFields.push(`allow_overlap = $${paramIdx++}`);
|
|
1215
|
-
params.push(updates.allowOverlap);
|
|
1216
|
-
}
|
|
1217
|
-
if (nextRunAt !== void 0) {
|
|
1218
|
-
updateFields.push(`next_run_at = $${paramIdx++}`);
|
|
1219
|
-
params.push(nextRunAt);
|
|
1220
|
-
}
|
|
1221
|
-
if (updateFields.length === 0) {
|
|
1222
|
-
log(`No fields to update for cron schedule ${id}`);
|
|
1223
|
-
return;
|
|
1224
|
-
}
|
|
1225
|
-
updateFields.push(`updated_at = NOW()`);
|
|
1226
|
-
params.push(id);
|
|
1227
|
-
const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
|
|
1228
|
-
await client.query(query, params);
|
|
1229
|
-
log(`Edited cron schedule ${id}`);
|
|
1631
|
+
let deletedInBatch;
|
|
1632
|
+
do {
|
|
1633
|
+
const client = await this.pool.connect();
|
|
1634
|
+
try {
|
|
1635
|
+
const result = await client.query(
|
|
1636
|
+
`
|
|
1637
|
+
DELETE FROM job_queue
|
|
1638
|
+
WHERE id IN (
|
|
1639
|
+
SELECT id FROM job_queue
|
|
1640
|
+
WHERE status = 'completed'
|
|
1641
|
+
AND updated_at < NOW() - INTERVAL '1 day' * $1::int
|
|
1642
|
+
LIMIT $2
|
|
1643
|
+
)
|
|
1644
|
+
`,
|
|
1645
|
+
[daysToKeep, batchSize]
|
|
1646
|
+
);
|
|
1647
|
+
deletedInBatch = result.rowCount || 0;
|
|
1648
|
+
totalDeleted += deletedInBatch;
|
|
1649
|
+
} finally {
|
|
1650
|
+
client.release();
|
|
1651
|
+
}
|
|
1652
|
+
} while (deletedInBatch === batchSize);
|
|
1653
|
+
log(`Deleted ${totalDeleted} old jobs`);
|
|
1654
|
+
return totalDeleted;
|
|
1230
1655
|
} catch (error) {
|
|
1231
|
-
log(`Error
|
|
1656
|
+
log(`Error cleaning up old jobs: ${error}`);
|
|
1232
1657
|
throw error;
|
|
1233
|
-
} finally {
|
|
1234
|
-
client.release();
|
|
1235
1658
|
}
|
|
1236
1659
|
}
|
|
1237
1660
|
/**
|
|
1238
|
-
*
|
|
1239
|
-
*
|
|
1661
|
+
* Delete job events older than the given number of days.
|
|
1662
|
+
* Deletes in batches of 1000 to avoid long-running transactions
|
|
1663
|
+
* and excessive WAL bloat at scale.
|
|
1664
|
+
*
|
|
1665
|
+
* @param daysToKeep - Number of days to retain events (default 30).
|
|
1666
|
+
* @param batchSize - Number of rows to delete per batch (default 1000).
|
|
1667
|
+
* @returns Total number of deleted events.
|
|
1240
1668
|
*/
|
|
1241
|
-
async
|
|
1242
|
-
|
|
1669
|
+
async cleanupOldJobEvents(daysToKeep = 30, batchSize = 1e3) {
|
|
1670
|
+
let totalDeleted = 0;
|
|
1243
1671
|
try {
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1672
|
+
let deletedInBatch;
|
|
1673
|
+
do {
|
|
1674
|
+
const client = await this.pool.connect();
|
|
1675
|
+
try {
|
|
1676
|
+
const result = await client.query(
|
|
1677
|
+
`
|
|
1678
|
+
DELETE FROM job_events
|
|
1679
|
+
WHERE id IN (
|
|
1680
|
+
SELECT id FROM job_events
|
|
1681
|
+
WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
|
|
1682
|
+
LIMIT $2
|
|
1683
|
+
)
|
|
1684
|
+
`,
|
|
1685
|
+
[daysToKeep, batchSize]
|
|
1686
|
+
);
|
|
1687
|
+
deletedInBatch = result.rowCount || 0;
|
|
1688
|
+
totalDeleted += deletedInBatch;
|
|
1689
|
+
} finally {
|
|
1690
|
+
client.release();
|
|
1691
|
+
}
|
|
1692
|
+
} while (deletedInBatch === batchSize);
|
|
1693
|
+
log(`Deleted ${totalDeleted} old job events`);
|
|
1694
|
+
return totalDeleted;
|
|
1262
1695
|
} catch (error) {
|
|
1263
|
-
|
|
1264
|
-
log("cron_schedules table does not exist, skipping cron enqueue");
|
|
1265
|
-
return [];
|
|
1266
|
-
}
|
|
1267
|
-
log(`Error getting due cron schedules: ${error}`);
|
|
1696
|
+
log(`Error cleaning up old job events: ${error}`);
|
|
1268
1697
|
throw error;
|
|
1269
|
-
} finally {
|
|
1270
|
-
client.release();
|
|
1271
1698
|
}
|
|
1272
1699
|
}
|
|
1273
|
-
|
|
1274
|
-
* Update a cron schedule after a job has been enqueued.
|
|
1275
|
-
* Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
|
|
1276
|
-
*/
|
|
1277
|
-
async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
|
|
1700
|
+
async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
|
|
1278
1701
|
const client = await this.pool.connect();
|
|
1279
1702
|
try {
|
|
1280
|
-
await client.query(
|
|
1281
|
-
`
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1703
|
+
const result = await client.query(
|
|
1704
|
+
`
|
|
1705
|
+
UPDATE job_queue
|
|
1706
|
+
SET status = 'pending', locked_at = NULL, locked_by = NULL, updated_at = NOW()
|
|
1707
|
+
WHERE status = 'processing'
|
|
1708
|
+
AND locked_at < NOW() - GREATEST(
|
|
1709
|
+
INTERVAL '1 minute' * $1::int,
|
|
1710
|
+
INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
|
|
1711
|
+
)
|
|
1712
|
+
RETURNING id
|
|
1713
|
+
`,
|
|
1714
|
+
[maxProcessingTimeMinutes]
|
|
1291
1715
|
);
|
|
1716
|
+
log(`Reclaimed ${result.rowCount} stuck jobs`);
|
|
1717
|
+
return result.rowCount || 0;
|
|
1292
1718
|
} catch (error) {
|
|
1293
|
-
log(`Error
|
|
1719
|
+
log(`Error reclaiming stuck jobs: ${error}`);
|
|
1294
1720
|
throw error;
|
|
1295
1721
|
} finally {
|
|
1296
1722
|
client.release();
|
|
1297
1723
|
}
|
|
1298
1724
|
}
|
|
1299
1725
|
// ── Internal helpers ──────────────────────────────────────────────────
|
|
1300
|
-
|
|
1726
|
+
/**
|
|
1727
|
+
* Batch-insert multiple job events in a single query.
|
|
1728
|
+
* More efficient than individual recordJobEvent calls.
|
|
1729
|
+
*/
|
|
1730
|
+
async recordJobEventsBatch(events) {
|
|
1731
|
+
if (events.length === 0) return;
|
|
1301
1732
|
const client = await this.pool.connect();
|
|
1302
1733
|
try {
|
|
1303
|
-
|
|
1304
|
-
const params = [
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
}
|
|
1314
|
-
await client.query(
|
|
1315
|
-
`UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
|
|
1316
|
-
params
|
|
1317
|
-
);
|
|
1318
|
-
} finally {
|
|
1319
|
-
client.release();
|
|
1320
|
-
}
|
|
1321
|
-
}
|
|
1322
|
-
};
|
|
1323
|
-
var recordJobEvent = async (pool, jobId, eventType, metadata) => new PostgresBackend(pool).recordJobEvent(jobId, eventType, metadata);
|
|
1324
|
-
var waitJob = async (pool, jobId, options) => {
|
|
1325
|
-
const client = await pool.connect();
|
|
1326
|
-
try {
|
|
1327
|
-
const result = await client.query(
|
|
1328
|
-
`
|
|
1329
|
-
UPDATE job_queue
|
|
1330
|
-
SET status = 'waiting',
|
|
1331
|
-
wait_until = $2,
|
|
1332
|
-
wait_token_id = $3,
|
|
1333
|
-
step_data = $4,
|
|
1334
|
-
locked_at = NULL,
|
|
1335
|
-
locked_by = NULL,
|
|
1336
|
-
updated_at = NOW()
|
|
1337
|
-
WHERE id = $1 AND status = 'processing'
|
|
1338
|
-
`,
|
|
1339
|
-
[
|
|
1340
|
-
jobId,
|
|
1341
|
-
options.waitUntil ?? null,
|
|
1342
|
-
options.waitTokenId ?? null,
|
|
1343
|
-
JSON.stringify(options.stepData)
|
|
1344
|
-
]
|
|
1345
|
-
);
|
|
1346
|
-
if (result.rowCount === 0) {
|
|
1347
|
-
log(
|
|
1348
|
-
`Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
|
|
1349
|
-
);
|
|
1350
|
-
return;
|
|
1351
|
-
}
|
|
1352
|
-
await recordJobEvent(pool, jobId, "waiting" /* Waiting */, {
|
|
1353
|
-
waitUntil: options.waitUntil?.toISOString() ?? null,
|
|
1354
|
-
waitTokenId: options.waitTokenId ?? null
|
|
1355
|
-
});
|
|
1356
|
-
log(`Job ${jobId} set to waiting`);
|
|
1357
|
-
} catch (error) {
|
|
1358
|
-
log(`Error setting job ${jobId} to waiting: ${error}`);
|
|
1359
|
-
throw error;
|
|
1360
|
-
} finally {
|
|
1361
|
-
client.release();
|
|
1362
|
-
}
|
|
1363
|
-
};
|
|
1364
|
-
var updateStepData = async (pool, jobId, stepData) => {
|
|
1365
|
-
const client = await pool.connect();
|
|
1366
|
-
try {
|
|
1367
|
-
await client.query(
|
|
1368
|
-
`UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
|
|
1369
|
-
[jobId, JSON.stringify(stepData)]
|
|
1370
|
-
);
|
|
1371
|
-
} catch (error) {
|
|
1372
|
-
log(`Error updating step_data for job ${jobId}: ${error}`);
|
|
1373
|
-
} finally {
|
|
1374
|
-
client.release();
|
|
1375
|
-
}
|
|
1376
|
-
};
|
|
1377
|
-
var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
|
|
1378
|
-
function parseTimeoutString(timeout) {
|
|
1379
|
-
const match = timeout.match(/^(\d+)(s|m|h|d)$/);
|
|
1380
|
-
if (!match) {
|
|
1381
|
-
throw new Error(
|
|
1382
|
-
`Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
|
|
1383
|
-
);
|
|
1384
|
-
}
|
|
1385
|
-
const value = parseInt(match[1], 10);
|
|
1386
|
-
const unit = match[2];
|
|
1387
|
-
let ms;
|
|
1388
|
-
switch (unit) {
|
|
1389
|
-
case "s":
|
|
1390
|
-
ms = value * 1e3;
|
|
1391
|
-
break;
|
|
1392
|
-
case "m":
|
|
1393
|
-
ms = value * 60 * 1e3;
|
|
1394
|
-
break;
|
|
1395
|
-
case "h":
|
|
1396
|
-
ms = value * 60 * 60 * 1e3;
|
|
1397
|
-
break;
|
|
1398
|
-
case "d":
|
|
1399
|
-
ms = value * 24 * 60 * 60 * 1e3;
|
|
1400
|
-
break;
|
|
1401
|
-
default:
|
|
1402
|
-
throw new Error(`Unknown timeout unit: "${unit}"`);
|
|
1403
|
-
}
|
|
1404
|
-
if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
|
|
1405
|
-
throw new Error(
|
|
1406
|
-
`Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
|
|
1407
|
-
);
|
|
1408
|
-
}
|
|
1409
|
-
return ms;
|
|
1410
|
-
}
|
|
1411
|
-
var createWaitpoint = async (pool, jobId, options) => {
|
|
1412
|
-
const client = await pool.connect();
|
|
1413
|
-
try {
|
|
1414
|
-
const id = `wp_${crypto.randomUUID()}`;
|
|
1415
|
-
let timeoutAt = null;
|
|
1416
|
-
if (options?.timeout) {
|
|
1417
|
-
const ms = parseTimeoutString(options.timeout);
|
|
1418
|
-
timeoutAt = new Date(Date.now() + ms);
|
|
1419
|
-
}
|
|
1420
|
-
await client.query(
|
|
1421
|
-
`INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
|
|
1422
|
-
[id, jobId, timeoutAt, options?.tags ?? null]
|
|
1423
|
-
);
|
|
1424
|
-
log(`Created waitpoint ${id} for job ${jobId}`);
|
|
1425
|
-
return { id };
|
|
1426
|
-
} catch (error) {
|
|
1427
|
-
log(`Error creating waitpoint: ${error}`);
|
|
1428
|
-
throw error;
|
|
1429
|
-
} finally {
|
|
1430
|
-
client.release();
|
|
1431
|
-
}
|
|
1432
|
-
};
|
|
1433
|
-
var completeWaitpoint = async (pool, tokenId, data) => {
|
|
1434
|
-
const client = await pool.connect();
|
|
1435
|
-
try {
|
|
1436
|
-
await client.query("BEGIN");
|
|
1437
|
-
const wpResult = await client.query(
|
|
1438
|
-
`UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
|
|
1439
|
-
WHERE id = $1 AND status = 'waiting'
|
|
1440
|
-
RETURNING job_id`,
|
|
1441
|
-
[tokenId, data != null ? JSON.stringify(data) : null]
|
|
1442
|
-
);
|
|
1443
|
-
if (wpResult.rows.length === 0) {
|
|
1444
|
-
await client.query("ROLLBACK");
|
|
1445
|
-
log(`Waitpoint ${tokenId} not found or already completed`);
|
|
1446
|
-
return;
|
|
1447
|
-
}
|
|
1448
|
-
const jobId = wpResult.rows[0].job_id;
|
|
1449
|
-
if (jobId != null) {
|
|
1734
|
+
const values = [];
|
|
1735
|
+
const params = [];
|
|
1736
|
+
let paramIdx = 1;
|
|
1737
|
+
for (const event of events) {
|
|
1738
|
+
values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
|
|
1739
|
+
params.push(
|
|
1740
|
+
event.jobId,
|
|
1741
|
+
event.eventType,
|
|
1742
|
+
event.metadata ? JSON.stringify(event.metadata) : null
|
|
1743
|
+
);
|
|
1744
|
+
}
|
|
1450
1745
|
await client.query(
|
|
1451
|
-
`
|
|
1452
|
-
|
|
1453
|
-
WHERE id = $1 AND status = 'waiting'`,
|
|
1454
|
-
[jobId]
|
|
1746
|
+
`INSERT INTO job_events (job_id, event_type, metadata) VALUES ${values.join(", ")}`,
|
|
1747
|
+
params
|
|
1455
1748
|
);
|
|
1749
|
+
} catch (error) {
|
|
1750
|
+
log(`Error recording batch job events: ${error}`);
|
|
1751
|
+
} finally {
|
|
1752
|
+
client.release();
|
|
1456
1753
|
}
|
|
1457
|
-
await client.query("COMMIT");
|
|
1458
|
-
log(`Completed waitpoint ${tokenId} for job ${jobId}`);
|
|
1459
|
-
} catch (error) {
|
|
1460
|
-
await client.query("ROLLBACK");
|
|
1461
|
-
log(`Error completing waitpoint ${tokenId}: ${error}`);
|
|
1462
|
-
throw error;
|
|
1463
|
-
} finally {
|
|
1464
|
-
client.release();
|
|
1465
|
-
}
|
|
1466
|
-
};
|
|
1467
|
-
var getWaitpoint = async (pool, tokenId) => {
|
|
1468
|
-
const client = await pool.connect();
|
|
1469
|
-
try {
|
|
1470
|
-
const result = await client.query(
|
|
1471
|
-
`SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
|
|
1472
|
-
[tokenId]
|
|
1473
|
-
);
|
|
1474
|
-
if (result.rows.length === 0) return null;
|
|
1475
|
-
return result.rows[0];
|
|
1476
|
-
} catch (error) {
|
|
1477
|
-
log(`Error getting waitpoint ${tokenId}: ${error}`);
|
|
1478
|
-
throw error;
|
|
1479
|
-
} finally {
|
|
1480
|
-
client.release();
|
|
1481
1754
|
}
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1755
|
+
// ── Cron schedules ──────────────────────────────────────────────────
|
|
1756
|
+
/** Create a cron schedule and return its ID. */
|
|
1757
|
+
async addCronSchedule(input) {
|
|
1758
|
+
const client = await this.pool.connect();
|
|
1759
|
+
try {
|
|
1760
|
+
const result = await client.query(
|
|
1761
|
+
`INSERT INTO cron_schedules
|
|
1762
|
+
(schedule_name, cron_expression, job_type, payload, max_attempts,
|
|
1763
|
+
priority, timeout_ms, force_kill_on_timeout, tags, timezone,
|
|
1764
|
+
allow_overlap, next_run_at)
|
|
1765
|
+
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
|
|
1766
|
+
RETURNING id`,
|
|
1767
|
+
[
|
|
1768
|
+
input.scheduleName,
|
|
1769
|
+
input.cronExpression,
|
|
1770
|
+
input.jobType,
|
|
1771
|
+
input.payload,
|
|
1772
|
+
input.maxAttempts,
|
|
1773
|
+
input.priority,
|
|
1774
|
+
input.timeoutMs,
|
|
1775
|
+
input.forceKillOnTimeout,
|
|
1776
|
+
input.tags ?? null,
|
|
1777
|
+
input.timezone,
|
|
1778
|
+
input.allowOverlap,
|
|
1779
|
+
input.nextRunAt
|
|
1780
|
+
]
|
|
1781
|
+
);
|
|
1782
|
+
const id = result.rows[0].id;
|
|
1783
|
+
log(`Added cron schedule ${id}: "${input.scheduleName}"`);
|
|
1784
|
+
return id;
|
|
1785
|
+
} catch (error) {
|
|
1786
|
+
if (error?.code === "23505") {
|
|
1787
|
+
throw new Error(
|
|
1788
|
+
`Cron schedule with name "${input.scheduleName}" already exists`
|
|
1500
1789
|
);
|
|
1501
1790
|
}
|
|
1791
|
+
log(`Error adding cron schedule: ${error}`);
|
|
1792
|
+
throw error;
|
|
1793
|
+
} finally {
|
|
1794
|
+
client.release();
|
|
1502
1795
|
}
|
|
1503
|
-
await client.query("COMMIT");
|
|
1504
|
-
const count = result.rowCount || 0;
|
|
1505
|
-
if (count > 0) {
|
|
1506
|
-
log(`Expired ${count} timed-out waitpoints`);
|
|
1507
|
-
}
|
|
1508
|
-
return count;
|
|
1509
|
-
} catch (error) {
|
|
1510
|
-
await client.query("ROLLBACK");
|
|
1511
|
-
log(`Error expiring timed-out waitpoints: ${error}`);
|
|
1512
|
-
throw error;
|
|
1513
|
-
} finally {
|
|
1514
|
-
client.release();
|
|
1515
|
-
}
|
|
1516
|
-
};
|
|
1517
|
-
function tryExtractPool(backend) {
|
|
1518
|
-
if (backend instanceof PostgresBackend) {
|
|
1519
|
-
return backend.getPool();
|
|
1520
1796
|
}
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
waitUntil: async () => {
|
|
1537
|
-
throw waitError();
|
|
1538
|
-
},
|
|
1539
|
-
createToken: async () => {
|
|
1540
|
-
throw waitError();
|
|
1541
|
-
},
|
|
1542
|
-
waitForToken: async () => {
|
|
1543
|
-
throw waitError();
|
|
1544
|
-
},
|
|
1545
|
-
setProgress: async (percent) => {
|
|
1546
|
-
if (percent < 0 || percent > 100)
|
|
1547
|
-
throw new Error("Progress must be between 0 and 100");
|
|
1548
|
-
await backend.updateProgress(jobId, Math.round(percent));
|
|
1549
|
-
}
|
|
1550
|
-
};
|
|
1551
|
-
}
|
|
1552
|
-
function validateHandlerSerializable(handler, jobType) {
|
|
1553
|
-
try {
|
|
1554
|
-
const handlerString = handler.toString();
|
|
1555
|
-
if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
|
|
1556
|
-
throw new Error(
|
|
1557
|
-
`Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
|
|
1558
|
-
);
|
|
1559
|
-
}
|
|
1560
|
-
if (handlerString.includes("[native code]")) {
|
|
1561
|
-
throw new Error(
|
|
1562
|
-
`Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
|
|
1797
|
+
/** Get a cron schedule by ID. */
|
|
1798
|
+
async getCronSchedule(id) {
|
|
1799
|
+
const client = await this.pool.connect();
|
|
1800
|
+
try {
|
|
1801
|
+
const result = await client.query(
|
|
1802
|
+
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1803
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1804
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1805
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1806
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1807
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1808
|
+
next_run_at AS "nextRunAt",
|
|
1809
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1810
|
+
FROM cron_schedules WHERE id = $1`,
|
|
1811
|
+
[id]
|
|
1563
1812
|
);
|
|
1813
|
+
if (result.rows.length === 0) return null;
|
|
1814
|
+
return result.rows[0];
|
|
1815
|
+
} catch (error) {
|
|
1816
|
+
log(`Error getting cron schedule ${id}: ${error}`);
|
|
1817
|
+
throw error;
|
|
1818
|
+
} finally {
|
|
1819
|
+
client.release();
|
|
1564
1820
|
}
|
|
1821
|
+
}
|
|
1822
|
+
/** Get a cron schedule by its unique name. */
|
|
1823
|
+
async getCronScheduleByName(name) {
|
|
1824
|
+
const client = await this.pool.connect();
|
|
1565
1825
|
try {
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1826
|
+
const result = await client.query(
|
|
1827
|
+
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1828
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1829
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1830
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1831
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1832
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1833
|
+
next_run_at AS "nextRunAt",
|
|
1834
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1835
|
+
FROM cron_schedules WHERE schedule_name = $1`,
|
|
1836
|
+
[name]
|
|
1570
1837
|
);
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1838
|
+
if (result.rows.length === 0) return null;
|
|
1839
|
+
return result.rows[0];
|
|
1840
|
+
} catch (error) {
|
|
1841
|
+
log(`Error getting cron schedule by name "${name}": ${error}`);
|
|
1574
1842
|
throw error;
|
|
1843
|
+
} finally {
|
|
1844
|
+
client.release();
|
|
1575
1845
|
}
|
|
1576
|
-
throw new Error(
|
|
1577
|
-
`Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
|
|
1578
|
-
);
|
|
1579
1846
|
}
|
|
1580
|
-
|
|
1581
|
-
async
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1586
|
-
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
1597
|
-
}, timeoutMs);
|
|
1598
|
-
|
|
1599
|
-
try {
|
|
1600
|
-
// Execute the handler
|
|
1601
|
-
// Note: This uses Function constructor which requires the handler to be serializable.
|
|
1602
|
-
// The handler should be validated before reaching this point.
|
|
1603
|
-
let handlerFn;
|
|
1604
|
-
try {
|
|
1605
|
-
// Wrap handlerCode in parentheses to ensure it's treated as an expression
|
|
1606
|
-
// This handles both arrow functions and regular functions
|
|
1607
|
-
const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
|
|
1608
|
-
? handlerCode
|
|
1609
|
-
: '(' + handlerCode + ')';
|
|
1610
|
-
handlerFn = new Function('return ' + wrappedCode)();
|
|
1611
|
-
} catch (parseError) {
|
|
1612
|
-
clearTimeout(timeoutId);
|
|
1613
|
-
parentPort.postMessage({
|
|
1614
|
-
type: 'error',
|
|
1615
|
-
error: {
|
|
1616
|
-
message: 'Handler cannot be deserialized in worker thread. ' +
|
|
1617
|
-
'Ensure your handler is a standalone function without closures over external variables. ' +
|
|
1618
|
-
'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
|
|
1619
|
-
stack: parseError instanceof Error ? parseError.stack : undefined,
|
|
1620
|
-
name: 'SerializationError',
|
|
1621
|
-
},
|
|
1622
|
-
});
|
|
1623
|
-
return;
|
|
1624
|
-
}
|
|
1625
|
-
|
|
1626
|
-
// Ensure handlerFn is actually a function
|
|
1627
|
-
if (typeof handlerFn !== 'function') {
|
|
1628
|
-
clearTimeout(timeoutId);
|
|
1629
|
-
parentPort.postMessage({
|
|
1630
|
-
type: 'error',
|
|
1631
|
-
error: {
|
|
1632
|
-
message: 'Handler deserialization did not produce a function. ' +
|
|
1633
|
-
'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
|
|
1634
|
-
name: 'SerializationError',
|
|
1635
|
-
},
|
|
1636
|
-
});
|
|
1637
|
-
return;
|
|
1638
|
-
}
|
|
1639
|
-
|
|
1640
|
-
handlerFn(payload, signal)
|
|
1641
|
-
.then(() => {
|
|
1642
|
-
clearTimeout(timeoutId);
|
|
1643
|
-
parentPort.postMessage({ type: 'success' });
|
|
1644
|
-
})
|
|
1645
|
-
.catch((error) => {
|
|
1646
|
-
clearTimeout(timeoutId);
|
|
1647
|
-
parentPort.postMessage({
|
|
1648
|
-
type: 'error',
|
|
1649
|
-
error: {
|
|
1650
|
-
message: error.message,
|
|
1651
|
-
stack: error.stack,
|
|
1652
|
-
name: error.name,
|
|
1653
|
-
},
|
|
1654
|
-
});
|
|
1655
|
-
});
|
|
1656
|
-
} catch (error) {
|
|
1657
|
-
clearTimeout(timeoutId);
|
|
1658
|
-
parentPort.postMessage({
|
|
1659
|
-
type: 'error',
|
|
1660
|
-
error: {
|
|
1661
|
-
message: error.message,
|
|
1662
|
-
stack: error.stack,
|
|
1663
|
-
name: error.name,
|
|
1664
|
-
},
|
|
1665
|
-
});
|
|
1666
|
-
}
|
|
1667
|
-
})();
|
|
1668
|
-
`;
|
|
1669
|
-
const worker = new worker_threads.Worker(workerCode, {
|
|
1670
|
-
eval: true,
|
|
1671
|
-
workerData: {
|
|
1672
|
-
handlerCode: handler.toString(),
|
|
1673
|
-
payload,
|
|
1674
|
-
timeoutMs
|
|
1847
|
+
/** List cron schedules, optionally filtered by status. */
|
|
1848
|
+
async listCronSchedules(status) {
|
|
1849
|
+
const client = await this.pool.connect();
|
|
1850
|
+
try {
|
|
1851
|
+
let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1852
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1853
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1854
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1855
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1856
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1857
|
+
next_run_at AS "nextRunAt",
|
|
1858
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1859
|
+
FROM cron_schedules`;
|
|
1860
|
+
const params = [];
|
|
1861
|
+
if (status) {
|
|
1862
|
+
query += ` WHERE status = $1`;
|
|
1863
|
+
params.push(status);
|
|
1675
1864
|
}
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1865
|
+
query += ` ORDER BY created_at ASC`;
|
|
1866
|
+
const result = await client.query(query, params);
|
|
1867
|
+
return result.rows;
|
|
1868
|
+
} catch (error) {
|
|
1869
|
+
log(`Error listing cron schedules: ${error}`);
|
|
1870
|
+
throw error;
|
|
1871
|
+
} finally {
|
|
1872
|
+
client.release();
|
|
1873
|
+
}
|
|
1874
|
+
}
|
|
1875
|
+
/** Delete a cron schedule by ID. */
|
|
1876
|
+
async removeCronSchedule(id) {
|
|
1877
|
+
const client = await this.pool.connect();
|
|
1878
|
+
try {
|
|
1879
|
+
await client.query(`DELETE FROM cron_schedules WHERE id = $1`, [id]);
|
|
1880
|
+
log(`Removed cron schedule ${id}`);
|
|
1881
|
+
} catch (error) {
|
|
1882
|
+
log(`Error removing cron schedule ${id}: ${error}`);
|
|
1883
|
+
throw error;
|
|
1884
|
+
} finally {
|
|
1885
|
+
client.release();
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
/** Pause a cron schedule. */
|
|
1889
|
+
async pauseCronSchedule(id) {
|
|
1890
|
+
const client = await this.pool.connect();
|
|
1891
|
+
try {
|
|
1892
|
+
await client.query(
|
|
1893
|
+
`UPDATE cron_schedules SET status = 'paused', updated_at = NOW() WHERE id = $1`,
|
|
1894
|
+
[id]
|
|
1895
|
+
);
|
|
1896
|
+
log(`Paused cron schedule ${id}`);
|
|
1897
|
+
} catch (error) {
|
|
1898
|
+
log(`Error pausing cron schedule ${id}: ${error}`);
|
|
1899
|
+
throw error;
|
|
1900
|
+
} finally {
|
|
1901
|
+
client.release();
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
/** Resume a paused cron schedule. */
|
|
1905
|
+
async resumeCronSchedule(id) {
|
|
1906
|
+
const client = await this.pool.connect();
|
|
1907
|
+
try {
|
|
1908
|
+
await client.query(
|
|
1909
|
+
`UPDATE cron_schedules SET status = 'active', updated_at = NOW() WHERE id = $1`,
|
|
1910
|
+
[id]
|
|
1911
|
+
);
|
|
1912
|
+
log(`Resumed cron schedule ${id}`);
|
|
1913
|
+
} catch (error) {
|
|
1914
|
+
log(`Error resuming cron schedule ${id}: ${error}`);
|
|
1915
|
+
throw error;
|
|
1916
|
+
} finally {
|
|
1917
|
+
client.release();
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
/** Edit a cron schedule. */
|
|
1921
|
+
async editCronSchedule(id, updates, nextRunAt) {
|
|
1922
|
+
const client = await this.pool.connect();
|
|
1923
|
+
try {
|
|
1924
|
+
const updateFields = [];
|
|
1925
|
+
const params = [];
|
|
1926
|
+
let paramIdx = 1;
|
|
1927
|
+
if (updates.cronExpression !== void 0) {
|
|
1928
|
+
updateFields.push(`cron_expression = $${paramIdx++}`);
|
|
1929
|
+
params.push(updates.cronExpression);
|
|
1694
1930
|
}
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
resolved = true;
|
|
1699
|
-
reject(error);
|
|
1700
|
-
});
|
|
1701
|
-
worker.on("exit", (code) => {
|
|
1702
|
-
if (resolved) return;
|
|
1703
|
-
if (code !== 0) {
|
|
1704
|
-
resolved = true;
|
|
1705
|
-
reject(new Error(`Worker stopped with exit code ${code}`));
|
|
1931
|
+
if (updates.payload !== void 0) {
|
|
1932
|
+
updateFields.push(`payload = $${paramIdx++}`);
|
|
1933
|
+
params.push(updates.payload);
|
|
1706
1934
|
}
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
resolved = true;
|
|
1711
|
-
worker.terminate().then(() => {
|
|
1712
|
-
const timeoutError = new Error(
|
|
1713
|
-
`Job timed out after ${timeoutMs} ms and was forcefully terminated`
|
|
1714
|
-
);
|
|
1715
|
-
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
1716
|
-
reject(timeoutError);
|
|
1717
|
-
}).catch((err) => {
|
|
1718
|
-
reject(err);
|
|
1719
|
-
});
|
|
1935
|
+
if (updates.maxAttempts !== void 0) {
|
|
1936
|
+
updateFields.push(`max_attempts = $${paramIdx++}`);
|
|
1937
|
+
params.push(updates.maxAttempts);
|
|
1720
1938
|
}
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
function calculateWaitUntil(duration) {
|
|
1725
|
-
const now = Date.now();
|
|
1726
|
-
let ms = 0;
|
|
1727
|
-
if (duration.seconds) ms += duration.seconds * 1e3;
|
|
1728
|
-
if (duration.minutes) ms += duration.minutes * 60 * 1e3;
|
|
1729
|
-
if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
|
|
1730
|
-
if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
|
|
1731
|
-
if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
|
|
1732
|
-
if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
|
|
1733
|
-
if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
|
|
1734
|
-
if (ms <= 0) {
|
|
1735
|
-
throw new Error(
|
|
1736
|
-
"waitFor duration must be positive. Provide at least one positive duration field."
|
|
1737
|
-
);
|
|
1738
|
-
}
|
|
1739
|
-
return new Date(now + ms);
|
|
1740
|
-
}
|
|
1741
|
-
async function resolveCompletedWaits(pool, stepData) {
|
|
1742
|
-
for (const key of Object.keys(stepData)) {
|
|
1743
|
-
if (!key.startsWith("__wait_")) continue;
|
|
1744
|
-
const entry = stepData[key];
|
|
1745
|
-
if (!entry || typeof entry !== "object" || entry.completed) continue;
|
|
1746
|
-
if (entry.type === "duration" || entry.type === "date") {
|
|
1747
|
-
stepData[key] = { ...entry, completed: true };
|
|
1748
|
-
} else if (entry.type === "token" && entry.tokenId) {
|
|
1749
|
-
const wp = await getWaitpoint(pool, entry.tokenId);
|
|
1750
|
-
if (wp && wp.status === "completed") {
|
|
1751
|
-
stepData[key] = {
|
|
1752
|
-
...entry,
|
|
1753
|
-
completed: true,
|
|
1754
|
-
result: { ok: true, output: wp.output }
|
|
1755
|
-
};
|
|
1756
|
-
} else if (wp && wp.status === "timed_out") {
|
|
1757
|
-
stepData[key] = {
|
|
1758
|
-
...entry,
|
|
1759
|
-
completed: true,
|
|
1760
|
-
result: { ok: false, error: "Token timed out" }
|
|
1761
|
-
};
|
|
1939
|
+
if (updates.priority !== void 0) {
|
|
1940
|
+
updateFields.push(`priority = $${paramIdx++}`);
|
|
1941
|
+
params.push(updates.priority);
|
|
1762
1942
|
}
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
function buildWaitContext(backend, pool, jobId, stepData, baseCtx) {
|
|
1767
|
-
let waitCounter = 0;
|
|
1768
|
-
const ctx = {
|
|
1769
|
-
prolong: baseCtx.prolong,
|
|
1770
|
-
onTimeout: baseCtx.onTimeout,
|
|
1771
|
-
run: async (stepName, fn) => {
|
|
1772
|
-
const cached = stepData[stepName];
|
|
1773
|
-
if (cached && typeof cached === "object" && cached.__completed) {
|
|
1774
|
-
log(`Step "${stepName}" replayed from cache for job ${jobId}`);
|
|
1775
|
-
return cached.result;
|
|
1943
|
+
if (updates.timeoutMs !== void 0) {
|
|
1944
|
+
updateFields.push(`timeout_ms = $${paramIdx++}`);
|
|
1945
|
+
params.push(updates.timeoutMs);
|
|
1776
1946
|
}
|
|
1777
|
-
|
|
1778
|
-
|
|
1779
|
-
|
|
1780
|
-
return result;
|
|
1781
|
-
},
|
|
1782
|
-
waitFor: async (duration) => {
|
|
1783
|
-
const waitKey = `__wait_${waitCounter++}`;
|
|
1784
|
-
const cached = stepData[waitKey];
|
|
1785
|
-
if (cached && typeof cached === "object" && cached.completed) {
|
|
1786
|
-
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
1787
|
-
return;
|
|
1947
|
+
if (updates.forceKillOnTimeout !== void 0) {
|
|
1948
|
+
updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
|
|
1949
|
+
params.push(updates.forceKillOnTimeout);
|
|
1788
1950
|
}
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
},
|
|
1793
|
-
waitUntil: async (date) => {
|
|
1794
|
-
const waitKey = `__wait_${waitCounter++}`;
|
|
1795
|
-
const cached = stepData[waitKey];
|
|
1796
|
-
if (cached && typeof cached === "object" && cached.completed) {
|
|
1797
|
-
log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
|
|
1798
|
-
return;
|
|
1951
|
+
if (updates.tags !== void 0) {
|
|
1952
|
+
updateFields.push(`tags = $${paramIdx++}`);
|
|
1953
|
+
params.push(updates.tags);
|
|
1799
1954
|
}
|
|
1800
|
-
|
|
1801
|
-
|
|
1802
|
-
|
|
1803
|
-
createToken: async (options) => {
|
|
1804
|
-
const token = await createWaitpoint(pool, jobId, options);
|
|
1805
|
-
return token;
|
|
1806
|
-
},
|
|
1807
|
-
waitForToken: async (tokenId) => {
|
|
1808
|
-
const waitKey = `__wait_${waitCounter++}`;
|
|
1809
|
-
const cached = stepData[waitKey];
|
|
1810
|
-
if (cached && typeof cached === "object" && cached.completed) {
|
|
1811
|
-
log(
|
|
1812
|
-
`Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
|
|
1813
|
-
);
|
|
1814
|
-
return cached.result;
|
|
1955
|
+
if (updates.timezone !== void 0) {
|
|
1956
|
+
updateFields.push(`timezone = $${paramIdx++}`);
|
|
1957
|
+
params.push(updates.timezone);
|
|
1815
1958
|
}
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
ok: true,
|
|
1820
|
-
output: wp.output
|
|
1821
|
-
};
|
|
1822
|
-
stepData[waitKey] = {
|
|
1823
|
-
type: "token",
|
|
1824
|
-
tokenId,
|
|
1825
|
-
completed: true,
|
|
1826
|
-
result
|
|
1827
|
-
};
|
|
1828
|
-
await updateStepData(pool, jobId, stepData);
|
|
1829
|
-
return result;
|
|
1959
|
+
if (updates.allowOverlap !== void 0) {
|
|
1960
|
+
updateFields.push(`allow_overlap = $${paramIdx++}`);
|
|
1961
|
+
params.push(updates.allowOverlap);
|
|
1830
1962
|
}
|
|
1831
|
-
if (
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
tokenId,
|
|
1839
|
-
completed: true,
|
|
1840
|
-
result
|
|
1841
|
-
};
|
|
1842
|
-
await updateStepData(pool, jobId, stepData);
|
|
1843
|
-
return result;
|
|
1963
|
+
if (nextRunAt !== void 0) {
|
|
1964
|
+
updateFields.push(`next_run_at = $${paramIdx++}`);
|
|
1965
|
+
params.push(nextRunAt);
|
|
1966
|
+
}
|
|
1967
|
+
if (updateFields.length === 0) {
|
|
1968
|
+
log(`No fields to update for cron schedule ${id}`);
|
|
1969
|
+
return;
|
|
1844
1970
|
}
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1971
|
+
updateFields.push(`updated_at = NOW()`);
|
|
1972
|
+
params.push(id);
|
|
1973
|
+
const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
|
|
1974
|
+
await client.query(query, params);
|
|
1975
|
+
log(`Edited cron schedule ${id}`);
|
|
1976
|
+
} catch (error) {
|
|
1977
|
+
log(`Error editing cron schedule ${id}: ${error}`);
|
|
1978
|
+
throw error;
|
|
1979
|
+
} finally {
|
|
1980
|
+
client.release();
|
|
1852
1981
|
}
|
|
1853
|
-
};
|
|
1854
|
-
return ctx;
|
|
1855
|
-
}
|
|
1856
|
-
async function processJobWithHandlers(backend, job, jobHandlers) {
|
|
1857
|
-
const handler = jobHandlers[job.jobType];
|
|
1858
|
-
if (!handler) {
|
|
1859
|
-
await backend.setPendingReasonForUnpickedJobs(
|
|
1860
|
-
`No handler registered for job type: ${job.jobType}`,
|
|
1861
|
-
job.jobType
|
|
1862
|
-
);
|
|
1863
|
-
await backend.failJob(
|
|
1864
|
-
job.id,
|
|
1865
|
-
new Error(`No handler registered for job type: ${job.jobType}`),
|
|
1866
|
-
"no_handler" /* NoHandler */
|
|
1867
|
-
);
|
|
1868
|
-
return;
|
|
1869
|
-
}
|
|
1870
|
-
const stepData = { ...job.stepData || {} };
|
|
1871
|
-
const pool = tryExtractPool(backend);
|
|
1872
|
-
const hasStepHistory = Object.keys(stepData).some(
|
|
1873
|
-
(k) => k.startsWith("__wait_")
|
|
1874
|
-
);
|
|
1875
|
-
if (hasStepHistory && pool) {
|
|
1876
|
-
await resolveCompletedWaits(pool, stepData);
|
|
1877
|
-
await updateStepData(pool, job.id, stepData);
|
|
1878
1982
|
}
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
const timeoutError = new Error(`Job timed out after ${ms} ms`);
|
|
1909
|
-
timeoutError.failureReason = "timeout" /* Timeout */;
|
|
1910
|
-
if (timeoutReject) {
|
|
1911
|
-
timeoutReject(timeoutError);
|
|
1912
|
-
}
|
|
1913
|
-
}, ms);
|
|
1914
|
-
};
|
|
1915
|
-
const hasTimeout = timeoutMs != null && timeoutMs > 0;
|
|
1916
|
-
const baseCtx = hasTimeout ? {
|
|
1917
|
-
prolong: (ms) => {
|
|
1918
|
-
const duration = ms ?? timeoutMs;
|
|
1919
|
-
if (duration != null && duration > 0) {
|
|
1920
|
-
armTimeout(duration);
|
|
1921
|
-
backend.prolongJob(job.id).catch(() => {
|
|
1922
|
-
});
|
|
1923
|
-
}
|
|
1924
|
-
},
|
|
1925
|
-
onTimeout: (callback) => {
|
|
1926
|
-
onTimeoutCallback = callback;
|
|
1927
|
-
}
|
|
1928
|
-
} : {
|
|
1929
|
-
prolong: () => {
|
|
1930
|
-
log("prolong() called but ignored: job has no timeout set");
|
|
1931
|
-
},
|
|
1932
|
-
onTimeout: () => {
|
|
1933
|
-
log("onTimeout() called but ignored: job has no timeout set");
|
|
1934
|
-
}
|
|
1935
|
-
};
|
|
1936
|
-
const ctx = pool ? buildWaitContext(backend, pool, job.id, stepData, baseCtx) : buildBasicContext(backend, job.id, baseCtx);
|
|
1937
|
-
if (forceKillOnTimeout && !hasTimeout) {
|
|
1938
|
-
log(
|
|
1939
|
-
`forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
|
|
1940
|
-
);
|
|
1941
|
-
}
|
|
1942
|
-
const jobPromise = handler(job.payload, controller.signal, ctx);
|
|
1943
|
-
if (hasTimeout) {
|
|
1944
|
-
await Promise.race([
|
|
1945
|
-
jobPromise,
|
|
1946
|
-
new Promise((_, reject) => {
|
|
1947
|
-
timeoutReject = reject;
|
|
1948
|
-
armTimeout(timeoutMs);
|
|
1949
|
-
})
|
|
1950
|
-
]);
|
|
1951
|
-
} else {
|
|
1952
|
-
await jobPromise;
|
|
1983
|
+
/**
|
|
1984
|
+
* Atomically fetch all active cron schedules whose nextRunAt <= NOW().
|
|
1985
|
+
* Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
|
|
1986
|
+
*/
|
|
1987
|
+
async getDueCronSchedules() {
|
|
1988
|
+
const client = await this.pool.connect();
|
|
1989
|
+
try {
|
|
1990
|
+
const result = await client.query(
|
|
1991
|
+
`SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
|
|
1992
|
+
job_type AS "jobType", payload, max_attempts AS "maxAttempts",
|
|
1993
|
+
priority, timeout_ms AS "timeoutMs",
|
|
1994
|
+
force_kill_on_timeout AS "forceKillOnTimeout", tags,
|
|
1995
|
+
timezone, allow_overlap AS "allowOverlap", status,
|
|
1996
|
+
last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
|
|
1997
|
+
next_run_at AS "nextRunAt",
|
|
1998
|
+
created_at AS "createdAt", updated_at AS "updatedAt"
|
|
1999
|
+
FROM cron_schedules
|
|
2000
|
+
WHERE status = 'active'
|
|
2001
|
+
AND next_run_at IS NOT NULL
|
|
2002
|
+
AND next_run_at <= NOW()
|
|
2003
|
+
ORDER BY next_run_at ASC
|
|
2004
|
+
FOR UPDATE SKIP LOCKED`
|
|
2005
|
+
);
|
|
2006
|
+
log(`Found ${result.rows.length} due cron schedules`);
|
|
2007
|
+
return result.rows;
|
|
2008
|
+
} catch (error) {
|
|
2009
|
+
if (error?.code === "42P01") {
|
|
2010
|
+
log("cron_schedules table does not exist, skipping cron enqueue");
|
|
2011
|
+
return [];
|
|
1953
2012
|
}
|
|
2013
|
+
log(`Error getting due cron schedules: ${error}`);
|
|
2014
|
+
throw error;
|
|
2015
|
+
} finally {
|
|
2016
|
+
client.release();
|
|
1954
2017
|
}
|
|
1955
|
-
|
|
1956
|
-
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
|
|
1962
|
-
|
|
1963
|
-
|
|
1964
|
-
|
|
1965
|
-
|
|
1966
|
-
|
|
1967
|
-
|
|
1968
|
-
|
|
1969
|
-
|
|
2018
|
+
}
|
|
2019
|
+
/**
|
|
2020
|
+
* Update a cron schedule after a job has been enqueued.
|
|
2021
|
+
* Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
|
|
2022
|
+
*/
|
|
2023
|
+
async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
|
|
2024
|
+
const client = await this.pool.connect();
|
|
2025
|
+
try {
|
|
2026
|
+
await client.query(
|
|
2027
|
+
`UPDATE cron_schedules
|
|
2028
|
+
SET last_enqueued_at = $2,
|
|
2029
|
+
last_job_id = $3,
|
|
2030
|
+
next_run_at = $4,
|
|
2031
|
+
updated_at = NOW()
|
|
2032
|
+
WHERE id = $1`,
|
|
2033
|
+
[id, lastEnqueuedAt, lastJobId, nextRunAt]
|
|
2034
|
+
);
|
|
1970
2035
|
log(
|
|
1971
|
-
`
|
|
2036
|
+
`Updated cron schedule ${id}: lastJobId=${lastJobId}, nextRunAt=${nextRunAt?.toISOString() ?? "null"}`
|
|
1972
2037
|
);
|
|
1973
|
-
|
|
1974
|
-
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
return;
|
|
1979
|
-
}
|
|
1980
|
-
console.error(`Error processing job ${job.id}:`, error);
|
|
1981
|
-
let failureReason = "handler_error" /* HandlerError */;
|
|
1982
|
-
if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
|
|
1983
|
-
failureReason = "timeout" /* Timeout */;
|
|
2038
|
+
} catch (error) {
|
|
2039
|
+
log(`Error updating cron schedule ${id} after enqueue: ${error}`);
|
|
2040
|
+
throw error;
|
|
2041
|
+
} finally {
|
|
2042
|
+
client.release();
|
|
1984
2043
|
}
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
|
|
1995
|
-
|
|
1996
|
-
|
|
1997
|
-
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
|
|
2008
|
-
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
onError(err instanceof Error ? err : new Error(String(err)));
|
|
2022
|
-
}
|
|
2023
|
-
next();
|
|
2024
|
-
});
|
|
2025
|
-
}
|
|
2026
|
-
};
|
|
2027
|
-
next();
|
|
2028
|
-
});
|
|
2029
|
-
}
|
|
2030
|
-
var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
|
|
2031
|
-
const {
|
|
2032
|
-
workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
|
|
2033
|
-
batchSize = 10,
|
|
2034
|
-
pollInterval = 5e3,
|
|
2035
|
-
onError = (error) => console.error("Job processor error:", error),
|
|
2036
|
-
jobType,
|
|
2037
|
-
concurrency = 3
|
|
2038
|
-
} = options;
|
|
2039
|
-
let running = false;
|
|
2040
|
-
let intervalId = null;
|
|
2041
|
-
let currentBatchPromise = null;
|
|
2042
|
-
setLogContext(options.verbose ?? false);
|
|
2043
|
-
const processJobs = async () => {
|
|
2044
|
-
if (!running) return 0;
|
|
2045
|
-
if (onBeforeBatch) {
|
|
2046
|
-
try {
|
|
2047
|
-
await onBeforeBatch();
|
|
2048
|
-
} catch (hookError) {
|
|
2049
|
-
log(`onBeforeBatch hook error: ${hookError}`);
|
|
2050
|
-
if (onError) {
|
|
2051
|
-
onError(
|
|
2052
|
-
hookError instanceof Error ? hookError : new Error(String(hookError))
|
|
2053
|
-
);
|
|
2054
|
-
}
|
|
2044
|
+
}
|
|
2045
|
+
// ── Wait / step-data support ────────────────────────────────────────
|
|
2046
|
+
/**
|
|
2047
|
+
* Transition a job from 'processing' to 'waiting' status.
|
|
2048
|
+
* Persists step data so the handler can resume from where it left off.
|
|
2049
|
+
*
|
|
2050
|
+
* @param jobId - The job to pause.
|
|
2051
|
+
* @param options - Wait configuration including optional waitUntil date, token ID, and step data.
|
|
2052
|
+
*/
|
|
2053
|
+
async waitJob(jobId, options) {
|
|
2054
|
+
const client = await this.pool.connect();
|
|
2055
|
+
try {
|
|
2056
|
+
const result = await client.query(
|
|
2057
|
+
`
|
|
2058
|
+
UPDATE job_queue
|
|
2059
|
+
SET status = 'waiting',
|
|
2060
|
+
wait_until = $2,
|
|
2061
|
+
wait_token_id = $3,
|
|
2062
|
+
step_data = $4,
|
|
2063
|
+
locked_at = NULL,
|
|
2064
|
+
locked_by = NULL,
|
|
2065
|
+
updated_at = NOW()
|
|
2066
|
+
WHERE id = $1 AND status = 'processing'
|
|
2067
|
+
`,
|
|
2068
|
+
[
|
|
2069
|
+
jobId,
|
|
2070
|
+
options.waitUntil ?? null,
|
|
2071
|
+
options.waitTokenId ?? null,
|
|
2072
|
+
JSON.stringify(options.stepData)
|
|
2073
|
+
]
|
|
2074
|
+
);
|
|
2075
|
+
if (result.rowCount === 0) {
|
|
2076
|
+
log(
|
|
2077
|
+
`Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
|
|
2078
|
+
);
|
|
2079
|
+
return;
|
|
2055
2080
|
}
|
|
2081
|
+
await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
|
|
2082
|
+
waitUntil: options.waitUntil?.toISOString() ?? null,
|
|
2083
|
+
waitTokenId: options.waitTokenId ?? null
|
|
2084
|
+
});
|
|
2085
|
+
log(`Job ${jobId} set to waiting`);
|
|
2086
|
+
} catch (error) {
|
|
2087
|
+
log(`Error setting job ${jobId} to waiting: ${error}`);
|
|
2088
|
+
throw error;
|
|
2089
|
+
} finally {
|
|
2090
|
+
client.release();
|
|
2056
2091
|
}
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2092
|
+
}
|
|
2093
|
+
/**
|
|
2094
|
+
* Persist step data for a job. Called after each ctx.run() step completes.
|
|
2095
|
+
* Best-effort: does not throw to avoid killing the running handler.
|
|
2096
|
+
*
|
|
2097
|
+
* @param jobId - The job to update.
|
|
2098
|
+
* @param stepData - The step data to persist.
|
|
2099
|
+
*/
|
|
2100
|
+
async updateStepData(jobId, stepData) {
|
|
2101
|
+
const client = await this.pool.connect();
|
|
2060
2102
|
try {
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
batchSize,
|
|
2065
|
-
jobType,
|
|
2066
|
-
handlers,
|
|
2067
|
-
concurrency,
|
|
2068
|
-
onError
|
|
2103
|
+
await client.query(
|
|
2104
|
+
`UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
|
|
2105
|
+
[jobId, JSON.stringify(stepData)]
|
|
2069
2106
|
);
|
|
2070
|
-
return processed;
|
|
2071
2107
|
} catch (error) {
|
|
2072
|
-
|
|
2108
|
+
log(`Error updating step_data for job ${jobId}: ${error}`);
|
|
2109
|
+
} finally {
|
|
2110
|
+
client.release();
|
|
2073
2111
|
}
|
|
2074
|
-
return 0;
|
|
2075
|
-
};
|
|
2076
|
-
return {
|
|
2077
|
-
/**
|
|
2078
|
-
* Start the job processor in the background.
|
|
2079
|
-
* - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
|
|
2080
|
-
* - You have to call the stop method to stop the processor.
|
|
2081
|
-
*/
|
|
2082
|
-
startInBackground: () => {
|
|
2083
|
-
if (running) return;
|
|
2084
|
-
log(`Starting job processor with workerId: ${workerId}`);
|
|
2085
|
-
running = true;
|
|
2086
|
-
const scheduleNext = (immediate) => {
|
|
2087
|
-
if (!running) return;
|
|
2088
|
-
if (immediate) {
|
|
2089
|
-
intervalId = setTimeout(loop, 0);
|
|
2090
|
-
} else {
|
|
2091
|
-
intervalId = setTimeout(loop, pollInterval);
|
|
2092
|
-
}
|
|
2093
|
-
};
|
|
2094
|
-
const loop = async () => {
|
|
2095
|
-
if (!running) return;
|
|
2096
|
-
currentBatchPromise = processJobs();
|
|
2097
|
-
const processed = await currentBatchPromise;
|
|
2098
|
-
currentBatchPromise = null;
|
|
2099
|
-
scheduleNext(processed === batchSize);
|
|
2100
|
-
};
|
|
2101
|
-
loop();
|
|
2102
|
-
},
|
|
2103
|
-
/**
|
|
2104
|
-
* Stop the job processor that runs in the background.
|
|
2105
|
-
* Does not wait for in-flight jobs.
|
|
2106
|
-
*/
|
|
2107
|
-
stop: () => {
|
|
2108
|
-
log(`Stopping job processor with workerId: ${workerId}`);
|
|
2109
|
-
running = false;
|
|
2110
|
-
if (intervalId) {
|
|
2111
|
-
clearTimeout(intervalId);
|
|
2112
|
-
intervalId = null;
|
|
2113
|
-
}
|
|
2114
|
-
},
|
|
2115
|
-
/**
|
|
2116
|
-
* Stop the job processor and wait for all in-flight jobs to complete.
|
|
2117
|
-
* Useful for graceful shutdown (e.g., SIGTERM handling).
|
|
2118
|
-
*/
|
|
2119
|
-
stopAndDrain: async (drainTimeoutMs = 3e4) => {
|
|
2120
|
-
log(`Stopping and draining job processor with workerId: ${workerId}`);
|
|
2121
|
-
running = false;
|
|
2122
|
-
if (intervalId) {
|
|
2123
|
-
clearTimeout(intervalId);
|
|
2124
|
-
intervalId = null;
|
|
2125
|
-
}
|
|
2126
|
-
if (currentBatchPromise) {
|
|
2127
|
-
await Promise.race([
|
|
2128
|
-
currentBatchPromise.catch(() => {
|
|
2129
|
-
}),
|
|
2130
|
-
new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
|
|
2131
|
-
]);
|
|
2132
|
-
currentBatchPromise = null;
|
|
2133
|
-
}
|
|
2134
|
-
log(`Job processor ${workerId} drained`);
|
|
2135
|
-
},
|
|
2136
|
-
/**
|
|
2137
|
-
* Start the job processor synchronously.
|
|
2138
|
-
* - This will process all jobs immediately and then stop.
|
|
2139
|
-
* - The pollInterval is ignored.
|
|
2140
|
-
*/
|
|
2141
|
-
start: async () => {
|
|
2142
|
-
log(`Starting job processor with workerId: ${workerId}`);
|
|
2143
|
-
running = true;
|
|
2144
|
-
const processed = await processJobs();
|
|
2145
|
-
running = false;
|
|
2146
|
-
return processed;
|
|
2147
|
-
},
|
|
2148
|
-
isRunning: () => running
|
|
2149
|
-
};
|
|
2150
|
-
};
|
|
2151
|
-
function loadPemOrFile(value) {
|
|
2152
|
-
if (!value) return void 0;
|
|
2153
|
-
if (value.startsWith("file://")) {
|
|
2154
|
-
const filePath = value.slice(7);
|
|
2155
|
-
return fs__default.default.readFileSync(filePath, "utf8");
|
|
2156
2112
|
}
|
|
2157
|
-
|
|
2158
|
-
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2113
|
+
/**
|
|
2114
|
+
* Create a waitpoint token in the database.
|
|
2115
|
+
*
|
|
2116
|
+
* @param jobId - The job ID to associate with the token (null if created outside a handler).
|
|
2117
|
+
* @param options - Optional timeout string (e.g. '10m', '1h') and tags.
|
|
2118
|
+
* @returns The created waitpoint with its unique ID.
|
|
2119
|
+
*/
|
|
2120
|
+
async createWaitpoint(jobId, options) {
|
|
2121
|
+
const client = await this.pool.connect();
|
|
2165
2122
|
try {
|
|
2166
|
-
const
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2123
|
+
const id = `wp_${crypto.randomUUID()}`;
|
|
2124
|
+
let timeoutAt = null;
|
|
2125
|
+
if (options?.timeout) {
|
|
2126
|
+
const ms = parseTimeoutString(options.timeout);
|
|
2127
|
+
timeoutAt = new Date(Date.now() + ms);
|
|
2171
2128
|
}
|
|
2172
|
-
|
|
2173
|
-
|
|
2174
|
-
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
|
|
2178
|
-
|
|
2129
|
+
await client.query(
|
|
2130
|
+
`INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
|
|
2131
|
+
[id, jobId, timeoutAt, options?.tags ?? null]
|
|
2132
|
+
);
|
|
2133
|
+
log(`Created waitpoint ${id} for job ${jobId}`);
|
|
2134
|
+
return { id };
|
|
2135
|
+
} catch (error) {
|
|
2136
|
+
log(`Error creating waitpoint: ${error}`);
|
|
2137
|
+
throw error;
|
|
2138
|
+
} finally {
|
|
2139
|
+
client.release();
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
/**
|
|
2143
|
+
* Complete a waitpoint token and move the associated job back to 'pending'.
|
|
2144
|
+
*
|
|
2145
|
+
* @param tokenId - The waitpoint token ID to complete.
|
|
2146
|
+
* @param data - Optional data to pass to the waiting handler.
|
|
2147
|
+
*/
|
|
2148
|
+
async completeWaitpoint(tokenId, data) {
|
|
2149
|
+
const client = await this.pool.connect();
|
|
2150
|
+
try {
|
|
2151
|
+
await client.query("BEGIN");
|
|
2152
|
+
const wpResult = await client.query(
|
|
2153
|
+
`UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
|
|
2154
|
+
WHERE id = $1 AND status = 'waiting'
|
|
2155
|
+
RETURNING job_id`,
|
|
2156
|
+
[tokenId, data != null ? JSON.stringify(data) : null]
|
|
2157
|
+
);
|
|
2158
|
+
if (wpResult.rows.length === 0) {
|
|
2159
|
+
await client.query("ROLLBACK");
|
|
2160
|
+
log(`Waitpoint ${tokenId} not found or already completed`);
|
|
2161
|
+
return;
|
|
2179
2162
|
}
|
|
2180
|
-
|
|
2181
|
-
if (
|
|
2182
|
-
|
|
2163
|
+
const jobId = wpResult.rows[0].job_id;
|
|
2164
|
+
if (jobId != null) {
|
|
2165
|
+
await client.query(
|
|
2166
|
+
`UPDATE job_queue
|
|
2167
|
+
SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
|
|
2168
|
+
WHERE id = $1 AND status = 'waiting'`,
|
|
2169
|
+
[jobId]
|
|
2170
|
+
);
|
|
2183
2171
|
}
|
|
2172
|
+
await client.query("COMMIT");
|
|
2173
|
+
log(`Completed waitpoint ${tokenId} for job ${jobId}`);
|
|
2174
|
+
} catch (error) {
|
|
2175
|
+
await client.query("ROLLBACK");
|
|
2176
|
+
log(`Error completing waitpoint ${tokenId}: ${error}`);
|
|
2177
|
+
throw error;
|
|
2178
|
+
} finally {
|
|
2179
|
+
client.release();
|
|
2184
2180
|
}
|
|
2185
2181
|
}
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2182
|
+
/**
|
|
2183
|
+
* Retrieve a waitpoint token by its ID.
|
|
2184
|
+
*
|
|
2185
|
+
* @param tokenId - The waitpoint token ID to look up.
|
|
2186
|
+
* @returns The waitpoint record, or null if not found.
|
|
2187
|
+
*/
|
|
2188
|
+
async getWaitpoint(tokenId) {
|
|
2189
|
+
const client = await this.pool.connect();
|
|
2190
|
+
try {
|
|
2191
|
+
const result = await client.query(
|
|
2192
|
+
`SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
|
|
2193
|
+
[tokenId]
|
|
2194
|
+
);
|
|
2195
|
+
if (result.rows.length === 0) return null;
|
|
2196
|
+
return result.rows[0];
|
|
2197
|
+
} catch (error) {
|
|
2198
|
+
log(`Error getting waitpoint ${tokenId}: ${error}`);
|
|
2199
|
+
throw error;
|
|
2200
|
+
} finally {
|
|
2201
|
+
client.release();
|
|
2193
2202
|
}
|
|
2194
|
-
const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
|
|
2195
|
-
ssl = {
|
|
2196
|
-
...ssl,
|
|
2197
|
-
...caValue ? { ca: caValue } : {},
|
|
2198
|
-
cert: loadPemOrFile(
|
|
2199
|
-
typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
|
|
2200
|
-
),
|
|
2201
|
-
key: loadPemOrFile(
|
|
2202
|
-
typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
|
|
2203
|
-
),
|
|
2204
|
-
rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
|
|
2205
|
-
};
|
|
2206
2203
|
}
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2204
|
+
/**
|
|
2205
|
+
* Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
|
|
2206
|
+
*
|
|
2207
|
+
* @returns The number of tokens that were expired.
|
|
2208
|
+
*/
|
|
2209
|
+
async expireTimedOutWaitpoints() {
|
|
2210
|
+
const client = await this.pool.connect();
|
|
2211
|
+
try {
|
|
2212
|
+
await client.query("BEGIN");
|
|
2213
|
+
const result = await client.query(
|
|
2214
|
+
`UPDATE waitpoints
|
|
2215
|
+
SET status = 'timed_out'
|
|
2216
|
+
WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
|
|
2217
|
+
RETURNING id, job_id`
|
|
2218
|
+
);
|
|
2219
|
+
for (const row of result.rows) {
|
|
2220
|
+
if (row.job_id != null) {
|
|
2221
|
+
await client.query(
|
|
2222
|
+
`UPDATE job_queue
|
|
2223
|
+
SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
|
|
2224
|
+
WHERE id = $1 AND status = 'waiting'`,
|
|
2225
|
+
[row.job_id]
|
|
2226
|
+
);
|
|
2227
|
+
}
|
|
2228
|
+
}
|
|
2229
|
+
await client.query("COMMIT");
|
|
2230
|
+
const count = result.rowCount || 0;
|
|
2231
|
+
if (count > 0) {
|
|
2232
|
+
log(`Expired ${count} timed-out waitpoints`);
|
|
2233
|
+
}
|
|
2234
|
+
return count;
|
|
2235
|
+
} catch (error) {
|
|
2236
|
+
await client.query("ROLLBACK");
|
|
2237
|
+
log(`Error expiring timed-out waitpoints: ${error}`);
|
|
2238
|
+
throw error;
|
|
2239
|
+
} finally {
|
|
2240
|
+
client.release();
|
|
2241
|
+
}
|
|
2221
2242
|
}
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2243
|
+
// ── Internal helpers ──────────────────────────────────────────────────
|
|
2244
|
+
async setPendingReasonForUnpickedJobs(reason, jobType) {
|
|
2245
|
+
const client = await this.pool.connect();
|
|
2246
|
+
try {
|
|
2247
|
+
let jobTypeFilter = "";
|
|
2248
|
+
const params = [reason];
|
|
2249
|
+
if (jobType) {
|
|
2250
|
+
if (Array.isArray(jobType)) {
|
|
2251
|
+
jobTypeFilter = ` AND job_type = ANY($2)`;
|
|
2252
|
+
params.push(jobType);
|
|
2253
|
+
} else {
|
|
2254
|
+
jobTypeFilter = ` AND job_type = $2`;
|
|
2255
|
+
params.push(jobType);
|
|
2256
|
+
}
|
|
2257
|
+
}
|
|
2258
|
+
await client.query(
|
|
2259
|
+
`UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
|
|
2260
|
+
params
|
|
2261
|
+
);
|
|
2262
|
+
} finally {
|
|
2263
|
+
client.release();
|
|
2264
|
+
}
|
|
2230
2265
|
}
|
|
2231
|
-
return pool;
|
|
2232
2266
|
};
|
|
2233
2267
|
|
|
2234
2268
|
// src/backends/redis-scripts.ts
|
|
@@ -2285,7 +2319,10 @@ redis.call('HMSET', jobKey,
|
|
|
2285
2319
|
'lastFailedAt', 'null',
|
|
2286
2320
|
'lastCancelledAt', 'null',
|
|
2287
2321
|
'tags', tagsJson,
|
|
2288
|
-
'idempotencyKey', idempotencyKey
|
|
2322
|
+
'idempotencyKey', idempotencyKey,
|
|
2323
|
+
'waitUntil', 'null',
|
|
2324
|
+
'waitTokenId', 'null',
|
|
2325
|
+
'stepData', 'null'
|
|
2289
2326
|
)
|
|
2290
2327
|
|
|
2291
2328
|
-- Status index
|
|
@@ -2368,7 +2405,25 @@ for _, jobId in ipairs(retries) do
|
|
|
2368
2405
|
redis.call('ZREM', prefix .. 'retry', jobId)
|
|
2369
2406
|
end
|
|
2370
2407
|
|
|
2371
|
-
-- 3.
|
|
2408
|
+
-- 3. Move ready waiting jobs (time-based, no token) into queue
|
|
2409
|
+
local waitingJobs = redis.call('ZRANGEBYSCORE', prefix .. 'waiting', '-inf', nowMs, 'LIMIT', 0, 200)
|
|
2410
|
+
for _, jobId in ipairs(waitingJobs) do
|
|
2411
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2412
|
+
local status = redis.call('HGET', jk, 'status')
|
|
2413
|
+
local waitTokenId = redis.call('HGET', jk, 'waitTokenId')
|
|
2414
|
+
if status == 'waiting' and (waitTokenId == false or waitTokenId == 'null') then
|
|
2415
|
+
local pri = tonumber(redis.call('HGET', jk, 'priority') or '0')
|
|
2416
|
+
local ca = tonumber(redis.call('HGET', jk, 'createdAt'))
|
|
2417
|
+
local score = pri * ${SCORE_RANGE} + (${SCORE_RANGE} - ca)
|
|
2418
|
+
redis.call('ZADD', prefix .. 'queue', score, jobId)
|
|
2419
|
+
redis.call('SREM', prefix .. 'status:waiting', jobId)
|
|
2420
|
+
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2421
|
+
redis.call('HMSET', jk, 'status', 'pending', 'waitUntil', 'null')
|
|
2422
|
+
end
|
|
2423
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2424
|
+
end
|
|
2425
|
+
|
|
2426
|
+
-- 4. Parse job type filter
|
|
2372
2427
|
local filterTypes = nil
|
|
2373
2428
|
if jobTypeFilter ~= "null" then
|
|
2374
2429
|
-- Could be a JSON array or a plain string
|
|
@@ -2381,7 +2436,7 @@ if jobTypeFilter ~= "null" then
|
|
|
2381
2436
|
end
|
|
2382
2437
|
end
|
|
2383
2438
|
|
|
2384
|
-
--
|
|
2439
|
+
-- 5. Pop candidates from queue (highest score first)
|
|
2385
2440
|
-- We pop more than batchSize because some may be filtered out
|
|
2386
2441
|
local popCount = batchSize * 3
|
|
2387
2442
|
local candidates = redis.call('ZPOPMAX', prefix .. 'queue', popCount)
|
|
@@ -2465,7 +2520,10 @@ local jk = prefix .. 'job:' .. jobId
|
|
|
2465
2520
|
redis.call('HMSET', jk,
|
|
2466
2521
|
'status', 'completed',
|
|
2467
2522
|
'updatedAt', nowMs,
|
|
2468
|
-
'completedAt', nowMs
|
|
2523
|
+
'completedAt', nowMs,
|
|
2524
|
+
'stepData', 'null',
|
|
2525
|
+
'waitUntil', 'null',
|
|
2526
|
+
'waitTokenId', 'null'
|
|
2469
2527
|
)
|
|
2470
2528
|
redis.call('SREM', prefix .. 'status:processing', jobId)
|
|
2471
2529
|
redis.call('SADD', prefix .. 'status:completed', jobId)
|
|
@@ -2524,6 +2582,7 @@ local nowMs = tonumber(ARGV[2])
|
|
|
2524
2582
|
local jk = prefix .. 'job:' .. jobId
|
|
2525
2583
|
|
|
2526
2584
|
local oldStatus = redis.call('HGET', jk, 'status')
|
|
2585
|
+
if oldStatus ~= 'failed' and oldStatus ~= 'processing' then return 0 end
|
|
2527
2586
|
|
|
2528
2587
|
redis.call('HMSET', jk,
|
|
2529
2588
|
'status', 'pending',
|
|
@@ -2535,9 +2594,7 @@ redis.call('HMSET', jk,
|
|
|
2535
2594
|
)
|
|
2536
2595
|
|
|
2537
2596
|
-- Remove from old status, add to pending
|
|
2538
|
-
|
|
2539
|
-
redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
|
|
2540
|
-
end
|
|
2597
|
+
redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
|
|
2541
2598
|
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2542
2599
|
|
|
2543
2600
|
-- Remove from retry sorted set if present
|
|
@@ -2558,18 +2615,21 @@ local nowMs = ARGV[2]
|
|
|
2558
2615
|
local jk = prefix .. 'job:' .. jobId
|
|
2559
2616
|
|
|
2560
2617
|
local status = redis.call('HGET', jk, 'status')
|
|
2561
|
-
if status ~= 'pending' then return 0 end
|
|
2618
|
+
if status ~= 'pending' and status ~= 'waiting' then return 0 end
|
|
2562
2619
|
|
|
2563
2620
|
redis.call('HMSET', jk,
|
|
2564
2621
|
'status', 'cancelled',
|
|
2565
2622
|
'updatedAt', nowMs,
|
|
2566
|
-
'lastCancelledAt', nowMs
|
|
2623
|
+
'lastCancelledAt', nowMs,
|
|
2624
|
+
'waitUntil', 'null',
|
|
2625
|
+
'waitTokenId', 'null'
|
|
2567
2626
|
)
|
|
2568
|
-
redis.call('SREM', prefix .. 'status:
|
|
2627
|
+
redis.call('SREM', prefix .. 'status:' .. status, jobId)
|
|
2569
2628
|
redis.call('SADD', prefix .. 'status:cancelled', jobId)
|
|
2570
|
-
-- Remove from queue / delayed
|
|
2629
|
+
-- Remove from queue / delayed / waiting
|
|
2571
2630
|
redis.call('ZREM', prefix .. 'queue', jobId)
|
|
2572
2631
|
redis.call('ZREM', prefix .. 'delayed', jobId)
|
|
2632
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2573
2633
|
|
|
2574
2634
|
return 1
|
|
2575
2635
|
`;
|
|
@@ -2637,18 +2697,16 @@ end
|
|
|
2637
2697
|
|
|
2638
2698
|
return count
|
|
2639
2699
|
`;
|
|
2640
|
-
var
|
|
2700
|
+
var CLEANUP_OLD_JOBS_BATCH_SCRIPT = `
|
|
2641
2701
|
local prefix = KEYS[1]
|
|
2642
2702
|
local cutoffMs = tonumber(ARGV[1])
|
|
2643
|
-
|
|
2644
|
-
local completed = redis.call('SMEMBERS', prefix .. 'status:completed')
|
|
2645
2703
|
local count = 0
|
|
2646
2704
|
|
|
2647
|
-
for
|
|
2705
|
+
for i = 2, #ARGV do
|
|
2706
|
+
local jobId = ARGV[i]
|
|
2648
2707
|
local jk = prefix .. 'job:' .. jobId
|
|
2649
2708
|
local updatedAt = tonumber(redis.call('HGET', jk, 'updatedAt'))
|
|
2650
2709
|
if updatedAt and updatedAt < cutoffMs then
|
|
2651
|
-
-- Remove all indexes
|
|
2652
2710
|
local jobType = redis.call('HGET', jk, 'jobType')
|
|
2653
2711
|
local tagsJson = redis.call('HGET', jk, 'tags')
|
|
2654
2712
|
local idempotencyKey = redis.call('HGET', jk, 'idempotencyKey')
|
|
@@ -2671,7 +2729,6 @@ for _, jobId in ipairs(completed) do
|
|
|
2671
2729
|
if idempotencyKey and idempotencyKey ~= 'null' then
|
|
2672
2730
|
redis.call('DEL', prefix .. 'idempotency:' .. idempotencyKey)
|
|
2673
2731
|
end
|
|
2674
|
-
-- Delete events
|
|
2675
2732
|
redis.call('DEL', prefix .. 'events:' .. jobId)
|
|
2676
2733
|
|
|
2677
2734
|
count = count + 1
|
|
@@ -2680,8 +2737,158 @@ end
|
|
|
2680
2737
|
|
|
2681
2738
|
return count
|
|
2682
2739
|
`;
|
|
2740
|
+
var WAIT_JOB_SCRIPT = `
|
|
2741
|
+
local prefix = KEYS[1]
|
|
2742
|
+
local jobId = ARGV[1]
|
|
2743
|
+
local waitUntilMs = ARGV[2]
|
|
2744
|
+
local waitTokenId = ARGV[3]
|
|
2745
|
+
local stepDataJson = ARGV[4]
|
|
2746
|
+
local nowMs = ARGV[5]
|
|
2747
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2748
|
+
|
|
2749
|
+
local status = redis.call('HGET', jk, 'status')
|
|
2750
|
+
if status ~= 'processing' then return 0 end
|
|
2751
|
+
|
|
2752
|
+
redis.call('HMSET', jk,
|
|
2753
|
+
'status', 'waiting',
|
|
2754
|
+
'waitUntil', waitUntilMs,
|
|
2755
|
+
'waitTokenId', waitTokenId,
|
|
2756
|
+
'stepData', stepDataJson,
|
|
2757
|
+
'lockedAt', 'null',
|
|
2758
|
+
'lockedBy', 'null',
|
|
2759
|
+
'updatedAt', nowMs
|
|
2760
|
+
)
|
|
2761
|
+
redis.call('SREM', prefix .. 'status:processing', jobId)
|
|
2762
|
+
redis.call('SADD', prefix .. 'status:waiting', jobId)
|
|
2763
|
+
|
|
2764
|
+
-- Add to waiting sorted set if time-based wait
|
|
2765
|
+
if waitUntilMs ~= 'null' then
|
|
2766
|
+
redis.call('ZADD', prefix .. 'waiting', tonumber(waitUntilMs), jobId)
|
|
2767
|
+
end
|
|
2768
|
+
|
|
2769
|
+
return 1
|
|
2770
|
+
`;
|
|
2771
|
+
var COMPLETE_WAITPOINT_SCRIPT = `
|
|
2772
|
+
local prefix = KEYS[1]
|
|
2773
|
+
local tokenId = ARGV[1]
|
|
2774
|
+
local outputJson = ARGV[2]
|
|
2775
|
+
local nowMs = ARGV[3]
|
|
2776
|
+
local wpk = prefix .. 'waitpoint:' .. tokenId
|
|
2777
|
+
|
|
2778
|
+
local wpStatus = redis.call('HGET', wpk, 'status')
|
|
2779
|
+
if not wpStatus or wpStatus ~= 'waiting' then return 0 end
|
|
2780
|
+
|
|
2781
|
+
redis.call('HMSET', wpk,
|
|
2782
|
+
'status', 'completed',
|
|
2783
|
+
'output', outputJson,
|
|
2784
|
+
'completedAt', nowMs
|
|
2785
|
+
)
|
|
2786
|
+
|
|
2787
|
+
-- Move associated job back to pending
|
|
2788
|
+
local jobId = redis.call('HGET', wpk, 'jobId')
|
|
2789
|
+
if jobId and jobId ~= 'null' then
|
|
2790
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2791
|
+
local jobStatus = redis.call('HGET', jk, 'status')
|
|
2792
|
+
if jobStatus == 'waiting' then
|
|
2793
|
+
redis.call('HMSET', jk,
|
|
2794
|
+
'status', 'pending',
|
|
2795
|
+
'waitTokenId', 'null',
|
|
2796
|
+
'waitUntil', 'null',
|
|
2797
|
+
'updatedAt', nowMs
|
|
2798
|
+
)
|
|
2799
|
+
redis.call('SREM', prefix .. 'status:waiting', jobId)
|
|
2800
|
+
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2801
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2802
|
+
|
|
2803
|
+
-- Re-add to queue
|
|
2804
|
+
local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
|
|
2805
|
+
local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
|
|
2806
|
+
local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
|
|
2807
|
+
redis.call('ZADD', prefix .. 'queue', score, jobId)
|
|
2808
|
+
end
|
|
2809
|
+
end
|
|
2810
|
+
|
|
2811
|
+
return 1
|
|
2812
|
+
`;
|
|
2813
|
+
var EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT = `
|
|
2814
|
+
local prefix = KEYS[1]
|
|
2815
|
+
local nowMs = tonumber(ARGV[1])
|
|
2816
|
+
|
|
2817
|
+
local expiredIds = redis.call('ZRANGEBYSCORE', prefix .. 'waitpoint_timeout', '-inf', nowMs)
|
|
2818
|
+
local count = 0
|
|
2819
|
+
|
|
2820
|
+
for _, tokenId in ipairs(expiredIds) do
|
|
2821
|
+
local wpk = prefix .. 'waitpoint:' .. tokenId
|
|
2822
|
+
local wpStatus = redis.call('HGET', wpk, 'status')
|
|
2823
|
+
if wpStatus == 'waiting' then
|
|
2824
|
+
redis.call('HMSET', wpk,
|
|
2825
|
+
'status', 'timed_out'
|
|
2826
|
+
)
|
|
2827
|
+
|
|
2828
|
+
-- Move associated job back to pending
|
|
2829
|
+
local jobId = redis.call('HGET', wpk, 'jobId')
|
|
2830
|
+
if jobId and jobId ~= 'null' then
|
|
2831
|
+
local jk = prefix .. 'job:' .. jobId
|
|
2832
|
+
local jobStatus = redis.call('HGET', jk, 'status')
|
|
2833
|
+
if jobStatus == 'waiting' then
|
|
2834
|
+
redis.call('HMSET', jk,
|
|
2835
|
+
'status', 'pending',
|
|
2836
|
+
'waitTokenId', 'null',
|
|
2837
|
+
'waitUntil', 'null',
|
|
2838
|
+
'updatedAt', nowMs
|
|
2839
|
+
)
|
|
2840
|
+
redis.call('SREM', prefix .. 'status:waiting', jobId)
|
|
2841
|
+
redis.call('SADD', prefix .. 'status:pending', jobId)
|
|
2842
|
+
redis.call('ZREM', prefix .. 'waiting', jobId)
|
|
2843
|
+
|
|
2844
|
+
local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
|
|
2845
|
+
local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
|
|
2846
|
+
local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
|
|
2847
|
+
redis.call('ZADD', prefix .. 'queue', score, jobId)
|
|
2848
|
+
end
|
|
2849
|
+
end
|
|
2850
|
+
|
|
2851
|
+
count = count + 1
|
|
2852
|
+
end
|
|
2853
|
+
redis.call('ZREM', prefix .. 'waitpoint_timeout', tokenId)
|
|
2854
|
+
end
|
|
2683
2855
|
|
|
2684
|
-
|
|
2856
|
+
return count
|
|
2857
|
+
`;
|
|
2858
|
+
var MAX_TIMEOUT_MS2 = 365 * 24 * 60 * 60 * 1e3;
|
|
2859
|
+
function parseTimeoutString2(timeout) {
|
|
2860
|
+
const match = timeout.match(/^(\d+)(s|m|h|d)$/);
|
|
2861
|
+
if (!match) {
|
|
2862
|
+
throw new Error(
|
|
2863
|
+
`Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
|
|
2864
|
+
);
|
|
2865
|
+
}
|
|
2866
|
+
const value = parseInt(match[1], 10);
|
|
2867
|
+
const unit = match[2];
|
|
2868
|
+
let ms;
|
|
2869
|
+
switch (unit) {
|
|
2870
|
+
case "s":
|
|
2871
|
+
ms = value * 1e3;
|
|
2872
|
+
break;
|
|
2873
|
+
case "m":
|
|
2874
|
+
ms = value * 60 * 1e3;
|
|
2875
|
+
break;
|
|
2876
|
+
case "h":
|
|
2877
|
+
ms = value * 60 * 60 * 1e3;
|
|
2878
|
+
break;
|
|
2879
|
+
case "d":
|
|
2880
|
+
ms = value * 24 * 60 * 60 * 1e3;
|
|
2881
|
+
break;
|
|
2882
|
+
default:
|
|
2883
|
+
throw new Error(`Unknown timeout unit: "${unit}"`);
|
|
2884
|
+
}
|
|
2885
|
+
if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS2) {
|
|
2886
|
+
throw new Error(
|
|
2887
|
+
`Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
|
|
2888
|
+
);
|
|
2889
|
+
}
|
|
2890
|
+
return ms;
|
|
2891
|
+
}
|
|
2685
2892
|
function hashToObject(arr) {
|
|
2686
2893
|
const obj = {};
|
|
2687
2894
|
for (let i = 0; i < arr.length; i += 2) {
|
|
@@ -2747,9 +2954,20 @@ function deserializeJob(h) {
|
|
|
2747
2954
|
lastCancelledAt: dateOrNull(h.lastCancelledAt),
|
|
2748
2955
|
tags,
|
|
2749
2956
|
idempotencyKey: nullish(h.idempotencyKey),
|
|
2750
|
-
progress: numOrNull(h.progress)
|
|
2957
|
+
progress: numOrNull(h.progress),
|
|
2958
|
+
waitUntil: dateOrNull(h.waitUntil),
|
|
2959
|
+
waitTokenId: nullish(h.waitTokenId),
|
|
2960
|
+
stepData: parseStepData(h.stepData)
|
|
2751
2961
|
};
|
|
2752
2962
|
}
|
|
2963
|
+
function parseStepData(raw) {
|
|
2964
|
+
if (!raw || raw === "null") return void 0;
|
|
2965
|
+
try {
|
|
2966
|
+
return JSON.parse(raw);
|
|
2967
|
+
} catch {
|
|
2968
|
+
return void 0;
|
|
2969
|
+
}
|
|
2970
|
+
}
|
|
2753
2971
|
var RedisBackend = class {
|
|
2754
2972
|
constructor(redisConfig) {
|
|
2755
2973
|
let IORedis;
|
|
@@ -2905,8 +3123,14 @@ var RedisBackend = class {
|
|
|
2905
3123
|
if (filters.runAt) {
|
|
2906
3124
|
jobs = this.filterByRunAt(jobs, filters.runAt);
|
|
2907
3125
|
}
|
|
3126
|
+
if (filters.cursor !== void 0) {
|
|
3127
|
+
jobs = jobs.filter((j) => j.id < filters.cursor);
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
jobs.sort((a, b) => b.id - a.id);
|
|
3131
|
+
if (filters?.cursor !== void 0) {
|
|
3132
|
+
return jobs.slice(0, limit);
|
|
2908
3133
|
}
|
|
2909
|
-
jobs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
|
|
2910
3134
|
return jobs.slice(offset, offset + limit);
|
|
2911
3135
|
}
|
|
2912
3136
|
async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
|
|
@@ -3138,35 +3362,302 @@ var RedisBackend = class {
|
|
|
3138
3362
|
log(`Edited ${count} pending jobs`);
|
|
3139
3363
|
return count;
|
|
3140
3364
|
}
|
|
3141
|
-
|
|
3365
|
+
/**
|
|
3366
|
+
* Delete completed jobs older than the given number of days.
|
|
3367
|
+
* Uses SSCAN to iterate the completed set in batches, avoiding
|
|
3368
|
+
* loading all IDs into memory and preventing long Redis blocks.
|
|
3369
|
+
*
|
|
3370
|
+
* @param daysToKeep - Number of days to retain completed jobs (default 30).
|
|
3371
|
+
* @param batchSize - Number of IDs to scan per SSCAN iteration (default 200).
|
|
3372
|
+
* @returns Total number of deleted jobs.
|
|
3373
|
+
*/
|
|
3374
|
+
async cleanupOldJobs(daysToKeep = 30, batchSize = 200) {
|
|
3375
|
+
const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
|
|
3376
|
+
const setKey = `${this.prefix}status:completed`;
|
|
3377
|
+
let totalDeleted = 0;
|
|
3378
|
+
let cursor = "0";
|
|
3379
|
+
do {
|
|
3380
|
+
const [nextCursor, ids] = await this.client.sscan(
|
|
3381
|
+
setKey,
|
|
3382
|
+
cursor,
|
|
3383
|
+
"COUNT",
|
|
3384
|
+
batchSize
|
|
3385
|
+
);
|
|
3386
|
+
cursor = nextCursor;
|
|
3387
|
+
if (ids.length > 0) {
|
|
3388
|
+
const result = await this.client.eval(
|
|
3389
|
+
CLEANUP_OLD_JOBS_BATCH_SCRIPT,
|
|
3390
|
+
1,
|
|
3391
|
+
this.prefix,
|
|
3392
|
+
cutoffMs,
|
|
3393
|
+
...ids
|
|
3394
|
+
);
|
|
3395
|
+
totalDeleted += Number(result);
|
|
3396
|
+
}
|
|
3397
|
+
} while (cursor !== "0");
|
|
3398
|
+
log(`Deleted ${totalDeleted} old jobs`);
|
|
3399
|
+
return totalDeleted;
|
|
3400
|
+
}
|
|
3401
|
+
/**
|
|
3402
|
+
* Delete job events older than the given number of days.
|
|
3403
|
+
* Iterates all event lists and removes events whose createdAt is before the cutoff.
|
|
3404
|
+
* Also removes orphaned event lists (where the job no longer exists).
|
|
3405
|
+
*
|
|
3406
|
+
* @param daysToKeep - Number of days to retain events (default 30).
|
|
3407
|
+
* @param batchSize - Number of event keys to scan per SCAN iteration (default 200).
|
|
3408
|
+
* @returns Total number of deleted events.
|
|
3409
|
+
*/
|
|
3410
|
+
async cleanupOldJobEvents(daysToKeep = 30, batchSize = 200) {
|
|
3142
3411
|
const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
|
|
3412
|
+
const pattern = `${this.prefix}events:*`;
|
|
3413
|
+
let totalDeleted = 0;
|
|
3414
|
+
let cursor = "0";
|
|
3415
|
+
do {
|
|
3416
|
+
const [nextCursor, keys] = await this.client.scan(
|
|
3417
|
+
cursor,
|
|
3418
|
+
"MATCH",
|
|
3419
|
+
pattern,
|
|
3420
|
+
"COUNT",
|
|
3421
|
+
batchSize
|
|
3422
|
+
);
|
|
3423
|
+
cursor = nextCursor;
|
|
3424
|
+
for (const key of keys) {
|
|
3425
|
+
const jobIdStr = key.slice(`${this.prefix}events:`.length);
|
|
3426
|
+
const jobExists = await this.client.exists(
|
|
3427
|
+
`${this.prefix}job:${jobIdStr}`
|
|
3428
|
+
);
|
|
3429
|
+
if (!jobExists) {
|
|
3430
|
+
const len = await this.client.llen(key);
|
|
3431
|
+
await this.client.del(key);
|
|
3432
|
+
totalDeleted += len;
|
|
3433
|
+
continue;
|
|
3434
|
+
}
|
|
3435
|
+
const events = await this.client.lrange(key, 0, -1);
|
|
3436
|
+
const kept = [];
|
|
3437
|
+
for (const raw of events) {
|
|
3438
|
+
try {
|
|
3439
|
+
const e = JSON.parse(raw);
|
|
3440
|
+
if (e.createdAt >= cutoffMs) {
|
|
3441
|
+
kept.push(raw);
|
|
3442
|
+
} else {
|
|
3443
|
+
totalDeleted++;
|
|
3444
|
+
}
|
|
3445
|
+
} catch {
|
|
3446
|
+
totalDeleted++;
|
|
3447
|
+
}
|
|
3448
|
+
}
|
|
3449
|
+
if (kept.length === 0) {
|
|
3450
|
+
await this.client.del(key);
|
|
3451
|
+
} else if (kept.length < events.length) {
|
|
3452
|
+
const pipeline = this.client.pipeline();
|
|
3453
|
+
pipeline.del(key);
|
|
3454
|
+
for (const raw of kept) {
|
|
3455
|
+
pipeline.rpush(key, raw);
|
|
3456
|
+
}
|
|
3457
|
+
await pipeline.exec();
|
|
3458
|
+
}
|
|
3459
|
+
}
|
|
3460
|
+
} while (cursor !== "0");
|
|
3461
|
+
log(`Deleted ${totalDeleted} old job events`);
|
|
3462
|
+
return totalDeleted;
|
|
3463
|
+
}
|
|
3464
|
+
async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
|
|
3465
|
+
const maxAgeMs = maxProcessingTimeMinutes * 60 * 1e3;
|
|
3466
|
+
const now = this.nowMs();
|
|
3143
3467
|
const result = await this.client.eval(
|
|
3144
|
-
|
|
3468
|
+
RECLAIM_STUCK_JOBS_SCRIPT,
|
|
3145
3469
|
1,
|
|
3146
3470
|
this.prefix,
|
|
3147
|
-
|
|
3471
|
+
maxAgeMs,
|
|
3472
|
+
now
|
|
3148
3473
|
);
|
|
3149
|
-
log(`
|
|
3474
|
+
log(`Reclaimed ${result} stuck jobs`);
|
|
3150
3475
|
return Number(result);
|
|
3151
3476
|
}
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3477
|
+
// ── Wait / step-data support ────────────────────────────────────────
|
|
3478
|
+
/**
|
|
3479
|
+
* Transition a job from 'processing' to 'waiting' status.
|
|
3480
|
+
* Persists step data so the handler can resume from where it left off.
|
|
3481
|
+
*
|
|
3482
|
+
* @param jobId - The job to pause.
|
|
3483
|
+
* @param options - Wait configuration including optional waitUntil date, token ID, and step data.
|
|
3484
|
+
*/
|
|
3485
|
+
async waitJob(jobId, options) {
|
|
3486
|
+
const now = this.nowMs();
|
|
3487
|
+
const waitUntilMs = options.waitUntil ? options.waitUntil.getTime().toString() : "null";
|
|
3488
|
+
const waitTokenId = options.waitTokenId ?? "null";
|
|
3489
|
+
const stepDataJson = JSON.stringify(options.stepData);
|
|
3490
|
+
const result = await this.client.eval(
|
|
3491
|
+
WAIT_JOB_SCRIPT,
|
|
3492
|
+
1,
|
|
3493
|
+
this.prefix,
|
|
3494
|
+
jobId,
|
|
3495
|
+
waitUntilMs,
|
|
3496
|
+
waitTokenId,
|
|
3497
|
+
stepDataJson,
|
|
3498
|
+
now
|
|
3155
3499
|
);
|
|
3156
|
-
|
|
3500
|
+
if (Number(result) === 0) {
|
|
3501
|
+
log(
|
|
3502
|
+
`Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
|
|
3503
|
+
);
|
|
3504
|
+
return;
|
|
3505
|
+
}
|
|
3506
|
+
await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
|
|
3507
|
+
waitUntil: options.waitUntil?.toISOString() ?? null,
|
|
3508
|
+
waitTokenId: options.waitTokenId ?? null
|
|
3509
|
+
});
|
|
3510
|
+
log(`Job ${jobId} set to waiting`);
|
|
3157
3511
|
}
|
|
3158
|
-
|
|
3159
|
-
|
|
3512
|
+
/**
|
|
3513
|
+
* Persist step data for a job. Called after each ctx.run() step completes.
|
|
3514
|
+
* Best-effort: does not throw to avoid killing the running handler.
|
|
3515
|
+
*
|
|
3516
|
+
* @param jobId - The job to update.
|
|
3517
|
+
* @param stepData - The step data to persist.
|
|
3518
|
+
*/
|
|
3519
|
+
async updateStepData(jobId, stepData) {
|
|
3520
|
+
try {
|
|
3521
|
+
const now = this.nowMs();
|
|
3522
|
+
await this.client.hset(
|
|
3523
|
+
`${this.prefix}job:${jobId}`,
|
|
3524
|
+
"stepData",
|
|
3525
|
+
JSON.stringify(stepData),
|
|
3526
|
+
"updatedAt",
|
|
3527
|
+
now.toString()
|
|
3528
|
+
);
|
|
3529
|
+
} catch (error) {
|
|
3530
|
+
log(`Error updating stepData for job ${jobId}: ${error}`);
|
|
3531
|
+
}
|
|
3532
|
+
}
|
|
3533
|
+
/**
|
|
3534
|
+
* Create a waitpoint token.
|
|
3535
|
+
*
|
|
3536
|
+
* @param jobId - The job ID to associate with the token (null if created outside a handler).
|
|
3537
|
+
* @param options - Optional timeout string (e.g. '10m', '1h') and tags.
|
|
3538
|
+
* @returns The created waitpoint with its unique ID.
|
|
3539
|
+
*/
|
|
3540
|
+
async createWaitpoint(jobId, options) {
|
|
3541
|
+
const id = `wp_${crypto.randomUUID()}`;
|
|
3542
|
+
const now = this.nowMs();
|
|
3543
|
+
let timeoutAt = null;
|
|
3544
|
+
if (options?.timeout) {
|
|
3545
|
+
const ms = parseTimeoutString2(options.timeout);
|
|
3546
|
+
timeoutAt = now + ms;
|
|
3547
|
+
}
|
|
3548
|
+
const key = `${this.prefix}waitpoint:${id}`;
|
|
3549
|
+
const fields = [
|
|
3550
|
+
"id",
|
|
3551
|
+
id,
|
|
3552
|
+
"jobId",
|
|
3553
|
+
jobId !== null ? jobId.toString() : "null",
|
|
3554
|
+
"status",
|
|
3555
|
+
"waiting",
|
|
3556
|
+
"output",
|
|
3557
|
+
"null",
|
|
3558
|
+
"timeoutAt",
|
|
3559
|
+
timeoutAt !== null ? timeoutAt.toString() : "null",
|
|
3560
|
+
"createdAt",
|
|
3561
|
+
now.toString(),
|
|
3562
|
+
"completedAt",
|
|
3563
|
+
"null",
|
|
3564
|
+
"tags",
|
|
3565
|
+
options?.tags ? JSON.stringify(options.tags) : "null"
|
|
3566
|
+
];
|
|
3567
|
+
await this.client.hmset(key, ...fields);
|
|
3568
|
+
if (timeoutAt !== null) {
|
|
3569
|
+
await this.client.zadd(`${this.prefix}waitpoint_timeout`, timeoutAt, id);
|
|
3570
|
+
}
|
|
3571
|
+
log(`Created waitpoint ${id} for job ${jobId}`);
|
|
3572
|
+
return { id };
|
|
3573
|
+
}
|
|
3574
|
+
/**
|
|
3575
|
+
* Complete a waitpoint token and move the associated job back to 'pending'.
|
|
3576
|
+
*
|
|
3577
|
+
* @param tokenId - The waitpoint token ID to complete.
|
|
3578
|
+
* @param data - Optional data to pass to the waiting handler.
|
|
3579
|
+
*/
|
|
3580
|
+
async completeWaitpoint(tokenId, data) {
|
|
3160
3581
|
const now = this.nowMs();
|
|
3582
|
+
const outputJson = data != null ? JSON.stringify(data) : "null";
|
|
3161
3583
|
const result = await this.client.eval(
|
|
3162
|
-
|
|
3584
|
+
COMPLETE_WAITPOINT_SCRIPT,
|
|
3163
3585
|
1,
|
|
3164
3586
|
this.prefix,
|
|
3165
|
-
|
|
3587
|
+
tokenId,
|
|
3588
|
+
outputJson,
|
|
3166
3589
|
now
|
|
3167
3590
|
);
|
|
3168
|
-
|
|
3169
|
-
|
|
3591
|
+
if (Number(result) === 0) {
|
|
3592
|
+
log(`Waitpoint ${tokenId} not found or already completed`);
|
|
3593
|
+
return;
|
|
3594
|
+
}
|
|
3595
|
+
log(`Completed waitpoint ${tokenId}`);
|
|
3596
|
+
}
|
|
3597
|
+
/**
|
|
3598
|
+
* Retrieve a waitpoint token by its ID.
|
|
3599
|
+
*
|
|
3600
|
+
* @param tokenId - The waitpoint token ID to look up.
|
|
3601
|
+
* @returns The waitpoint record, or null if not found.
|
|
3602
|
+
*/
|
|
3603
|
+
async getWaitpoint(tokenId) {
|
|
3604
|
+
const data = await this.client.hgetall(
|
|
3605
|
+
`${this.prefix}waitpoint:${tokenId}`
|
|
3606
|
+
);
|
|
3607
|
+
if (!data || Object.keys(data).length === 0) return null;
|
|
3608
|
+
const nullish = (v) => v === void 0 || v === "null" || v === "" ? null : v;
|
|
3609
|
+
const numOrNull = (v) => {
|
|
3610
|
+
const n = nullish(v);
|
|
3611
|
+
return n === null ? null : Number(n);
|
|
3612
|
+
};
|
|
3613
|
+
const dateOrNull = (v) => {
|
|
3614
|
+
const n = numOrNull(v);
|
|
3615
|
+
return n === null ? null : new Date(n);
|
|
3616
|
+
};
|
|
3617
|
+
let output = null;
|
|
3618
|
+
if (data.output && data.output !== "null") {
|
|
3619
|
+
try {
|
|
3620
|
+
output = JSON.parse(data.output);
|
|
3621
|
+
} catch {
|
|
3622
|
+
output = data.output;
|
|
3623
|
+
}
|
|
3624
|
+
}
|
|
3625
|
+
let tags = null;
|
|
3626
|
+
if (data.tags && data.tags !== "null") {
|
|
3627
|
+
try {
|
|
3628
|
+
tags = JSON.parse(data.tags);
|
|
3629
|
+
} catch {
|
|
3630
|
+
}
|
|
3631
|
+
}
|
|
3632
|
+
return {
|
|
3633
|
+
id: data.id,
|
|
3634
|
+
jobId: numOrNull(data.jobId),
|
|
3635
|
+
status: data.status,
|
|
3636
|
+
output,
|
|
3637
|
+
timeoutAt: dateOrNull(data.timeoutAt),
|
|
3638
|
+
createdAt: new Date(Number(data.createdAt)),
|
|
3639
|
+
completedAt: dateOrNull(data.completedAt),
|
|
3640
|
+
tags
|
|
3641
|
+
};
|
|
3642
|
+
}
|
|
3643
|
+
/**
|
|
3644
|
+
* Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
|
|
3645
|
+
*
|
|
3646
|
+
* @returns The number of tokens that were expired.
|
|
3647
|
+
*/
|
|
3648
|
+
async expireTimedOutWaitpoints() {
|
|
3649
|
+
const now = this.nowMs();
|
|
3650
|
+
const result = await this.client.eval(
|
|
3651
|
+
EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT,
|
|
3652
|
+
1,
|
|
3653
|
+
this.prefix,
|
|
3654
|
+
now
|
|
3655
|
+
);
|
|
3656
|
+
const count = Number(result);
|
|
3657
|
+
if (count > 0) {
|
|
3658
|
+
log(`Expired ${count} timed-out waitpoints`);
|
|
3659
|
+
}
|
|
3660
|
+
return count;
|
|
3170
3661
|
}
|
|
3171
3662
|
// ── Internal helpers ──────────────────────────────────────────────────
|
|
3172
3663
|
async setPendingReasonForUnpickedJobs(reason, jobType) {
|
|
@@ -3715,10 +4206,9 @@ var initJobQueue = (config) => {
|
|
|
3715
4206
|
const backendType = config.backend ?? "postgres";
|
|
3716
4207
|
setLogContext(config.verbose ?? false);
|
|
3717
4208
|
let backend;
|
|
3718
|
-
let pool;
|
|
3719
4209
|
if (backendType === "postgres") {
|
|
3720
4210
|
const pgConfig = config;
|
|
3721
|
-
pool = createPool(pgConfig.databaseConfig);
|
|
4211
|
+
const pool = createPool(pgConfig.databaseConfig);
|
|
3722
4212
|
backend = new PostgresBackend(pool);
|
|
3723
4213
|
} else if (backendType === "redis") {
|
|
3724
4214
|
const redisConfig = config.redisConfig;
|
|
@@ -3726,14 +4216,6 @@ var initJobQueue = (config) => {
|
|
|
3726
4216
|
} else {
|
|
3727
4217
|
throw new Error(`Unknown backend: ${backendType}`);
|
|
3728
4218
|
}
|
|
3729
|
-
const requirePool = () => {
|
|
3730
|
-
if (!pool) {
|
|
3731
|
-
throw new Error(
|
|
3732
|
-
'Wait/Token features require the PostgreSQL backend. Configure with backend: "postgres" to use these features.'
|
|
3733
|
-
);
|
|
3734
|
-
}
|
|
3735
|
-
return pool;
|
|
3736
|
-
};
|
|
3737
4219
|
const enqueueDueCronJobsImpl = async () => {
|
|
3738
4220
|
const dueSchedules = await backend.getDueCronSchedules();
|
|
3739
4221
|
let count = 0;
|
|
@@ -3800,8 +4282,8 @@ var initJobQueue = (config) => {
|
|
|
3800
4282
|
config.verbose ?? false
|
|
3801
4283
|
),
|
|
3802
4284
|
retryJob: (jobId) => backend.retryJob(jobId),
|
|
3803
|
-
cleanupOldJobs: (daysToKeep) => backend.cleanupOldJobs(daysToKeep),
|
|
3804
|
-
cleanupOldJobEvents: (daysToKeep) => backend.cleanupOldJobEvents(daysToKeep),
|
|
4285
|
+
cleanupOldJobs: (daysToKeep, batchSize) => backend.cleanupOldJobs(daysToKeep, batchSize),
|
|
4286
|
+
cleanupOldJobEvents: (daysToKeep, batchSize) => backend.cleanupOldJobEvents(daysToKeep, batchSize),
|
|
3805
4287
|
cancelJob: withLogContext(
|
|
3806
4288
|
(jobId) => backend.cancelJob(jobId),
|
|
3807
4289
|
config.verbose ?? false
|
|
@@ -3838,21 +4320,21 @@ var initJobQueue = (config) => {
|
|
|
3838
4320
|
(jobId) => backend.getJobEvents(jobId),
|
|
3839
4321
|
config.verbose ?? false
|
|
3840
4322
|
),
|
|
3841
|
-
// Wait / Token support (
|
|
4323
|
+
// Wait / Token support (works with all backends)
|
|
3842
4324
|
createToken: withLogContext(
|
|
3843
|
-
(options) => createWaitpoint(
|
|
4325
|
+
(options) => backend.createWaitpoint(null, options),
|
|
3844
4326
|
config.verbose ?? false
|
|
3845
4327
|
),
|
|
3846
4328
|
completeToken: withLogContext(
|
|
3847
|
-
(tokenId, data) => completeWaitpoint(
|
|
4329
|
+
(tokenId, data) => backend.completeWaitpoint(tokenId, data),
|
|
3848
4330
|
config.verbose ?? false
|
|
3849
4331
|
),
|
|
3850
4332
|
getToken: withLogContext(
|
|
3851
|
-
(tokenId) => getWaitpoint(
|
|
4333
|
+
(tokenId) => backend.getWaitpoint(tokenId),
|
|
3852
4334
|
config.verbose ?? false
|
|
3853
4335
|
),
|
|
3854
4336
|
expireTimedOutTokens: withLogContext(
|
|
3855
|
-
() => expireTimedOutWaitpoints(
|
|
4337
|
+
() => backend.expireTimedOutWaitpoints(),
|
|
3856
4338
|
config.verbose ?? false
|
|
3857
4339
|
),
|
|
3858
4340
|
// Cron schedule operations
|
|
@@ -3933,7 +4415,7 @@ var initJobQueue = (config) => {
|
|
|
3933
4415
|
),
|
|
3934
4416
|
// Advanced access
|
|
3935
4417
|
getPool: () => {
|
|
3936
|
-
if (
|
|
4418
|
+
if (!(backend instanceof PostgresBackend)) {
|
|
3937
4419
|
throw new Error(
|
|
3938
4420
|
"getPool() is only available with the PostgreSQL backend."
|
|
3939
4421
|
);
|