@nicnocquee/dataqueue 1.31.0 → 1.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,11 +1,11 @@
1
1
  'use strict';
2
2
 
3
- var async_hooks = require('async_hooks');
4
- var crypto = require('crypto');
5
3
  var worker_threads = require('worker_threads');
4
+ var async_hooks = require('async_hooks');
6
5
  var pg = require('pg');
7
6
  var pgConnectionString = require('pg-connection-string');
8
7
  var fs = require('fs');
8
+ var crypto = require('crypto');
9
9
  var module$1 = require('module');
10
10
  var croner = require('croner');
11
11
 
@@ -14,6 +14,8 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
14
14
 
15
15
  var fs__default = /*#__PURE__*/_interopDefault(fs);
16
16
 
17
+ // src/processor.ts
18
+
17
19
  // src/types.ts
18
20
  var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
19
21
  JobEventType2["Added"] = "added";
@@ -27,11 +29,11 @@ var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
27
29
  JobEventType2["Waiting"] = "waiting";
28
30
  return JobEventType2;
29
31
  })(JobEventType || {});
30
- var FailureReason = /* @__PURE__ */ ((FailureReason5) => {
31
- FailureReason5["Timeout"] = "timeout";
32
- FailureReason5["HandlerError"] = "handler_error";
33
- FailureReason5["NoHandler"] = "no_handler";
34
- return FailureReason5;
32
+ var FailureReason = /* @__PURE__ */ ((FailureReason4) => {
33
+ FailureReason4["Timeout"] = "timeout";
34
+ FailureReason4["HandlerError"] = "handler_error";
35
+ FailureReason4["NoHandler"] = "no_handler";
36
+ return FailureReason4;
35
37
  })(FailureReason || {});
36
38
  var WaitSignal = class extends Error {
37
39
  constructor(type, waitUntil, tokenId, stepData) {
@@ -58,804 +60,942 @@ var log = (message) => {
58
60
  }
59
61
  };
60
62
 
61
- // src/backends/postgres.ts
62
- var PostgresBackend = class {
63
- constructor(pool) {
64
- this.pool = pool;
65
- }
66
- /** Expose the raw pool for advanced usage. */
67
- getPool() {
68
- return this.pool;
69
- }
70
- // ── Events ──────────────────────────────────────────────────────────
71
- async recordJobEvent(jobId, eventType, metadata) {
72
- const client = await this.pool.connect();
73
- try {
74
- await client.query(
75
- `INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
76
- [jobId, eventType, metadata ? JSON.stringify(metadata) : null]
77
- );
78
- } catch (error) {
79
- log(`Error recording job event for job ${jobId}: ${error}`);
80
- } finally {
81
- client.release();
82
- }
83
- }
84
- async getJobEvents(jobId) {
85
- const client = await this.pool.connect();
86
- try {
87
- const res = await client.query(
88
- `SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
89
- [jobId]
90
- );
91
- return res.rows;
92
- } finally {
93
- client.release();
94
- }
95
- }
96
- // ── Job CRUD ──────────────────────────────────────────────────────────
97
- async addJob({
98
- jobType,
99
- payload,
100
- maxAttempts = 3,
101
- priority = 0,
102
- runAt = null,
103
- timeoutMs = void 0,
104
- forceKillOnTimeout = false,
105
- tags = void 0,
106
- idempotencyKey = void 0
107
- }) {
108
- const client = await this.pool.connect();
109
- try {
110
- let result;
111
- const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
112
- if (runAt) {
113
- result = await client.query(
114
- `INSERT INTO job_queue
115
- (job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
116
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
117
- ${onConflict}
118
- RETURNING id`,
119
- [
120
- jobType,
121
- payload,
122
- maxAttempts,
123
- priority,
124
- runAt,
125
- timeoutMs ?? null,
126
- forceKillOnTimeout ?? false,
127
- tags ?? null,
128
- idempotencyKey ?? null
129
- ]
130
- );
131
- } else {
132
- result = await client.query(
133
- `INSERT INTO job_queue
134
- (job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
135
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
136
- ${onConflict}
137
- RETURNING id`,
138
- [
139
- jobType,
140
- payload,
141
- maxAttempts,
142
- priority,
143
- timeoutMs ?? null,
144
- forceKillOnTimeout ?? false,
145
- tags ?? null,
146
- idempotencyKey ?? null
147
- ]
148
- );
149
- }
150
- if (result.rows.length === 0 && idempotencyKey) {
151
- const existing = await client.query(
152
- `SELECT id FROM job_queue WHERE idempotency_key = $1`,
153
- [idempotencyKey]
154
- );
155
- if (existing.rows.length > 0) {
156
- log(
157
- `Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
158
- );
159
- return existing.rows[0].id;
160
- }
161
- throw new Error(
162
- `Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
163
- );
164
- }
165
- const jobId = result.rows[0].id;
166
- log(
167
- `Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
63
+ // src/processor.ts
64
+ function validateHandlerSerializable(handler, jobType) {
65
+ try {
66
+ const handlerString = handler.toString();
67
+ if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
68
+ throw new Error(
69
+ `Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
168
70
  );
169
- await this.recordJobEvent(jobId, "added" /* Added */, {
170
- jobType,
171
- payload,
172
- tags,
173
- idempotencyKey
174
- });
175
- return jobId;
176
- } catch (error) {
177
- log(`Error adding job: ${error}`);
178
- throw error;
179
- } finally {
180
- client.release();
181
71
  }
182
- }
183
- async getJob(id) {
184
- const client = await this.pool.connect();
185
- try {
186
- const result = await client.query(
187
- `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
188
- [id]
72
+ if (handlerString.includes("[native code]")) {
73
+ throw new Error(
74
+ `Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
189
75
  );
190
- if (result.rows.length === 0) {
191
- log(`Job ${id} not found`);
192
- return null;
193
- }
194
- log(`Found job ${id}`);
195
- const job = result.rows[0];
196
- return {
197
- ...job,
198
- payload: job.payload,
199
- timeoutMs: job.timeoutMs,
200
- forceKillOnTimeout: job.forceKillOnTimeout,
201
- failureReason: job.failureReason
202
- };
203
- } catch (error) {
204
- log(`Error getting job ${id}: ${error}`);
205
- throw error;
206
- } finally {
207
- client.release();
208
76
  }
209
- }
210
- async getJobsByStatus(status, limit = 100, offset = 0) {
211
- const client = await this.pool.connect();
212
77
  try {
213
- const result = await client.query(
214
- `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
215
- [status, limit, offset]
78
+ new Function("return " + handlerString);
79
+ } catch (parseError) {
80
+ throw new Error(
81
+ `Handler for job type "${jobType}" cannot be serialized: ${parseError instanceof Error ? parseError.message : String(parseError)}. When using forceKillOnTimeout, handlers must be serializable functions without closures over external variables.`
216
82
  );
217
- log(`Found ${result.rows.length} jobs by status ${status}`);
218
- return result.rows.map((job) => ({
219
- ...job,
220
- payload: job.payload,
221
- timeoutMs: job.timeoutMs,
222
- forceKillOnTimeout: job.forceKillOnTimeout,
223
- failureReason: job.failureReason
224
- }));
225
- } catch (error) {
226
- log(`Error getting jobs by status ${status}: ${error}`);
227
- throw error;
228
- } finally {
229
- client.release();
230
83
  }
231
- }
232
- async getAllJobs(limit = 100, offset = 0) {
233
- const client = await this.pool.connect();
234
- try {
235
- const result = await client.query(
236
- `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
237
- [limit, offset]
238
- );
239
- log(`Found ${result.rows.length} jobs (all)`);
240
- return result.rows.map((job) => ({
241
- ...job,
242
- payload: job.payload,
243
- timeoutMs: job.timeoutMs,
244
- forceKillOnTimeout: job.forceKillOnTimeout
245
- }));
246
- } catch (error) {
247
- log(`Error getting all jobs: ${error}`);
84
+ } catch (error) {
85
+ if (error instanceof Error) {
248
86
  throw error;
249
- } finally {
250
- client.release();
251
87
  }
88
+ throw new Error(
89
+ `Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
90
+ );
252
91
  }
253
- async getJobs(filters, limit = 100, offset = 0) {
254
- const client = await this.pool.connect();
255
- try {
256
- let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue`;
257
- const params = [];
258
- const where = [];
259
- let paramIdx = 1;
260
- if (filters) {
261
- if (filters.jobType) {
262
- where.push(`job_type = $${paramIdx++}`);
263
- params.push(filters.jobType);
264
- }
265
- if (filters.priority !== void 0) {
266
- where.push(`priority = $${paramIdx++}`);
267
- params.push(filters.priority);
268
- }
269
- if (filters.runAt) {
270
- if (filters.runAt instanceof Date) {
271
- where.push(`run_at = $${paramIdx++}`);
272
- params.push(filters.runAt);
273
- } else if (typeof filters.runAt === "object" && (filters.runAt.gt !== void 0 || filters.runAt.gte !== void 0 || filters.runAt.lt !== void 0 || filters.runAt.lte !== void 0 || filters.runAt.eq !== void 0)) {
274
- const ops = filters.runAt;
275
- if (ops.gt) {
276
- where.push(`run_at > $${paramIdx++}`);
277
- params.push(ops.gt);
278
- }
279
- if (ops.gte) {
280
- where.push(`run_at >= $${paramIdx++}`);
281
- params.push(ops.gte);
282
- }
283
- if (ops.lt) {
284
- where.push(`run_at < $${paramIdx++}`);
285
- params.push(ops.lt);
286
- }
287
- if (ops.lte) {
288
- where.push(`run_at <= $${paramIdx++}`);
289
- params.push(ops.lte);
290
- }
291
- if (ops.eq) {
292
- where.push(`run_at = $${paramIdx++}`);
293
- params.push(ops.eq);
294
- }
295
- }
296
- }
297
- if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
298
- const mode = filters.tags.mode || "all";
299
- const tagValues = filters.tags.values;
300
- switch (mode) {
301
- case "exact":
302
- where.push(`tags = $${paramIdx++}`);
303
- params.push(tagValues);
304
- break;
305
- case "all":
306
- where.push(`tags @> $${paramIdx++}`);
307
- params.push(tagValues);
308
- break;
309
- case "any":
310
- where.push(`tags && $${paramIdx++}`);
311
- params.push(tagValues);
312
- break;
313
- case "none":
314
- where.push(`NOT (tags && $${paramIdx++})`);
315
- params.push(tagValues);
316
- break;
317
- default:
318
- where.push(`tags @> $${paramIdx++}`);
319
- params.push(tagValues);
92
+ }
93
+ async function runHandlerInWorker(handler, payload, timeoutMs, jobType) {
94
+ validateHandlerSerializable(handler, jobType);
95
+ return new Promise((resolve, reject) => {
96
+ const workerCode = `
97
+ (function() {
98
+ const { parentPort, workerData } = require('worker_threads');
99
+ const { handlerCode, payload, timeoutMs } = workerData;
100
+
101
+ // Create an AbortController for the handler
102
+ const controller = new AbortController();
103
+ const signal = controller.signal;
104
+
105
+ // Set up timeout
106
+ const timeoutId = setTimeout(() => {
107
+ controller.abort();
108
+ parentPort.postMessage({ type: 'timeout' });
109
+ }, timeoutMs);
110
+
111
+ try {
112
+ // Execute the handler
113
+ // Note: This uses Function constructor which requires the handler to be serializable.
114
+ // The handler should be validated before reaching this point.
115
+ let handlerFn;
116
+ try {
117
+ // Wrap handlerCode in parentheses to ensure it's treated as an expression
118
+ // This handles both arrow functions and regular functions
119
+ const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
120
+ ? handlerCode
121
+ : '(' + handlerCode + ')';
122
+ handlerFn = new Function('return ' + wrappedCode)();
123
+ } catch (parseError) {
124
+ clearTimeout(timeoutId);
125
+ parentPort.postMessage({
126
+ type: 'error',
127
+ error: {
128
+ message: 'Handler cannot be deserialized in worker thread. ' +
129
+ 'Ensure your handler is a standalone function without closures over external variables. ' +
130
+ 'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
131
+ stack: parseError instanceof Error ? parseError.stack : undefined,
132
+ name: 'SerializationError',
133
+ },
134
+ });
135
+ return;
320
136
  }
137
+
138
+ // Ensure handlerFn is actually a function
139
+ if (typeof handlerFn !== 'function') {
140
+ clearTimeout(timeoutId);
141
+ parentPort.postMessage({
142
+ type: 'error',
143
+ error: {
144
+ message: 'Handler deserialization did not produce a function. ' +
145
+ 'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
146
+ name: 'SerializationError',
147
+ },
148
+ });
149
+ return;
150
+ }
151
+
152
+ handlerFn(payload, signal)
153
+ .then(() => {
154
+ clearTimeout(timeoutId);
155
+ parentPort.postMessage({ type: 'success' });
156
+ })
157
+ .catch((error) => {
158
+ clearTimeout(timeoutId);
159
+ parentPort.postMessage({
160
+ type: 'error',
161
+ error: {
162
+ message: error.message,
163
+ stack: error.stack,
164
+ name: error.name,
165
+ },
166
+ });
167
+ });
168
+ } catch (error) {
169
+ clearTimeout(timeoutId);
170
+ parentPort.postMessage({
171
+ type: 'error',
172
+ error: {
173
+ message: error.message,
174
+ stack: error.stack,
175
+ name: error.name,
176
+ },
177
+ });
321
178
  }
322
- if (filters.cursor !== void 0) {
323
- where.push(`id < $${paramIdx++}`);
324
- params.push(filters.cursor);
325
- }
179
+ })();
180
+ `;
181
+ const worker = new worker_threads.Worker(workerCode, {
182
+ eval: true,
183
+ workerData: {
184
+ handlerCode: handler.toString(),
185
+ payload,
186
+ timeoutMs
326
187
  }
327
- if (where.length > 0) {
328
- query += ` WHERE ${where.join(" AND ")}`;
188
+ });
189
+ let resolved = false;
190
+ worker.on("message", (message) => {
191
+ if (resolved) return;
192
+ resolved = true;
193
+ if (message.type === "success") {
194
+ resolve();
195
+ } else if (message.type === "timeout") {
196
+ const timeoutError = new Error(
197
+ `Job timed out after ${timeoutMs} ms and was forcefully terminated`
198
+ );
199
+ timeoutError.failureReason = "timeout" /* Timeout */;
200
+ reject(timeoutError);
201
+ } else if (message.type === "error") {
202
+ const error = new Error(message.error.message);
203
+ error.stack = message.error.stack;
204
+ error.name = message.error.name;
205
+ reject(error);
329
206
  }
330
- paramIdx = params.length + 1;
331
- query += ` ORDER BY id DESC LIMIT $${paramIdx++}`;
332
- if (!filters?.cursor) {
333
- query += ` OFFSET $${paramIdx}`;
334
- params.push(limit, offset);
335
- } else {
336
- params.push(limit);
207
+ });
208
+ worker.on("error", (error) => {
209
+ if (resolved) return;
210
+ resolved = true;
211
+ reject(error);
212
+ });
213
+ worker.on("exit", (code) => {
214
+ if (resolved) return;
215
+ if (code !== 0) {
216
+ resolved = true;
217
+ reject(new Error(`Worker stopped with exit code ${code}`));
337
218
  }
338
- const result = await client.query(query, params);
339
- log(`Found ${result.rows.length} jobs`);
340
- return result.rows.map((job) => ({
341
- ...job,
342
- payload: job.payload,
343
- timeoutMs: job.timeoutMs,
344
- forceKillOnTimeout: job.forceKillOnTimeout,
345
- failureReason: job.failureReason
346
- }));
347
- } catch (error) {
348
- log(`Error getting jobs: ${error}`);
349
- throw error;
350
- } finally {
351
- client.release();
352
- }
219
+ });
220
+ setTimeout(() => {
221
+ if (!resolved) {
222
+ resolved = true;
223
+ worker.terminate().then(() => {
224
+ const timeoutError = new Error(
225
+ `Job timed out after ${timeoutMs} ms and was forcefully terminated`
226
+ );
227
+ timeoutError.failureReason = "timeout" /* Timeout */;
228
+ reject(timeoutError);
229
+ }).catch((err) => {
230
+ reject(err);
231
+ });
232
+ }
233
+ }, timeoutMs + 100);
234
+ });
235
+ }
236
+ function calculateWaitUntil(duration) {
237
+ const now = Date.now();
238
+ let ms = 0;
239
+ if (duration.seconds) ms += duration.seconds * 1e3;
240
+ if (duration.minutes) ms += duration.minutes * 60 * 1e3;
241
+ if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
242
+ if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
243
+ if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
244
+ if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
245
+ if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
246
+ if (ms <= 0) {
247
+ throw new Error(
248
+ "waitFor duration must be positive. Provide at least one positive duration field."
249
+ );
353
250
  }
354
- async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
355
- const client = await this.pool.connect();
356
- try {
357
- let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
358
- FROM job_queue`;
359
- let params = [];
360
- switch (mode) {
361
- case "exact":
362
- query += " WHERE tags = $1";
363
- params = [tags];
364
- break;
365
- case "all":
366
- query += " WHERE tags @> $1";
367
- params = [tags];
368
- break;
369
- case "any":
370
- query += " WHERE tags && $1";
371
- params = [tags];
372
- break;
373
- case "none":
374
- query += " WHERE NOT (tags && $1)";
375
- params = [tags];
376
- break;
377
- default:
378
- query += " WHERE tags @> $1";
379
- params = [tags];
251
+ return new Date(now + ms);
252
+ }
253
+ async function resolveCompletedWaits(backend, stepData) {
254
+ for (const key of Object.keys(stepData)) {
255
+ if (!key.startsWith("__wait_")) continue;
256
+ const entry = stepData[key];
257
+ if (!entry || typeof entry !== "object" || entry.completed) continue;
258
+ if (entry.type === "duration" || entry.type === "date") {
259
+ stepData[key] = { ...entry, completed: true };
260
+ } else if (entry.type === "token" && entry.tokenId) {
261
+ const wp = await backend.getWaitpoint(entry.tokenId);
262
+ if (wp && wp.status === "completed") {
263
+ stepData[key] = {
264
+ ...entry,
265
+ completed: true,
266
+ result: { ok: true, output: wp.output }
267
+ };
268
+ } else if (wp && wp.status === "timed_out") {
269
+ stepData[key] = {
270
+ ...entry,
271
+ completed: true,
272
+ result: { ok: false, error: "Token timed out" }
273
+ };
380
274
  }
381
- query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
382
- params.push(limit, offset);
383
- const result = await client.query(query, params);
384
- log(
385
- `Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
386
- );
387
- return result.rows.map((job) => ({
388
- ...job,
389
- payload: job.payload,
390
- timeoutMs: job.timeoutMs,
391
- forceKillOnTimeout: job.forceKillOnTimeout,
392
- failureReason: job.failureReason
393
- }));
394
- } catch (error) {
395
- log(
396
- `Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
397
- );
398
- throw error;
399
- } finally {
400
- client.release();
401
275
  }
402
276
  }
403
- // ── Processing lifecycle ──────────────────────────────────────────────
404
- async getNextBatch(workerId, batchSize = 10, jobType) {
405
- const client = await this.pool.connect();
406
- try {
407
- await client.query("BEGIN");
408
- let jobTypeFilter = "";
409
- const params = [workerId, batchSize];
410
- if (jobType) {
411
- if (Array.isArray(jobType)) {
412
- jobTypeFilter = ` AND job_type = ANY($3)`;
413
- params.push(jobType);
414
- } else {
415
- jobTypeFilter = ` AND job_type = $3`;
416
- params.push(jobType);
417
- }
277
+ }
278
+ function buildWaitContext(backend, jobId, stepData, baseCtx) {
279
+ let waitCounter = 0;
280
+ const ctx = {
281
+ prolong: baseCtx.prolong,
282
+ onTimeout: baseCtx.onTimeout,
283
+ run: async (stepName, fn) => {
284
+ const cached = stepData[stepName];
285
+ if (cached && typeof cached === "object" && cached.__completed) {
286
+ log(`Step "${stepName}" replayed from cache for job ${jobId}`);
287
+ return cached.result;
418
288
  }
419
- const result = await client.query(
420
- `
421
- UPDATE job_queue
422
- SET status = 'processing',
423
- locked_at = NOW(),
424
- locked_by = $1,
425
- attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
426
- updated_at = NOW(),
427
- pending_reason = NULL,
428
- started_at = COALESCE(started_at, NOW()),
429
- last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
430
- wait_until = NULL
431
- WHERE id IN (
432
- SELECT id FROM job_queue
433
- WHERE (
434
- (
435
- (status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
436
- AND (attempts < max_attempts)
437
- AND run_at <= NOW()
438
- )
439
- OR (
440
- status = 'waiting'
441
- AND wait_until IS NOT NULL
442
- AND wait_until <= NOW()
443
- AND wait_token_id IS NULL
444
- )
445
- )
446
- ${jobTypeFilter}
447
- ORDER BY priority DESC, created_at ASC
448
- LIMIT $2
449
- FOR UPDATE SKIP LOCKED
450
- )
451
- RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
452
- `,
453
- params
454
- );
455
- log(`Found ${result.rows.length} jobs to process`);
456
- await client.query("COMMIT");
457
- if (result.rows.length > 0) {
458
- await this.recordJobEventsBatch(
459
- result.rows.map((row) => ({
460
- jobId: row.id,
461
- eventType: "processing" /* Processing */
462
- }))
463
- );
289
+ const result = await fn();
290
+ stepData[stepName] = { __completed: true, result };
291
+ await backend.updateStepData(jobId, stepData);
292
+ return result;
293
+ },
294
+ waitFor: async (duration) => {
295
+ const waitKey = `__wait_${waitCounter++}`;
296
+ const cached = stepData[waitKey];
297
+ if (cached && typeof cached === "object" && cached.completed) {
298
+ log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
299
+ return;
464
300
  }
465
- return result.rows.map((job) => ({
466
- ...job,
467
- payload: job.payload,
468
- timeoutMs: job.timeoutMs,
469
- forceKillOnTimeout: job.forceKillOnTimeout
470
- }));
471
- } catch (error) {
472
- log(`Error getting next batch: ${error}`);
473
- await client.query("ROLLBACK");
474
- throw error;
475
- } finally {
476
- client.release();
477
- }
478
- }
479
- async completeJob(jobId) {
480
- const client = await this.pool.connect();
481
- try {
482
- const result = await client.query(
483
- `
484
- UPDATE job_queue
485
- SET status = 'completed', updated_at = NOW(), completed_at = NOW(),
486
- step_data = NULL, wait_until = NULL, wait_token_id = NULL
487
- WHERE id = $1 AND status = 'processing'
488
- `,
489
- [jobId]
490
- );
491
- if (result.rowCount === 0) {
301
+ const waitUntilDate = calculateWaitUntil(duration);
302
+ stepData[waitKey] = { type: "duration", completed: false };
303
+ throw new WaitSignal("duration", waitUntilDate, void 0, stepData);
304
+ },
305
+ waitUntil: async (date) => {
306
+ const waitKey = `__wait_${waitCounter++}`;
307
+ const cached = stepData[waitKey];
308
+ if (cached && typeof cached === "object" && cached.completed) {
309
+ log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
310
+ return;
311
+ }
312
+ stepData[waitKey] = { type: "date", completed: false };
313
+ throw new WaitSignal("date", date, void 0, stepData);
314
+ },
315
+ createToken: async (options) => {
316
+ const token = await backend.createWaitpoint(jobId, options);
317
+ return token;
318
+ },
319
+ waitForToken: async (tokenId) => {
320
+ const waitKey = `__wait_${waitCounter++}`;
321
+ const cached = stepData[waitKey];
322
+ if (cached && typeof cached === "object" && cached.completed) {
492
323
  log(
493
- `Job ${jobId} could not be completed (not in processing state or does not exist)`
324
+ `Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
494
325
  );
326
+ return cached.result;
495
327
  }
496
- await this.recordJobEvent(jobId, "completed" /* Completed */);
497
- log(`Completed job ${jobId}`);
498
- } catch (error) {
499
- log(`Error completing job ${jobId}: ${error}`);
500
- throw error;
501
- } finally {
502
- client.release();
328
+ const wp = await backend.getWaitpoint(tokenId);
329
+ if (wp && wp.status === "completed") {
330
+ const result = {
331
+ ok: true,
332
+ output: wp.output
333
+ };
334
+ stepData[waitKey] = {
335
+ type: "token",
336
+ tokenId,
337
+ completed: true,
338
+ result
339
+ };
340
+ await backend.updateStepData(jobId, stepData);
341
+ return result;
342
+ }
343
+ if (wp && wp.status === "timed_out") {
344
+ const result = {
345
+ ok: false,
346
+ error: "Token timed out"
347
+ };
348
+ stepData[waitKey] = {
349
+ type: "token",
350
+ tokenId,
351
+ completed: true,
352
+ result
353
+ };
354
+ await backend.updateStepData(jobId, stepData);
355
+ return result;
356
+ }
357
+ stepData[waitKey] = { type: "token", tokenId, completed: false };
358
+ throw new WaitSignal("token", void 0, tokenId, stepData);
359
+ },
360
+ setProgress: async (percent) => {
361
+ if (percent < 0 || percent > 100)
362
+ throw new Error("Progress must be between 0 and 100");
363
+ await backend.updateProgress(jobId, Math.round(percent));
503
364
  }
365
+ };
366
+ return ctx;
367
+ }
368
+ async function processJobWithHandlers(backend, job, jobHandlers) {
369
+ const handler = jobHandlers[job.jobType];
370
+ if (!handler) {
371
+ await backend.setPendingReasonForUnpickedJobs(
372
+ `No handler registered for job type: ${job.jobType}`,
373
+ job.jobType
374
+ );
375
+ await backend.failJob(
376
+ job.id,
377
+ new Error(`No handler registered for job type: ${job.jobType}`),
378
+ "no_handler" /* NoHandler */
379
+ );
380
+ return;
504
381
  }
505
- async failJob(jobId, error, failureReason) {
506
- const client = await this.pool.connect();
507
- try {
508
- const result = await client.query(
509
- `
510
- UPDATE job_queue
511
- SET status = 'failed',
512
- updated_at = NOW(),
513
- next_attempt_at = CASE
514
- WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
515
- ELSE NULL
516
- END,
517
- error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
518
- failure_reason = $3,
519
- last_failed_at = NOW()
520
- WHERE id = $1 AND status IN ('processing', 'pending')
521
- `,
522
- [
523
- jobId,
524
- JSON.stringify([
525
- {
526
- message: error.message || String(error),
527
- timestamp: (/* @__PURE__ */ new Date()).toISOString()
382
+ const stepData = { ...job.stepData || {} };
383
+ const hasStepHistory = Object.keys(stepData).some(
384
+ (k) => k.startsWith("__wait_")
385
+ );
386
+ if (hasStepHistory) {
387
+ await resolveCompletedWaits(backend, stepData);
388
+ await backend.updateStepData(job.id, stepData);
389
+ }
390
+ const timeoutMs = job.timeoutMs ?? void 0;
391
+ const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
392
+ let timeoutId;
393
+ const controller = new AbortController();
394
+ try {
395
+ if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
396
+ await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
397
+ } else {
398
+ let onTimeoutCallback;
399
+ let timeoutReject;
400
+ const armTimeout = (ms) => {
401
+ if (timeoutId) clearTimeout(timeoutId);
402
+ timeoutId = setTimeout(() => {
403
+ if (onTimeoutCallback) {
404
+ try {
405
+ const extension = onTimeoutCallback();
406
+ if (typeof extension === "number" && extension > 0) {
407
+ backend.prolongJob(job.id).catch(() => {
408
+ });
409
+ armTimeout(extension);
410
+ return;
411
+ }
412
+ } catch (callbackError) {
413
+ log(
414
+ `onTimeout callback threw for job ${job.id}: ${callbackError}`
415
+ );
528
416
  }
529
- ]),
530
- failureReason ?? null
531
- ]
532
- );
533
- if (result.rowCount === 0) {
417
+ }
418
+ controller.abort();
419
+ const timeoutError = new Error(`Job timed out after ${ms} ms`);
420
+ timeoutError.failureReason = "timeout" /* Timeout */;
421
+ if (timeoutReject) {
422
+ timeoutReject(timeoutError);
423
+ }
424
+ }, ms);
425
+ };
426
+ const hasTimeout = timeoutMs != null && timeoutMs > 0;
427
+ const baseCtx = hasTimeout ? {
428
+ prolong: (ms) => {
429
+ const duration = ms ?? timeoutMs;
430
+ if (duration != null && duration > 0) {
431
+ armTimeout(duration);
432
+ backend.prolongJob(job.id).catch(() => {
433
+ });
434
+ }
435
+ },
436
+ onTimeout: (callback) => {
437
+ onTimeoutCallback = callback;
438
+ }
439
+ } : {
440
+ prolong: () => {
441
+ log("prolong() called but ignored: job has no timeout set");
442
+ },
443
+ onTimeout: () => {
444
+ log("onTimeout() called but ignored: job has no timeout set");
445
+ }
446
+ };
447
+ const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
448
+ if (forceKillOnTimeout && !hasTimeout) {
534
449
  log(
535
- `Job ${jobId} could not be failed (not in processing/pending state or does not exist)`
450
+ `forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
536
451
  );
537
452
  }
538
- await this.recordJobEvent(jobId, "failed" /* Failed */, {
539
- message: error.message || String(error),
540
- failureReason
453
+ const jobPromise = handler(job.payload, controller.signal, ctx);
454
+ if (hasTimeout) {
455
+ await Promise.race([
456
+ jobPromise,
457
+ new Promise((_, reject) => {
458
+ timeoutReject = reject;
459
+ armTimeout(timeoutMs);
460
+ })
461
+ ]);
462
+ } else {
463
+ await jobPromise;
464
+ }
465
+ }
466
+ if (timeoutId) clearTimeout(timeoutId);
467
+ await backend.completeJob(job.id);
468
+ } catch (error) {
469
+ if (timeoutId) clearTimeout(timeoutId);
470
+ if (error instanceof WaitSignal) {
471
+ log(
472
+ `Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? "none"}, tokenId=${error.tokenId ?? "none"}`
473
+ );
474
+ await backend.waitJob(job.id, {
475
+ waitUntil: error.waitUntil,
476
+ waitTokenId: error.tokenId,
477
+ stepData: error.stepData
541
478
  });
542
- log(`Failed job ${jobId}`);
543
- } catch (err) {
544
- log(`Error failing job ${jobId}: ${err}`);
545
- throw err;
546
- } finally {
547
- client.release();
479
+ return;
480
+ }
481
+ console.error(`Error processing job ${job.id}:`, error);
482
+ let failureReason = "handler_error" /* HandlerError */;
483
+ if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
484
+ failureReason = "timeout" /* Timeout */;
485
+ }
486
+ await backend.failJob(
487
+ job.id,
488
+ error instanceof Error ? error : new Error(String(error)),
489
+ failureReason
490
+ );
491
+ }
492
+ }
493
+ async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError) {
494
+ const jobs = await backend.getNextBatch(
495
+ workerId,
496
+ batchSize,
497
+ jobType
498
+ );
499
+ if (!concurrency || concurrency >= jobs.length) {
500
+ await Promise.all(
501
+ jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers))
502
+ );
503
+ return jobs.length;
504
+ }
505
+ let idx = 0;
506
+ let running = 0;
507
+ let finished = 0;
508
+ return new Promise((resolve, reject) => {
509
+ const next = () => {
510
+ if (finished === jobs.length) return resolve(jobs.length);
511
+ while (running < concurrency && idx < jobs.length) {
512
+ const job = jobs[idx++];
513
+ running++;
514
+ processJobWithHandlers(backend, job, jobHandlers).then(() => {
515
+ running--;
516
+ finished++;
517
+ next();
518
+ }).catch((err) => {
519
+ running--;
520
+ finished++;
521
+ if (onError) {
522
+ onError(err instanceof Error ? err : new Error(String(err)));
523
+ }
524
+ next();
525
+ });
526
+ }
527
+ };
528
+ next();
529
+ });
530
+ }
531
+ var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
532
+ const {
533
+ workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
534
+ batchSize = 10,
535
+ pollInterval = 5e3,
536
+ onError = (error) => console.error("Job processor error:", error),
537
+ jobType,
538
+ concurrency = 3
539
+ } = options;
540
+ let running = false;
541
+ let intervalId = null;
542
+ let currentBatchPromise = null;
543
+ setLogContext(options.verbose ?? false);
544
+ const processJobs = async () => {
545
+ if (!running) return 0;
546
+ if (onBeforeBatch) {
547
+ try {
548
+ await onBeforeBatch();
549
+ } catch (hookError) {
550
+ log(`onBeforeBatch hook error: ${hookError}`);
551
+ if (onError) {
552
+ onError(
553
+ hookError instanceof Error ? hookError : new Error(String(hookError))
554
+ );
555
+ }
556
+ }
548
557
  }
549
- }
550
- async prolongJob(jobId) {
551
- const client = await this.pool.connect();
558
+ log(
559
+ `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(",") : jobType}` : ""}`
560
+ );
552
561
  try {
553
- await client.query(
554
- `
555
- UPDATE job_queue
556
- SET locked_at = NOW(), updated_at = NOW()
557
- WHERE id = $1 AND status = 'processing'
558
- `,
559
- [jobId]
562
+ const processed = await processBatchWithHandlers(
563
+ backend,
564
+ workerId,
565
+ batchSize,
566
+ jobType,
567
+ handlers,
568
+ concurrency,
569
+ onError
560
570
  );
561
- await this.recordJobEvent(jobId, "prolonged" /* Prolonged */);
562
- log(`Prolonged job ${jobId}`);
571
+ return processed;
563
572
  } catch (error) {
564
- log(`Error prolonging job ${jobId}: ${error}`);
565
- } finally {
566
- client.release();
573
+ onError(error instanceof Error ? error : new Error(String(error)));
567
574
  }
575
+ return 0;
576
+ };
577
+ return {
578
+ /**
579
+ * Start the job processor in the background.
580
+ * - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
581
+ * - You have to call the stop method to stop the processor.
582
+ */
583
+ startInBackground: () => {
584
+ if (running) return;
585
+ log(`Starting job processor with workerId: ${workerId}`);
586
+ running = true;
587
+ const scheduleNext = (immediate) => {
588
+ if (!running) return;
589
+ if (immediate) {
590
+ intervalId = setTimeout(loop, 0);
591
+ } else {
592
+ intervalId = setTimeout(loop, pollInterval);
593
+ }
594
+ };
595
+ const loop = async () => {
596
+ if (!running) return;
597
+ currentBatchPromise = processJobs();
598
+ const processed = await currentBatchPromise;
599
+ currentBatchPromise = null;
600
+ scheduleNext(processed === batchSize);
601
+ };
602
+ loop();
603
+ },
604
+ /**
605
+ * Stop the job processor that runs in the background.
606
+ * Does not wait for in-flight jobs.
607
+ */
608
+ stop: () => {
609
+ log(`Stopping job processor with workerId: ${workerId}`);
610
+ running = false;
611
+ if (intervalId) {
612
+ clearTimeout(intervalId);
613
+ intervalId = null;
614
+ }
615
+ },
616
+ /**
617
+ * Stop the job processor and wait for all in-flight jobs to complete.
618
+ * Useful for graceful shutdown (e.g., SIGTERM handling).
619
+ */
620
+ stopAndDrain: async (drainTimeoutMs = 3e4) => {
621
+ log(`Stopping and draining job processor with workerId: ${workerId}`);
622
+ running = false;
623
+ if (intervalId) {
624
+ clearTimeout(intervalId);
625
+ intervalId = null;
626
+ }
627
+ if (currentBatchPromise) {
628
+ await Promise.race([
629
+ currentBatchPromise.catch(() => {
630
+ }),
631
+ new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
632
+ ]);
633
+ currentBatchPromise = null;
634
+ }
635
+ log(`Job processor ${workerId} drained`);
636
+ },
637
+ /**
638
+ * Start the job processor synchronously.
639
+ * - This will process all jobs immediately and then stop.
640
+ * - The pollInterval is ignored.
641
+ */
642
+ start: async () => {
643
+ log(`Starting job processor with workerId: ${workerId}`);
644
+ running = true;
645
+ const processed = await processJobs();
646
+ running = false;
647
+ return processed;
648
+ },
649
+ isRunning: () => running
650
+ };
651
+ };
652
+ function loadPemOrFile(value) {
653
+ if (!value) return void 0;
654
+ if (value.startsWith("file://")) {
655
+ const filePath = value.slice(7);
656
+ return fs__default.default.readFileSync(filePath, "utf8");
568
657
  }
569
- // ── Progress ──────────────────────────────────────────────────────────
570
- async updateProgress(jobId, progress) {
571
- const client = await this.pool.connect();
658
+ return value;
659
+ }
660
+ var createPool = (config) => {
661
+ let searchPath;
662
+ let ssl = void 0;
663
+ let customCA;
664
+ let sslmode;
665
+ if (config.connectionString) {
572
666
  try {
573
- await client.query(
574
- `UPDATE job_queue SET progress = $2, updated_at = NOW() WHERE id = $1`,
575
- [jobId, progress]
576
- );
577
- log(`Updated progress for job ${jobId}: ${progress}%`);
578
- } catch (error) {
579
- log(`Error updating progress for job ${jobId}: ${error}`);
580
- } finally {
581
- client.release();
667
+ const url = new URL(config.connectionString);
668
+ searchPath = url.searchParams.get("search_path") || void 0;
669
+ sslmode = url.searchParams.get("sslmode") || void 0;
670
+ if (sslmode === "no-verify") {
671
+ ssl = { rejectUnauthorized: false };
672
+ }
673
+ } catch (e) {
674
+ const parsed = pgConnectionString.parse(config.connectionString);
675
+ if (parsed.options) {
676
+ const match = parsed.options.match(/search_path=([^\s]+)/);
677
+ if (match) {
678
+ searchPath = match[1];
679
+ }
680
+ }
681
+ sslmode = typeof parsed.sslmode === "string" ? parsed.sslmode : void 0;
682
+ if (sslmode === "no-verify") {
683
+ ssl = { rejectUnauthorized: false };
684
+ }
582
685
  }
583
686
  }
584
- // ── Job management ────────────────────────────────────────────────────
585
- async retryJob(jobId) {
586
- const client = await this.pool.connect();
587
- try {
588
- const result = await client.query(
589
- `
590
- UPDATE job_queue
591
- SET status = 'pending',
592
- updated_at = NOW(),
593
- locked_at = NULL,
594
- locked_by = NULL,
595
- next_attempt_at = NOW(),
596
- last_retried_at = NOW()
597
- WHERE id = $1 AND status IN ('failed', 'processing')
598
- `,
599
- [jobId]
687
+ if (config.ssl) {
688
+ if (typeof config.ssl.ca === "string") {
689
+ customCA = config.ssl.ca;
690
+ } else if (typeof process.env.PGSSLROOTCERT === "string") {
691
+ customCA = process.env.PGSSLROOTCERT;
692
+ } else {
693
+ customCA = void 0;
694
+ }
695
+ const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
696
+ ssl = {
697
+ ...ssl,
698
+ ...caValue ? { ca: caValue } : {},
699
+ cert: loadPemOrFile(
700
+ typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
701
+ ),
702
+ key: loadPemOrFile(
703
+ typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
704
+ ),
705
+ rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
706
+ };
707
+ }
708
+ if (sslmode && customCA) {
709
+ const warning = `
710
+
711
+ \x1B[33m**************************************************
712
+ \u26A0\uFE0F WARNING: SSL CONFIGURATION ISSUE
713
+ **************************************************
714
+ Both sslmode ('${sslmode}') is set in the connection string
715
+ and a custom CA is provided (via config.ssl.ca or PGSSLROOTCERT).
716
+ This combination may cause connection failures or unexpected behavior.
717
+
718
+ Recommended: Remove sslmode from the connection string when using a custom CA.
719
+ **************************************************\x1B[0m
720
+ `;
721
+ console.warn(warning);
722
+ }
723
+ const pool = new pg.Pool({
724
+ ...config,
725
+ ...ssl ? { ssl } : {}
726
+ });
727
+ if (searchPath) {
728
+ pool.on("connect", (client) => {
729
+ client.query(`SET search_path TO ${searchPath}`);
730
+ });
731
+ }
732
+ return pool;
733
+ };
734
+ var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
735
+ function parseTimeoutString(timeout) {
736
+ const match = timeout.match(/^(\d+)(s|m|h|d)$/);
737
+ if (!match) {
738
+ throw new Error(
739
+ `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
740
+ );
741
+ }
742
+ const value = parseInt(match[1], 10);
743
+ const unit = match[2];
744
+ let ms;
745
+ switch (unit) {
746
+ case "s":
747
+ ms = value * 1e3;
748
+ break;
749
+ case "m":
750
+ ms = value * 60 * 1e3;
751
+ break;
752
+ case "h":
753
+ ms = value * 60 * 60 * 1e3;
754
+ break;
755
+ case "d":
756
+ ms = value * 24 * 60 * 60 * 1e3;
757
+ break;
758
+ default:
759
+ throw new Error(`Unknown timeout unit: "${unit}"`);
760
+ }
761
+ if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
762
+ throw new Error(
763
+ `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
764
+ );
765
+ }
766
+ return ms;
767
+ }
768
+ var PostgresBackend = class {
769
+ constructor(pool) {
770
+ this.pool = pool;
771
+ }
772
+ /** Expose the raw pool for advanced usage. */
773
+ getPool() {
774
+ return this.pool;
775
+ }
776
+ // ── Events ──────────────────────────────────────────────────────────
777
+ async recordJobEvent(jobId, eventType, metadata) {
778
+ const client = await this.pool.connect();
779
+ try {
780
+ await client.query(
781
+ `INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
782
+ [jobId, eventType, metadata ? JSON.stringify(metadata) : null]
600
783
  );
601
- if (result.rowCount === 0) {
602
- log(
603
- `Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
604
- );
605
- }
606
- await this.recordJobEvent(jobId, "retried" /* Retried */);
607
- log(`Retried job ${jobId}`);
608
784
  } catch (error) {
609
- log(`Error retrying job ${jobId}: ${error}`);
610
- throw error;
785
+ log(`Error recording job event for job ${jobId}: ${error}`);
611
786
  } finally {
612
787
  client.release();
613
788
  }
614
789
  }
615
- async cancelJob(jobId) {
790
+ async getJobEvents(jobId) {
616
791
  const client = await this.pool.connect();
617
792
  try {
618
- await client.query(
619
- `
620
- UPDATE job_queue
621
- SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
622
- wait_until = NULL, wait_token_id = NULL
623
- WHERE id = $1 AND status IN ('pending', 'waiting')
624
- `,
793
+ const res = await client.query(
794
+ `SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
625
795
  [jobId]
626
796
  );
627
- await this.recordJobEvent(jobId, "cancelled" /* Cancelled */);
628
- log(`Cancelled job ${jobId}`);
629
- } catch (error) {
630
- log(`Error cancelling job ${jobId}: ${error}`);
631
- throw error;
797
+ return res.rows;
632
798
  } finally {
633
799
  client.release();
634
800
  }
635
801
  }
636
- async cancelAllUpcomingJobs(filters) {
802
+ // ── Job CRUD ──────────────────────────────────────────────────────────
803
+ async addJob({
804
+ jobType,
805
+ payload,
806
+ maxAttempts = 3,
807
+ priority = 0,
808
+ runAt = null,
809
+ timeoutMs = void 0,
810
+ forceKillOnTimeout = false,
811
+ tags = void 0,
812
+ idempotencyKey = void 0
813
+ }) {
637
814
  const client = await this.pool.connect();
638
815
  try {
639
- let query = `
640
- UPDATE job_queue
641
- SET status = 'cancelled', updated_at = NOW()
642
- WHERE status = 'pending'`;
643
- const params = [];
644
- let paramIdx = 1;
645
- if (filters) {
646
- if (filters.jobType) {
647
- query += ` AND job_type = $${paramIdx++}`;
648
- params.push(filters.jobType);
649
- }
650
- if (filters.priority !== void 0) {
651
- query += ` AND priority = $${paramIdx++}`;
652
- params.push(filters.priority);
653
- }
654
- if (filters.runAt) {
655
- if (filters.runAt instanceof Date) {
656
- query += ` AND run_at = $${paramIdx++}`;
657
- params.push(filters.runAt);
658
- } else if (typeof filters.runAt === "object") {
659
- const ops = filters.runAt;
660
- if (ops.gt) {
661
- query += ` AND run_at > $${paramIdx++}`;
662
- params.push(ops.gt);
663
- }
664
- if (ops.gte) {
665
- query += ` AND run_at >= $${paramIdx++}`;
666
- params.push(ops.gte);
667
- }
668
- if (ops.lt) {
669
- query += ` AND run_at < $${paramIdx++}`;
670
- params.push(ops.lt);
671
- }
672
- if (ops.lte) {
673
- query += ` AND run_at <= $${paramIdx++}`;
674
- params.push(ops.lte);
675
- }
676
- if (ops.eq) {
677
- query += ` AND run_at = $${paramIdx++}`;
678
- params.push(ops.eq);
679
- }
680
- }
681
- }
682
- if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
683
- const mode = filters.tags.mode || "all";
684
- const tagValues = filters.tags.values;
685
- switch (mode) {
686
- case "exact":
687
- query += ` AND tags = $${paramIdx++}`;
688
- params.push(tagValues);
689
- break;
690
- case "all":
691
- query += ` AND tags @> $${paramIdx++}`;
692
- params.push(tagValues);
693
- break;
694
- case "any":
695
- query += ` AND tags && $${paramIdx++}`;
696
- params.push(tagValues);
697
- break;
698
- case "none":
699
- query += ` AND NOT (tags && $${paramIdx++})`;
700
- params.push(tagValues);
701
- break;
702
- default:
703
- query += ` AND tags @> $${paramIdx++}`;
704
- params.push(tagValues);
705
- }
816
+ let result;
817
+ const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
818
+ if (runAt) {
819
+ result = await client.query(
820
+ `INSERT INTO job_queue
821
+ (job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
822
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
823
+ ${onConflict}
824
+ RETURNING id`,
825
+ [
826
+ jobType,
827
+ payload,
828
+ maxAttempts,
829
+ priority,
830
+ runAt,
831
+ timeoutMs ?? null,
832
+ forceKillOnTimeout ?? false,
833
+ tags ?? null,
834
+ idempotencyKey ?? null
835
+ ]
836
+ );
837
+ } else {
838
+ result = await client.query(
839
+ `INSERT INTO job_queue
840
+ (job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
841
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
842
+ ${onConflict}
843
+ RETURNING id`,
844
+ [
845
+ jobType,
846
+ payload,
847
+ maxAttempts,
848
+ priority,
849
+ timeoutMs ?? null,
850
+ forceKillOnTimeout ?? false,
851
+ tags ?? null,
852
+ idempotencyKey ?? null
853
+ ]
854
+ );
855
+ }
856
+ if (result.rows.length === 0 && idempotencyKey) {
857
+ const existing = await client.query(
858
+ `SELECT id FROM job_queue WHERE idempotency_key = $1`,
859
+ [idempotencyKey]
860
+ );
861
+ if (existing.rows.length > 0) {
862
+ log(
863
+ `Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
864
+ );
865
+ return existing.rows[0].id;
706
866
  }
867
+ throw new Error(
868
+ `Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
869
+ );
707
870
  }
708
- query += "\nRETURNING id";
709
- const result = await client.query(query, params);
710
- log(`Cancelled ${result.rowCount} jobs`);
711
- return result.rowCount || 0;
871
+ const jobId = result.rows[0].id;
872
+ log(
873
+ `Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
874
+ );
875
+ await this.recordJobEvent(jobId, "added" /* Added */, {
876
+ jobType,
877
+ payload,
878
+ tags,
879
+ idempotencyKey
880
+ });
881
+ return jobId;
712
882
  } catch (error) {
713
- log(`Error cancelling upcoming jobs: ${error}`);
883
+ log(`Error adding job: ${error}`);
714
884
  throw error;
715
885
  } finally {
716
886
  client.release();
717
887
  }
718
888
  }
719
- async editJob(jobId, updates) {
889
+ async getJob(id) {
720
890
  const client = await this.pool.connect();
721
891
  try {
722
- const updateFields = [];
723
- const params = [];
724
- let paramIdx = 1;
725
- if (updates.payload !== void 0) {
726
- updateFields.push(`payload = $${paramIdx++}`);
727
- params.push(updates.payload);
728
- }
729
- if (updates.maxAttempts !== void 0) {
730
- updateFields.push(`max_attempts = $${paramIdx++}`);
731
- params.push(updates.maxAttempts);
732
- }
733
- if (updates.priority !== void 0) {
734
- updateFields.push(`priority = $${paramIdx++}`);
735
- params.push(updates.priority);
736
- }
737
- if (updates.runAt !== void 0) {
738
- if (updates.runAt === null) {
739
- updateFields.push(`run_at = NOW()`);
740
- } else {
741
- updateFields.push(`run_at = $${paramIdx++}`);
742
- params.push(updates.runAt);
743
- }
744
- }
745
- if (updates.timeoutMs !== void 0) {
746
- updateFields.push(`timeout_ms = $${paramIdx++}`);
747
- params.push(updates.timeoutMs ?? null);
748
- }
749
- if (updates.tags !== void 0) {
750
- updateFields.push(`tags = $${paramIdx++}`);
751
- params.push(updates.tags ?? null);
752
- }
753
- if (updateFields.length === 0) {
754
- log(`No fields to update for job ${jobId}`);
755
- return;
892
+ const result = await client.query(
893
+ `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
894
+ [id]
895
+ );
896
+ if (result.rows.length === 0) {
897
+ log(`Job ${id} not found`);
898
+ return null;
756
899
  }
757
- updateFields.push(`updated_at = NOW()`);
758
- params.push(jobId);
759
- const query = `
760
- UPDATE job_queue
761
- SET ${updateFields.join(", ")}
762
- WHERE id = $${paramIdx} AND status = 'pending'
763
- `;
764
- await client.query(query, params);
765
- const metadata = {};
766
- if (updates.payload !== void 0) metadata.payload = updates.payload;
767
- if (updates.maxAttempts !== void 0)
768
- metadata.maxAttempts = updates.maxAttempts;
769
- if (updates.priority !== void 0) metadata.priority = updates.priority;
770
- if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
771
- if (updates.timeoutMs !== void 0)
772
- metadata.timeoutMs = updates.timeoutMs;
773
- if (updates.tags !== void 0) metadata.tags = updates.tags;
774
- await this.recordJobEvent(jobId, "edited" /* Edited */, metadata);
775
- log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
900
+ log(`Found job ${id}`);
901
+ const job = result.rows[0];
902
+ return {
903
+ ...job,
904
+ payload: job.payload,
905
+ timeoutMs: job.timeoutMs,
906
+ forceKillOnTimeout: job.forceKillOnTimeout,
907
+ failureReason: job.failureReason
908
+ };
909
+ } catch (error) {
910
+ log(`Error getting job ${id}: ${error}`);
911
+ throw error;
912
+ } finally {
913
+ client.release();
914
+ }
915
+ }
916
+ async getJobsByStatus(status, limit = 100, offset = 0) {
917
+ const client = await this.pool.connect();
918
+ try {
919
+ const result = await client.query(
920
+ `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
921
+ [status, limit, offset]
922
+ );
923
+ log(`Found ${result.rows.length} jobs by status ${status}`);
924
+ return result.rows.map((job) => ({
925
+ ...job,
926
+ payload: job.payload,
927
+ timeoutMs: job.timeoutMs,
928
+ forceKillOnTimeout: job.forceKillOnTimeout,
929
+ failureReason: job.failureReason
930
+ }));
776
931
  } catch (error) {
777
- log(`Error editing job ${jobId}: ${error}`);
932
+ log(`Error getting jobs by status ${status}: ${error}`);
778
933
  throw error;
779
934
  } finally {
780
935
  client.release();
781
936
  }
782
937
  }
783
- async editAllPendingJobs(filters = void 0, updates) {
938
+ async getAllJobs(limit = 100, offset = 0) {
784
939
  const client = await this.pool.connect();
785
940
  try {
786
- const updateFields = [];
941
+ const result = await client.query(
942
+ `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
943
+ [limit, offset]
944
+ );
945
+ log(`Found ${result.rows.length} jobs (all)`);
946
+ return result.rows.map((job) => ({
947
+ ...job,
948
+ payload: job.payload,
949
+ timeoutMs: job.timeoutMs,
950
+ forceKillOnTimeout: job.forceKillOnTimeout
951
+ }));
952
+ } catch (error) {
953
+ log(`Error getting all jobs: ${error}`);
954
+ throw error;
955
+ } finally {
956
+ client.release();
957
+ }
958
+ }
959
+ async getJobs(filters, limit = 100, offset = 0) {
960
+ const client = await this.pool.connect();
961
+ try {
962
+ let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue`;
787
963
  const params = [];
964
+ const where = [];
788
965
  let paramIdx = 1;
789
- if (updates.payload !== void 0) {
790
- updateFields.push(`payload = $${paramIdx++}`);
791
- params.push(updates.payload);
792
- }
793
- if (updates.maxAttempts !== void 0) {
794
- updateFields.push(`max_attempts = $${paramIdx++}`);
795
- params.push(updates.maxAttempts);
796
- }
797
- if (updates.priority !== void 0) {
798
- updateFields.push(`priority = $${paramIdx++}`);
799
- params.push(updates.priority);
800
- }
801
- if (updates.runAt !== void 0) {
802
- if (updates.runAt === null) {
803
- updateFields.push(`run_at = NOW()`);
804
- } else {
805
- updateFields.push(`run_at = $${paramIdx++}`);
806
- params.push(updates.runAt);
807
- }
808
- }
809
- if (updates.timeoutMs !== void 0) {
810
- updateFields.push(`timeout_ms = $${paramIdx++}`);
811
- params.push(updates.timeoutMs ?? null);
812
- }
813
- if (updates.tags !== void 0) {
814
- updateFields.push(`tags = $${paramIdx++}`);
815
- params.push(updates.tags ?? null);
816
- }
817
- if (updateFields.length === 0) {
818
- log(`No fields to update for batch edit`);
819
- return 0;
820
- }
821
- updateFields.push(`updated_at = NOW()`);
822
- let query = `
823
- UPDATE job_queue
824
- SET ${updateFields.join(", ")}
825
- WHERE status = 'pending'`;
826
966
  if (filters) {
827
967
  if (filters.jobType) {
828
- query += ` AND job_type = $${paramIdx++}`;
968
+ where.push(`job_type = $${paramIdx++}`);
829
969
  params.push(filters.jobType);
830
970
  }
831
971
  if (filters.priority !== void 0) {
832
- query += ` AND priority = $${paramIdx++}`;
972
+ where.push(`priority = $${paramIdx++}`);
833
973
  params.push(filters.priority);
834
974
  }
835
975
  if (filters.runAt) {
836
976
  if (filters.runAt instanceof Date) {
837
- query += ` AND run_at = $${paramIdx++}`;
977
+ where.push(`run_at = $${paramIdx++}`);
838
978
  params.push(filters.runAt);
839
- } else if (typeof filters.runAt === "object") {
979
+ } else if (typeof filters.runAt === "object" && (filters.runAt.gt !== void 0 || filters.runAt.gte !== void 0 || filters.runAt.lt !== void 0 || filters.runAt.lte !== void 0 || filters.runAt.eq !== void 0)) {
840
980
  const ops = filters.runAt;
841
981
  if (ops.gt) {
842
- query += ` AND run_at > $${paramIdx++}`;
982
+ where.push(`run_at > $${paramIdx++}`);
843
983
  params.push(ops.gt);
844
984
  }
845
985
  if (ops.gte) {
846
- query += ` AND run_at >= $${paramIdx++}`;
986
+ where.push(`run_at >= $${paramIdx++}`);
847
987
  params.push(ops.gte);
848
988
  }
849
989
  if (ops.lt) {
850
- query += ` AND run_at < $${paramIdx++}`;
990
+ where.push(`run_at < $${paramIdx++}`);
851
991
  params.push(ops.lt);
852
992
  }
853
993
  if (ops.lte) {
854
- query += ` AND run_at <= $${paramIdx++}`;
994
+ where.push(`run_at <= $${paramIdx++}`);
855
995
  params.push(ops.lte);
856
996
  }
857
997
  if (ops.eq) {
858
- query += ` AND run_at = $${paramIdx++}`;
998
+ where.push(`run_at = $${paramIdx++}`);
859
999
  params.push(ops.eq);
860
1000
  }
861
1001
  }
@@ -865,1370 +1005,1264 @@ var PostgresBackend = class {
865
1005
  const tagValues = filters.tags.values;
866
1006
  switch (mode) {
867
1007
  case "exact":
868
- query += ` AND tags = $${paramIdx++}`;
1008
+ where.push(`tags = $${paramIdx++}`);
869
1009
  params.push(tagValues);
870
1010
  break;
871
1011
  case "all":
872
- query += ` AND tags @> $${paramIdx++}`;
1012
+ where.push(`tags @> $${paramIdx++}`);
873
1013
  params.push(tagValues);
874
1014
  break;
875
1015
  case "any":
876
- query += ` AND tags && $${paramIdx++}`;
1016
+ where.push(`tags && $${paramIdx++}`);
877
1017
  params.push(tagValues);
878
1018
  break;
879
1019
  case "none":
880
- query += ` AND NOT (tags && $${paramIdx++})`;
1020
+ where.push(`NOT (tags && $${paramIdx++})`);
881
1021
  params.push(tagValues);
882
1022
  break;
883
1023
  default:
884
- query += ` AND tags @> $${paramIdx++}`;
1024
+ where.push(`tags @> $${paramIdx++}`);
885
1025
  params.push(tagValues);
886
1026
  }
887
1027
  }
1028
+ if (filters.cursor !== void 0) {
1029
+ where.push(`id < $${paramIdx++}`);
1030
+ params.push(filters.cursor);
1031
+ }
1032
+ }
1033
+ if (where.length > 0) {
1034
+ query += ` WHERE ${where.join(" AND ")}`;
1035
+ }
1036
+ paramIdx = params.length + 1;
1037
+ query += ` ORDER BY id DESC LIMIT $${paramIdx++}`;
1038
+ if (!filters?.cursor) {
1039
+ query += ` OFFSET $${paramIdx}`;
1040
+ params.push(limit, offset);
1041
+ } else {
1042
+ params.push(limit);
888
1043
  }
889
- query += "\nRETURNING id";
890
1044
  const result = await client.query(query, params);
891
- const editedCount = result.rowCount || 0;
892
- const metadata = {};
893
- if (updates.payload !== void 0) metadata.payload = updates.payload;
894
- if (updates.maxAttempts !== void 0)
895
- metadata.maxAttempts = updates.maxAttempts;
896
- if (updates.priority !== void 0) metadata.priority = updates.priority;
897
- if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
898
- if (updates.timeoutMs !== void 0)
899
- metadata.timeoutMs = updates.timeoutMs;
900
- if (updates.tags !== void 0) metadata.tags = updates.tags;
901
- for (const row of result.rows) {
902
- await this.recordJobEvent(row.id, "edited" /* Edited */, metadata);
1045
+ log(`Found ${result.rows.length} jobs`);
1046
+ return result.rows.map((job) => ({
1047
+ ...job,
1048
+ payload: job.payload,
1049
+ timeoutMs: job.timeoutMs,
1050
+ forceKillOnTimeout: job.forceKillOnTimeout,
1051
+ failureReason: job.failureReason
1052
+ }));
1053
+ } catch (error) {
1054
+ log(`Error getting jobs: ${error}`);
1055
+ throw error;
1056
+ } finally {
1057
+ client.release();
1058
+ }
1059
+ }
1060
+ async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
1061
+ const client = await this.pool.connect();
1062
+ try {
1063
+ let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
1064
+ FROM job_queue`;
1065
+ let params = [];
1066
+ switch (mode) {
1067
+ case "exact":
1068
+ query += " WHERE tags = $1";
1069
+ params = [tags];
1070
+ break;
1071
+ case "all":
1072
+ query += " WHERE tags @> $1";
1073
+ params = [tags];
1074
+ break;
1075
+ case "any":
1076
+ query += " WHERE tags && $1";
1077
+ params = [tags];
1078
+ break;
1079
+ case "none":
1080
+ query += " WHERE NOT (tags && $1)";
1081
+ params = [tags];
1082
+ break;
1083
+ default:
1084
+ query += " WHERE tags @> $1";
1085
+ params = [tags];
1086
+ }
1087
+ query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
1088
+ params.push(limit, offset);
1089
+ const result = await client.query(query, params);
1090
+ log(
1091
+ `Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
1092
+ );
1093
+ return result.rows.map((job) => ({
1094
+ ...job,
1095
+ payload: job.payload,
1096
+ timeoutMs: job.timeoutMs,
1097
+ forceKillOnTimeout: job.forceKillOnTimeout,
1098
+ failureReason: job.failureReason
1099
+ }));
1100
+ } catch (error) {
1101
+ log(
1102
+ `Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
1103
+ );
1104
+ throw error;
1105
+ } finally {
1106
+ client.release();
1107
+ }
1108
+ }
1109
+ // ── Processing lifecycle ──────────────────────────────────────────────
1110
+ async getNextBatch(workerId, batchSize = 10, jobType) {
1111
+ const client = await this.pool.connect();
1112
+ try {
1113
+ await client.query("BEGIN");
1114
+ let jobTypeFilter = "";
1115
+ const params = [workerId, batchSize];
1116
+ if (jobType) {
1117
+ if (Array.isArray(jobType)) {
1118
+ jobTypeFilter = ` AND job_type = ANY($3)`;
1119
+ params.push(jobType);
1120
+ } else {
1121
+ jobTypeFilter = ` AND job_type = $3`;
1122
+ params.push(jobType);
1123
+ }
1124
+ }
1125
+ const result = await client.query(
1126
+ `
1127
+ UPDATE job_queue
1128
+ SET status = 'processing',
1129
+ locked_at = NOW(),
1130
+ locked_by = $1,
1131
+ attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
1132
+ updated_at = NOW(),
1133
+ pending_reason = NULL,
1134
+ started_at = COALESCE(started_at, NOW()),
1135
+ last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
1136
+ wait_until = NULL
1137
+ WHERE id IN (
1138
+ SELECT id FROM job_queue
1139
+ WHERE (
1140
+ (
1141
+ (status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
1142
+ AND (attempts < max_attempts)
1143
+ AND run_at <= NOW()
1144
+ )
1145
+ OR (
1146
+ status = 'waiting'
1147
+ AND wait_until IS NOT NULL
1148
+ AND wait_until <= NOW()
1149
+ AND wait_token_id IS NULL
1150
+ )
1151
+ )
1152
+ ${jobTypeFilter}
1153
+ ORDER BY priority DESC, created_at ASC
1154
+ LIMIT $2
1155
+ FOR UPDATE SKIP LOCKED
1156
+ )
1157
+ RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
1158
+ `,
1159
+ params
1160
+ );
1161
+ log(`Found ${result.rows.length} jobs to process`);
1162
+ await client.query("COMMIT");
1163
+ if (result.rows.length > 0) {
1164
+ await this.recordJobEventsBatch(
1165
+ result.rows.map((row) => ({
1166
+ jobId: row.id,
1167
+ eventType: "processing" /* Processing */
1168
+ }))
1169
+ );
903
1170
  }
904
- log(`Edited ${editedCount} pending jobs: ${JSON.stringify(metadata)}`);
905
- return editedCount;
1171
+ return result.rows.map((job) => ({
1172
+ ...job,
1173
+ payload: job.payload,
1174
+ timeoutMs: job.timeoutMs,
1175
+ forceKillOnTimeout: job.forceKillOnTimeout
1176
+ }));
906
1177
  } catch (error) {
907
- log(`Error editing pending jobs: ${error}`);
1178
+ log(`Error getting next batch: ${error}`);
1179
+ await client.query("ROLLBACK");
908
1180
  throw error;
909
1181
  } finally {
910
1182
  client.release();
911
1183
  }
912
1184
  }
913
- async cleanupOldJobs(daysToKeep = 30) {
1185
+ async completeJob(jobId) {
914
1186
  const client = await this.pool.connect();
915
1187
  try {
916
1188
  const result = await client.query(
917
1189
  `
918
- DELETE FROM job_queue
919
- WHERE status = 'completed'
920
- AND updated_at < NOW() - INTERVAL '1 day' * $1::int
921
- RETURNING id
1190
+ UPDATE job_queue
1191
+ SET status = 'completed', updated_at = NOW(), completed_at = NOW(),
1192
+ step_data = NULL, wait_until = NULL, wait_token_id = NULL
1193
+ WHERE id = $1 AND status = 'processing'
922
1194
  `,
923
- [daysToKeep]
1195
+ [jobId]
924
1196
  );
925
- log(`Deleted ${result.rowCount} old jobs`);
926
- return result.rowCount || 0;
1197
+ if (result.rowCount === 0) {
1198
+ log(
1199
+ `Job ${jobId} could not be completed (not in processing state or does not exist)`
1200
+ );
1201
+ }
1202
+ await this.recordJobEvent(jobId, "completed" /* Completed */);
1203
+ log(`Completed job ${jobId}`);
927
1204
  } catch (error) {
928
- log(`Error cleaning up old jobs: ${error}`);
1205
+ log(`Error completing job ${jobId}: ${error}`);
929
1206
  throw error;
930
1207
  } finally {
931
1208
  client.release();
932
1209
  }
933
1210
  }
934
- async cleanupOldJobEvents(daysToKeep = 30) {
1211
+ async failJob(jobId, error, failureReason) {
935
1212
  const client = await this.pool.connect();
936
1213
  try {
937
1214
  const result = await client.query(
938
1215
  `
939
- DELETE FROM job_events
940
- WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
941
- RETURNING id
1216
+ UPDATE job_queue
1217
+ SET status = 'failed',
1218
+ updated_at = NOW(),
1219
+ next_attempt_at = CASE
1220
+ WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
1221
+ ELSE NULL
1222
+ END,
1223
+ error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
1224
+ failure_reason = $3,
1225
+ last_failed_at = NOW()
1226
+ WHERE id = $1 AND status IN ('processing', 'pending')
942
1227
  `,
943
- [daysToKeep]
1228
+ [
1229
+ jobId,
1230
+ JSON.stringify([
1231
+ {
1232
+ message: error.message || String(error),
1233
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
1234
+ }
1235
+ ]),
1236
+ failureReason ?? null
1237
+ ]
944
1238
  );
945
- log(`Deleted ${result.rowCount} old job events`);
946
- return result.rowCount || 0;
947
- } catch (error) {
948
- log(`Error cleaning up old job events: ${error}`);
949
- throw error;
1239
+ if (result.rowCount === 0) {
1240
+ log(
1241
+ `Job ${jobId} could not be failed (not in processing/pending state or does not exist)`
1242
+ );
1243
+ }
1244
+ await this.recordJobEvent(jobId, "failed" /* Failed */, {
1245
+ message: error.message || String(error),
1246
+ failureReason
1247
+ });
1248
+ log(`Failed job ${jobId}`);
1249
+ } catch (err) {
1250
+ log(`Error failing job ${jobId}: ${err}`);
1251
+ throw err;
950
1252
  } finally {
951
1253
  client.release();
952
1254
  }
953
1255
  }
954
- async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
1256
+ async prolongJob(jobId) {
955
1257
  const client = await this.pool.connect();
956
1258
  try {
957
- const result = await client.query(
1259
+ await client.query(
958
1260
  `
959
1261
  UPDATE job_queue
960
- SET status = 'pending', locked_at = NULL, locked_by = NULL, updated_at = NOW()
961
- WHERE status = 'processing'
962
- AND locked_at < NOW() - GREATEST(
963
- INTERVAL '1 minute' * $1::int,
964
- INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
965
- )
966
- RETURNING id
967
- `,
968
- [maxProcessingTimeMinutes]
1262
+ SET locked_at = NOW(), updated_at = NOW()
1263
+ WHERE id = $1 AND status = 'processing'
1264
+ `,
1265
+ [jobId]
969
1266
  );
970
- log(`Reclaimed ${result.rowCount} stuck jobs`);
971
- return result.rowCount || 0;
1267
+ await this.recordJobEvent(jobId, "prolonged" /* Prolonged */);
1268
+ log(`Prolonged job ${jobId}`);
972
1269
  } catch (error) {
973
- log(`Error reclaiming stuck jobs: ${error}`);
974
- throw error;
1270
+ log(`Error prolonging job ${jobId}: ${error}`);
975
1271
  } finally {
976
1272
  client.release();
977
1273
  }
978
1274
  }
979
- // ── Internal helpers ──────────────────────────────────────────────────
980
- /**
981
- * Batch-insert multiple job events in a single query.
982
- * More efficient than individual recordJobEvent calls.
983
- */
984
- async recordJobEventsBatch(events) {
985
- if (events.length === 0) return;
1275
+ // ── Progress ──────────────────────────────────────────────────────────
1276
+ async updateProgress(jobId, progress) {
986
1277
  const client = await this.pool.connect();
987
1278
  try {
988
- const values = [];
989
- const params = [];
990
- let paramIdx = 1;
991
- for (const event of events) {
992
- values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
993
- params.push(
994
- event.jobId,
995
- event.eventType,
996
- event.metadata ? JSON.stringify(event.metadata) : null
997
- );
998
- }
999
1279
  await client.query(
1000
- `INSERT INTO job_events (job_id, event_type, metadata) VALUES ${values.join(", ")}`,
1001
- params
1280
+ `UPDATE job_queue SET progress = $2, updated_at = NOW() WHERE id = $1`,
1281
+ [jobId, progress]
1002
1282
  );
1283
+ log(`Updated progress for job ${jobId}: ${progress}%`);
1003
1284
  } catch (error) {
1004
- log(`Error recording batch job events: ${error}`);
1285
+ log(`Error updating progress for job ${jobId}: ${error}`);
1005
1286
  } finally {
1006
1287
  client.release();
1007
1288
  }
1008
1289
  }
1009
- // ── Cron schedules ──────────────────────────────────────────────────
1010
- /** Create a cron schedule and return its ID. */
1011
- async addCronSchedule(input) {
1290
+ // ── Job management ────────────────────────────────────────────────────
1291
+ async retryJob(jobId) {
1012
1292
  const client = await this.pool.connect();
1013
1293
  try {
1014
1294
  const result = await client.query(
1015
- `INSERT INTO cron_schedules
1016
- (schedule_name, cron_expression, job_type, payload, max_attempts,
1017
- priority, timeout_ms, force_kill_on_timeout, tags, timezone,
1018
- allow_overlap, next_run_at)
1019
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
1020
- RETURNING id`,
1021
- [
1022
- input.scheduleName,
1023
- input.cronExpression,
1024
- input.jobType,
1025
- input.payload,
1026
- input.maxAttempts,
1027
- input.priority,
1028
- input.timeoutMs,
1029
- input.forceKillOnTimeout,
1030
- input.tags ?? null,
1031
- input.timezone,
1032
- input.allowOverlap,
1033
- input.nextRunAt
1034
- ]
1295
+ `
1296
+ UPDATE job_queue
1297
+ SET status = 'pending',
1298
+ updated_at = NOW(),
1299
+ locked_at = NULL,
1300
+ locked_by = NULL,
1301
+ next_attempt_at = NOW(),
1302
+ last_retried_at = NOW()
1303
+ WHERE id = $1 AND status IN ('failed', 'processing')
1304
+ `,
1305
+ [jobId]
1035
1306
  );
1036
- const id = result.rows[0].id;
1037
- log(`Added cron schedule ${id}: "${input.scheduleName}"`);
1038
- return id;
1039
- } catch (error) {
1040
- if (error?.code === "23505") {
1041
- throw new Error(
1042
- `Cron schedule with name "${input.scheduleName}" already exists`
1307
+ if (result.rowCount === 0) {
1308
+ log(
1309
+ `Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
1043
1310
  );
1044
1311
  }
1045
- log(`Error adding cron schedule: ${error}`);
1046
- throw error;
1047
- } finally {
1048
- client.release();
1049
- }
1050
- }
1051
- /** Get a cron schedule by ID. */
1052
- async getCronSchedule(id) {
1053
- const client = await this.pool.connect();
1054
- try {
1055
- const result = await client.query(
1056
- `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1057
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1058
- priority, timeout_ms AS "timeoutMs",
1059
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1060
- timezone, allow_overlap AS "allowOverlap", status,
1061
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1062
- next_run_at AS "nextRunAt",
1063
- created_at AS "createdAt", updated_at AS "updatedAt"
1064
- FROM cron_schedules WHERE id = $1`,
1065
- [id]
1066
- );
1067
- if (result.rows.length === 0) return null;
1068
- return result.rows[0];
1312
+ await this.recordJobEvent(jobId, "retried" /* Retried */);
1313
+ log(`Retried job ${jobId}`);
1069
1314
  } catch (error) {
1070
- log(`Error getting cron schedule ${id}: ${error}`);
1315
+ log(`Error retrying job ${jobId}: ${error}`);
1071
1316
  throw error;
1072
1317
  } finally {
1073
1318
  client.release();
1074
1319
  }
1075
1320
  }
1076
- /** Get a cron schedule by its unique name. */
1077
- async getCronScheduleByName(name) {
1321
+ async cancelJob(jobId) {
1078
1322
  const client = await this.pool.connect();
1079
1323
  try {
1080
- const result = await client.query(
1081
- `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1082
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1083
- priority, timeout_ms AS "timeoutMs",
1084
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1085
- timezone, allow_overlap AS "allowOverlap", status,
1086
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1087
- next_run_at AS "nextRunAt",
1088
- created_at AS "createdAt", updated_at AS "updatedAt"
1089
- FROM cron_schedules WHERE schedule_name = $1`,
1090
- [name]
1324
+ await client.query(
1325
+ `
1326
+ UPDATE job_queue
1327
+ SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
1328
+ wait_until = NULL, wait_token_id = NULL
1329
+ WHERE id = $1 AND status IN ('pending', 'waiting')
1330
+ `,
1331
+ [jobId]
1091
1332
  );
1092
- if (result.rows.length === 0) return null;
1093
- return result.rows[0];
1094
- } catch (error) {
1095
- log(`Error getting cron schedule by name "${name}": ${error}`);
1096
- throw error;
1097
- } finally {
1098
- client.release();
1099
- }
1100
- }
1101
- /** List cron schedules, optionally filtered by status. */
1102
- async listCronSchedules(status) {
1103
- const client = await this.pool.connect();
1104
- try {
1105
- let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1106
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1107
- priority, timeout_ms AS "timeoutMs",
1108
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1109
- timezone, allow_overlap AS "allowOverlap", status,
1110
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1111
- next_run_at AS "nextRunAt",
1112
- created_at AS "createdAt", updated_at AS "updatedAt"
1113
- FROM cron_schedules`;
1114
- const params = [];
1115
- if (status) {
1116
- query += ` WHERE status = $1`;
1117
- params.push(status);
1118
- }
1119
- query += ` ORDER BY created_at ASC`;
1120
- const result = await client.query(query, params);
1121
- return result.rows;
1333
+ await this.recordJobEvent(jobId, "cancelled" /* Cancelled */);
1334
+ log(`Cancelled job ${jobId}`);
1122
1335
  } catch (error) {
1123
- log(`Error listing cron schedules: ${error}`);
1336
+ log(`Error cancelling job ${jobId}: ${error}`);
1124
1337
  throw error;
1125
1338
  } finally {
1126
1339
  client.release();
1127
1340
  }
1128
1341
  }
1129
- /** Delete a cron schedule by ID. */
1130
- async removeCronSchedule(id) {
1342
+ async cancelAllUpcomingJobs(filters) {
1131
1343
  const client = await this.pool.connect();
1132
1344
  try {
1133
- await client.query(`DELETE FROM cron_schedules WHERE id = $1`, [id]);
1134
- log(`Removed cron schedule ${id}`);
1345
+ let query = `
1346
+ UPDATE job_queue
1347
+ SET status = 'cancelled', updated_at = NOW()
1348
+ WHERE status = 'pending'`;
1349
+ const params = [];
1350
+ let paramIdx = 1;
1351
+ if (filters) {
1352
+ if (filters.jobType) {
1353
+ query += ` AND job_type = $${paramIdx++}`;
1354
+ params.push(filters.jobType);
1355
+ }
1356
+ if (filters.priority !== void 0) {
1357
+ query += ` AND priority = $${paramIdx++}`;
1358
+ params.push(filters.priority);
1359
+ }
1360
+ if (filters.runAt) {
1361
+ if (filters.runAt instanceof Date) {
1362
+ query += ` AND run_at = $${paramIdx++}`;
1363
+ params.push(filters.runAt);
1364
+ } else if (typeof filters.runAt === "object") {
1365
+ const ops = filters.runAt;
1366
+ if (ops.gt) {
1367
+ query += ` AND run_at > $${paramIdx++}`;
1368
+ params.push(ops.gt);
1369
+ }
1370
+ if (ops.gte) {
1371
+ query += ` AND run_at >= $${paramIdx++}`;
1372
+ params.push(ops.gte);
1373
+ }
1374
+ if (ops.lt) {
1375
+ query += ` AND run_at < $${paramIdx++}`;
1376
+ params.push(ops.lt);
1377
+ }
1378
+ if (ops.lte) {
1379
+ query += ` AND run_at <= $${paramIdx++}`;
1380
+ params.push(ops.lte);
1381
+ }
1382
+ if (ops.eq) {
1383
+ query += ` AND run_at = $${paramIdx++}`;
1384
+ params.push(ops.eq);
1385
+ }
1386
+ }
1387
+ }
1388
+ if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
1389
+ const mode = filters.tags.mode || "all";
1390
+ const tagValues = filters.tags.values;
1391
+ switch (mode) {
1392
+ case "exact":
1393
+ query += ` AND tags = $${paramIdx++}`;
1394
+ params.push(tagValues);
1395
+ break;
1396
+ case "all":
1397
+ query += ` AND tags @> $${paramIdx++}`;
1398
+ params.push(tagValues);
1399
+ break;
1400
+ case "any":
1401
+ query += ` AND tags && $${paramIdx++}`;
1402
+ params.push(tagValues);
1403
+ break;
1404
+ case "none":
1405
+ query += ` AND NOT (tags && $${paramIdx++})`;
1406
+ params.push(tagValues);
1407
+ break;
1408
+ default:
1409
+ query += ` AND tags @> $${paramIdx++}`;
1410
+ params.push(tagValues);
1411
+ }
1412
+ }
1413
+ }
1414
+ query += "\nRETURNING id";
1415
+ const result = await client.query(query, params);
1416
+ log(`Cancelled ${result.rowCount} jobs`);
1417
+ return result.rowCount || 0;
1135
1418
  } catch (error) {
1136
- log(`Error removing cron schedule ${id}: ${error}`);
1419
+ log(`Error cancelling upcoming jobs: ${error}`);
1137
1420
  throw error;
1138
1421
  } finally {
1139
1422
  client.release();
1140
1423
  }
1141
1424
  }
1142
- /** Pause a cron schedule. */
1143
- async pauseCronSchedule(id) {
1425
+ async editJob(jobId, updates) {
1144
1426
  const client = await this.pool.connect();
1145
1427
  try {
1146
- await client.query(
1147
- `UPDATE cron_schedules SET status = 'paused', updated_at = NOW() WHERE id = $1`,
1148
- [id]
1149
- );
1150
- log(`Paused cron schedule ${id}`);
1428
+ const updateFields = [];
1429
+ const params = [];
1430
+ let paramIdx = 1;
1431
+ if (updates.payload !== void 0) {
1432
+ updateFields.push(`payload = $${paramIdx++}`);
1433
+ params.push(updates.payload);
1434
+ }
1435
+ if (updates.maxAttempts !== void 0) {
1436
+ updateFields.push(`max_attempts = $${paramIdx++}`);
1437
+ params.push(updates.maxAttempts);
1438
+ }
1439
+ if (updates.priority !== void 0) {
1440
+ updateFields.push(`priority = $${paramIdx++}`);
1441
+ params.push(updates.priority);
1442
+ }
1443
+ if (updates.runAt !== void 0) {
1444
+ if (updates.runAt === null) {
1445
+ updateFields.push(`run_at = NOW()`);
1446
+ } else {
1447
+ updateFields.push(`run_at = $${paramIdx++}`);
1448
+ params.push(updates.runAt);
1449
+ }
1450
+ }
1451
+ if (updates.timeoutMs !== void 0) {
1452
+ updateFields.push(`timeout_ms = $${paramIdx++}`);
1453
+ params.push(updates.timeoutMs ?? null);
1454
+ }
1455
+ if (updates.tags !== void 0) {
1456
+ updateFields.push(`tags = $${paramIdx++}`);
1457
+ params.push(updates.tags ?? null);
1458
+ }
1459
+ if (updateFields.length === 0) {
1460
+ log(`No fields to update for job ${jobId}`);
1461
+ return;
1462
+ }
1463
+ updateFields.push(`updated_at = NOW()`);
1464
+ params.push(jobId);
1465
+ const query = `
1466
+ UPDATE job_queue
1467
+ SET ${updateFields.join(", ")}
1468
+ WHERE id = $${paramIdx} AND status = 'pending'
1469
+ `;
1470
+ await client.query(query, params);
1471
+ const metadata = {};
1472
+ if (updates.payload !== void 0) metadata.payload = updates.payload;
1473
+ if (updates.maxAttempts !== void 0)
1474
+ metadata.maxAttempts = updates.maxAttempts;
1475
+ if (updates.priority !== void 0) metadata.priority = updates.priority;
1476
+ if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
1477
+ if (updates.timeoutMs !== void 0)
1478
+ metadata.timeoutMs = updates.timeoutMs;
1479
+ if (updates.tags !== void 0) metadata.tags = updates.tags;
1480
+ await this.recordJobEvent(jobId, "edited" /* Edited */, metadata);
1481
+ log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
1151
1482
  } catch (error) {
1152
- log(`Error pausing cron schedule ${id}: ${error}`);
1483
+ log(`Error editing job ${jobId}: ${error}`);
1153
1484
  throw error;
1154
1485
  } finally {
1155
1486
  client.release();
1156
1487
  }
1157
1488
  }
1158
- /** Resume a paused cron schedule. */
1159
- async resumeCronSchedule(id) {
1489
+ async editAllPendingJobs(filters = void 0, updates) {
1160
1490
  const client = await this.pool.connect();
1161
1491
  try {
1162
- await client.query(
1163
- `UPDATE cron_schedules SET status = 'active', updated_at = NOW() WHERE id = $1`,
1164
- [id]
1165
- );
1166
- log(`Resumed cron schedule ${id}`);
1492
+ const updateFields = [];
1493
+ const params = [];
1494
+ let paramIdx = 1;
1495
+ if (updates.payload !== void 0) {
1496
+ updateFields.push(`payload = $${paramIdx++}`);
1497
+ params.push(updates.payload);
1498
+ }
1499
+ if (updates.maxAttempts !== void 0) {
1500
+ updateFields.push(`max_attempts = $${paramIdx++}`);
1501
+ params.push(updates.maxAttempts);
1502
+ }
1503
+ if (updates.priority !== void 0) {
1504
+ updateFields.push(`priority = $${paramIdx++}`);
1505
+ params.push(updates.priority);
1506
+ }
1507
+ if (updates.runAt !== void 0) {
1508
+ if (updates.runAt === null) {
1509
+ updateFields.push(`run_at = NOW()`);
1510
+ } else {
1511
+ updateFields.push(`run_at = $${paramIdx++}`);
1512
+ params.push(updates.runAt);
1513
+ }
1514
+ }
1515
+ if (updates.timeoutMs !== void 0) {
1516
+ updateFields.push(`timeout_ms = $${paramIdx++}`);
1517
+ params.push(updates.timeoutMs ?? null);
1518
+ }
1519
+ if (updates.tags !== void 0) {
1520
+ updateFields.push(`tags = $${paramIdx++}`);
1521
+ params.push(updates.tags ?? null);
1522
+ }
1523
+ if (updateFields.length === 0) {
1524
+ log(`No fields to update for batch edit`);
1525
+ return 0;
1526
+ }
1527
+ updateFields.push(`updated_at = NOW()`);
1528
+ let query = `
1529
+ UPDATE job_queue
1530
+ SET ${updateFields.join(", ")}
1531
+ WHERE status = 'pending'`;
1532
+ if (filters) {
1533
+ if (filters.jobType) {
1534
+ query += ` AND job_type = $${paramIdx++}`;
1535
+ params.push(filters.jobType);
1536
+ }
1537
+ if (filters.priority !== void 0) {
1538
+ query += ` AND priority = $${paramIdx++}`;
1539
+ params.push(filters.priority);
1540
+ }
1541
+ if (filters.runAt) {
1542
+ if (filters.runAt instanceof Date) {
1543
+ query += ` AND run_at = $${paramIdx++}`;
1544
+ params.push(filters.runAt);
1545
+ } else if (typeof filters.runAt === "object") {
1546
+ const ops = filters.runAt;
1547
+ if (ops.gt) {
1548
+ query += ` AND run_at > $${paramIdx++}`;
1549
+ params.push(ops.gt);
1550
+ }
1551
+ if (ops.gte) {
1552
+ query += ` AND run_at >= $${paramIdx++}`;
1553
+ params.push(ops.gte);
1554
+ }
1555
+ if (ops.lt) {
1556
+ query += ` AND run_at < $${paramIdx++}`;
1557
+ params.push(ops.lt);
1558
+ }
1559
+ if (ops.lte) {
1560
+ query += ` AND run_at <= $${paramIdx++}`;
1561
+ params.push(ops.lte);
1562
+ }
1563
+ if (ops.eq) {
1564
+ query += ` AND run_at = $${paramIdx++}`;
1565
+ params.push(ops.eq);
1566
+ }
1567
+ }
1568
+ }
1569
+ if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
1570
+ const mode = filters.tags.mode || "all";
1571
+ const tagValues = filters.tags.values;
1572
+ switch (mode) {
1573
+ case "exact":
1574
+ query += ` AND tags = $${paramIdx++}`;
1575
+ params.push(tagValues);
1576
+ break;
1577
+ case "all":
1578
+ query += ` AND tags @> $${paramIdx++}`;
1579
+ params.push(tagValues);
1580
+ break;
1581
+ case "any":
1582
+ query += ` AND tags && $${paramIdx++}`;
1583
+ params.push(tagValues);
1584
+ break;
1585
+ case "none":
1586
+ query += ` AND NOT (tags && $${paramIdx++})`;
1587
+ params.push(tagValues);
1588
+ break;
1589
+ default:
1590
+ query += ` AND tags @> $${paramIdx++}`;
1591
+ params.push(tagValues);
1592
+ }
1593
+ }
1594
+ }
1595
+ query += "\nRETURNING id";
1596
+ const result = await client.query(query, params);
1597
+ const editedCount = result.rowCount || 0;
1598
+ const metadata = {};
1599
+ if (updates.payload !== void 0) metadata.payload = updates.payload;
1600
+ if (updates.maxAttempts !== void 0)
1601
+ metadata.maxAttempts = updates.maxAttempts;
1602
+ if (updates.priority !== void 0) metadata.priority = updates.priority;
1603
+ if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
1604
+ if (updates.timeoutMs !== void 0)
1605
+ metadata.timeoutMs = updates.timeoutMs;
1606
+ if (updates.tags !== void 0) metadata.tags = updates.tags;
1607
+ for (const row of result.rows) {
1608
+ await this.recordJobEvent(row.id, "edited" /* Edited */, metadata);
1609
+ }
1610
+ log(`Edited ${editedCount} pending jobs: ${JSON.stringify(metadata)}`);
1611
+ return editedCount;
1167
1612
  } catch (error) {
1168
- log(`Error resuming cron schedule ${id}: ${error}`);
1613
+ log(`Error editing pending jobs: ${error}`);
1169
1614
  throw error;
1170
1615
  } finally {
1171
1616
  client.release();
1172
1617
  }
1173
1618
  }
1174
- /** Edit a cron schedule. */
1175
- async editCronSchedule(id, updates, nextRunAt) {
1176
- const client = await this.pool.connect();
1619
+ /**
1620
+ * Delete completed jobs older than the given number of days.
1621
+ * Deletes in batches of 1000 to avoid long-running transactions
1622
+ * and excessive WAL bloat at scale.
1623
+ *
1624
+ * @param daysToKeep - Number of days to retain completed jobs (default 30).
1625
+ * @param batchSize - Number of rows to delete per batch (default 1000).
1626
+ * @returns Total number of deleted jobs.
1627
+ */
1628
+ async cleanupOldJobs(daysToKeep = 30, batchSize = 1e3) {
1629
+ let totalDeleted = 0;
1177
1630
  try {
1178
- const updateFields = [];
1179
- const params = [];
1180
- let paramIdx = 1;
1181
- if (updates.cronExpression !== void 0) {
1182
- updateFields.push(`cron_expression = $${paramIdx++}`);
1183
- params.push(updates.cronExpression);
1184
- }
1185
- if (updates.payload !== void 0) {
1186
- updateFields.push(`payload = $${paramIdx++}`);
1187
- params.push(updates.payload);
1188
- }
1189
- if (updates.maxAttempts !== void 0) {
1190
- updateFields.push(`max_attempts = $${paramIdx++}`);
1191
- params.push(updates.maxAttempts);
1192
- }
1193
- if (updates.priority !== void 0) {
1194
- updateFields.push(`priority = $${paramIdx++}`);
1195
- params.push(updates.priority);
1196
- }
1197
- if (updates.timeoutMs !== void 0) {
1198
- updateFields.push(`timeout_ms = $${paramIdx++}`);
1199
- params.push(updates.timeoutMs);
1200
- }
1201
- if (updates.forceKillOnTimeout !== void 0) {
1202
- updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
1203
- params.push(updates.forceKillOnTimeout);
1204
- }
1205
- if (updates.tags !== void 0) {
1206
- updateFields.push(`tags = $${paramIdx++}`);
1207
- params.push(updates.tags);
1208
- }
1209
- if (updates.timezone !== void 0) {
1210
- updateFields.push(`timezone = $${paramIdx++}`);
1211
- params.push(updates.timezone);
1212
- }
1213
- if (updates.allowOverlap !== void 0) {
1214
- updateFields.push(`allow_overlap = $${paramIdx++}`);
1215
- params.push(updates.allowOverlap);
1216
- }
1217
- if (nextRunAt !== void 0) {
1218
- updateFields.push(`next_run_at = $${paramIdx++}`);
1219
- params.push(nextRunAt);
1220
- }
1221
- if (updateFields.length === 0) {
1222
- log(`No fields to update for cron schedule ${id}`);
1223
- return;
1224
- }
1225
- updateFields.push(`updated_at = NOW()`);
1226
- params.push(id);
1227
- const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
1228
- await client.query(query, params);
1229
- log(`Edited cron schedule ${id}`);
1631
+ let deletedInBatch;
1632
+ do {
1633
+ const client = await this.pool.connect();
1634
+ try {
1635
+ const result = await client.query(
1636
+ `
1637
+ DELETE FROM job_queue
1638
+ WHERE id IN (
1639
+ SELECT id FROM job_queue
1640
+ WHERE status = 'completed'
1641
+ AND updated_at < NOW() - INTERVAL '1 day' * $1::int
1642
+ LIMIT $2
1643
+ )
1644
+ `,
1645
+ [daysToKeep, batchSize]
1646
+ );
1647
+ deletedInBatch = result.rowCount || 0;
1648
+ totalDeleted += deletedInBatch;
1649
+ } finally {
1650
+ client.release();
1651
+ }
1652
+ } while (deletedInBatch === batchSize);
1653
+ log(`Deleted ${totalDeleted} old jobs`);
1654
+ return totalDeleted;
1230
1655
  } catch (error) {
1231
- log(`Error editing cron schedule ${id}: ${error}`);
1656
+ log(`Error cleaning up old jobs: ${error}`);
1232
1657
  throw error;
1233
- } finally {
1234
- client.release();
1235
1658
  }
1236
1659
  }
1237
1660
  /**
1238
- * Atomically fetch all active cron schedules whose nextRunAt <= NOW().
1239
- * Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
1661
+ * Delete job events older than the given number of days.
1662
+ * Deletes in batches of 1000 to avoid long-running transactions
1663
+ * and excessive WAL bloat at scale.
1664
+ *
1665
+ * @param daysToKeep - Number of days to retain events (default 30).
1666
+ * @param batchSize - Number of rows to delete per batch (default 1000).
1667
+ * @returns Total number of deleted events.
1240
1668
  */
1241
- async getDueCronSchedules() {
1242
- const client = await this.pool.connect();
1669
+ async cleanupOldJobEvents(daysToKeep = 30, batchSize = 1e3) {
1670
+ let totalDeleted = 0;
1243
1671
  try {
1244
- const result = await client.query(
1245
- `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1246
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1247
- priority, timeout_ms AS "timeoutMs",
1248
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1249
- timezone, allow_overlap AS "allowOverlap", status,
1250
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1251
- next_run_at AS "nextRunAt",
1252
- created_at AS "createdAt", updated_at AS "updatedAt"
1253
- FROM cron_schedules
1254
- WHERE status = 'active'
1255
- AND next_run_at IS NOT NULL
1256
- AND next_run_at <= NOW()
1257
- ORDER BY next_run_at ASC
1258
- FOR UPDATE SKIP LOCKED`
1259
- );
1260
- log(`Found ${result.rows.length} due cron schedules`);
1261
- return result.rows;
1672
+ let deletedInBatch;
1673
+ do {
1674
+ const client = await this.pool.connect();
1675
+ try {
1676
+ const result = await client.query(
1677
+ `
1678
+ DELETE FROM job_events
1679
+ WHERE id IN (
1680
+ SELECT id FROM job_events
1681
+ WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
1682
+ LIMIT $2
1683
+ )
1684
+ `,
1685
+ [daysToKeep, batchSize]
1686
+ );
1687
+ deletedInBatch = result.rowCount || 0;
1688
+ totalDeleted += deletedInBatch;
1689
+ } finally {
1690
+ client.release();
1691
+ }
1692
+ } while (deletedInBatch === batchSize);
1693
+ log(`Deleted ${totalDeleted} old job events`);
1694
+ return totalDeleted;
1262
1695
  } catch (error) {
1263
- if (error?.code === "42P01") {
1264
- log("cron_schedules table does not exist, skipping cron enqueue");
1265
- return [];
1266
- }
1267
- log(`Error getting due cron schedules: ${error}`);
1696
+ log(`Error cleaning up old job events: ${error}`);
1268
1697
  throw error;
1269
- } finally {
1270
- client.release();
1271
1698
  }
1272
1699
  }
1273
- /**
1274
- * Update a cron schedule after a job has been enqueued.
1275
- * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
1276
- */
1277
- async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
1700
+ async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
1278
1701
  const client = await this.pool.connect();
1279
1702
  try {
1280
- await client.query(
1281
- `UPDATE cron_schedules
1282
- SET last_enqueued_at = $2,
1283
- last_job_id = $3,
1284
- next_run_at = $4,
1285
- updated_at = NOW()
1286
- WHERE id = $1`,
1287
- [id, lastEnqueuedAt, lastJobId, nextRunAt]
1288
- );
1289
- log(
1290
- `Updated cron schedule ${id}: lastJobId=${lastJobId}, nextRunAt=${nextRunAt?.toISOString() ?? "null"}`
1703
+ const result = await client.query(
1704
+ `
1705
+ UPDATE job_queue
1706
+ SET status = 'pending', locked_at = NULL, locked_by = NULL, updated_at = NOW()
1707
+ WHERE status = 'processing'
1708
+ AND locked_at < NOW() - GREATEST(
1709
+ INTERVAL '1 minute' * $1::int,
1710
+ INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
1711
+ )
1712
+ RETURNING id
1713
+ `,
1714
+ [maxProcessingTimeMinutes]
1291
1715
  );
1716
+ log(`Reclaimed ${result.rowCount} stuck jobs`);
1717
+ return result.rowCount || 0;
1292
1718
  } catch (error) {
1293
- log(`Error updating cron schedule ${id} after enqueue: ${error}`);
1719
+ log(`Error reclaiming stuck jobs: ${error}`);
1294
1720
  throw error;
1295
1721
  } finally {
1296
1722
  client.release();
1297
1723
  }
1298
1724
  }
1299
1725
  // ── Internal helpers ──────────────────────────────────────────────────
1300
- async setPendingReasonForUnpickedJobs(reason, jobType) {
1726
+ /**
1727
+ * Batch-insert multiple job events in a single query.
1728
+ * More efficient than individual recordJobEvent calls.
1729
+ */
1730
+ async recordJobEventsBatch(events) {
1731
+ if (events.length === 0) return;
1301
1732
  const client = await this.pool.connect();
1302
1733
  try {
1303
- let jobTypeFilter = "";
1304
- const params = [reason];
1305
- if (jobType) {
1306
- if (Array.isArray(jobType)) {
1307
- jobTypeFilter = ` AND job_type = ANY($2)`;
1308
- params.push(jobType);
1309
- } else {
1310
- jobTypeFilter = ` AND job_type = $2`;
1311
- params.push(jobType);
1312
- }
1313
- }
1314
- await client.query(
1315
- `UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
1316
- params
1317
- );
1318
- } finally {
1319
- client.release();
1320
- }
1321
- }
1322
- };
1323
- var recordJobEvent = async (pool, jobId, eventType, metadata) => new PostgresBackend(pool).recordJobEvent(jobId, eventType, metadata);
1324
- var waitJob = async (pool, jobId, options) => {
1325
- const client = await pool.connect();
1326
- try {
1327
- const result = await client.query(
1328
- `
1329
- UPDATE job_queue
1330
- SET status = 'waiting',
1331
- wait_until = $2,
1332
- wait_token_id = $3,
1333
- step_data = $4,
1334
- locked_at = NULL,
1335
- locked_by = NULL,
1336
- updated_at = NOW()
1337
- WHERE id = $1 AND status = 'processing'
1338
- `,
1339
- [
1340
- jobId,
1341
- options.waitUntil ?? null,
1342
- options.waitTokenId ?? null,
1343
- JSON.stringify(options.stepData)
1344
- ]
1345
- );
1346
- if (result.rowCount === 0) {
1347
- log(
1348
- `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
1349
- );
1350
- return;
1351
- }
1352
- await recordJobEvent(pool, jobId, "waiting" /* Waiting */, {
1353
- waitUntil: options.waitUntil?.toISOString() ?? null,
1354
- waitTokenId: options.waitTokenId ?? null
1355
- });
1356
- log(`Job ${jobId} set to waiting`);
1357
- } catch (error) {
1358
- log(`Error setting job ${jobId} to waiting: ${error}`);
1359
- throw error;
1360
- } finally {
1361
- client.release();
1362
- }
1363
- };
1364
- var updateStepData = async (pool, jobId, stepData) => {
1365
- const client = await pool.connect();
1366
- try {
1367
- await client.query(
1368
- `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
1369
- [jobId, JSON.stringify(stepData)]
1370
- );
1371
- } catch (error) {
1372
- log(`Error updating step_data for job ${jobId}: ${error}`);
1373
- } finally {
1374
- client.release();
1375
- }
1376
- };
1377
- var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
1378
- function parseTimeoutString(timeout) {
1379
- const match = timeout.match(/^(\d+)(s|m|h|d)$/);
1380
- if (!match) {
1381
- throw new Error(
1382
- `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
1383
- );
1384
- }
1385
- const value = parseInt(match[1], 10);
1386
- const unit = match[2];
1387
- let ms;
1388
- switch (unit) {
1389
- case "s":
1390
- ms = value * 1e3;
1391
- break;
1392
- case "m":
1393
- ms = value * 60 * 1e3;
1394
- break;
1395
- case "h":
1396
- ms = value * 60 * 60 * 1e3;
1397
- break;
1398
- case "d":
1399
- ms = value * 24 * 60 * 60 * 1e3;
1400
- break;
1401
- default:
1402
- throw new Error(`Unknown timeout unit: "${unit}"`);
1403
- }
1404
- if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
1405
- throw new Error(
1406
- `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
1407
- );
1408
- }
1409
- return ms;
1410
- }
1411
- var createWaitpoint = async (pool, jobId, options) => {
1412
- const client = await pool.connect();
1413
- try {
1414
- const id = `wp_${crypto.randomUUID()}`;
1415
- let timeoutAt = null;
1416
- if (options?.timeout) {
1417
- const ms = parseTimeoutString(options.timeout);
1418
- timeoutAt = new Date(Date.now() + ms);
1419
- }
1420
- await client.query(
1421
- `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
1422
- [id, jobId, timeoutAt, options?.tags ?? null]
1423
- );
1424
- log(`Created waitpoint ${id} for job ${jobId}`);
1425
- return { id };
1426
- } catch (error) {
1427
- log(`Error creating waitpoint: ${error}`);
1428
- throw error;
1429
- } finally {
1430
- client.release();
1431
- }
1432
- };
1433
- var completeWaitpoint = async (pool, tokenId, data) => {
1434
- const client = await pool.connect();
1435
- try {
1436
- await client.query("BEGIN");
1437
- const wpResult = await client.query(
1438
- `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
1439
- WHERE id = $1 AND status = 'waiting'
1440
- RETURNING job_id`,
1441
- [tokenId, data != null ? JSON.stringify(data) : null]
1442
- );
1443
- if (wpResult.rows.length === 0) {
1444
- await client.query("ROLLBACK");
1445
- log(`Waitpoint ${tokenId} not found or already completed`);
1446
- return;
1447
- }
1448
- const jobId = wpResult.rows[0].job_id;
1449
- if (jobId != null) {
1734
+ const values = [];
1735
+ const params = [];
1736
+ let paramIdx = 1;
1737
+ for (const event of events) {
1738
+ values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
1739
+ params.push(
1740
+ event.jobId,
1741
+ event.eventType,
1742
+ event.metadata ? JSON.stringify(event.metadata) : null
1743
+ );
1744
+ }
1450
1745
  await client.query(
1451
- `UPDATE job_queue
1452
- SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
1453
- WHERE id = $1 AND status = 'waiting'`,
1454
- [jobId]
1746
+ `INSERT INTO job_events (job_id, event_type, metadata) VALUES ${values.join(", ")}`,
1747
+ params
1455
1748
  );
1749
+ } catch (error) {
1750
+ log(`Error recording batch job events: ${error}`);
1751
+ } finally {
1752
+ client.release();
1456
1753
  }
1457
- await client.query("COMMIT");
1458
- log(`Completed waitpoint ${tokenId} for job ${jobId}`);
1459
- } catch (error) {
1460
- await client.query("ROLLBACK");
1461
- log(`Error completing waitpoint ${tokenId}: ${error}`);
1462
- throw error;
1463
- } finally {
1464
- client.release();
1465
- }
1466
- };
1467
- var getWaitpoint = async (pool, tokenId) => {
1468
- const client = await pool.connect();
1469
- try {
1470
- const result = await client.query(
1471
- `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
1472
- [tokenId]
1473
- );
1474
- if (result.rows.length === 0) return null;
1475
- return result.rows[0];
1476
- } catch (error) {
1477
- log(`Error getting waitpoint ${tokenId}: ${error}`);
1478
- throw error;
1479
- } finally {
1480
- client.release();
1481
1754
  }
1482
- };
1483
- var expireTimedOutWaitpoints = async (pool) => {
1484
- const client = await pool.connect();
1485
- try {
1486
- await client.query("BEGIN");
1487
- const result = await client.query(
1488
- `UPDATE waitpoints
1489
- SET status = 'timed_out'
1490
- WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
1491
- RETURNING id, job_id`
1492
- );
1493
- for (const row of result.rows) {
1494
- if (row.job_id != null) {
1495
- await client.query(
1496
- `UPDATE job_queue
1497
- SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
1498
- WHERE id = $1 AND status = 'waiting'`,
1499
- [row.job_id]
1755
+ // ── Cron schedules ──────────────────────────────────────────────────
1756
+ /** Create a cron schedule and return its ID. */
1757
+ async addCronSchedule(input) {
1758
+ const client = await this.pool.connect();
1759
+ try {
1760
+ const result = await client.query(
1761
+ `INSERT INTO cron_schedules
1762
+ (schedule_name, cron_expression, job_type, payload, max_attempts,
1763
+ priority, timeout_ms, force_kill_on_timeout, tags, timezone,
1764
+ allow_overlap, next_run_at)
1765
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
1766
+ RETURNING id`,
1767
+ [
1768
+ input.scheduleName,
1769
+ input.cronExpression,
1770
+ input.jobType,
1771
+ input.payload,
1772
+ input.maxAttempts,
1773
+ input.priority,
1774
+ input.timeoutMs,
1775
+ input.forceKillOnTimeout,
1776
+ input.tags ?? null,
1777
+ input.timezone,
1778
+ input.allowOverlap,
1779
+ input.nextRunAt
1780
+ ]
1781
+ );
1782
+ const id = result.rows[0].id;
1783
+ log(`Added cron schedule ${id}: "${input.scheduleName}"`);
1784
+ return id;
1785
+ } catch (error) {
1786
+ if (error?.code === "23505") {
1787
+ throw new Error(
1788
+ `Cron schedule with name "${input.scheduleName}" already exists`
1500
1789
  );
1501
1790
  }
1791
+ log(`Error adding cron schedule: ${error}`);
1792
+ throw error;
1793
+ } finally {
1794
+ client.release();
1502
1795
  }
1503
- await client.query("COMMIT");
1504
- const count = result.rowCount || 0;
1505
- if (count > 0) {
1506
- log(`Expired ${count} timed-out waitpoints`);
1507
- }
1508
- return count;
1509
- } catch (error) {
1510
- await client.query("ROLLBACK");
1511
- log(`Error expiring timed-out waitpoints: ${error}`);
1512
- throw error;
1513
- } finally {
1514
- client.release();
1515
- }
1516
- };
1517
- function tryExtractPool(backend) {
1518
- if (backend instanceof PostgresBackend) {
1519
- return backend.getPool();
1520
1796
  }
1521
- return null;
1522
- }
1523
- function buildBasicContext(backend, jobId, baseCtx) {
1524
- const waitError = () => new Error(
1525
- "Wait features (waitFor, waitUntil, createToken, waitForToken, ctx.run) are currently only supported with the PostgreSQL backend."
1526
- );
1527
- return {
1528
- prolong: baseCtx.prolong,
1529
- onTimeout: baseCtx.onTimeout,
1530
- run: async (_stepName, fn) => {
1531
- return fn();
1532
- },
1533
- waitFor: async () => {
1534
- throw waitError();
1535
- },
1536
- waitUntil: async () => {
1537
- throw waitError();
1538
- },
1539
- createToken: async () => {
1540
- throw waitError();
1541
- },
1542
- waitForToken: async () => {
1543
- throw waitError();
1544
- },
1545
- setProgress: async (percent) => {
1546
- if (percent < 0 || percent > 100)
1547
- throw new Error("Progress must be between 0 and 100");
1548
- await backend.updateProgress(jobId, Math.round(percent));
1549
- }
1550
- };
1551
- }
1552
- function validateHandlerSerializable(handler, jobType) {
1553
- try {
1554
- const handlerString = handler.toString();
1555
- if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
1556
- throw new Error(
1557
- `Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
1558
- );
1559
- }
1560
- if (handlerString.includes("[native code]")) {
1561
- throw new Error(
1562
- `Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
1797
+ /** Get a cron schedule by ID. */
1798
+ async getCronSchedule(id) {
1799
+ const client = await this.pool.connect();
1800
+ try {
1801
+ const result = await client.query(
1802
+ `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1803
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1804
+ priority, timeout_ms AS "timeoutMs",
1805
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1806
+ timezone, allow_overlap AS "allowOverlap", status,
1807
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1808
+ next_run_at AS "nextRunAt",
1809
+ created_at AS "createdAt", updated_at AS "updatedAt"
1810
+ FROM cron_schedules WHERE id = $1`,
1811
+ [id]
1563
1812
  );
1813
+ if (result.rows.length === 0) return null;
1814
+ return result.rows[0];
1815
+ } catch (error) {
1816
+ log(`Error getting cron schedule ${id}: ${error}`);
1817
+ throw error;
1818
+ } finally {
1819
+ client.release();
1564
1820
  }
1821
+ }
1822
+ /** Get a cron schedule by its unique name. */
1823
+ async getCronScheduleByName(name) {
1824
+ const client = await this.pool.connect();
1565
1825
  try {
1566
- new Function("return " + handlerString);
1567
- } catch (parseError) {
1568
- throw new Error(
1569
- `Handler for job type "${jobType}" cannot be serialized: ${parseError instanceof Error ? parseError.message : String(parseError)}. When using forceKillOnTimeout, handlers must be serializable functions without closures over external variables.`
1826
+ const result = await client.query(
1827
+ `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1828
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1829
+ priority, timeout_ms AS "timeoutMs",
1830
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1831
+ timezone, allow_overlap AS "allowOverlap", status,
1832
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1833
+ next_run_at AS "nextRunAt",
1834
+ created_at AS "createdAt", updated_at AS "updatedAt"
1835
+ FROM cron_schedules WHERE schedule_name = $1`,
1836
+ [name]
1570
1837
  );
1571
- }
1572
- } catch (error) {
1573
- if (error instanceof Error) {
1838
+ if (result.rows.length === 0) return null;
1839
+ return result.rows[0];
1840
+ } catch (error) {
1841
+ log(`Error getting cron schedule by name "${name}": ${error}`);
1574
1842
  throw error;
1843
+ } finally {
1844
+ client.release();
1575
1845
  }
1576
- throw new Error(
1577
- `Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
1578
- );
1579
1846
  }
1580
- }
1581
- async function runHandlerInWorker(handler, payload, timeoutMs, jobType) {
1582
- validateHandlerSerializable(handler, jobType);
1583
- return new Promise((resolve, reject) => {
1584
- const workerCode = `
1585
- (function() {
1586
- const { parentPort, workerData } = require('worker_threads');
1587
- const { handlerCode, payload, timeoutMs } = workerData;
1588
-
1589
- // Create an AbortController for the handler
1590
- const controller = new AbortController();
1591
- const signal = controller.signal;
1592
-
1593
- // Set up timeout
1594
- const timeoutId = setTimeout(() => {
1595
- controller.abort();
1596
- parentPort.postMessage({ type: 'timeout' });
1597
- }, timeoutMs);
1598
-
1599
- try {
1600
- // Execute the handler
1601
- // Note: This uses Function constructor which requires the handler to be serializable.
1602
- // The handler should be validated before reaching this point.
1603
- let handlerFn;
1604
- try {
1605
- // Wrap handlerCode in parentheses to ensure it's treated as an expression
1606
- // This handles both arrow functions and regular functions
1607
- const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
1608
- ? handlerCode
1609
- : '(' + handlerCode + ')';
1610
- handlerFn = new Function('return ' + wrappedCode)();
1611
- } catch (parseError) {
1612
- clearTimeout(timeoutId);
1613
- parentPort.postMessage({
1614
- type: 'error',
1615
- error: {
1616
- message: 'Handler cannot be deserialized in worker thread. ' +
1617
- 'Ensure your handler is a standalone function without closures over external variables. ' +
1618
- 'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
1619
- stack: parseError instanceof Error ? parseError.stack : undefined,
1620
- name: 'SerializationError',
1621
- },
1622
- });
1623
- return;
1624
- }
1625
-
1626
- // Ensure handlerFn is actually a function
1627
- if (typeof handlerFn !== 'function') {
1628
- clearTimeout(timeoutId);
1629
- parentPort.postMessage({
1630
- type: 'error',
1631
- error: {
1632
- message: 'Handler deserialization did not produce a function. ' +
1633
- 'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
1634
- name: 'SerializationError',
1635
- },
1636
- });
1637
- return;
1638
- }
1639
-
1640
- handlerFn(payload, signal)
1641
- .then(() => {
1642
- clearTimeout(timeoutId);
1643
- parentPort.postMessage({ type: 'success' });
1644
- })
1645
- .catch((error) => {
1646
- clearTimeout(timeoutId);
1647
- parentPort.postMessage({
1648
- type: 'error',
1649
- error: {
1650
- message: error.message,
1651
- stack: error.stack,
1652
- name: error.name,
1653
- },
1654
- });
1655
- });
1656
- } catch (error) {
1657
- clearTimeout(timeoutId);
1658
- parentPort.postMessage({
1659
- type: 'error',
1660
- error: {
1661
- message: error.message,
1662
- stack: error.stack,
1663
- name: error.name,
1664
- },
1665
- });
1666
- }
1667
- })();
1668
- `;
1669
- const worker = new worker_threads.Worker(workerCode, {
1670
- eval: true,
1671
- workerData: {
1672
- handlerCode: handler.toString(),
1673
- payload,
1674
- timeoutMs
1847
+ /** List cron schedules, optionally filtered by status. */
1848
+ async listCronSchedules(status) {
1849
+ const client = await this.pool.connect();
1850
+ try {
1851
+ let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1852
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1853
+ priority, timeout_ms AS "timeoutMs",
1854
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1855
+ timezone, allow_overlap AS "allowOverlap", status,
1856
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1857
+ next_run_at AS "nextRunAt",
1858
+ created_at AS "createdAt", updated_at AS "updatedAt"
1859
+ FROM cron_schedules`;
1860
+ const params = [];
1861
+ if (status) {
1862
+ query += ` WHERE status = $1`;
1863
+ params.push(status);
1675
1864
  }
1676
- });
1677
- let resolved = false;
1678
- worker.on("message", (message) => {
1679
- if (resolved) return;
1680
- resolved = true;
1681
- if (message.type === "success") {
1682
- resolve();
1683
- } else if (message.type === "timeout") {
1684
- const timeoutError = new Error(
1685
- `Job timed out after ${timeoutMs} ms and was forcefully terminated`
1686
- );
1687
- timeoutError.failureReason = "timeout" /* Timeout */;
1688
- reject(timeoutError);
1689
- } else if (message.type === "error") {
1690
- const error = new Error(message.error.message);
1691
- error.stack = message.error.stack;
1692
- error.name = message.error.name;
1693
- reject(error);
1865
+ query += ` ORDER BY created_at ASC`;
1866
+ const result = await client.query(query, params);
1867
+ return result.rows;
1868
+ } catch (error) {
1869
+ log(`Error listing cron schedules: ${error}`);
1870
+ throw error;
1871
+ } finally {
1872
+ client.release();
1873
+ }
1874
+ }
1875
+ /** Delete a cron schedule by ID. */
1876
+ async removeCronSchedule(id) {
1877
+ const client = await this.pool.connect();
1878
+ try {
1879
+ await client.query(`DELETE FROM cron_schedules WHERE id = $1`, [id]);
1880
+ log(`Removed cron schedule ${id}`);
1881
+ } catch (error) {
1882
+ log(`Error removing cron schedule ${id}: ${error}`);
1883
+ throw error;
1884
+ } finally {
1885
+ client.release();
1886
+ }
1887
+ }
1888
+ /** Pause a cron schedule. */
1889
+ async pauseCronSchedule(id) {
1890
+ const client = await this.pool.connect();
1891
+ try {
1892
+ await client.query(
1893
+ `UPDATE cron_schedules SET status = 'paused', updated_at = NOW() WHERE id = $1`,
1894
+ [id]
1895
+ );
1896
+ log(`Paused cron schedule ${id}`);
1897
+ } catch (error) {
1898
+ log(`Error pausing cron schedule ${id}: ${error}`);
1899
+ throw error;
1900
+ } finally {
1901
+ client.release();
1902
+ }
1903
+ }
1904
+ /** Resume a paused cron schedule. */
1905
+ async resumeCronSchedule(id) {
1906
+ const client = await this.pool.connect();
1907
+ try {
1908
+ await client.query(
1909
+ `UPDATE cron_schedules SET status = 'active', updated_at = NOW() WHERE id = $1`,
1910
+ [id]
1911
+ );
1912
+ log(`Resumed cron schedule ${id}`);
1913
+ } catch (error) {
1914
+ log(`Error resuming cron schedule ${id}: ${error}`);
1915
+ throw error;
1916
+ } finally {
1917
+ client.release();
1918
+ }
1919
+ }
1920
+ /** Edit a cron schedule. */
1921
+ async editCronSchedule(id, updates, nextRunAt) {
1922
+ const client = await this.pool.connect();
1923
+ try {
1924
+ const updateFields = [];
1925
+ const params = [];
1926
+ let paramIdx = 1;
1927
+ if (updates.cronExpression !== void 0) {
1928
+ updateFields.push(`cron_expression = $${paramIdx++}`);
1929
+ params.push(updates.cronExpression);
1694
1930
  }
1695
- });
1696
- worker.on("error", (error) => {
1697
- if (resolved) return;
1698
- resolved = true;
1699
- reject(error);
1700
- });
1701
- worker.on("exit", (code) => {
1702
- if (resolved) return;
1703
- if (code !== 0) {
1704
- resolved = true;
1705
- reject(new Error(`Worker stopped with exit code ${code}`));
1931
+ if (updates.payload !== void 0) {
1932
+ updateFields.push(`payload = $${paramIdx++}`);
1933
+ params.push(updates.payload);
1706
1934
  }
1707
- });
1708
- setTimeout(() => {
1709
- if (!resolved) {
1710
- resolved = true;
1711
- worker.terminate().then(() => {
1712
- const timeoutError = new Error(
1713
- `Job timed out after ${timeoutMs} ms and was forcefully terminated`
1714
- );
1715
- timeoutError.failureReason = "timeout" /* Timeout */;
1716
- reject(timeoutError);
1717
- }).catch((err) => {
1718
- reject(err);
1719
- });
1935
+ if (updates.maxAttempts !== void 0) {
1936
+ updateFields.push(`max_attempts = $${paramIdx++}`);
1937
+ params.push(updates.maxAttempts);
1720
1938
  }
1721
- }, timeoutMs + 100);
1722
- });
1723
- }
1724
- function calculateWaitUntil(duration) {
1725
- const now = Date.now();
1726
- let ms = 0;
1727
- if (duration.seconds) ms += duration.seconds * 1e3;
1728
- if (duration.minutes) ms += duration.minutes * 60 * 1e3;
1729
- if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
1730
- if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
1731
- if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
1732
- if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
1733
- if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
1734
- if (ms <= 0) {
1735
- throw new Error(
1736
- "waitFor duration must be positive. Provide at least one positive duration field."
1737
- );
1738
- }
1739
- return new Date(now + ms);
1740
- }
1741
- async function resolveCompletedWaits(pool, stepData) {
1742
- for (const key of Object.keys(stepData)) {
1743
- if (!key.startsWith("__wait_")) continue;
1744
- const entry = stepData[key];
1745
- if (!entry || typeof entry !== "object" || entry.completed) continue;
1746
- if (entry.type === "duration" || entry.type === "date") {
1747
- stepData[key] = { ...entry, completed: true };
1748
- } else if (entry.type === "token" && entry.tokenId) {
1749
- const wp = await getWaitpoint(pool, entry.tokenId);
1750
- if (wp && wp.status === "completed") {
1751
- stepData[key] = {
1752
- ...entry,
1753
- completed: true,
1754
- result: { ok: true, output: wp.output }
1755
- };
1756
- } else if (wp && wp.status === "timed_out") {
1757
- stepData[key] = {
1758
- ...entry,
1759
- completed: true,
1760
- result: { ok: false, error: "Token timed out" }
1761
- };
1939
+ if (updates.priority !== void 0) {
1940
+ updateFields.push(`priority = $${paramIdx++}`);
1941
+ params.push(updates.priority);
1762
1942
  }
1763
- }
1764
- }
1765
- }
1766
- function buildWaitContext(backend, pool, jobId, stepData, baseCtx) {
1767
- let waitCounter = 0;
1768
- const ctx = {
1769
- prolong: baseCtx.prolong,
1770
- onTimeout: baseCtx.onTimeout,
1771
- run: async (stepName, fn) => {
1772
- const cached = stepData[stepName];
1773
- if (cached && typeof cached === "object" && cached.__completed) {
1774
- log(`Step "${stepName}" replayed from cache for job ${jobId}`);
1775
- return cached.result;
1943
+ if (updates.timeoutMs !== void 0) {
1944
+ updateFields.push(`timeout_ms = $${paramIdx++}`);
1945
+ params.push(updates.timeoutMs);
1776
1946
  }
1777
- const result = await fn();
1778
- stepData[stepName] = { __completed: true, result };
1779
- await updateStepData(pool, jobId, stepData);
1780
- return result;
1781
- },
1782
- waitFor: async (duration) => {
1783
- const waitKey = `__wait_${waitCounter++}`;
1784
- const cached = stepData[waitKey];
1785
- if (cached && typeof cached === "object" && cached.completed) {
1786
- log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
1787
- return;
1947
+ if (updates.forceKillOnTimeout !== void 0) {
1948
+ updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
1949
+ params.push(updates.forceKillOnTimeout);
1788
1950
  }
1789
- const waitUntilDate = calculateWaitUntil(duration);
1790
- stepData[waitKey] = { type: "duration", completed: false };
1791
- throw new WaitSignal("duration", waitUntilDate, void 0, stepData);
1792
- },
1793
- waitUntil: async (date) => {
1794
- const waitKey = `__wait_${waitCounter++}`;
1795
- const cached = stepData[waitKey];
1796
- if (cached && typeof cached === "object" && cached.completed) {
1797
- log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
1798
- return;
1951
+ if (updates.tags !== void 0) {
1952
+ updateFields.push(`tags = $${paramIdx++}`);
1953
+ params.push(updates.tags);
1799
1954
  }
1800
- stepData[waitKey] = { type: "date", completed: false };
1801
- throw new WaitSignal("date", date, void 0, stepData);
1802
- },
1803
- createToken: async (options) => {
1804
- const token = await createWaitpoint(pool, jobId, options);
1805
- return token;
1806
- },
1807
- waitForToken: async (tokenId) => {
1808
- const waitKey = `__wait_${waitCounter++}`;
1809
- const cached = stepData[waitKey];
1810
- if (cached && typeof cached === "object" && cached.completed) {
1811
- log(
1812
- `Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
1813
- );
1814
- return cached.result;
1955
+ if (updates.timezone !== void 0) {
1956
+ updateFields.push(`timezone = $${paramIdx++}`);
1957
+ params.push(updates.timezone);
1815
1958
  }
1816
- const wp = await getWaitpoint(pool, tokenId);
1817
- if (wp && wp.status === "completed") {
1818
- const result = {
1819
- ok: true,
1820
- output: wp.output
1821
- };
1822
- stepData[waitKey] = {
1823
- type: "token",
1824
- tokenId,
1825
- completed: true,
1826
- result
1827
- };
1828
- await updateStepData(pool, jobId, stepData);
1829
- return result;
1959
+ if (updates.allowOverlap !== void 0) {
1960
+ updateFields.push(`allow_overlap = $${paramIdx++}`);
1961
+ params.push(updates.allowOverlap);
1830
1962
  }
1831
- if (wp && wp.status === "timed_out") {
1832
- const result = {
1833
- ok: false,
1834
- error: "Token timed out"
1835
- };
1836
- stepData[waitKey] = {
1837
- type: "token",
1838
- tokenId,
1839
- completed: true,
1840
- result
1841
- };
1842
- await updateStepData(pool, jobId, stepData);
1843
- return result;
1963
+ if (nextRunAt !== void 0) {
1964
+ updateFields.push(`next_run_at = $${paramIdx++}`);
1965
+ params.push(nextRunAt);
1966
+ }
1967
+ if (updateFields.length === 0) {
1968
+ log(`No fields to update for cron schedule ${id}`);
1969
+ return;
1844
1970
  }
1845
- stepData[waitKey] = { type: "token", tokenId, completed: false };
1846
- throw new WaitSignal("token", void 0, tokenId, stepData);
1847
- },
1848
- setProgress: async (percent) => {
1849
- if (percent < 0 || percent > 100)
1850
- throw new Error("Progress must be between 0 and 100");
1851
- await backend.updateProgress(jobId, Math.round(percent));
1971
+ updateFields.push(`updated_at = NOW()`);
1972
+ params.push(id);
1973
+ const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
1974
+ await client.query(query, params);
1975
+ log(`Edited cron schedule ${id}`);
1976
+ } catch (error) {
1977
+ log(`Error editing cron schedule ${id}: ${error}`);
1978
+ throw error;
1979
+ } finally {
1980
+ client.release();
1852
1981
  }
1853
- };
1854
- return ctx;
1855
- }
1856
- async function processJobWithHandlers(backend, job, jobHandlers) {
1857
- const handler = jobHandlers[job.jobType];
1858
- if (!handler) {
1859
- await backend.setPendingReasonForUnpickedJobs(
1860
- `No handler registered for job type: ${job.jobType}`,
1861
- job.jobType
1862
- );
1863
- await backend.failJob(
1864
- job.id,
1865
- new Error(`No handler registered for job type: ${job.jobType}`),
1866
- "no_handler" /* NoHandler */
1867
- );
1868
- return;
1869
- }
1870
- const stepData = { ...job.stepData || {} };
1871
- const pool = tryExtractPool(backend);
1872
- const hasStepHistory = Object.keys(stepData).some(
1873
- (k) => k.startsWith("__wait_")
1874
- );
1875
- if (hasStepHistory && pool) {
1876
- await resolveCompletedWaits(pool, stepData);
1877
- await updateStepData(pool, job.id, stepData);
1878
1982
  }
1879
- const timeoutMs = job.timeoutMs ?? void 0;
1880
- const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
1881
- let timeoutId;
1882
- const controller = new AbortController();
1883
- try {
1884
- if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
1885
- await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
1886
- } else {
1887
- let onTimeoutCallback;
1888
- let timeoutReject;
1889
- const armTimeout = (ms) => {
1890
- if (timeoutId) clearTimeout(timeoutId);
1891
- timeoutId = setTimeout(() => {
1892
- if (onTimeoutCallback) {
1893
- try {
1894
- const extension = onTimeoutCallback();
1895
- if (typeof extension === "number" && extension > 0) {
1896
- backend.prolongJob(job.id).catch(() => {
1897
- });
1898
- armTimeout(extension);
1899
- return;
1900
- }
1901
- } catch (callbackError) {
1902
- log(
1903
- `onTimeout callback threw for job ${job.id}: ${callbackError}`
1904
- );
1905
- }
1906
- }
1907
- controller.abort();
1908
- const timeoutError = new Error(`Job timed out after ${ms} ms`);
1909
- timeoutError.failureReason = "timeout" /* Timeout */;
1910
- if (timeoutReject) {
1911
- timeoutReject(timeoutError);
1912
- }
1913
- }, ms);
1914
- };
1915
- const hasTimeout = timeoutMs != null && timeoutMs > 0;
1916
- const baseCtx = hasTimeout ? {
1917
- prolong: (ms) => {
1918
- const duration = ms ?? timeoutMs;
1919
- if (duration != null && duration > 0) {
1920
- armTimeout(duration);
1921
- backend.prolongJob(job.id).catch(() => {
1922
- });
1923
- }
1924
- },
1925
- onTimeout: (callback) => {
1926
- onTimeoutCallback = callback;
1927
- }
1928
- } : {
1929
- prolong: () => {
1930
- log("prolong() called but ignored: job has no timeout set");
1931
- },
1932
- onTimeout: () => {
1933
- log("onTimeout() called but ignored: job has no timeout set");
1934
- }
1935
- };
1936
- const ctx = pool ? buildWaitContext(backend, pool, job.id, stepData, baseCtx) : buildBasicContext(backend, job.id, baseCtx);
1937
- if (forceKillOnTimeout && !hasTimeout) {
1938
- log(
1939
- `forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
1940
- );
1941
- }
1942
- const jobPromise = handler(job.payload, controller.signal, ctx);
1943
- if (hasTimeout) {
1944
- await Promise.race([
1945
- jobPromise,
1946
- new Promise((_, reject) => {
1947
- timeoutReject = reject;
1948
- armTimeout(timeoutMs);
1949
- })
1950
- ]);
1951
- } else {
1952
- await jobPromise;
1983
+ /**
1984
+ * Atomically fetch all active cron schedules whose nextRunAt <= NOW().
1985
+ * Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
1986
+ */
1987
+ async getDueCronSchedules() {
1988
+ const client = await this.pool.connect();
1989
+ try {
1990
+ const result = await client.query(
1991
+ `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1992
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1993
+ priority, timeout_ms AS "timeoutMs",
1994
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1995
+ timezone, allow_overlap AS "allowOverlap", status,
1996
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1997
+ next_run_at AS "nextRunAt",
1998
+ created_at AS "createdAt", updated_at AS "updatedAt"
1999
+ FROM cron_schedules
2000
+ WHERE status = 'active'
2001
+ AND next_run_at IS NOT NULL
2002
+ AND next_run_at <= NOW()
2003
+ ORDER BY next_run_at ASC
2004
+ FOR UPDATE SKIP LOCKED`
2005
+ );
2006
+ log(`Found ${result.rows.length} due cron schedules`);
2007
+ return result.rows;
2008
+ } catch (error) {
2009
+ if (error?.code === "42P01") {
2010
+ log("cron_schedules table does not exist, skipping cron enqueue");
2011
+ return [];
1953
2012
  }
2013
+ log(`Error getting due cron schedules: ${error}`);
2014
+ throw error;
2015
+ } finally {
2016
+ client.release();
1954
2017
  }
1955
- if (timeoutId) clearTimeout(timeoutId);
1956
- await backend.completeJob(job.id);
1957
- } catch (error) {
1958
- if (timeoutId) clearTimeout(timeoutId);
1959
- if (error instanceof WaitSignal) {
1960
- if (!pool) {
1961
- await backend.failJob(
1962
- job.id,
1963
- new Error(
1964
- "WaitSignal received but wait features require the PostgreSQL backend."
1965
- ),
1966
- "handler_error" /* HandlerError */
1967
- );
1968
- return;
1969
- }
2018
+ }
2019
+ /**
2020
+ * Update a cron schedule after a job has been enqueued.
2021
+ * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
2022
+ */
2023
+ async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
2024
+ const client = await this.pool.connect();
2025
+ try {
2026
+ await client.query(
2027
+ `UPDATE cron_schedules
2028
+ SET last_enqueued_at = $2,
2029
+ last_job_id = $3,
2030
+ next_run_at = $4,
2031
+ updated_at = NOW()
2032
+ WHERE id = $1`,
2033
+ [id, lastEnqueuedAt, lastJobId, nextRunAt]
2034
+ );
1970
2035
  log(
1971
- `Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? "none"}, tokenId=${error.tokenId ?? "none"}`
2036
+ `Updated cron schedule ${id}: lastJobId=${lastJobId}, nextRunAt=${nextRunAt?.toISOString() ?? "null"}`
1972
2037
  );
1973
- await waitJob(pool, job.id, {
1974
- waitUntil: error.waitUntil,
1975
- waitTokenId: error.tokenId,
1976
- stepData: error.stepData
1977
- });
1978
- return;
1979
- }
1980
- console.error(`Error processing job ${job.id}:`, error);
1981
- let failureReason = "handler_error" /* HandlerError */;
1982
- if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
1983
- failureReason = "timeout" /* Timeout */;
2038
+ } catch (error) {
2039
+ log(`Error updating cron schedule ${id} after enqueue: ${error}`);
2040
+ throw error;
2041
+ } finally {
2042
+ client.release();
1984
2043
  }
1985
- await backend.failJob(
1986
- job.id,
1987
- error instanceof Error ? error : new Error(String(error)),
1988
- failureReason
1989
- );
1990
- }
1991
- }
1992
- async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError) {
1993
- const jobs = await backend.getNextBatch(
1994
- workerId,
1995
- batchSize,
1996
- jobType
1997
- );
1998
- if (!concurrency || concurrency >= jobs.length) {
1999
- await Promise.all(
2000
- jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers))
2001
- );
2002
- return jobs.length;
2003
- }
2004
- let idx = 0;
2005
- let running = 0;
2006
- let finished = 0;
2007
- return new Promise((resolve, reject) => {
2008
- const next = () => {
2009
- if (finished === jobs.length) return resolve(jobs.length);
2010
- while (running < concurrency && idx < jobs.length) {
2011
- const job = jobs[idx++];
2012
- running++;
2013
- processJobWithHandlers(backend, job, jobHandlers).then(() => {
2014
- running--;
2015
- finished++;
2016
- next();
2017
- }).catch((err) => {
2018
- running--;
2019
- finished++;
2020
- if (onError) {
2021
- onError(err instanceof Error ? err : new Error(String(err)));
2022
- }
2023
- next();
2024
- });
2025
- }
2026
- };
2027
- next();
2028
- });
2029
- }
2030
- var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
2031
- const {
2032
- workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
2033
- batchSize = 10,
2034
- pollInterval = 5e3,
2035
- onError = (error) => console.error("Job processor error:", error),
2036
- jobType,
2037
- concurrency = 3
2038
- } = options;
2039
- let running = false;
2040
- let intervalId = null;
2041
- let currentBatchPromise = null;
2042
- setLogContext(options.verbose ?? false);
2043
- const processJobs = async () => {
2044
- if (!running) return 0;
2045
- if (onBeforeBatch) {
2046
- try {
2047
- await onBeforeBatch();
2048
- } catch (hookError) {
2049
- log(`onBeforeBatch hook error: ${hookError}`);
2050
- if (onError) {
2051
- onError(
2052
- hookError instanceof Error ? hookError : new Error(String(hookError))
2053
- );
2054
- }
2044
+ }
2045
+ // ── Wait / step-data support ────────────────────────────────────────
2046
+ /**
2047
+ * Transition a job from 'processing' to 'waiting' status.
2048
+ * Persists step data so the handler can resume from where it left off.
2049
+ *
2050
+ * @param jobId - The job to pause.
2051
+ * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
2052
+ */
2053
+ async waitJob(jobId, options) {
2054
+ const client = await this.pool.connect();
2055
+ try {
2056
+ const result = await client.query(
2057
+ `
2058
+ UPDATE job_queue
2059
+ SET status = 'waiting',
2060
+ wait_until = $2,
2061
+ wait_token_id = $3,
2062
+ step_data = $4,
2063
+ locked_at = NULL,
2064
+ locked_by = NULL,
2065
+ updated_at = NOW()
2066
+ WHERE id = $1 AND status = 'processing'
2067
+ `,
2068
+ [
2069
+ jobId,
2070
+ options.waitUntil ?? null,
2071
+ options.waitTokenId ?? null,
2072
+ JSON.stringify(options.stepData)
2073
+ ]
2074
+ );
2075
+ if (result.rowCount === 0) {
2076
+ log(
2077
+ `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
2078
+ );
2079
+ return;
2055
2080
  }
2081
+ await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
2082
+ waitUntil: options.waitUntil?.toISOString() ?? null,
2083
+ waitTokenId: options.waitTokenId ?? null
2084
+ });
2085
+ log(`Job ${jobId} set to waiting`);
2086
+ } catch (error) {
2087
+ log(`Error setting job ${jobId} to waiting: ${error}`);
2088
+ throw error;
2089
+ } finally {
2090
+ client.release();
2056
2091
  }
2057
- log(
2058
- `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(",") : jobType}` : ""}`
2059
- );
2092
+ }
2093
+ /**
2094
+ * Persist step data for a job. Called after each ctx.run() step completes.
2095
+ * Best-effort: does not throw to avoid killing the running handler.
2096
+ *
2097
+ * @param jobId - The job to update.
2098
+ * @param stepData - The step data to persist.
2099
+ */
2100
+ async updateStepData(jobId, stepData) {
2101
+ const client = await this.pool.connect();
2060
2102
  try {
2061
- const processed = await processBatchWithHandlers(
2062
- backend,
2063
- workerId,
2064
- batchSize,
2065
- jobType,
2066
- handlers,
2067
- concurrency,
2068
- onError
2103
+ await client.query(
2104
+ `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
2105
+ [jobId, JSON.stringify(stepData)]
2069
2106
  );
2070
- return processed;
2071
2107
  } catch (error) {
2072
- onError(error instanceof Error ? error : new Error(String(error)));
2108
+ log(`Error updating step_data for job ${jobId}: ${error}`);
2109
+ } finally {
2110
+ client.release();
2073
2111
  }
2074
- return 0;
2075
- };
2076
- return {
2077
- /**
2078
- * Start the job processor in the background.
2079
- * - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
2080
- * - You have to call the stop method to stop the processor.
2081
- */
2082
- startInBackground: () => {
2083
- if (running) return;
2084
- log(`Starting job processor with workerId: ${workerId}`);
2085
- running = true;
2086
- const scheduleNext = (immediate) => {
2087
- if (!running) return;
2088
- if (immediate) {
2089
- intervalId = setTimeout(loop, 0);
2090
- } else {
2091
- intervalId = setTimeout(loop, pollInterval);
2092
- }
2093
- };
2094
- const loop = async () => {
2095
- if (!running) return;
2096
- currentBatchPromise = processJobs();
2097
- const processed = await currentBatchPromise;
2098
- currentBatchPromise = null;
2099
- scheduleNext(processed === batchSize);
2100
- };
2101
- loop();
2102
- },
2103
- /**
2104
- * Stop the job processor that runs in the background.
2105
- * Does not wait for in-flight jobs.
2106
- */
2107
- stop: () => {
2108
- log(`Stopping job processor with workerId: ${workerId}`);
2109
- running = false;
2110
- if (intervalId) {
2111
- clearTimeout(intervalId);
2112
- intervalId = null;
2113
- }
2114
- },
2115
- /**
2116
- * Stop the job processor and wait for all in-flight jobs to complete.
2117
- * Useful for graceful shutdown (e.g., SIGTERM handling).
2118
- */
2119
- stopAndDrain: async (drainTimeoutMs = 3e4) => {
2120
- log(`Stopping and draining job processor with workerId: ${workerId}`);
2121
- running = false;
2122
- if (intervalId) {
2123
- clearTimeout(intervalId);
2124
- intervalId = null;
2125
- }
2126
- if (currentBatchPromise) {
2127
- await Promise.race([
2128
- currentBatchPromise.catch(() => {
2129
- }),
2130
- new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
2131
- ]);
2132
- currentBatchPromise = null;
2133
- }
2134
- log(`Job processor ${workerId} drained`);
2135
- },
2136
- /**
2137
- * Start the job processor synchronously.
2138
- * - This will process all jobs immediately and then stop.
2139
- * - The pollInterval is ignored.
2140
- */
2141
- start: async () => {
2142
- log(`Starting job processor with workerId: ${workerId}`);
2143
- running = true;
2144
- const processed = await processJobs();
2145
- running = false;
2146
- return processed;
2147
- },
2148
- isRunning: () => running
2149
- };
2150
- };
2151
- function loadPemOrFile(value) {
2152
- if (!value) return void 0;
2153
- if (value.startsWith("file://")) {
2154
- const filePath = value.slice(7);
2155
- return fs__default.default.readFileSync(filePath, "utf8");
2156
2112
  }
2157
- return value;
2158
- }
2159
- var createPool = (config) => {
2160
- let searchPath;
2161
- let ssl = void 0;
2162
- let customCA;
2163
- let sslmode;
2164
- if (config.connectionString) {
2113
+ /**
2114
+ * Create a waitpoint token in the database.
2115
+ *
2116
+ * @param jobId - The job ID to associate with the token (null if created outside a handler).
2117
+ * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
2118
+ * @returns The created waitpoint with its unique ID.
2119
+ */
2120
+ async createWaitpoint(jobId, options) {
2121
+ const client = await this.pool.connect();
2165
2122
  try {
2166
- const url = new URL(config.connectionString);
2167
- searchPath = url.searchParams.get("search_path") || void 0;
2168
- sslmode = url.searchParams.get("sslmode") || void 0;
2169
- if (sslmode === "no-verify") {
2170
- ssl = { rejectUnauthorized: false };
2123
+ const id = `wp_${crypto.randomUUID()}`;
2124
+ let timeoutAt = null;
2125
+ if (options?.timeout) {
2126
+ const ms = parseTimeoutString(options.timeout);
2127
+ timeoutAt = new Date(Date.now() + ms);
2171
2128
  }
2172
- } catch (e) {
2173
- const parsed = pgConnectionString.parse(config.connectionString);
2174
- if (parsed.options) {
2175
- const match = parsed.options.match(/search_path=([^\s]+)/);
2176
- if (match) {
2177
- searchPath = match[1];
2178
- }
2129
+ await client.query(
2130
+ `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
2131
+ [id, jobId, timeoutAt, options?.tags ?? null]
2132
+ );
2133
+ log(`Created waitpoint ${id} for job ${jobId}`);
2134
+ return { id };
2135
+ } catch (error) {
2136
+ log(`Error creating waitpoint: ${error}`);
2137
+ throw error;
2138
+ } finally {
2139
+ client.release();
2140
+ }
2141
+ }
2142
+ /**
2143
+ * Complete a waitpoint token and move the associated job back to 'pending'.
2144
+ *
2145
+ * @param tokenId - The waitpoint token ID to complete.
2146
+ * @param data - Optional data to pass to the waiting handler.
2147
+ */
2148
+ async completeWaitpoint(tokenId, data) {
2149
+ const client = await this.pool.connect();
2150
+ try {
2151
+ await client.query("BEGIN");
2152
+ const wpResult = await client.query(
2153
+ `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
2154
+ WHERE id = $1 AND status = 'waiting'
2155
+ RETURNING job_id`,
2156
+ [tokenId, data != null ? JSON.stringify(data) : null]
2157
+ );
2158
+ if (wpResult.rows.length === 0) {
2159
+ await client.query("ROLLBACK");
2160
+ log(`Waitpoint ${tokenId} not found or already completed`);
2161
+ return;
2179
2162
  }
2180
- sslmode = typeof parsed.sslmode === "string" ? parsed.sslmode : void 0;
2181
- if (sslmode === "no-verify") {
2182
- ssl = { rejectUnauthorized: false };
2163
+ const jobId = wpResult.rows[0].job_id;
2164
+ if (jobId != null) {
2165
+ await client.query(
2166
+ `UPDATE job_queue
2167
+ SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
2168
+ WHERE id = $1 AND status = 'waiting'`,
2169
+ [jobId]
2170
+ );
2183
2171
  }
2172
+ await client.query("COMMIT");
2173
+ log(`Completed waitpoint ${tokenId} for job ${jobId}`);
2174
+ } catch (error) {
2175
+ await client.query("ROLLBACK");
2176
+ log(`Error completing waitpoint ${tokenId}: ${error}`);
2177
+ throw error;
2178
+ } finally {
2179
+ client.release();
2184
2180
  }
2185
2181
  }
2186
- if (config.ssl) {
2187
- if (typeof config.ssl.ca === "string") {
2188
- customCA = config.ssl.ca;
2189
- } else if (typeof process.env.PGSSLROOTCERT === "string") {
2190
- customCA = process.env.PGSSLROOTCERT;
2191
- } else {
2192
- customCA = void 0;
2182
+ /**
2183
+ * Retrieve a waitpoint token by its ID.
2184
+ *
2185
+ * @param tokenId - The waitpoint token ID to look up.
2186
+ * @returns The waitpoint record, or null if not found.
2187
+ */
2188
+ async getWaitpoint(tokenId) {
2189
+ const client = await this.pool.connect();
2190
+ try {
2191
+ const result = await client.query(
2192
+ `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
2193
+ [tokenId]
2194
+ );
2195
+ if (result.rows.length === 0) return null;
2196
+ return result.rows[0];
2197
+ } catch (error) {
2198
+ log(`Error getting waitpoint ${tokenId}: ${error}`);
2199
+ throw error;
2200
+ } finally {
2201
+ client.release();
2193
2202
  }
2194
- const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
2195
- ssl = {
2196
- ...ssl,
2197
- ...caValue ? { ca: caValue } : {},
2198
- cert: loadPemOrFile(
2199
- typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
2200
- ),
2201
- key: loadPemOrFile(
2202
- typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
2203
- ),
2204
- rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
2205
- };
2206
2203
  }
2207
- if (sslmode && customCA) {
2208
- const warning = `
2209
-
2210
- \x1B[33m**************************************************
2211
- \u26A0\uFE0F WARNING: SSL CONFIGURATION ISSUE
2212
- **************************************************
2213
- Both sslmode ('${sslmode}') is set in the connection string
2214
- and a custom CA is provided (via config.ssl.ca or PGSSLROOTCERT).
2215
- This combination may cause connection failures or unexpected behavior.
2216
-
2217
- Recommended: Remove sslmode from the connection string when using a custom CA.
2218
- **************************************************\x1B[0m
2219
- `;
2220
- console.warn(warning);
2204
+ /**
2205
+ * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
2206
+ *
2207
+ * @returns The number of tokens that were expired.
2208
+ */
2209
+ async expireTimedOutWaitpoints() {
2210
+ const client = await this.pool.connect();
2211
+ try {
2212
+ await client.query("BEGIN");
2213
+ const result = await client.query(
2214
+ `UPDATE waitpoints
2215
+ SET status = 'timed_out'
2216
+ WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
2217
+ RETURNING id, job_id`
2218
+ );
2219
+ for (const row of result.rows) {
2220
+ if (row.job_id != null) {
2221
+ await client.query(
2222
+ `UPDATE job_queue
2223
+ SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
2224
+ WHERE id = $1 AND status = 'waiting'`,
2225
+ [row.job_id]
2226
+ );
2227
+ }
2228
+ }
2229
+ await client.query("COMMIT");
2230
+ const count = result.rowCount || 0;
2231
+ if (count > 0) {
2232
+ log(`Expired ${count} timed-out waitpoints`);
2233
+ }
2234
+ return count;
2235
+ } catch (error) {
2236
+ await client.query("ROLLBACK");
2237
+ log(`Error expiring timed-out waitpoints: ${error}`);
2238
+ throw error;
2239
+ } finally {
2240
+ client.release();
2241
+ }
2221
2242
  }
2222
- const pool = new pg.Pool({
2223
- ...config,
2224
- ...ssl ? { ssl } : {}
2225
- });
2226
- if (searchPath) {
2227
- pool.on("connect", (client) => {
2228
- client.query(`SET search_path TO ${searchPath}`);
2229
- });
2243
+ // ── Internal helpers ──────────────────────────────────────────────────
2244
+ async setPendingReasonForUnpickedJobs(reason, jobType) {
2245
+ const client = await this.pool.connect();
2246
+ try {
2247
+ let jobTypeFilter = "";
2248
+ const params = [reason];
2249
+ if (jobType) {
2250
+ if (Array.isArray(jobType)) {
2251
+ jobTypeFilter = ` AND job_type = ANY($2)`;
2252
+ params.push(jobType);
2253
+ } else {
2254
+ jobTypeFilter = ` AND job_type = $2`;
2255
+ params.push(jobType);
2256
+ }
2257
+ }
2258
+ await client.query(
2259
+ `UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
2260
+ params
2261
+ );
2262
+ } finally {
2263
+ client.release();
2264
+ }
2230
2265
  }
2231
- return pool;
2232
2266
  };
2233
2267
 
2234
2268
  // src/backends/redis-scripts.ts
@@ -2285,7 +2319,10 @@ redis.call('HMSET', jobKey,
2285
2319
  'lastFailedAt', 'null',
2286
2320
  'lastCancelledAt', 'null',
2287
2321
  'tags', tagsJson,
2288
- 'idempotencyKey', idempotencyKey
2322
+ 'idempotencyKey', idempotencyKey,
2323
+ 'waitUntil', 'null',
2324
+ 'waitTokenId', 'null',
2325
+ 'stepData', 'null'
2289
2326
  )
2290
2327
 
2291
2328
  -- Status index
@@ -2368,7 +2405,25 @@ for _, jobId in ipairs(retries) do
2368
2405
  redis.call('ZREM', prefix .. 'retry', jobId)
2369
2406
  end
2370
2407
 
2371
- -- 3. Parse job type filter
2408
+ -- 3. Move ready waiting jobs (time-based, no token) into queue
2409
+ local waitingJobs = redis.call('ZRANGEBYSCORE', prefix .. 'waiting', '-inf', nowMs, 'LIMIT', 0, 200)
2410
+ for _, jobId in ipairs(waitingJobs) do
2411
+ local jk = prefix .. 'job:' .. jobId
2412
+ local status = redis.call('HGET', jk, 'status')
2413
+ local waitTokenId = redis.call('HGET', jk, 'waitTokenId')
2414
+ if status == 'waiting' and (waitTokenId == false or waitTokenId == 'null') then
2415
+ local pri = tonumber(redis.call('HGET', jk, 'priority') or '0')
2416
+ local ca = tonumber(redis.call('HGET', jk, 'createdAt'))
2417
+ local score = pri * ${SCORE_RANGE} + (${SCORE_RANGE} - ca)
2418
+ redis.call('ZADD', prefix .. 'queue', score, jobId)
2419
+ redis.call('SREM', prefix .. 'status:waiting', jobId)
2420
+ redis.call('SADD', prefix .. 'status:pending', jobId)
2421
+ redis.call('HMSET', jk, 'status', 'pending', 'waitUntil', 'null')
2422
+ end
2423
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2424
+ end
2425
+
2426
+ -- 4. Parse job type filter
2372
2427
  local filterTypes = nil
2373
2428
  if jobTypeFilter ~= "null" then
2374
2429
  -- Could be a JSON array or a plain string
@@ -2381,7 +2436,7 @@ if jobTypeFilter ~= "null" then
2381
2436
  end
2382
2437
  end
2383
2438
 
2384
- -- 4. Pop candidates from queue (highest score first)
2439
+ -- 5. Pop candidates from queue (highest score first)
2385
2440
  -- We pop more than batchSize because some may be filtered out
2386
2441
  local popCount = batchSize * 3
2387
2442
  local candidates = redis.call('ZPOPMAX', prefix .. 'queue', popCount)
@@ -2465,7 +2520,10 @@ local jk = prefix .. 'job:' .. jobId
2465
2520
  redis.call('HMSET', jk,
2466
2521
  'status', 'completed',
2467
2522
  'updatedAt', nowMs,
2468
- 'completedAt', nowMs
2523
+ 'completedAt', nowMs,
2524
+ 'stepData', 'null',
2525
+ 'waitUntil', 'null',
2526
+ 'waitTokenId', 'null'
2469
2527
  )
2470
2528
  redis.call('SREM', prefix .. 'status:processing', jobId)
2471
2529
  redis.call('SADD', prefix .. 'status:completed', jobId)
@@ -2524,6 +2582,7 @@ local nowMs = tonumber(ARGV[2])
2524
2582
  local jk = prefix .. 'job:' .. jobId
2525
2583
 
2526
2584
  local oldStatus = redis.call('HGET', jk, 'status')
2585
+ if oldStatus ~= 'failed' and oldStatus ~= 'processing' then return 0 end
2527
2586
 
2528
2587
  redis.call('HMSET', jk,
2529
2588
  'status', 'pending',
@@ -2535,9 +2594,7 @@ redis.call('HMSET', jk,
2535
2594
  )
2536
2595
 
2537
2596
  -- Remove from old status, add to pending
2538
- if oldStatus then
2539
- redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
2540
- end
2597
+ redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
2541
2598
  redis.call('SADD', prefix .. 'status:pending', jobId)
2542
2599
 
2543
2600
  -- Remove from retry sorted set if present
@@ -2558,18 +2615,21 @@ local nowMs = ARGV[2]
2558
2615
  local jk = prefix .. 'job:' .. jobId
2559
2616
 
2560
2617
  local status = redis.call('HGET', jk, 'status')
2561
- if status ~= 'pending' then return 0 end
2618
+ if status ~= 'pending' and status ~= 'waiting' then return 0 end
2562
2619
 
2563
2620
  redis.call('HMSET', jk,
2564
2621
  'status', 'cancelled',
2565
2622
  'updatedAt', nowMs,
2566
- 'lastCancelledAt', nowMs
2623
+ 'lastCancelledAt', nowMs,
2624
+ 'waitUntil', 'null',
2625
+ 'waitTokenId', 'null'
2567
2626
  )
2568
- redis.call('SREM', prefix .. 'status:pending', jobId)
2627
+ redis.call('SREM', prefix .. 'status:' .. status, jobId)
2569
2628
  redis.call('SADD', prefix .. 'status:cancelled', jobId)
2570
- -- Remove from queue / delayed
2629
+ -- Remove from queue / delayed / waiting
2571
2630
  redis.call('ZREM', prefix .. 'queue', jobId)
2572
2631
  redis.call('ZREM', prefix .. 'delayed', jobId)
2632
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2573
2633
 
2574
2634
  return 1
2575
2635
  `;
@@ -2637,18 +2697,16 @@ end
2637
2697
 
2638
2698
  return count
2639
2699
  `;
2640
- var CLEANUP_OLD_JOBS_SCRIPT = `
2700
+ var CLEANUP_OLD_JOBS_BATCH_SCRIPT = `
2641
2701
  local prefix = KEYS[1]
2642
2702
  local cutoffMs = tonumber(ARGV[1])
2643
-
2644
- local completed = redis.call('SMEMBERS', prefix .. 'status:completed')
2645
2703
  local count = 0
2646
2704
 
2647
- for _, jobId in ipairs(completed) do
2705
+ for i = 2, #ARGV do
2706
+ local jobId = ARGV[i]
2648
2707
  local jk = prefix .. 'job:' .. jobId
2649
2708
  local updatedAt = tonumber(redis.call('HGET', jk, 'updatedAt'))
2650
2709
  if updatedAt and updatedAt < cutoffMs then
2651
- -- Remove all indexes
2652
2710
  local jobType = redis.call('HGET', jk, 'jobType')
2653
2711
  local tagsJson = redis.call('HGET', jk, 'tags')
2654
2712
  local idempotencyKey = redis.call('HGET', jk, 'idempotencyKey')
@@ -2671,7 +2729,6 @@ for _, jobId in ipairs(completed) do
2671
2729
  if idempotencyKey and idempotencyKey ~= 'null' then
2672
2730
  redis.call('DEL', prefix .. 'idempotency:' .. idempotencyKey)
2673
2731
  end
2674
- -- Delete events
2675
2732
  redis.call('DEL', prefix .. 'events:' .. jobId)
2676
2733
 
2677
2734
  count = count + 1
@@ -2680,8 +2737,158 @@ end
2680
2737
 
2681
2738
  return count
2682
2739
  `;
2740
+ var WAIT_JOB_SCRIPT = `
2741
+ local prefix = KEYS[1]
2742
+ local jobId = ARGV[1]
2743
+ local waitUntilMs = ARGV[2]
2744
+ local waitTokenId = ARGV[3]
2745
+ local stepDataJson = ARGV[4]
2746
+ local nowMs = ARGV[5]
2747
+ local jk = prefix .. 'job:' .. jobId
2748
+
2749
+ local status = redis.call('HGET', jk, 'status')
2750
+ if status ~= 'processing' then return 0 end
2751
+
2752
+ redis.call('HMSET', jk,
2753
+ 'status', 'waiting',
2754
+ 'waitUntil', waitUntilMs,
2755
+ 'waitTokenId', waitTokenId,
2756
+ 'stepData', stepDataJson,
2757
+ 'lockedAt', 'null',
2758
+ 'lockedBy', 'null',
2759
+ 'updatedAt', nowMs
2760
+ )
2761
+ redis.call('SREM', prefix .. 'status:processing', jobId)
2762
+ redis.call('SADD', prefix .. 'status:waiting', jobId)
2763
+
2764
+ -- Add to waiting sorted set if time-based wait
2765
+ if waitUntilMs ~= 'null' then
2766
+ redis.call('ZADD', prefix .. 'waiting', tonumber(waitUntilMs), jobId)
2767
+ end
2768
+
2769
+ return 1
2770
+ `;
2771
+ var COMPLETE_WAITPOINT_SCRIPT = `
2772
+ local prefix = KEYS[1]
2773
+ local tokenId = ARGV[1]
2774
+ local outputJson = ARGV[2]
2775
+ local nowMs = ARGV[3]
2776
+ local wpk = prefix .. 'waitpoint:' .. tokenId
2777
+
2778
+ local wpStatus = redis.call('HGET', wpk, 'status')
2779
+ if not wpStatus or wpStatus ~= 'waiting' then return 0 end
2780
+
2781
+ redis.call('HMSET', wpk,
2782
+ 'status', 'completed',
2783
+ 'output', outputJson,
2784
+ 'completedAt', nowMs
2785
+ )
2786
+
2787
+ -- Move associated job back to pending
2788
+ local jobId = redis.call('HGET', wpk, 'jobId')
2789
+ if jobId and jobId ~= 'null' then
2790
+ local jk = prefix .. 'job:' .. jobId
2791
+ local jobStatus = redis.call('HGET', jk, 'status')
2792
+ if jobStatus == 'waiting' then
2793
+ redis.call('HMSET', jk,
2794
+ 'status', 'pending',
2795
+ 'waitTokenId', 'null',
2796
+ 'waitUntil', 'null',
2797
+ 'updatedAt', nowMs
2798
+ )
2799
+ redis.call('SREM', prefix .. 'status:waiting', jobId)
2800
+ redis.call('SADD', prefix .. 'status:pending', jobId)
2801
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2802
+
2803
+ -- Re-add to queue
2804
+ local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
2805
+ local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
2806
+ local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
2807
+ redis.call('ZADD', prefix .. 'queue', score, jobId)
2808
+ end
2809
+ end
2810
+
2811
+ return 1
2812
+ `;
2813
+ var EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT = `
2814
+ local prefix = KEYS[1]
2815
+ local nowMs = tonumber(ARGV[1])
2816
+
2817
+ local expiredIds = redis.call('ZRANGEBYSCORE', prefix .. 'waitpoint_timeout', '-inf', nowMs)
2818
+ local count = 0
2819
+
2820
+ for _, tokenId in ipairs(expiredIds) do
2821
+ local wpk = prefix .. 'waitpoint:' .. tokenId
2822
+ local wpStatus = redis.call('HGET', wpk, 'status')
2823
+ if wpStatus == 'waiting' then
2824
+ redis.call('HMSET', wpk,
2825
+ 'status', 'timed_out'
2826
+ )
2827
+
2828
+ -- Move associated job back to pending
2829
+ local jobId = redis.call('HGET', wpk, 'jobId')
2830
+ if jobId and jobId ~= 'null' then
2831
+ local jk = prefix .. 'job:' .. jobId
2832
+ local jobStatus = redis.call('HGET', jk, 'status')
2833
+ if jobStatus == 'waiting' then
2834
+ redis.call('HMSET', jk,
2835
+ 'status', 'pending',
2836
+ 'waitTokenId', 'null',
2837
+ 'waitUntil', 'null',
2838
+ 'updatedAt', nowMs
2839
+ )
2840
+ redis.call('SREM', prefix .. 'status:waiting', jobId)
2841
+ redis.call('SADD', prefix .. 'status:pending', jobId)
2842
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2843
+
2844
+ local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
2845
+ local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
2846
+ local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
2847
+ redis.call('ZADD', prefix .. 'queue', score, jobId)
2848
+ end
2849
+ end
2850
+
2851
+ count = count + 1
2852
+ end
2853
+ redis.call('ZREM', prefix .. 'waitpoint_timeout', tokenId)
2854
+ end
2683
2855
 
2684
- // src/backends/redis.ts
2856
+ return count
2857
+ `;
2858
+ var MAX_TIMEOUT_MS2 = 365 * 24 * 60 * 60 * 1e3;
2859
+ function parseTimeoutString2(timeout) {
2860
+ const match = timeout.match(/^(\d+)(s|m|h|d)$/);
2861
+ if (!match) {
2862
+ throw new Error(
2863
+ `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
2864
+ );
2865
+ }
2866
+ const value = parseInt(match[1], 10);
2867
+ const unit = match[2];
2868
+ let ms;
2869
+ switch (unit) {
2870
+ case "s":
2871
+ ms = value * 1e3;
2872
+ break;
2873
+ case "m":
2874
+ ms = value * 60 * 1e3;
2875
+ break;
2876
+ case "h":
2877
+ ms = value * 60 * 60 * 1e3;
2878
+ break;
2879
+ case "d":
2880
+ ms = value * 24 * 60 * 60 * 1e3;
2881
+ break;
2882
+ default:
2883
+ throw new Error(`Unknown timeout unit: "${unit}"`);
2884
+ }
2885
+ if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS2) {
2886
+ throw new Error(
2887
+ `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
2888
+ );
2889
+ }
2890
+ return ms;
2891
+ }
2685
2892
  function hashToObject(arr) {
2686
2893
  const obj = {};
2687
2894
  for (let i = 0; i < arr.length; i += 2) {
@@ -2747,9 +2954,20 @@ function deserializeJob(h) {
2747
2954
  lastCancelledAt: dateOrNull(h.lastCancelledAt),
2748
2955
  tags,
2749
2956
  idempotencyKey: nullish(h.idempotencyKey),
2750
- progress: numOrNull(h.progress)
2957
+ progress: numOrNull(h.progress),
2958
+ waitUntil: dateOrNull(h.waitUntil),
2959
+ waitTokenId: nullish(h.waitTokenId),
2960
+ stepData: parseStepData(h.stepData)
2751
2961
  };
2752
2962
  }
2963
+ function parseStepData(raw) {
2964
+ if (!raw || raw === "null") return void 0;
2965
+ try {
2966
+ return JSON.parse(raw);
2967
+ } catch {
2968
+ return void 0;
2969
+ }
2970
+ }
2753
2971
  var RedisBackend = class {
2754
2972
  constructor(redisConfig) {
2755
2973
  let IORedis;
@@ -2905,8 +3123,14 @@ var RedisBackend = class {
2905
3123
  if (filters.runAt) {
2906
3124
  jobs = this.filterByRunAt(jobs, filters.runAt);
2907
3125
  }
3126
+ if (filters.cursor !== void 0) {
3127
+ jobs = jobs.filter((j) => j.id < filters.cursor);
3128
+ }
3129
+ }
3130
+ jobs.sort((a, b) => b.id - a.id);
3131
+ if (filters?.cursor !== void 0) {
3132
+ return jobs.slice(0, limit);
2908
3133
  }
2909
- jobs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
2910
3134
  return jobs.slice(offset, offset + limit);
2911
3135
  }
2912
3136
  async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
@@ -3138,35 +3362,302 @@ var RedisBackend = class {
3138
3362
  log(`Edited ${count} pending jobs`);
3139
3363
  return count;
3140
3364
  }
3141
- async cleanupOldJobs(daysToKeep = 30) {
3365
+ /**
3366
+ * Delete completed jobs older than the given number of days.
3367
+ * Uses SSCAN to iterate the completed set in batches, avoiding
3368
+ * loading all IDs into memory and preventing long Redis blocks.
3369
+ *
3370
+ * @param daysToKeep - Number of days to retain completed jobs (default 30).
3371
+ * @param batchSize - Number of IDs to scan per SSCAN iteration (default 200).
3372
+ * @returns Total number of deleted jobs.
3373
+ */
3374
+ async cleanupOldJobs(daysToKeep = 30, batchSize = 200) {
3375
+ const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
3376
+ const setKey = `${this.prefix}status:completed`;
3377
+ let totalDeleted = 0;
3378
+ let cursor = "0";
3379
+ do {
3380
+ const [nextCursor, ids] = await this.client.sscan(
3381
+ setKey,
3382
+ cursor,
3383
+ "COUNT",
3384
+ batchSize
3385
+ );
3386
+ cursor = nextCursor;
3387
+ if (ids.length > 0) {
3388
+ const result = await this.client.eval(
3389
+ CLEANUP_OLD_JOBS_BATCH_SCRIPT,
3390
+ 1,
3391
+ this.prefix,
3392
+ cutoffMs,
3393
+ ...ids
3394
+ );
3395
+ totalDeleted += Number(result);
3396
+ }
3397
+ } while (cursor !== "0");
3398
+ log(`Deleted ${totalDeleted} old jobs`);
3399
+ return totalDeleted;
3400
+ }
3401
+ /**
3402
+ * Delete job events older than the given number of days.
3403
+ * Iterates all event lists and removes events whose createdAt is before the cutoff.
3404
+ * Also removes orphaned event lists (where the job no longer exists).
3405
+ *
3406
+ * @param daysToKeep - Number of days to retain events (default 30).
3407
+ * @param batchSize - Number of event keys to scan per SCAN iteration (default 200).
3408
+ * @returns Total number of deleted events.
3409
+ */
3410
+ async cleanupOldJobEvents(daysToKeep = 30, batchSize = 200) {
3142
3411
  const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
3412
+ const pattern = `${this.prefix}events:*`;
3413
+ let totalDeleted = 0;
3414
+ let cursor = "0";
3415
+ do {
3416
+ const [nextCursor, keys] = await this.client.scan(
3417
+ cursor,
3418
+ "MATCH",
3419
+ pattern,
3420
+ "COUNT",
3421
+ batchSize
3422
+ );
3423
+ cursor = nextCursor;
3424
+ for (const key of keys) {
3425
+ const jobIdStr = key.slice(`${this.prefix}events:`.length);
3426
+ const jobExists = await this.client.exists(
3427
+ `${this.prefix}job:${jobIdStr}`
3428
+ );
3429
+ if (!jobExists) {
3430
+ const len = await this.client.llen(key);
3431
+ await this.client.del(key);
3432
+ totalDeleted += len;
3433
+ continue;
3434
+ }
3435
+ const events = await this.client.lrange(key, 0, -1);
3436
+ const kept = [];
3437
+ for (const raw of events) {
3438
+ try {
3439
+ const e = JSON.parse(raw);
3440
+ if (e.createdAt >= cutoffMs) {
3441
+ kept.push(raw);
3442
+ } else {
3443
+ totalDeleted++;
3444
+ }
3445
+ } catch {
3446
+ totalDeleted++;
3447
+ }
3448
+ }
3449
+ if (kept.length === 0) {
3450
+ await this.client.del(key);
3451
+ } else if (kept.length < events.length) {
3452
+ const pipeline = this.client.pipeline();
3453
+ pipeline.del(key);
3454
+ for (const raw of kept) {
3455
+ pipeline.rpush(key, raw);
3456
+ }
3457
+ await pipeline.exec();
3458
+ }
3459
+ }
3460
+ } while (cursor !== "0");
3461
+ log(`Deleted ${totalDeleted} old job events`);
3462
+ return totalDeleted;
3463
+ }
3464
+ async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
3465
+ const maxAgeMs = maxProcessingTimeMinutes * 60 * 1e3;
3466
+ const now = this.nowMs();
3143
3467
  const result = await this.client.eval(
3144
- CLEANUP_OLD_JOBS_SCRIPT,
3468
+ RECLAIM_STUCK_JOBS_SCRIPT,
3145
3469
  1,
3146
3470
  this.prefix,
3147
- cutoffMs
3471
+ maxAgeMs,
3472
+ now
3148
3473
  );
3149
- log(`Deleted ${result} old jobs`);
3474
+ log(`Reclaimed ${result} stuck jobs`);
3150
3475
  return Number(result);
3151
3476
  }
3152
- async cleanupOldJobEvents(daysToKeep = 30) {
3153
- log(
3154
- `cleanupOldJobEvents is a no-op for Redis backend (events are cleaned up with their jobs)`
3477
+ // ── Wait / step-data support ────────────────────────────────────────
3478
+ /**
3479
+ * Transition a job from 'processing' to 'waiting' status.
3480
+ * Persists step data so the handler can resume from where it left off.
3481
+ *
3482
+ * @param jobId - The job to pause.
3483
+ * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
3484
+ */
3485
+ async waitJob(jobId, options) {
3486
+ const now = this.nowMs();
3487
+ const waitUntilMs = options.waitUntil ? options.waitUntil.getTime().toString() : "null";
3488
+ const waitTokenId = options.waitTokenId ?? "null";
3489
+ const stepDataJson = JSON.stringify(options.stepData);
3490
+ const result = await this.client.eval(
3491
+ WAIT_JOB_SCRIPT,
3492
+ 1,
3493
+ this.prefix,
3494
+ jobId,
3495
+ waitUntilMs,
3496
+ waitTokenId,
3497
+ stepDataJson,
3498
+ now
3155
3499
  );
3156
- return 0;
3500
+ if (Number(result) === 0) {
3501
+ log(
3502
+ `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
3503
+ );
3504
+ return;
3505
+ }
3506
+ await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
3507
+ waitUntil: options.waitUntil?.toISOString() ?? null,
3508
+ waitTokenId: options.waitTokenId ?? null
3509
+ });
3510
+ log(`Job ${jobId} set to waiting`);
3157
3511
  }
3158
- async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
3159
- const maxAgeMs = maxProcessingTimeMinutes * 60 * 1e3;
3512
+ /**
3513
+ * Persist step data for a job. Called after each ctx.run() step completes.
3514
+ * Best-effort: does not throw to avoid killing the running handler.
3515
+ *
3516
+ * @param jobId - The job to update.
3517
+ * @param stepData - The step data to persist.
3518
+ */
3519
+ async updateStepData(jobId, stepData) {
3520
+ try {
3521
+ const now = this.nowMs();
3522
+ await this.client.hset(
3523
+ `${this.prefix}job:${jobId}`,
3524
+ "stepData",
3525
+ JSON.stringify(stepData),
3526
+ "updatedAt",
3527
+ now.toString()
3528
+ );
3529
+ } catch (error) {
3530
+ log(`Error updating stepData for job ${jobId}: ${error}`);
3531
+ }
3532
+ }
3533
+ /**
3534
+ * Create a waitpoint token.
3535
+ *
3536
+ * @param jobId - The job ID to associate with the token (null if created outside a handler).
3537
+ * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
3538
+ * @returns The created waitpoint with its unique ID.
3539
+ */
3540
+ async createWaitpoint(jobId, options) {
3541
+ const id = `wp_${crypto.randomUUID()}`;
3542
+ const now = this.nowMs();
3543
+ let timeoutAt = null;
3544
+ if (options?.timeout) {
3545
+ const ms = parseTimeoutString2(options.timeout);
3546
+ timeoutAt = now + ms;
3547
+ }
3548
+ const key = `${this.prefix}waitpoint:${id}`;
3549
+ const fields = [
3550
+ "id",
3551
+ id,
3552
+ "jobId",
3553
+ jobId !== null ? jobId.toString() : "null",
3554
+ "status",
3555
+ "waiting",
3556
+ "output",
3557
+ "null",
3558
+ "timeoutAt",
3559
+ timeoutAt !== null ? timeoutAt.toString() : "null",
3560
+ "createdAt",
3561
+ now.toString(),
3562
+ "completedAt",
3563
+ "null",
3564
+ "tags",
3565
+ options?.tags ? JSON.stringify(options.tags) : "null"
3566
+ ];
3567
+ await this.client.hmset(key, ...fields);
3568
+ if (timeoutAt !== null) {
3569
+ await this.client.zadd(`${this.prefix}waitpoint_timeout`, timeoutAt, id);
3570
+ }
3571
+ log(`Created waitpoint ${id} for job ${jobId}`);
3572
+ return { id };
3573
+ }
3574
+ /**
3575
+ * Complete a waitpoint token and move the associated job back to 'pending'.
3576
+ *
3577
+ * @param tokenId - The waitpoint token ID to complete.
3578
+ * @param data - Optional data to pass to the waiting handler.
3579
+ */
3580
+ async completeWaitpoint(tokenId, data) {
3160
3581
  const now = this.nowMs();
3582
+ const outputJson = data != null ? JSON.stringify(data) : "null";
3161
3583
  const result = await this.client.eval(
3162
- RECLAIM_STUCK_JOBS_SCRIPT,
3584
+ COMPLETE_WAITPOINT_SCRIPT,
3163
3585
  1,
3164
3586
  this.prefix,
3165
- maxAgeMs,
3587
+ tokenId,
3588
+ outputJson,
3166
3589
  now
3167
3590
  );
3168
- log(`Reclaimed ${result} stuck jobs`);
3169
- return Number(result);
3591
+ if (Number(result) === 0) {
3592
+ log(`Waitpoint ${tokenId} not found or already completed`);
3593
+ return;
3594
+ }
3595
+ log(`Completed waitpoint ${tokenId}`);
3596
+ }
3597
+ /**
3598
+ * Retrieve a waitpoint token by its ID.
3599
+ *
3600
+ * @param tokenId - The waitpoint token ID to look up.
3601
+ * @returns The waitpoint record, or null if not found.
3602
+ */
3603
+ async getWaitpoint(tokenId) {
3604
+ const data = await this.client.hgetall(
3605
+ `${this.prefix}waitpoint:${tokenId}`
3606
+ );
3607
+ if (!data || Object.keys(data).length === 0) return null;
3608
+ const nullish = (v) => v === void 0 || v === "null" || v === "" ? null : v;
3609
+ const numOrNull = (v) => {
3610
+ const n = nullish(v);
3611
+ return n === null ? null : Number(n);
3612
+ };
3613
+ const dateOrNull = (v) => {
3614
+ const n = numOrNull(v);
3615
+ return n === null ? null : new Date(n);
3616
+ };
3617
+ let output = null;
3618
+ if (data.output && data.output !== "null") {
3619
+ try {
3620
+ output = JSON.parse(data.output);
3621
+ } catch {
3622
+ output = data.output;
3623
+ }
3624
+ }
3625
+ let tags = null;
3626
+ if (data.tags && data.tags !== "null") {
3627
+ try {
3628
+ tags = JSON.parse(data.tags);
3629
+ } catch {
3630
+ }
3631
+ }
3632
+ return {
3633
+ id: data.id,
3634
+ jobId: numOrNull(data.jobId),
3635
+ status: data.status,
3636
+ output,
3637
+ timeoutAt: dateOrNull(data.timeoutAt),
3638
+ createdAt: new Date(Number(data.createdAt)),
3639
+ completedAt: dateOrNull(data.completedAt),
3640
+ tags
3641
+ };
3642
+ }
3643
+ /**
3644
+ * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
3645
+ *
3646
+ * @returns The number of tokens that were expired.
3647
+ */
3648
+ async expireTimedOutWaitpoints() {
3649
+ const now = this.nowMs();
3650
+ const result = await this.client.eval(
3651
+ EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT,
3652
+ 1,
3653
+ this.prefix,
3654
+ now
3655
+ );
3656
+ const count = Number(result);
3657
+ if (count > 0) {
3658
+ log(`Expired ${count} timed-out waitpoints`);
3659
+ }
3660
+ return count;
3170
3661
  }
3171
3662
  // ── Internal helpers ──────────────────────────────────────────────────
3172
3663
  async setPendingReasonForUnpickedJobs(reason, jobType) {
@@ -3715,10 +4206,9 @@ var initJobQueue = (config) => {
3715
4206
  const backendType = config.backend ?? "postgres";
3716
4207
  setLogContext(config.verbose ?? false);
3717
4208
  let backend;
3718
- let pool;
3719
4209
  if (backendType === "postgres") {
3720
4210
  const pgConfig = config;
3721
- pool = createPool(pgConfig.databaseConfig);
4211
+ const pool = createPool(pgConfig.databaseConfig);
3722
4212
  backend = new PostgresBackend(pool);
3723
4213
  } else if (backendType === "redis") {
3724
4214
  const redisConfig = config.redisConfig;
@@ -3726,14 +4216,6 @@ var initJobQueue = (config) => {
3726
4216
  } else {
3727
4217
  throw new Error(`Unknown backend: ${backendType}`);
3728
4218
  }
3729
- const requirePool = () => {
3730
- if (!pool) {
3731
- throw new Error(
3732
- 'Wait/Token features require the PostgreSQL backend. Configure with backend: "postgres" to use these features.'
3733
- );
3734
- }
3735
- return pool;
3736
- };
3737
4219
  const enqueueDueCronJobsImpl = async () => {
3738
4220
  const dueSchedules = await backend.getDueCronSchedules();
3739
4221
  let count = 0;
@@ -3800,8 +4282,8 @@ var initJobQueue = (config) => {
3800
4282
  config.verbose ?? false
3801
4283
  ),
3802
4284
  retryJob: (jobId) => backend.retryJob(jobId),
3803
- cleanupOldJobs: (daysToKeep) => backend.cleanupOldJobs(daysToKeep),
3804
- cleanupOldJobEvents: (daysToKeep) => backend.cleanupOldJobEvents(daysToKeep),
4285
+ cleanupOldJobs: (daysToKeep, batchSize) => backend.cleanupOldJobs(daysToKeep, batchSize),
4286
+ cleanupOldJobEvents: (daysToKeep, batchSize) => backend.cleanupOldJobEvents(daysToKeep, batchSize),
3805
4287
  cancelJob: withLogContext(
3806
4288
  (jobId) => backend.cancelJob(jobId),
3807
4289
  config.verbose ?? false
@@ -3838,21 +4320,21 @@ var initJobQueue = (config) => {
3838
4320
  (jobId) => backend.getJobEvents(jobId),
3839
4321
  config.verbose ?? false
3840
4322
  ),
3841
- // Wait / Token support (PostgreSQL-only for now)
4323
+ // Wait / Token support (works with all backends)
3842
4324
  createToken: withLogContext(
3843
- (options) => createWaitpoint(requirePool(), null, options),
4325
+ (options) => backend.createWaitpoint(null, options),
3844
4326
  config.verbose ?? false
3845
4327
  ),
3846
4328
  completeToken: withLogContext(
3847
- (tokenId, data) => completeWaitpoint(requirePool(), tokenId, data),
4329
+ (tokenId, data) => backend.completeWaitpoint(tokenId, data),
3848
4330
  config.verbose ?? false
3849
4331
  ),
3850
4332
  getToken: withLogContext(
3851
- (tokenId) => getWaitpoint(requirePool(), tokenId),
4333
+ (tokenId) => backend.getWaitpoint(tokenId),
3852
4334
  config.verbose ?? false
3853
4335
  ),
3854
4336
  expireTimedOutTokens: withLogContext(
3855
- () => expireTimedOutWaitpoints(requirePool()),
4337
+ () => backend.expireTimedOutWaitpoints(),
3856
4338
  config.verbose ?? false
3857
4339
  ),
3858
4340
  // Cron schedule operations
@@ -3933,7 +4415,7 @@ var initJobQueue = (config) => {
3933
4415
  ),
3934
4416
  // Advanced access
3935
4417
  getPool: () => {
3936
- if (backendType !== "postgres") {
4418
+ if (!(backend instanceof PostgresBackend)) {
3937
4419
  throw new Error(
3938
4420
  "getPool() is only available with the PostgreSQL backend."
3939
4421
  );