@nicnocquee/dataqueue 1.31.0 → 1.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,12 +1,14 @@
1
- import { AsyncLocalStorage } from 'async_hooks';
2
- import { randomUUID } from 'crypto';
3
1
  import { Worker } from 'worker_threads';
2
+ import { AsyncLocalStorage } from 'async_hooks';
4
3
  import { Pool } from 'pg';
5
4
  import { parse } from 'pg-connection-string';
6
5
  import fs from 'fs';
6
+ import { randomUUID } from 'crypto';
7
7
  import { createRequire } from 'module';
8
8
  import { Cron } from 'croner';
9
9
 
10
+ // src/processor.ts
11
+
10
12
  // src/types.ts
11
13
  var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
12
14
  JobEventType2["Added"] = "added";
@@ -20,11 +22,11 @@ var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
20
22
  JobEventType2["Waiting"] = "waiting";
21
23
  return JobEventType2;
22
24
  })(JobEventType || {});
23
- var FailureReason = /* @__PURE__ */ ((FailureReason5) => {
24
- FailureReason5["Timeout"] = "timeout";
25
- FailureReason5["HandlerError"] = "handler_error";
26
- FailureReason5["NoHandler"] = "no_handler";
27
- return FailureReason5;
25
+ var FailureReason = /* @__PURE__ */ ((FailureReason4) => {
26
+ FailureReason4["Timeout"] = "timeout";
27
+ FailureReason4["HandlerError"] = "handler_error";
28
+ FailureReason4["NoHandler"] = "no_handler";
29
+ return FailureReason4;
28
30
  })(FailureReason || {});
29
31
  var WaitSignal = class extends Error {
30
32
  constructor(type, waitUntil, tokenId, stepData) {
@@ -51,804 +53,942 @@ var log = (message) => {
51
53
  }
52
54
  };
53
55
 
54
- // src/backends/postgres.ts
55
- var PostgresBackend = class {
56
- constructor(pool) {
57
- this.pool = pool;
58
- }
59
- /** Expose the raw pool for advanced usage. */
60
- getPool() {
61
- return this.pool;
62
- }
63
- // ── Events ──────────────────────────────────────────────────────────
64
- async recordJobEvent(jobId, eventType, metadata) {
65
- const client = await this.pool.connect();
66
- try {
67
- await client.query(
68
- `INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
69
- [jobId, eventType, metadata ? JSON.stringify(metadata) : null]
70
- );
71
- } catch (error) {
72
- log(`Error recording job event for job ${jobId}: ${error}`);
73
- } finally {
74
- client.release();
75
- }
76
- }
77
- async getJobEvents(jobId) {
78
- const client = await this.pool.connect();
79
- try {
80
- const res = await client.query(
81
- `SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
82
- [jobId]
83
- );
84
- return res.rows;
85
- } finally {
86
- client.release();
87
- }
88
- }
89
- // ── Job CRUD ──────────────────────────────────────────────────────────
90
- async addJob({
91
- jobType,
92
- payload,
93
- maxAttempts = 3,
94
- priority = 0,
95
- runAt = null,
96
- timeoutMs = void 0,
97
- forceKillOnTimeout = false,
98
- tags = void 0,
99
- idempotencyKey = void 0
100
- }) {
101
- const client = await this.pool.connect();
102
- try {
103
- let result;
104
- const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
105
- if (runAt) {
106
- result = await client.query(
107
- `INSERT INTO job_queue
108
- (job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
109
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
110
- ${onConflict}
111
- RETURNING id`,
112
- [
113
- jobType,
114
- payload,
115
- maxAttempts,
116
- priority,
117
- runAt,
118
- timeoutMs ?? null,
119
- forceKillOnTimeout ?? false,
120
- tags ?? null,
121
- idempotencyKey ?? null
122
- ]
123
- );
124
- } else {
125
- result = await client.query(
126
- `INSERT INTO job_queue
127
- (job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
128
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
129
- ${onConflict}
130
- RETURNING id`,
131
- [
132
- jobType,
133
- payload,
134
- maxAttempts,
135
- priority,
136
- timeoutMs ?? null,
137
- forceKillOnTimeout ?? false,
138
- tags ?? null,
139
- idempotencyKey ?? null
140
- ]
141
- );
142
- }
143
- if (result.rows.length === 0 && idempotencyKey) {
144
- const existing = await client.query(
145
- `SELECT id FROM job_queue WHERE idempotency_key = $1`,
146
- [idempotencyKey]
147
- );
148
- if (existing.rows.length > 0) {
149
- log(
150
- `Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
151
- );
152
- return existing.rows[0].id;
153
- }
154
- throw new Error(
155
- `Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
156
- );
157
- }
158
- const jobId = result.rows[0].id;
159
- log(
160
- `Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
56
+ // src/processor.ts
57
+ function validateHandlerSerializable(handler, jobType) {
58
+ try {
59
+ const handlerString = handler.toString();
60
+ if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
61
+ throw new Error(
62
+ `Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
161
63
  );
162
- await this.recordJobEvent(jobId, "added" /* Added */, {
163
- jobType,
164
- payload,
165
- tags,
166
- idempotencyKey
167
- });
168
- return jobId;
169
- } catch (error) {
170
- log(`Error adding job: ${error}`);
171
- throw error;
172
- } finally {
173
- client.release();
174
64
  }
175
- }
176
- async getJob(id) {
177
- const client = await this.pool.connect();
178
- try {
179
- const result = await client.query(
180
- `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
181
- [id]
65
+ if (handlerString.includes("[native code]")) {
66
+ throw new Error(
67
+ `Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
182
68
  );
183
- if (result.rows.length === 0) {
184
- log(`Job ${id} not found`);
185
- return null;
186
- }
187
- log(`Found job ${id}`);
188
- const job = result.rows[0];
189
- return {
190
- ...job,
191
- payload: job.payload,
192
- timeoutMs: job.timeoutMs,
193
- forceKillOnTimeout: job.forceKillOnTimeout,
194
- failureReason: job.failureReason
195
- };
196
- } catch (error) {
197
- log(`Error getting job ${id}: ${error}`);
198
- throw error;
199
- } finally {
200
- client.release();
201
69
  }
202
- }
203
- async getJobsByStatus(status, limit = 100, offset = 0) {
204
- const client = await this.pool.connect();
205
70
  try {
206
- const result = await client.query(
207
- `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
208
- [status, limit, offset]
71
+ new Function("return " + handlerString);
72
+ } catch (parseError) {
73
+ throw new Error(
74
+ `Handler for job type "${jobType}" cannot be serialized: ${parseError instanceof Error ? parseError.message : String(parseError)}. When using forceKillOnTimeout, handlers must be serializable functions without closures over external variables.`
209
75
  );
210
- log(`Found ${result.rows.length} jobs by status ${status}`);
211
- return result.rows.map((job) => ({
212
- ...job,
213
- payload: job.payload,
214
- timeoutMs: job.timeoutMs,
215
- forceKillOnTimeout: job.forceKillOnTimeout,
216
- failureReason: job.failureReason
217
- }));
218
- } catch (error) {
219
- log(`Error getting jobs by status ${status}: ${error}`);
220
- throw error;
221
- } finally {
222
- client.release();
223
76
  }
224
- }
225
- async getAllJobs(limit = 100, offset = 0) {
226
- const client = await this.pool.connect();
227
- try {
228
- const result = await client.query(
229
- `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
230
- [limit, offset]
231
- );
232
- log(`Found ${result.rows.length} jobs (all)`);
233
- return result.rows.map((job) => ({
234
- ...job,
235
- payload: job.payload,
236
- timeoutMs: job.timeoutMs,
237
- forceKillOnTimeout: job.forceKillOnTimeout
238
- }));
239
- } catch (error) {
240
- log(`Error getting all jobs: ${error}`);
77
+ } catch (error) {
78
+ if (error instanceof Error) {
241
79
  throw error;
242
- } finally {
243
- client.release();
244
80
  }
81
+ throw new Error(
82
+ `Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
83
+ );
245
84
  }
246
- async getJobs(filters, limit = 100, offset = 0) {
247
- const client = await this.pool.connect();
248
- try {
249
- let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue`;
250
- const params = [];
251
- const where = [];
252
- let paramIdx = 1;
253
- if (filters) {
254
- if (filters.jobType) {
255
- where.push(`job_type = $${paramIdx++}`);
256
- params.push(filters.jobType);
257
- }
258
- if (filters.priority !== void 0) {
259
- where.push(`priority = $${paramIdx++}`);
260
- params.push(filters.priority);
261
- }
262
- if (filters.runAt) {
263
- if (filters.runAt instanceof Date) {
264
- where.push(`run_at = $${paramIdx++}`);
265
- params.push(filters.runAt);
266
- } else if (typeof filters.runAt === "object" && (filters.runAt.gt !== void 0 || filters.runAt.gte !== void 0 || filters.runAt.lt !== void 0 || filters.runAt.lte !== void 0 || filters.runAt.eq !== void 0)) {
267
- const ops = filters.runAt;
268
- if (ops.gt) {
269
- where.push(`run_at > $${paramIdx++}`);
270
- params.push(ops.gt);
271
- }
272
- if (ops.gte) {
273
- where.push(`run_at >= $${paramIdx++}`);
274
- params.push(ops.gte);
275
- }
276
- if (ops.lt) {
277
- where.push(`run_at < $${paramIdx++}`);
278
- params.push(ops.lt);
279
- }
280
- if (ops.lte) {
281
- where.push(`run_at <= $${paramIdx++}`);
282
- params.push(ops.lte);
283
- }
284
- if (ops.eq) {
285
- where.push(`run_at = $${paramIdx++}`);
286
- params.push(ops.eq);
287
- }
288
- }
289
- }
290
- if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
291
- const mode = filters.tags.mode || "all";
292
- const tagValues = filters.tags.values;
293
- switch (mode) {
294
- case "exact":
295
- where.push(`tags = $${paramIdx++}`);
296
- params.push(tagValues);
297
- break;
298
- case "all":
299
- where.push(`tags @> $${paramIdx++}`);
300
- params.push(tagValues);
301
- break;
302
- case "any":
303
- where.push(`tags && $${paramIdx++}`);
304
- params.push(tagValues);
305
- break;
306
- case "none":
307
- where.push(`NOT (tags && $${paramIdx++})`);
308
- params.push(tagValues);
309
- break;
310
- default:
311
- where.push(`tags @> $${paramIdx++}`);
312
- params.push(tagValues);
85
+ }
86
+ async function runHandlerInWorker(handler, payload, timeoutMs, jobType) {
87
+ validateHandlerSerializable(handler, jobType);
88
+ return new Promise((resolve, reject) => {
89
+ const workerCode = `
90
+ (function() {
91
+ const { parentPort, workerData } = require('worker_threads');
92
+ const { handlerCode, payload, timeoutMs } = workerData;
93
+
94
+ // Create an AbortController for the handler
95
+ const controller = new AbortController();
96
+ const signal = controller.signal;
97
+
98
+ // Set up timeout
99
+ const timeoutId = setTimeout(() => {
100
+ controller.abort();
101
+ parentPort.postMessage({ type: 'timeout' });
102
+ }, timeoutMs);
103
+
104
+ try {
105
+ // Execute the handler
106
+ // Note: This uses Function constructor which requires the handler to be serializable.
107
+ // The handler should be validated before reaching this point.
108
+ let handlerFn;
109
+ try {
110
+ // Wrap handlerCode in parentheses to ensure it's treated as an expression
111
+ // This handles both arrow functions and regular functions
112
+ const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
113
+ ? handlerCode
114
+ : '(' + handlerCode + ')';
115
+ handlerFn = new Function('return ' + wrappedCode)();
116
+ } catch (parseError) {
117
+ clearTimeout(timeoutId);
118
+ parentPort.postMessage({
119
+ type: 'error',
120
+ error: {
121
+ message: 'Handler cannot be deserialized in worker thread. ' +
122
+ 'Ensure your handler is a standalone function without closures over external variables. ' +
123
+ 'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
124
+ stack: parseError instanceof Error ? parseError.stack : undefined,
125
+ name: 'SerializationError',
126
+ },
127
+ });
128
+ return;
313
129
  }
130
+
131
+ // Ensure handlerFn is actually a function
132
+ if (typeof handlerFn !== 'function') {
133
+ clearTimeout(timeoutId);
134
+ parentPort.postMessage({
135
+ type: 'error',
136
+ error: {
137
+ message: 'Handler deserialization did not produce a function. ' +
138
+ 'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
139
+ name: 'SerializationError',
140
+ },
141
+ });
142
+ return;
143
+ }
144
+
145
+ handlerFn(payload, signal)
146
+ .then(() => {
147
+ clearTimeout(timeoutId);
148
+ parentPort.postMessage({ type: 'success' });
149
+ })
150
+ .catch((error) => {
151
+ clearTimeout(timeoutId);
152
+ parentPort.postMessage({
153
+ type: 'error',
154
+ error: {
155
+ message: error.message,
156
+ stack: error.stack,
157
+ name: error.name,
158
+ },
159
+ });
160
+ });
161
+ } catch (error) {
162
+ clearTimeout(timeoutId);
163
+ parentPort.postMessage({
164
+ type: 'error',
165
+ error: {
166
+ message: error.message,
167
+ stack: error.stack,
168
+ name: error.name,
169
+ },
170
+ });
314
171
  }
315
- if (filters.cursor !== void 0) {
316
- where.push(`id < $${paramIdx++}`);
317
- params.push(filters.cursor);
318
- }
172
+ })();
173
+ `;
174
+ const worker = new Worker(workerCode, {
175
+ eval: true,
176
+ workerData: {
177
+ handlerCode: handler.toString(),
178
+ payload,
179
+ timeoutMs
319
180
  }
320
- if (where.length > 0) {
321
- query += ` WHERE ${where.join(" AND ")}`;
181
+ });
182
+ let resolved = false;
183
+ worker.on("message", (message) => {
184
+ if (resolved) return;
185
+ resolved = true;
186
+ if (message.type === "success") {
187
+ resolve();
188
+ } else if (message.type === "timeout") {
189
+ const timeoutError = new Error(
190
+ `Job timed out after ${timeoutMs} ms and was forcefully terminated`
191
+ );
192
+ timeoutError.failureReason = "timeout" /* Timeout */;
193
+ reject(timeoutError);
194
+ } else if (message.type === "error") {
195
+ const error = new Error(message.error.message);
196
+ error.stack = message.error.stack;
197
+ error.name = message.error.name;
198
+ reject(error);
322
199
  }
323
- paramIdx = params.length + 1;
324
- query += ` ORDER BY id DESC LIMIT $${paramIdx++}`;
325
- if (!filters?.cursor) {
326
- query += ` OFFSET $${paramIdx}`;
327
- params.push(limit, offset);
328
- } else {
329
- params.push(limit);
200
+ });
201
+ worker.on("error", (error) => {
202
+ if (resolved) return;
203
+ resolved = true;
204
+ reject(error);
205
+ });
206
+ worker.on("exit", (code) => {
207
+ if (resolved) return;
208
+ if (code !== 0) {
209
+ resolved = true;
210
+ reject(new Error(`Worker stopped with exit code ${code}`));
330
211
  }
331
- const result = await client.query(query, params);
332
- log(`Found ${result.rows.length} jobs`);
333
- return result.rows.map((job) => ({
334
- ...job,
335
- payload: job.payload,
336
- timeoutMs: job.timeoutMs,
337
- forceKillOnTimeout: job.forceKillOnTimeout,
338
- failureReason: job.failureReason
339
- }));
340
- } catch (error) {
341
- log(`Error getting jobs: ${error}`);
342
- throw error;
343
- } finally {
344
- client.release();
345
- }
212
+ });
213
+ setTimeout(() => {
214
+ if (!resolved) {
215
+ resolved = true;
216
+ worker.terminate().then(() => {
217
+ const timeoutError = new Error(
218
+ `Job timed out after ${timeoutMs} ms and was forcefully terminated`
219
+ );
220
+ timeoutError.failureReason = "timeout" /* Timeout */;
221
+ reject(timeoutError);
222
+ }).catch((err) => {
223
+ reject(err);
224
+ });
225
+ }
226
+ }, timeoutMs + 100);
227
+ });
228
+ }
229
+ function calculateWaitUntil(duration) {
230
+ const now = Date.now();
231
+ let ms = 0;
232
+ if (duration.seconds) ms += duration.seconds * 1e3;
233
+ if (duration.minutes) ms += duration.minutes * 60 * 1e3;
234
+ if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
235
+ if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
236
+ if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
237
+ if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
238
+ if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
239
+ if (ms <= 0) {
240
+ throw new Error(
241
+ "waitFor duration must be positive. Provide at least one positive duration field."
242
+ );
346
243
  }
347
- async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
348
- const client = await this.pool.connect();
349
- try {
350
- let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
351
- FROM job_queue`;
352
- let params = [];
353
- switch (mode) {
354
- case "exact":
355
- query += " WHERE tags = $1";
356
- params = [tags];
357
- break;
358
- case "all":
359
- query += " WHERE tags @> $1";
360
- params = [tags];
361
- break;
362
- case "any":
363
- query += " WHERE tags && $1";
364
- params = [tags];
365
- break;
366
- case "none":
367
- query += " WHERE NOT (tags && $1)";
368
- params = [tags];
369
- break;
370
- default:
371
- query += " WHERE tags @> $1";
372
- params = [tags];
244
+ return new Date(now + ms);
245
+ }
246
+ async function resolveCompletedWaits(backend, stepData) {
247
+ for (const key of Object.keys(stepData)) {
248
+ if (!key.startsWith("__wait_")) continue;
249
+ const entry = stepData[key];
250
+ if (!entry || typeof entry !== "object" || entry.completed) continue;
251
+ if (entry.type === "duration" || entry.type === "date") {
252
+ stepData[key] = { ...entry, completed: true };
253
+ } else if (entry.type === "token" && entry.tokenId) {
254
+ const wp = await backend.getWaitpoint(entry.tokenId);
255
+ if (wp && wp.status === "completed") {
256
+ stepData[key] = {
257
+ ...entry,
258
+ completed: true,
259
+ result: { ok: true, output: wp.output }
260
+ };
261
+ } else if (wp && wp.status === "timed_out") {
262
+ stepData[key] = {
263
+ ...entry,
264
+ completed: true,
265
+ result: { ok: false, error: "Token timed out" }
266
+ };
373
267
  }
374
- query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
375
- params.push(limit, offset);
376
- const result = await client.query(query, params);
377
- log(
378
- `Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
379
- );
380
- return result.rows.map((job) => ({
381
- ...job,
382
- payload: job.payload,
383
- timeoutMs: job.timeoutMs,
384
- forceKillOnTimeout: job.forceKillOnTimeout,
385
- failureReason: job.failureReason
386
- }));
387
- } catch (error) {
388
- log(
389
- `Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
390
- );
391
- throw error;
392
- } finally {
393
- client.release();
394
268
  }
395
269
  }
396
- // ── Processing lifecycle ──────────────────────────────────────────────
397
- async getNextBatch(workerId, batchSize = 10, jobType) {
398
- const client = await this.pool.connect();
399
- try {
400
- await client.query("BEGIN");
401
- let jobTypeFilter = "";
402
- const params = [workerId, batchSize];
403
- if (jobType) {
404
- if (Array.isArray(jobType)) {
405
- jobTypeFilter = ` AND job_type = ANY($3)`;
406
- params.push(jobType);
407
- } else {
408
- jobTypeFilter = ` AND job_type = $3`;
409
- params.push(jobType);
410
- }
270
+ }
271
+ function buildWaitContext(backend, jobId, stepData, baseCtx) {
272
+ let waitCounter = 0;
273
+ const ctx = {
274
+ prolong: baseCtx.prolong,
275
+ onTimeout: baseCtx.onTimeout,
276
+ run: async (stepName, fn) => {
277
+ const cached = stepData[stepName];
278
+ if (cached && typeof cached === "object" && cached.__completed) {
279
+ log(`Step "${stepName}" replayed from cache for job ${jobId}`);
280
+ return cached.result;
411
281
  }
412
- const result = await client.query(
413
- `
414
- UPDATE job_queue
415
- SET status = 'processing',
416
- locked_at = NOW(),
417
- locked_by = $1,
418
- attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
419
- updated_at = NOW(),
420
- pending_reason = NULL,
421
- started_at = COALESCE(started_at, NOW()),
422
- last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
423
- wait_until = NULL
424
- WHERE id IN (
425
- SELECT id FROM job_queue
426
- WHERE (
427
- (
428
- (status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
429
- AND (attempts < max_attempts)
430
- AND run_at <= NOW()
431
- )
432
- OR (
433
- status = 'waiting'
434
- AND wait_until IS NOT NULL
435
- AND wait_until <= NOW()
436
- AND wait_token_id IS NULL
437
- )
438
- )
439
- ${jobTypeFilter}
440
- ORDER BY priority DESC, created_at ASC
441
- LIMIT $2
442
- FOR UPDATE SKIP LOCKED
443
- )
444
- RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
445
- `,
446
- params
447
- );
448
- log(`Found ${result.rows.length} jobs to process`);
449
- await client.query("COMMIT");
450
- if (result.rows.length > 0) {
451
- await this.recordJobEventsBatch(
452
- result.rows.map((row) => ({
453
- jobId: row.id,
454
- eventType: "processing" /* Processing */
455
- }))
456
- );
282
+ const result = await fn();
283
+ stepData[stepName] = { __completed: true, result };
284
+ await backend.updateStepData(jobId, stepData);
285
+ return result;
286
+ },
287
+ waitFor: async (duration) => {
288
+ const waitKey = `__wait_${waitCounter++}`;
289
+ const cached = stepData[waitKey];
290
+ if (cached && typeof cached === "object" && cached.completed) {
291
+ log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
292
+ return;
457
293
  }
458
- return result.rows.map((job) => ({
459
- ...job,
460
- payload: job.payload,
461
- timeoutMs: job.timeoutMs,
462
- forceKillOnTimeout: job.forceKillOnTimeout
463
- }));
464
- } catch (error) {
465
- log(`Error getting next batch: ${error}`);
466
- await client.query("ROLLBACK");
467
- throw error;
468
- } finally {
469
- client.release();
470
- }
471
- }
472
- async completeJob(jobId) {
473
- const client = await this.pool.connect();
474
- try {
475
- const result = await client.query(
476
- `
477
- UPDATE job_queue
478
- SET status = 'completed', updated_at = NOW(), completed_at = NOW(),
479
- step_data = NULL, wait_until = NULL, wait_token_id = NULL
480
- WHERE id = $1 AND status = 'processing'
481
- `,
482
- [jobId]
483
- );
484
- if (result.rowCount === 0) {
294
+ const waitUntilDate = calculateWaitUntil(duration);
295
+ stepData[waitKey] = { type: "duration", completed: false };
296
+ throw new WaitSignal("duration", waitUntilDate, void 0, stepData);
297
+ },
298
+ waitUntil: async (date) => {
299
+ const waitKey = `__wait_${waitCounter++}`;
300
+ const cached = stepData[waitKey];
301
+ if (cached && typeof cached === "object" && cached.completed) {
302
+ log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
303
+ return;
304
+ }
305
+ stepData[waitKey] = { type: "date", completed: false };
306
+ throw new WaitSignal("date", date, void 0, stepData);
307
+ },
308
+ createToken: async (options) => {
309
+ const token = await backend.createWaitpoint(jobId, options);
310
+ return token;
311
+ },
312
+ waitForToken: async (tokenId) => {
313
+ const waitKey = `__wait_${waitCounter++}`;
314
+ const cached = stepData[waitKey];
315
+ if (cached && typeof cached === "object" && cached.completed) {
485
316
  log(
486
- `Job ${jobId} could not be completed (not in processing state or does not exist)`
317
+ `Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
487
318
  );
319
+ return cached.result;
488
320
  }
489
- await this.recordJobEvent(jobId, "completed" /* Completed */);
490
- log(`Completed job ${jobId}`);
491
- } catch (error) {
492
- log(`Error completing job ${jobId}: ${error}`);
493
- throw error;
494
- } finally {
495
- client.release();
321
+ const wp = await backend.getWaitpoint(tokenId);
322
+ if (wp && wp.status === "completed") {
323
+ const result = {
324
+ ok: true,
325
+ output: wp.output
326
+ };
327
+ stepData[waitKey] = {
328
+ type: "token",
329
+ tokenId,
330
+ completed: true,
331
+ result
332
+ };
333
+ await backend.updateStepData(jobId, stepData);
334
+ return result;
335
+ }
336
+ if (wp && wp.status === "timed_out") {
337
+ const result = {
338
+ ok: false,
339
+ error: "Token timed out"
340
+ };
341
+ stepData[waitKey] = {
342
+ type: "token",
343
+ tokenId,
344
+ completed: true,
345
+ result
346
+ };
347
+ await backend.updateStepData(jobId, stepData);
348
+ return result;
349
+ }
350
+ stepData[waitKey] = { type: "token", tokenId, completed: false };
351
+ throw new WaitSignal("token", void 0, tokenId, stepData);
352
+ },
353
+ setProgress: async (percent) => {
354
+ if (percent < 0 || percent > 100)
355
+ throw new Error("Progress must be between 0 and 100");
356
+ await backend.updateProgress(jobId, Math.round(percent));
496
357
  }
358
+ };
359
+ return ctx;
360
+ }
361
+ async function processJobWithHandlers(backend, job, jobHandlers) {
362
+ const handler = jobHandlers[job.jobType];
363
+ if (!handler) {
364
+ await backend.setPendingReasonForUnpickedJobs(
365
+ `No handler registered for job type: ${job.jobType}`,
366
+ job.jobType
367
+ );
368
+ await backend.failJob(
369
+ job.id,
370
+ new Error(`No handler registered for job type: ${job.jobType}`),
371
+ "no_handler" /* NoHandler */
372
+ );
373
+ return;
497
374
  }
498
- async failJob(jobId, error, failureReason) {
499
- const client = await this.pool.connect();
500
- try {
501
- const result = await client.query(
502
- `
503
- UPDATE job_queue
504
- SET status = 'failed',
505
- updated_at = NOW(),
506
- next_attempt_at = CASE
507
- WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
508
- ELSE NULL
509
- END,
510
- error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
511
- failure_reason = $3,
512
- last_failed_at = NOW()
513
- WHERE id = $1 AND status IN ('processing', 'pending')
514
- `,
515
- [
516
- jobId,
517
- JSON.stringify([
518
- {
519
- message: error.message || String(error),
520
- timestamp: (/* @__PURE__ */ new Date()).toISOString()
375
+ const stepData = { ...job.stepData || {} };
376
+ const hasStepHistory = Object.keys(stepData).some(
377
+ (k) => k.startsWith("__wait_")
378
+ );
379
+ if (hasStepHistory) {
380
+ await resolveCompletedWaits(backend, stepData);
381
+ await backend.updateStepData(job.id, stepData);
382
+ }
383
+ const timeoutMs = job.timeoutMs ?? void 0;
384
+ const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
385
+ let timeoutId;
386
+ const controller = new AbortController();
387
+ try {
388
+ if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
389
+ await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
390
+ } else {
391
+ let onTimeoutCallback;
392
+ let timeoutReject;
393
+ const armTimeout = (ms) => {
394
+ if (timeoutId) clearTimeout(timeoutId);
395
+ timeoutId = setTimeout(() => {
396
+ if (onTimeoutCallback) {
397
+ try {
398
+ const extension = onTimeoutCallback();
399
+ if (typeof extension === "number" && extension > 0) {
400
+ backend.prolongJob(job.id).catch(() => {
401
+ });
402
+ armTimeout(extension);
403
+ return;
404
+ }
405
+ } catch (callbackError) {
406
+ log(
407
+ `onTimeout callback threw for job ${job.id}: ${callbackError}`
408
+ );
521
409
  }
522
- ]),
523
- failureReason ?? null
524
- ]
525
- );
526
- if (result.rowCount === 0) {
410
+ }
411
+ controller.abort();
412
+ const timeoutError = new Error(`Job timed out after ${ms} ms`);
413
+ timeoutError.failureReason = "timeout" /* Timeout */;
414
+ if (timeoutReject) {
415
+ timeoutReject(timeoutError);
416
+ }
417
+ }, ms);
418
+ };
419
+ const hasTimeout = timeoutMs != null && timeoutMs > 0;
420
+ const baseCtx = hasTimeout ? {
421
+ prolong: (ms) => {
422
+ const duration = ms ?? timeoutMs;
423
+ if (duration != null && duration > 0) {
424
+ armTimeout(duration);
425
+ backend.prolongJob(job.id).catch(() => {
426
+ });
427
+ }
428
+ },
429
+ onTimeout: (callback) => {
430
+ onTimeoutCallback = callback;
431
+ }
432
+ } : {
433
+ prolong: () => {
434
+ log("prolong() called but ignored: job has no timeout set");
435
+ },
436
+ onTimeout: () => {
437
+ log("onTimeout() called but ignored: job has no timeout set");
438
+ }
439
+ };
440
+ const ctx = buildWaitContext(backend, job.id, stepData, baseCtx);
441
+ if (forceKillOnTimeout && !hasTimeout) {
527
442
  log(
528
- `Job ${jobId} could not be failed (not in processing/pending state or does not exist)`
443
+ `forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
529
444
  );
530
445
  }
531
- await this.recordJobEvent(jobId, "failed" /* Failed */, {
532
- message: error.message || String(error),
533
- failureReason
446
+ const jobPromise = handler(job.payload, controller.signal, ctx);
447
+ if (hasTimeout) {
448
+ await Promise.race([
449
+ jobPromise,
450
+ new Promise((_, reject) => {
451
+ timeoutReject = reject;
452
+ armTimeout(timeoutMs);
453
+ })
454
+ ]);
455
+ } else {
456
+ await jobPromise;
457
+ }
458
+ }
459
+ if (timeoutId) clearTimeout(timeoutId);
460
+ await backend.completeJob(job.id);
461
+ } catch (error) {
462
+ if (timeoutId) clearTimeout(timeoutId);
463
+ if (error instanceof WaitSignal) {
464
+ log(
465
+ `Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? "none"}, tokenId=${error.tokenId ?? "none"}`
466
+ );
467
+ await backend.waitJob(job.id, {
468
+ waitUntil: error.waitUntil,
469
+ waitTokenId: error.tokenId,
470
+ stepData: error.stepData
534
471
  });
535
- log(`Failed job ${jobId}`);
536
- } catch (err) {
537
- log(`Error failing job ${jobId}: ${err}`);
538
- throw err;
539
- } finally {
540
- client.release();
472
+ return;
473
+ }
474
+ console.error(`Error processing job ${job.id}:`, error);
475
+ let failureReason = "handler_error" /* HandlerError */;
476
+ if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
477
+ failureReason = "timeout" /* Timeout */;
478
+ }
479
+ await backend.failJob(
480
+ job.id,
481
+ error instanceof Error ? error : new Error(String(error)),
482
+ failureReason
483
+ );
484
+ }
485
+ }
486
+ async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError) {
487
+ const jobs = await backend.getNextBatch(
488
+ workerId,
489
+ batchSize,
490
+ jobType
491
+ );
492
+ if (!concurrency || concurrency >= jobs.length) {
493
+ await Promise.all(
494
+ jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers))
495
+ );
496
+ return jobs.length;
497
+ }
498
+ let idx = 0;
499
+ let running = 0;
500
+ let finished = 0;
501
+ return new Promise((resolve, reject) => {
502
+ const next = () => {
503
+ if (finished === jobs.length) return resolve(jobs.length);
504
+ while (running < concurrency && idx < jobs.length) {
505
+ const job = jobs[idx++];
506
+ running++;
507
+ processJobWithHandlers(backend, job, jobHandlers).then(() => {
508
+ running--;
509
+ finished++;
510
+ next();
511
+ }).catch((err) => {
512
+ running--;
513
+ finished++;
514
+ if (onError) {
515
+ onError(err instanceof Error ? err : new Error(String(err)));
516
+ }
517
+ next();
518
+ });
519
+ }
520
+ };
521
+ next();
522
+ });
523
+ }
524
+ var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
525
+ const {
526
+ workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
527
+ batchSize = 10,
528
+ pollInterval = 5e3,
529
+ onError = (error) => console.error("Job processor error:", error),
530
+ jobType,
531
+ concurrency = 3
532
+ } = options;
533
+ let running = false;
534
+ let intervalId = null;
535
+ let currentBatchPromise = null;
536
+ setLogContext(options.verbose ?? false);
537
+ const processJobs = async () => {
538
+ if (!running) return 0;
539
+ if (onBeforeBatch) {
540
+ try {
541
+ await onBeforeBatch();
542
+ } catch (hookError) {
543
+ log(`onBeforeBatch hook error: ${hookError}`);
544
+ if (onError) {
545
+ onError(
546
+ hookError instanceof Error ? hookError : new Error(String(hookError))
547
+ );
548
+ }
549
+ }
541
550
  }
542
- }
543
- async prolongJob(jobId) {
544
- const client = await this.pool.connect();
551
+ log(
552
+ `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(",") : jobType}` : ""}`
553
+ );
545
554
  try {
546
- await client.query(
547
- `
548
- UPDATE job_queue
549
- SET locked_at = NOW(), updated_at = NOW()
550
- WHERE id = $1 AND status = 'processing'
551
- `,
552
- [jobId]
555
+ const processed = await processBatchWithHandlers(
556
+ backend,
557
+ workerId,
558
+ batchSize,
559
+ jobType,
560
+ handlers,
561
+ concurrency,
562
+ onError
553
563
  );
554
- await this.recordJobEvent(jobId, "prolonged" /* Prolonged */);
555
- log(`Prolonged job ${jobId}`);
564
+ return processed;
556
565
  } catch (error) {
557
- log(`Error prolonging job ${jobId}: ${error}`);
558
- } finally {
559
- client.release();
566
+ onError(error instanceof Error ? error : new Error(String(error)));
560
567
  }
568
+ return 0;
569
+ };
570
+ return {
571
+ /**
572
+ * Start the job processor in the background.
573
+ * - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
574
+ * - You have to call the stop method to stop the processor.
575
+ */
576
+ startInBackground: () => {
577
+ if (running) return;
578
+ log(`Starting job processor with workerId: ${workerId}`);
579
+ running = true;
580
+ const scheduleNext = (immediate) => {
581
+ if (!running) return;
582
+ if (immediate) {
583
+ intervalId = setTimeout(loop, 0);
584
+ } else {
585
+ intervalId = setTimeout(loop, pollInterval);
586
+ }
587
+ };
588
+ const loop = async () => {
589
+ if (!running) return;
590
+ currentBatchPromise = processJobs();
591
+ const processed = await currentBatchPromise;
592
+ currentBatchPromise = null;
593
+ scheduleNext(processed === batchSize);
594
+ };
595
+ loop();
596
+ },
597
+ /**
598
+ * Stop the job processor that runs in the background.
599
+ * Does not wait for in-flight jobs.
600
+ */
601
+ stop: () => {
602
+ log(`Stopping job processor with workerId: ${workerId}`);
603
+ running = false;
604
+ if (intervalId) {
605
+ clearTimeout(intervalId);
606
+ intervalId = null;
607
+ }
608
+ },
609
+ /**
610
+ * Stop the job processor and wait for all in-flight jobs to complete.
611
+ * Useful for graceful shutdown (e.g., SIGTERM handling).
612
+ */
613
+ stopAndDrain: async (drainTimeoutMs = 3e4) => {
614
+ log(`Stopping and draining job processor with workerId: ${workerId}`);
615
+ running = false;
616
+ if (intervalId) {
617
+ clearTimeout(intervalId);
618
+ intervalId = null;
619
+ }
620
+ if (currentBatchPromise) {
621
+ await Promise.race([
622
+ currentBatchPromise.catch(() => {
623
+ }),
624
+ new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
625
+ ]);
626
+ currentBatchPromise = null;
627
+ }
628
+ log(`Job processor ${workerId} drained`);
629
+ },
630
+ /**
631
+ * Start the job processor synchronously.
632
+ * - This will process all jobs immediately and then stop.
633
+ * - The pollInterval is ignored.
634
+ */
635
+ start: async () => {
636
+ log(`Starting job processor with workerId: ${workerId}`);
637
+ running = true;
638
+ const processed = await processJobs();
639
+ running = false;
640
+ return processed;
641
+ },
642
+ isRunning: () => running
643
+ };
644
+ };
645
+ function loadPemOrFile(value) {
646
+ if (!value) return void 0;
647
+ if (value.startsWith("file://")) {
648
+ const filePath = value.slice(7);
649
+ return fs.readFileSync(filePath, "utf8");
561
650
  }
562
- // ── Progress ──────────────────────────────────────────────────────────
563
- async updateProgress(jobId, progress) {
564
- const client = await this.pool.connect();
651
+ return value;
652
+ }
653
+ var createPool = (config) => {
654
+ let searchPath;
655
+ let ssl = void 0;
656
+ let customCA;
657
+ let sslmode;
658
+ if (config.connectionString) {
565
659
  try {
566
- await client.query(
567
- `UPDATE job_queue SET progress = $2, updated_at = NOW() WHERE id = $1`,
568
- [jobId, progress]
569
- );
570
- log(`Updated progress for job ${jobId}: ${progress}%`);
571
- } catch (error) {
572
- log(`Error updating progress for job ${jobId}: ${error}`);
573
- } finally {
574
- client.release();
660
+ const url = new URL(config.connectionString);
661
+ searchPath = url.searchParams.get("search_path") || void 0;
662
+ sslmode = url.searchParams.get("sslmode") || void 0;
663
+ if (sslmode === "no-verify") {
664
+ ssl = { rejectUnauthorized: false };
665
+ }
666
+ } catch (e) {
667
+ const parsed = parse(config.connectionString);
668
+ if (parsed.options) {
669
+ const match = parsed.options.match(/search_path=([^\s]+)/);
670
+ if (match) {
671
+ searchPath = match[1];
672
+ }
673
+ }
674
+ sslmode = typeof parsed.sslmode === "string" ? parsed.sslmode : void 0;
675
+ if (sslmode === "no-verify") {
676
+ ssl = { rejectUnauthorized: false };
677
+ }
575
678
  }
576
679
  }
577
- // ── Job management ────────────────────────────────────────────────────
578
- async retryJob(jobId) {
579
- const client = await this.pool.connect();
580
- try {
581
- const result = await client.query(
582
- `
583
- UPDATE job_queue
584
- SET status = 'pending',
585
- updated_at = NOW(),
586
- locked_at = NULL,
587
- locked_by = NULL,
588
- next_attempt_at = NOW(),
589
- last_retried_at = NOW()
590
- WHERE id = $1 AND status IN ('failed', 'processing')
591
- `,
592
- [jobId]
680
+ if (config.ssl) {
681
+ if (typeof config.ssl.ca === "string") {
682
+ customCA = config.ssl.ca;
683
+ } else if (typeof process.env.PGSSLROOTCERT === "string") {
684
+ customCA = process.env.PGSSLROOTCERT;
685
+ } else {
686
+ customCA = void 0;
687
+ }
688
+ const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
689
+ ssl = {
690
+ ...ssl,
691
+ ...caValue ? { ca: caValue } : {},
692
+ cert: loadPemOrFile(
693
+ typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
694
+ ),
695
+ key: loadPemOrFile(
696
+ typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
697
+ ),
698
+ rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
699
+ };
700
+ }
701
+ if (sslmode && customCA) {
702
+ const warning = `
703
+
704
+ \x1B[33m**************************************************
705
+ \u26A0\uFE0F WARNING: SSL CONFIGURATION ISSUE
706
+ **************************************************
707
+ Both sslmode ('${sslmode}') is set in the connection string
708
+ and a custom CA is provided (via config.ssl.ca or PGSSLROOTCERT).
709
+ This combination may cause connection failures or unexpected behavior.
710
+
711
+ Recommended: Remove sslmode from the connection string when using a custom CA.
712
+ **************************************************\x1B[0m
713
+ `;
714
+ console.warn(warning);
715
+ }
716
+ const pool = new Pool({
717
+ ...config,
718
+ ...ssl ? { ssl } : {}
719
+ });
720
+ if (searchPath) {
721
+ pool.on("connect", (client) => {
722
+ client.query(`SET search_path TO ${searchPath}`);
723
+ });
724
+ }
725
+ return pool;
726
+ };
727
+ var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
728
+ function parseTimeoutString(timeout) {
729
+ const match = timeout.match(/^(\d+)(s|m|h|d)$/);
730
+ if (!match) {
731
+ throw new Error(
732
+ `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
733
+ );
734
+ }
735
+ const value = parseInt(match[1], 10);
736
+ const unit = match[2];
737
+ let ms;
738
+ switch (unit) {
739
+ case "s":
740
+ ms = value * 1e3;
741
+ break;
742
+ case "m":
743
+ ms = value * 60 * 1e3;
744
+ break;
745
+ case "h":
746
+ ms = value * 60 * 60 * 1e3;
747
+ break;
748
+ case "d":
749
+ ms = value * 24 * 60 * 60 * 1e3;
750
+ break;
751
+ default:
752
+ throw new Error(`Unknown timeout unit: "${unit}"`);
753
+ }
754
+ if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
755
+ throw new Error(
756
+ `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
757
+ );
758
+ }
759
+ return ms;
760
+ }
761
+ var PostgresBackend = class {
762
+ constructor(pool) {
763
+ this.pool = pool;
764
+ }
765
+ /** Expose the raw pool for advanced usage. */
766
+ getPool() {
767
+ return this.pool;
768
+ }
769
+ // ── Events ──────────────────────────────────────────────────────────
770
+ async recordJobEvent(jobId, eventType, metadata) {
771
+ const client = await this.pool.connect();
772
+ try {
773
+ await client.query(
774
+ `INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
775
+ [jobId, eventType, metadata ? JSON.stringify(metadata) : null]
593
776
  );
594
- if (result.rowCount === 0) {
595
- log(
596
- `Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
597
- );
598
- }
599
- await this.recordJobEvent(jobId, "retried" /* Retried */);
600
- log(`Retried job ${jobId}`);
601
777
  } catch (error) {
602
- log(`Error retrying job ${jobId}: ${error}`);
603
- throw error;
778
+ log(`Error recording job event for job ${jobId}: ${error}`);
604
779
  } finally {
605
780
  client.release();
606
781
  }
607
782
  }
608
- async cancelJob(jobId) {
783
+ async getJobEvents(jobId) {
609
784
  const client = await this.pool.connect();
610
785
  try {
611
- await client.query(
612
- `
613
- UPDATE job_queue
614
- SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
615
- wait_until = NULL, wait_token_id = NULL
616
- WHERE id = $1 AND status IN ('pending', 'waiting')
617
- `,
786
+ const res = await client.query(
787
+ `SELECT id, job_id AS "jobId", event_type AS "eventType", metadata, created_at AS "createdAt" FROM job_events WHERE job_id = $1 ORDER BY created_at ASC`,
618
788
  [jobId]
619
789
  );
620
- await this.recordJobEvent(jobId, "cancelled" /* Cancelled */);
621
- log(`Cancelled job ${jobId}`);
622
- } catch (error) {
623
- log(`Error cancelling job ${jobId}: ${error}`);
624
- throw error;
790
+ return res.rows;
625
791
  } finally {
626
792
  client.release();
627
793
  }
628
794
  }
629
- async cancelAllUpcomingJobs(filters) {
795
+ // ── Job CRUD ──────────────────────────────────────────────────────────
796
+ async addJob({
797
+ jobType,
798
+ payload,
799
+ maxAttempts = 3,
800
+ priority = 0,
801
+ runAt = null,
802
+ timeoutMs = void 0,
803
+ forceKillOnTimeout = false,
804
+ tags = void 0,
805
+ idempotencyKey = void 0
806
+ }) {
630
807
  const client = await this.pool.connect();
631
808
  try {
632
- let query = `
633
- UPDATE job_queue
634
- SET status = 'cancelled', updated_at = NOW()
635
- WHERE status = 'pending'`;
636
- const params = [];
637
- let paramIdx = 1;
638
- if (filters) {
639
- if (filters.jobType) {
640
- query += ` AND job_type = $${paramIdx++}`;
641
- params.push(filters.jobType);
642
- }
643
- if (filters.priority !== void 0) {
644
- query += ` AND priority = $${paramIdx++}`;
645
- params.push(filters.priority);
646
- }
647
- if (filters.runAt) {
648
- if (filters.runAt instanceof Date) {
649
- query += ` AND run_at = $${paramIdx++}`;
650
- params.push(filters.runAt);
651
- } else if (typeof filters.runAt === "object") {
652
- const ops = filters.runAt;
653
- if (ops.gt) {
654
- query += ` AND run_at > $${paramIdx++}`;
655
- params.push(ops.gt);
656
- }
657
- if (ops.gte) {
658
- query += ` AND run_at >= $${paramIdx++}`;
659
- params.push(ops.gte);
660
- }
661
- if (ops.lt) {
662
- query += ` AND run_at < $${paramIdx++}`;
663
- params.push(ops.lt);
664
- }
665
- if (ops.lte) {
666
- query += ` AND run_at <= $${paramIdx++}`;
667
- params.push(ops.lte);
668
- }
669
- if (ops.eq) {
670
- query += ` AND run_at = $${paramIdx++}`;
671
- params.push(ops.eq);
672
- }
673
- }
674
- }
675
- if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
676
- const mode = filters.tags.mode || "all";
677
- const tagValues = filters.tags.values;
678
- switch (mode) {
679
- case "exact":
680
- query += ` AND tags = $${paramIdx++}`;
681
- params.push(tagValues);
682
- break;
683
- case "all":
684
- query += ` AND tags @> $${paramIdx++}`;
685
- params.push(tagValues);
686
- break;
687
- case "any":
688
- query += ` AND tags && $${paramIdx++}`;
689
- params.push(tagValues);
690
- break;
691
- case "none":
692
- query += ` AND NOT (tags && $${paramIdx++})`;
693
- params.push(tagValues);
694
- break;
695
- default:
696
- query += ` AND tags @> $${paramIdx++}`;
697
- params.push(tagValues);
698
- }
809
+ let result;
810
+ const onConflict = idempotencyKey ? `ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL DO NOTHING` : "";
811
+ if (runAt) {
812
+ result = await client.query(
813
+ `INSERT INTO job_queue
814
+ (job_type, payload, max_attempts, priority, run_at, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
815
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
816
+ ${onConflict}
817
+ RETURNING id`,
818
+ [
819
+ jobType,
820
+ payload,
821
+ maxAttempts,
822
+ priority,
823
+ runAt,
824
+ timeoutMs ?? null,
825
+ forceKillOnTimeout ?? false,
826
+ tags ?? null,
827
+ idempotencyKey ?? null
828
+ ]
829
+ );
830
+ } else {
831
+ result = await client.query(
832
+ `INSERT INTO job_queue
833
+ (job_type, payload, max_attempts, priority, timeout_ms, force_kill_on_timeout, tags, idempotency_key)
834
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
835
+ ${onConflict}
836
+ RETURNING id`,
837
+ [
838
+ jobType,
839
+ payload,
840
+ maxAttempts,
841
+ priority,
842
+ timeoutMs ?? null,
843
+ forceKillOnTimeout ?? false,
844
+ tags ?? null,
845
+ idempotencyKey ?? null
846
+ ]
847
+ );
848
+ }
849
+ if (result.rows.length === 0 && idempotencyKey) {
850
+ const existing = await client.query(
851
+ `SELECT id FROM job_queue WHERE idempotency_key = $1`,
852
+ [idempotencyKey]
853
+ );
854
+ if (existing.rows.length > 0) {
855
+ log(
856
+ `Job with idempotency key "${idempotencyKey}" already exists (id: ${existing.rows[0].id}), returning existing job`
857
+ );
858
+ return existing.rows[0].id;
699
859
  }
860
+ throw new Error(
861
+ `Failed to insert job and could not find existing job with idempotency key "${idempotencyKey}"`
862
+ );
700
863
  }
701
- query += "\nRETURNING id";
702
- const result = await client.query(query, params);
703
- log(`Cancelled ${result.rowCount} jobs`);
704
- return result.rowCount || 0;
864
+ const jobId = result.rows[0].id;
865
+ log(
866
+ `Added job ${jobId}: payload ${JSON.stringify(payload)}, ${runAt ? `runAt ${runAt.toISOString()}, ` : ""}priority ${priority}, maxAttempts ${maxAttempts}, jobType ${jobType}, tags ${JSON.stringify(tags)}${idempotencyKey ? `, idempotencyKey "${idempotencyKey}"` : ""}`
867
+ );
868
+ await this.recordJobEvent(jobId, "added" /* Added */, {
869
+ jobType,
870
+ payload,
871
+ tags,
872
+ idempotencyKey
873
+ });
874
+ return jobId;
705
875
  } catch (error) {
706
- log(`Error cancelling upcoming jobs: ${error}`);
876
+ log(`Error adding job: ${error}`);
707
877
  throw error;
708
878
  } finally {
709
879
  client.release();
710
880
  }
711
881
  }
712
- async editJob(jobId, updates) {
882
+ async getJob(id) {
713
883
  const client = await this.pool.connect();
714
884
  try {
715
- const updateFields = [];
716
- const params = [];
717
- let paramIdx = 1;
718
- if (updates.payload !== void 0) {
719
- updateFields.push(`payload = $${paramIdx++}`);
720
- params.push(updates.payload);
721
- }
722
- if (updates.maxAttempts !== void 0) {
723
- updateFields.push(`max_attempts = $${paramIdx++}`);
724
- params.push(updates.maxAttempts);
725
- }
726
- if (updates.priority !== void 0) {
727
- updateFields.push(`priority = $${paramIdx++}`);
728
- params.push(updates.priority);
729
- }
730
- if (updates.runAt !== void 0) {
731
- if (updates.runAt === null) {
732
- updateFields.push(`run_at = NOW()`);
733
- } else {
734
- updateFields.push(`run_at = $${paramIdx++}`);
735
- params.push(updates.runAt);
736
- }
737
- }
738
- if (updates.timeoutMs !== void 0) {
739
- updateFields.push(`timeout_ms = $${paramIdx++}`);
740
- params.push(updates.timeoutMs ?? null);
741
- }
742
- if (updates.tags !== void 0) {
743
- updateFields.push(`tags = $${paramIdx++}`);
744
- params.push(updates.tags ?? null);
745
- }
746
- if (updateFields.length === 0) {
747
- log(`No fields to update for job ${jobId}`);
748
- return;
885
+ const result = await client.query(
886
+ `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE id = $1`,
887
+ [id]
888
+ );
889
+ if (result.rows.length === 0) {
890
+ log(`Job ${id} not found`);
891
+ return null;
749
892
  }
750
- updateFields.push(`updated_at = NOW()`);
751
- params.push(jobId);
752
- const query = `
753
- UPDATE job_queue
754
- SET ${updateFields.join(", ")}
755
- WHERE id = $${paramIdx} AND status = 'pending'
756
- `;
757
- await client.query(query, params);
758
- const metadata = {};
759
- if (updates.payload !== void 0) metadata.payload = updates.payload;
760
- if (updates.maxAttempts !== void 0)
761
- metadata.maxAttempts = updates.maxAttempts;
762
- if (updates.priority !== void 0) metadata.priority = updates.priority;
763
- if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
764
- if (updates.timeoutMs !== void 0)
765
- metadata.timeoutMs = updates.timeoutMs;
766
- if (updates.tags !== void 0) metadata.tags = updates.tags;
767
- await this.recordJobEvent(jobId, "edited" /* Edited */, metadata);
768
- log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
893
+ log(`Found job ${id}`);
894
+ const job = result.rows[0];
895
+ return {
896
+ ...job,
897
+ payload: job.payload,
898
+ timeoutMs: job.timeoutMs,
899
+ forceKillOnTimeout: job.forceKillOnTimeout,
900
+ failureReason: job.failureReason
901
+ };
902
+ } catch (error) {
903
+ log(`Error getting job ${id}: ${error}`);
904
+ throw error;
905
+ } finally {
906
+ client.release();
907
+ }
908
+ }
909
+ async getJobsByStatus(status, limit = 100, offset = 0) {
910
+ const client = await this.pool.connect();
911
+ try {
912
+ const result = await client.query(
913
+ `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3`,
914
+ [status, limit, offset]
915
+ );
916
+ log(`Found ${result.rows.length} jobs by status ${status}`);
917
+ return result.rows.map((job) => ({
918
+ ...job,
919
+ payload: job.payload,
920
+ timeoutMs: job.timeoutMs,
921
+ forceKillOnTimeout: job.forceKillOnTimeout,
922
+ failureReason: job.failureReason
923
+ }));
769
924
  } catch (error) {
770
- log(`Error editing job ${jobId}: ${error}`);
925
+ log(`Error getting jobs by status ${status}: ${error}`);
771
926
  throw error;
772
927
  } finally {
773
928
  client.release();
774
929
  }
775
930
  }
776
- async editAllPendingJobs(filters = void 0, updates) {
931
+ async getAllJobs(limit = 100, offset = 0) {
777
932
  const client = await this.pool.connect();
778
933
  try {
779
- const updateFields = [];
934
+ const result = await client.query(
935
+ `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2`,
936
+ [limit, offset]
937
+ );
938
+ log(`Found ${result.rows.length} jobs (all)`);
939
+ return result.rows.map((job) => ({
940
+ ...job,
941
+ payload: job.payload,
942
+ timeoutMs: job.timeoutMs,
943
+ forceKillOnTimeout: job.forceKillOnTimeout
944
+ }));
945
+ } catch (error) {
946
+ log(`Error getting all jobs: ${error}`);
947
+ throw error;
948
+ } finally {
949
+ client.release();
950
+ }
951
+ }
952
+ async getJobs(filters, limit = 100, offset = 0) {
953
+ const client = await this.pool.connect();
954
+ try {
955
+ let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress FROM job_queue`;
780
956
  const params = [];
957
+ const where = [];
781
958
  let paramIdx = 1;
782
- if (updates.payload !== void 0) {
783
- updateFields.push(`payload = $${paramIdx++}`);
784
- params.push(updates.payload);
785
- }
786
- if (updates.maxAttempts !== void 0) {
787
- updateFields.push(`max_attempts = $${paramIdx++}`);
788
- params.push(updates.maxAttempts);
789
- }
790
- if (updates.priority !== void 0) {
791
- updateFields.push(`priority = $${paramIdx++}`);
792
- params.push(updates.priority);
793
- }
794
- if (updates.runAt !== void 0) {
795
- if (updates.runAt === null) {
796
- updateFields.push(`run_at = NOW()`);
797
- } else {
798
- updateFields.push(`run_at = $${paramIdx++}`);
799
- params.push(updates.runAt);
800
- }
801
- }
802
- if (updates.timeoutMs !== void 0) {
803
- updateFields.push(`timeout_ms = $${paramIdx++}`);
804
- params.push(updates.timeoutMs ?? null);
805
- }
806
- if (updates.tags !== void 0) {
807
- updateFields.push(`tags = $${paramIdx++}`);
808
- params.push(updates.tags ?? null);
809
- }
810
- if (updateFields.length === 0) {
811
- log(`No fields to update for batch edit`);
812
- return 0;
813
- }
814
- updateFields.push(`updated_at = NOW()`);
815
- let query = `
816
- UPDATE job_queue
817
- SET ${updateFields.join(", ")}
818
- WHERE status = 'pending'`;
819
959
  if (filters) {
820
960
  if (filters.jobType) {
821
- query += ` AND job_type = $${paramIdx++}`;
961
+ where.push(`job_type = $${paramIdx++}`);
822
962
  params.push(filters.jobType);
823
963
  }
824
964
  if (filters.priority !== void 0) {
825
- query += ` AND priority = $${paramIdx++}`;
965
+ where.push(`priority = $${paramIdx++}`);
826
966
  params.push(filters.priority);
827
967
  }
828
968
  if (filters.runAt) {
829
969
  if (filters.runAt instanceof Date) {
830
- query += ` AND run_at = $${paramIdx++}`;
970
+ where.push(`run_at = $${paramIdx++}`);
831
971
  params.push(filters.runAt);
832
- } else if (typeof filters.runAt === "object") {
972
+ } else if (typeof filters.runAt === "object" && (filters.runAt.gt !== void 0 || filters.runAt.gte !== void 0 || filters.runAt.lt !== void 0 || filters.runAt.lte !== void 0 || filters.runAt.eq !== void 0)) {
833
973
  const ops = filters.runAt;
834
974
  if (ops.gt) {
835
- query += ` AND run_at > $${paramIdx++}`;
975
+ where.push(`run_at > $${paramIdx++}`);
836
976
  params.push(ops.gt);
837
977
  }
838
978
  if (ops.gte) {
839
- query += ` AND run_at >= $${paramIdx++}`;
979
+ where.push(`run_at >= $${paramIdx++}`);
840
980
  params.push(ops.gte);
841
981
  }
842
982
  if (ops.lt) {
843
- query += ` AND run_at < $${paramIdx++}`;
983
+ where.push(`run_at < $${paramIdx++}`);
844
984
  params.push(ops.lt);
845
985
  }
846
986
  if (ops.lte) {
847
- query += ` AND run_at <= $${paramIdx++}`;
987
+ where.push(`run_at <= $${paramIdx++}`);
848
988
  params.push(ops.lte);
849
989
  }
850
990
  if (ops.eq) {
851
- query += ` AND run_at = $${paramIdx++}`;
991
+ where.push(`run_at = $${paramIdx++}`);
852
992
  params.push(ops.eq);
853
993
  }
854
994
  }
@@ -858,1370 +998,1264 @@ var PostgresBackend = class {
858
998
  const tagValues = filters.tags.values;
859
999
  switch (mode) {
860
1000
  case "exact":
861
- query += ` AND tags = $${paramIdx++}`;
1001
+ where.push(`tags = $${paramIdx++}`);
862
1002
  params.push(tagValues);
863
1003
  break;
864
1004
  case "all":
865
- query += ` AND tags @> $${paramIdx++}`;
1005
+ where.push(`tags @> $${paramIdx++}`);
866
1006
  params.push(tagValues);
867
1007
  break;
868
1008
  case "any":
869
- query += ` AND tags && $${paramIdx++}`;
1009
+ where.push(`tags && $${paramIdx++}`);
870
1010
  params.push(tagValues);
871
1011
  break;
872
1012
  case "none":
873
- query += ` AND NOT (tags && $${paramIdx++})`;
1013
+ where.push(`NOT (tags && $${paramIdx++})`);
874
1014
  params.push(tagValues);
875
1015
  break;
876
1016
  default:
877
- query += ` AND tags @> $${paramIdx++}`;
1017
+ where.push(`tags @> $${paramIdx++}`);
878
1018
  params.push(tagValues);
879
1019
  }
880
1020
  }
1021
+ if (filters.cursor !== void 0) {
1022
+ where.push(`id < $${paramIdx++}`);
1023
+ params.push(filters.cursor);
1024
+ }
1025
+ }
1026
+ if (where.length > 0) {
1027
+ query += ` WHERE ${where.join(" AND ")}`;
1028
+ }
1029
+ paramIdx = params.length + 1;
1030
+ query += ` ORDER BY id DESC LIMIT $${paramIdx++}`;
1031
+ if (!filters?.cursor) {
1032
+ query += ` OFFSET $${paramIdx}`;
1033
+ params.push(limit, offset);
1034
+ } else {
1035
+ params.push(limit);
881
1036
  }
882
- query += "\nRETURNING id";
883
1037
  const result = await client.query(query, params);
884
- const editedCount = result.rowCount || 0;
885
- const metadata = {};
886
- if (updates.payload !== void 0) metadata.payload = updates.payload;
887
- if (updates.maxAttempts !== void 0)
888
- metadata.maxAttempts = updates.maxAttempts;
889
- if (updates.priority !== void 0) metadata.priority = updates.priority;
890
- if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
891
- if (updates.timeoutMs !== void 0)
892
- metadata.timeoutMs = updates.timeoutMs;
893
- if (updates.tags !== void 0) metadata.tags = updates.tags;
894
- for (const row of result.rows) {
895
- await this.recordJobEvent(row.id, "edited" /* Edited */, metadata);
1038
+ log(`Found ${result.rows.length} jobs`);
1039
+ return result.rows.map((job) => ({
1040
+ ...job,
1041
+ payload: job.payload,
1042
+ timeoutMs: job.timeoutMs,
1043
+ forceKillOnTimeout: job.forceKillOnTimeout,
1044
+ failureReason: job.failureReason
1045
+ }));
1046
+ } catch (error) {
1047
+ log(`Error getting jobs: ${error}`);
1048
+ throw error;
1049
+ } finally {
1050
+ client.release();
1051
+ }
1052
+ }
1053
+ async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
1054
+ const client = await this.pool.connect();
1055
+ try {
1056
+ let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags, idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
1057
+ FROM job_queue`;
1058
+ let params = [];
1059
+ switch (mode) {
1060
+ case "exact":
1061
+ query += " WHERE tags = $1";
1062
+ params = [tags];
1063
+ break;
1064
+ case "all":
1065
+ query += " WHERE tags @> $1";
1066
+ params = [tags];
1067
+ break;
1068
+ case "any":
1069
+ query += " WHERE tags && $1";
1070
+ params = [tags];
1071
+ break;
1072
+ case "none":
1073
+ query += " WHERE NOT (tags && $1)";
1074
+ params = [tags];
1075
+ break;
1076
+ default:
1077
+ query += " WHERE tags @> $1";
1078
+ params = [tags];
1079
+ }
1080
+ query += " ORDER BY created_at DESC LIMIT $2 OFFSET $3";
1081
+ params.push(limit, offset);
1082
+ const result = await client.query(query, params);
1083
+ log(
1084
+ `Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`
1085
+ );
1086
+ return result.rows.map((job) => ({
1087
+ ...job,
1088
+ payload: job.payload,
1089
+ timeoutMs: job.timeoutMs,
1090
+ forceKillOnTimeout: job.forceKillOnTimeout,
1091
+ failureReason: job.failureReason
1092
+ }));
1093
+ } catch (error) {
1094
+ log(
1095
+ `Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`
1096
+ );
1097
+ throw error;
1098
+ } finally {
1099
+ client.release();
1100
+ }
1101
+ }
1102
+ // ── Processing lifecycle ──────────────────────────────────────────────
1103
+ async getNextBatch(workerId, batchSize = 10, jobType) {
1104
+ const client = await this.pool.connect();
1105
+ try {
1106
+ await client.query("BEGIN");
1107
+ let jobTypeFilter = "";
1108
+ const params = [workerId, batchSize];
1109
+ if (jobType) {
1110
+ if (Array.isArray(jobType)) {
1111
+ jobTypeFilter = ` AND job_type = ANY($3)`;
1112
+ params.push(jobType);
1113
+ } else {
1114
+ jobTypeFilter = ` AND job_type = $3`;
1115
+ params.push(jobType);
1116
+ }
1117
+ }
1118
+ const result = await client.query(
1119
+ `
1120
+ UPDATE job_queue
1121
+ SET status = 'processing',
1122
+ locked_at = NOW(),
1123
+ locked_by = $1,
1124
+ attempts = CASE WHEN status = 'waiting' THEN attempts ELSE attempts + 1 END,
1125
+ updated_at = NOW(),
1126
+ pending_reason = NULL,
1127
+ started_at = COALESCE(started_at, NOW()),
1128
+ last_retried_at = CASE WHEN status != 'waiting' AND attempts > 0 THEN NOW() ELSE last_retried_at END,
1129
+ wait_until = NULL
1130
+ WHERE id IN (
1131
+ SELECT id FROM job_queue
1132
+ WHERE (
1133
+ (
1134
+ (status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
1135
+ AND (attempts < max_attempts)
1136
+ AND run_at <= NOW()
1137
+ )
1138
+ OR (
1139
+ status = 'waiting'
1140
+ AND wait_until IS NOT NULL
1141
+ AND wait_until <= NOW()
1142
+ AND wait_token_id IS NULL
1143
+ )
1144
+ )
1145
+ ${jobTypeFilter}
1146
+ ORDER BY priority DESC, created_at ASC
1147
+ LIMIT $2
1148
+ FOR UPDATE SKIP LOCKED
1149
+ )
1150
+ RETURNING id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", force_kill_on_timeout AS "forceKillOnTimeout", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", idempotency_key AS "idempotencyKey", wait_until AS "waitUntil", wait_token_id AS "waitTokenId", step_data AS "stepData", progress
1151
+ `,
1152
+ params
1153
+ );
1154
+ log(`Found ${result.rows.length} jobs to process`);
1155
+ await client.query("COMMIT");
1156
+ if (result.rows.length > 0) {
1157
+ await this.recordJobEventsBatch(
1158
+ result.rows.map((row) => ({
1159
+ jobId: row.id,
1160
+ eventType: "processing" /* Processing */
1161
+ }))
1162
+ );
896
1163
  }
897
- log(`Edited ${editedCount} pending jobs: ${JSON.stringify(metadata)}`);
898
- return editedCount;
1164
+ return result.rows.map((job) => ({
1165
+ ...job,
1166
+ payload: job.payload,
1167
+ timeoutMs: job.timeoutMs,
1168
+ forceKillOnTimeout: job.forceKillOnTimeout
1169
+ }));
899
1170
  } catch (error) {
900
- log(`Error editing pending jobs: ${error}`);
1171
+ log(`Error getting next batch: ${error}`);
1172
+ await client.query("ROLLBACK");
901
1173
  throw error;
902
1174
  } finally {
903
1175
  client.release();
904
1176
  }
905
1177
  }
906
- async cleanupOldJobs(daysToKeep = 30) {
1178
+ async completeJob(jobId) {
907
1179
  const client = await this.pool.connect();
908
1180
  try {
909
1181
  const result = await client.query(
910
1182
  `
911
- DELETE FROM job_queue
912
- WHERE status = 'completed'
913
- AND updated_at < NOW() - INTERVAL '1 day' * $1::int
914
- RETURNING id
1183
+ UPDATE job_queue
1184
+ SET status = 'completed', updated_at = NOW(), completed_at = NOW(),
1185
+ step_data = NULL, wait_until = NULL, wait_token_id = NULL
1186
+ WHERE id = $1 AND status = 'processing'
915
1187
  `,
916
- [daysToKeep]
1188
+ [jobId]
917
1189
  );
918
- log(`Deleted ${result.rowCount} old jobs`);
919
- return result.rowCount || 0;
1190
+ if (result.rowCount === 0) {
1191
+ log(
1192
+ `Job ${jobId} could not be completed (not in processing state or does not exist)`
1193
+ );
1194
+ }
1195
+ await this.recordJobEvent(jobId, "completed" /* Completed */);
1196
+ log(`Completed job ${jobId}`);
920
1197
  } catch (error) {
921
- log(`Error cleaning up old jobs: ${error}`);
1198
+ log(`Error completing job ${jobId}: ${error}`);
922
1199
  throw error;
923
1200
  } finally {
924
1201
  client.release();
925
1202
  }
926
1203
  }
927
- async cleanupOldJobEvents(daysToKeep = 30) {
1204
+ async failJob(jobId, error, failureReason) {
928
1205
  const client = await this.pool.connect();
929
1206
  try {
930
1207
  const result = await client.query(
931
1208
  `
932
- DELETE FROM job_events
933
- WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
934
- RETURNING id
1209
+ UPDATE job_queue
1210
+ SET status = 'failed',
1211
+ updated_at = NOW(),
1212
+ next_attempt_at = CASE
1213
+ WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
1214
+ ELSE NULL
1215
+ END,
1216
+ error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
1217
+ failure_reason = $3,
1218
+ last_failed_at = NOW()
1219
+ WHERE id = $1 AND status IN ('processing', 'pending')
935
1220
  `,
936
- [daysToKeep]
1221
+ [
1222
+ jobId,
1223
+ JSON.stringify([
1224
+ {
1225
+ message: error.message || String(error),
1226
+ timestamp: (/* @__PURE__ */ new Date()).toISOString()
1227
+ }
1228
+ ]),
1229
+ failureReason ?? null
1230
+ ]
937
1231
  );
938
- log(`Deleted ${result.rowCount} old job events`);
939
- return result.rowCount || 0;
940
- } catch (error) {
941
- log(`Error cleaning up old job events: ${error}`);
942
- throw error;
1232
+ if (result.rowCount === 0) {
1233
+ log(
1234
+ `Job ${jobId} could not be failed (not in processing/pending state or does not exist)`
1235
+ );
1236
+ }
1237
+ await this.recordJobEvent(jobId, "failed" /* Failed */, {
1238
+ message: error.message || String(error),
1239
+ failureReason
1240
+ });
1241
+ log(`Failed job ${jobId}`);
1242
+ } catch (err) {
1243
+ log(`Error failing job ${jobId}: ${err}`);
1244
+ throw err;
943
1245
  } finally {
944
1246
  client.release();
945
1247
  }
946
1248
  }
947
- async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
1249
+ async prolongJob(jobId) {
948
1250
  const client = await this.pool.connect();
949
1251
  try {
950
- const result = await client.query(
1252
+ await client.query(
951
1253
  `
952
1254
  UPDATE job_queue
953
- SET status = 'pending', locked_at = NULL, locked_by = NULL, updated_at = NOW()
954
- WHERE status = 'processing'
955
- AND locked_at < NOW() - GREATEST(
956
- INTERVAL '1 minute' * $1::int,
957
- INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
958
- )
959
- RETURNING id
960
- `,
961
- [maxProcessingTimeMinutes]
1255
+ SET locked_at = NOW(), updated_at = NOW()
1256
+ WHERE id = $1 AND status = 'processing'
1257
+ `,
1258
+ [jobId]
962
1259
  );
963
- log(`Reclaimed ${result.rowCount} stuck jobs`);
964
- return result.rowCount || 0;
1260
+ await this.recordJobEvent(jobId, "prolonged" /* Prolonged */);
1261
+ log(`Prolonged job ${jobId}`);
965
1262
  } catch (error) {
966
- log(`Error reclaiming stuck jobs: ${error}`);
967
- throw error;
1263
+ log(`Error prolonging job ${jobId}: ${error}`);
968
1264
  } finally {
969
1265
  client.release();
970
1266
  }
971
1267
  }
972
- // ── Internal helpers ──────────────────────────────────────────────────
973
- /**
974
- * Batch-insert multiple job events in a single query.
975
- * More efficient than individual recordJobEvent calls.
976
- */
977
- async recordJobEventsBatch(events) {
978
- if (events.length === 0) return;
1268
+ // ── Progress ──────────────────────────────────────────────────────────
1269
+ async updateProgress(jobId, progress) {
979
1270
  const client = await this.pool.connect();
980
1271
  try {
981
- const values = [];
982
- const params = [];
983
- let paramIdx = 1;
984
- for (const event of events) {
985
- values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
986
- params.push(
987
- event.jobId,
988
- event.eventType,
989
- event.metadata ? JSON.stringify(event.metadata) : null
990
- );
991
- }
992
1272
  await client.query(
993
- `INSERT INTO job_events (job_id, event_type, metadata) VALUES ${values.join(", ")}`,
994
- params
1273
+ `UPDATE job_queue SET progress = $2, updated_at = NOW() WHERE id = $1`,
1274
+ [jobId, progress]
995
1275
  );
1276
+ log(`Updated progress for job ${jobId}: ${progress}%`);
996
1277
  } catch (error) {
997
- log(`Error recording batch job events: ${error}`);
1278
+ log(`Error updating progress for job ${jobId}: ${error}`);
998
1279
  } finally {
999
1280
  client.release();
1000
1281
  }
1001
1282
  }
1002
- // ── Cron schedules ──────────────────────────────────────────────────
1003
- /** Create a cron schedule and return its ID. */
1004
- async addCronSchedule(input) {
1283
+ // ── Job management ────────────────────────────────────────────────────
1284
+ async retryJob(jobId) {
1005
1285
  const client = await this.pool.connect();
1006
1286
  try {
1007
1287
  const result = await client.query(
1008
- `INSERT INTO cron_schedules
1009
- (schedule_name, cron_expression, job_type, payload, max_attempts,
1010
- priority, timeout_ms, force_kill_on_timeout, tags, timezone,
1011
- allow_overlap, next_run_at)
1012
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
1013
- RETURNING id`,
1014
- [
1015
- input.scheduleName,
1016
- input.cronExpression,
1017
- input.jobType,
1018
- input.payload,
1019
- input.maxAttempts,
1020
- input.priority,
1021
- input.timeoutMs,
1022
- input.forceKillOnTimeout,
1023
- input.tags ?? null,
1024
- input.timezone,
1025
- input.allowOverlap,
1026
- input.nextRunAt
1027
- ]
1288
+ `
1289
+ UPDATE job_queue
1290
+ SET status = 'pending',
1291
+ updated_at = NOW(),
1292
+ locked_at = NULL,
1293
+ locked_by = NULL,
1294
+ next_attempt_at = NOW(),
1295
+ last_retried_at = NOW()
1296
+ WHERE id = $1 AND status IN ('failed', 'processing')
1297
+ `,
1298
+ [jobId]
1028
1299
  );
1029
- const id = result.rows[0].id;
1030
- log(`Added cron schedule ${id}: "${input.scheduleName}"`);
1031
- return id;
1032
- } catch (error) {
1033
- if (error?.code === "23505") {
1034
- throw new Error(
1035
- `Cron schedule with name "${input.scheduleName}" already exists`
1300
+ if (result.rowCount === 0) {
1301
+ log(
1302
+ `Job ${jobId} could not be retried (not in failed/processing state or does not exist)`
1036
1303
  );
1037
1304
  }
1038
- log(`Error adding cron schedule: ${error}`);
1039
- throw error;
1040
- } finally {
1041
- client.release();
1042
- }
1043
- }
1044
- /** Get a cron schedule by ID. */
1045
- async getCronSchedule(id) {
1046
- const client = await this.pool.connect();
1047
- try {
1048
- const result = await client.query(
1049
- `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1050
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1051
- priority, timeout_ms AS "timeoutMs",
1052
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1053
- timezone, allow_overlap AS "allowOverlap", status,
1054
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1055
- next_run_at AS "nextRunAt",
1056
- created_at AS "createdAt", updated_at AS "updatedAt"
1057
- FROM cron_schedules WHERE id = $1`,
1058
- [id]
1059
- );
1060
- if (result.rows.length === 0) return null;
1061
- return result.rows[0];
1305
+ await this.recordJobEvent(jobId, "retried" /* Retried */);
1306
+ log(`Retried job ${jobId}`);
1062
1307
  } catch (error) {
1063
- log(`Error getting cron schedule ${id}: ${error}`);
1308
+ log(`Error retrying job ${jobId}: ${error}`);
1064
1309
  throw error;
1065
1310
  } finally {
1066
1311
  client.release();
1067
1312
  }
1068
1313
  }
1069
- /** Get a cron schedule by its unique name. */
1070
- async getCronScheduleByName(name) {
1314
+ async cancelJob(jobId) {
1071
1315
  const client = await this.pool.connect();
1072
1316
  try {
1073
- const result = await client.query(
1074
- `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1075
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1076
- priority, timeout_ms AS "timeoutMs",
1077
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1078
- timezone, allow_overlap AS "allowOverlap", status,
1079
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1080
- next_run_at AS "nextRunAt",
1081
- created_at AS "createdAt", updated_at AS "updatedAt"
1082
- FROM cron_schedules WHERE schedule_name = $1`,
1083
- [name]
1317
+ await client.query(
1318
+ `
1319
+ UPDATE job_queue
1320
+ SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW(),
1321
+ wait_until = NULL, wait_token_id = NULL
1322
+ WHERE id = $1 AND status IN ('pending', 'waiting')
1323
+ `,
1324
+ [jobId]
1084
1325
  );
1085
- if (result.rows.length === 0) return null;
1086
- return result.rows[0];
1087
- } catch (error) {
1088
- log(`Error getting cron schedule by name "${name}": ${error}`);
1089
- throw error;
1090
- } finally {
1091
- client.release();
1092
- }
1093
- }
1094
- /** List cron schedules, optionally filtered by status. */
1095
- async listCronSchedules(status) {
1096
- const client = await this.pool.connect();
1097
- try {
1098
- let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1099
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1100
- priority, timeout_ms AS "timeoutMs",
1101
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1102
- timezone, allow_overlap AS "allowOverlap", status,
1103
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1104
- next_run_at AS "nextRunAt",
1105
- created_at AS "createdAt", updated_at AS "updatedAt"
1106
- FROM cron_schedules`;
1107
- const params = [];
1108
- if (status) {
1109
- query += ` WHERE status = $1`;
1110
- params.push(status);
1111
- }
1112
- query += ` ORDER BY created_at ASC`;
1113
- const result = await client.query(query, params);
1114
- return result.rows;
1326
+ await this.recordJobEvent(jobId, "cancelled" /* Cancelled */);
1327
+ log(`Cancelled job ${jobId}`);
1115
1328
  } catch (error) {
1116
- log(`Error listing cron schedules: ${error}`);
1329
+ log(`Error cancelling job ${jobId}: ${error}`);
1117
1330
  throw error;
1118
1331
  } finally {
1119
1332
  client.release();
1120
1333
  }
1121
1334
  }
1122
- /** Delete a cron schedule by ID. */
1123
- async removeCronSchedule(id) {
1335
+ async cancelAllUpcomingJobs(filters) {
1124
1336
  const client = await this.pool.connect();
1125
1337
  try {
1126
- await client.query(`DELETE FROM cron_schedules WHERE id = $1`, [id]);
1127
- log(`Removed cron schedule ${id}`);
1338
+ let query = `
1339
+ UPDATE job_queue
1340
+ SET status = 'cancelled', updated_at = NOW()
1341
+ WHERE status = 'pending'`;
1342
+ const params = [];
1343
+ let paramIdx = 1;
1344
+ if (filters) {
1345
+ if (filters.jobType) {
1346
+ query += ` AND job_type = $${paramIdx++}`;
1347
+ params.push(filters.jobType);
1348
+ }
1349
+ if (filters.priority !== void 0) {
1350
+ query += ` AND priority = $${paramIdx++}`;
1351
+ params.push(filters.priority);
1352
+ }
1353
+ if (filters.runAt) {
1354
+ if (filters.runAt instanceof Date) {
1355
+ query += ` AND run_at = $${paramIdx++}`;
1356
+ params.push(filters.runAt);
1357
+ } else if (typeof filters.runAt === "object") {
1358
+ const ops = filters.runAt;
1359
+ if (ops.gt) {
1360
+ query += ` AND run_at > $${paramIdx++}`;
1361
+ params.push(ops.gt);
1362
+ }
1363
+ if (ops.gte) {
1364
+ query += ` AND run_at >= $${paramIdx++}`;
1365
+ params.push(ops.gte);
1366
+ }
1367
+ if (ops.lt) {
1368
+ query += ` AND run_at < $${paramIdx++}`;
1369
+ params.push(ops.lt);
1370
+ }
1371
+ if (ops.lte) {
1372
+ query += ` AND run_at <= $${paramIdx++}`;
1373
+ params.push(ops.lte);
1374
+ }
1375
+ if (ops.eq) {
1376
+ query += ` AND run_at = $${paramIdx++}`;
1377
+ params.push(ops.eq);
1378
+ }
1379
+ }
1380
+ }
1381
+ if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
1382
+ const mode = filters.tags.mode || "all";
1383
+ const tagValues = filters.tags.values;
1384
+ switch (mode) {
1385
+ case "exact":
1386
+ query += ` AND tags = $${paramIdx++}`;
1387
+ params.push(tagValues);
1388
+ break;
1389
+ case "all":
1390
+ query += ` AND tags @> $${paramIdx++}`;
1391
+ params.push(tagValues);
1392
+ break;
1393
+ case "any":
1394
+ query += ` AND tags && $${paramIdx++}`;
1395
+ params.push(tagValues);
1396
+ break;
1397
+ case "none":
1398
+ query += ` AND NOT (tags && $${paramIdx++})`;
1399
+ params.push(tagValues);
1400
+ break;
1401
+ default:
1402
+ query += ` AND tags @> $${paramIdx++}`;
1403
+ params.push(tagValues);
1404
+ }
1405
+ }
1406
+ }
1407
+ query += "\nRETURNING id";
1408
+ const result = await client.query(query, params);
1409
+ log(`Cancelled ${result.rowCount} jobs`);
1410
+ return result.rowCount || 0;
1128
1411
  } catch (error) {
1129
- log(`Error removing cron schedule ${id}: ${error}`);
1412
+ log(`Error cancelling upcoming jobs: ${error}`);
1130
1413
  throw error;
1131
1414
  } finally {
1132
1415
  client.release();
1133
1416
  }
1134
1417
  }
1135
- /** Pause a cron schedule. */
1136
- async pauseCronSchedule(id) {
1418
+ async editJob(jobId, updates) {
1137
1419
  const client = await this.pool.connect();
1138
1420
  try {
1139
- await client.query(
1140
- `UPDATE cron_schedules SET status = 'paused', updated_at = NOW() WHERE id = $1`,
1141
- [id]
1142
- );
1143
- log(`Paused cron schedule ${id}`);
1421
+ const updateFields = [];
1422
+ const params = [];
1423
+ let paramIdx = 1;
1424
+ if (updates.payload !== void 0) {
1425
+ updateFields.push(`payload = $${paramIdx++}`);
1426
+ params.push(updates.payload);
1427
+ }
1428
+ if (updates.maxAttempts !== void 0) {
1429
+ updateFields.push(`max_attempts = $${paramIdx++}`);
1430
+ params.push(updates.maxAttempts);
1431
+ }
1432
+ if (updates.priority !== void 0) {
1433
+ updateFields.push(`priority = $${paramIdx++}`);
1434
+ params.push(updates.priority);
1435
+ }
1436
+ if (updates.runAt !== void 0) {
1437
+ if (updates.runAt === null) {
1438
+ updateFields.push(`run_at = NOW()`);
1439
+ } else {
1440
+ updateFields.push(`run_at = $${paramIdx++}`);
1441
+ params.push(updates.runAt);
1442
+ }
1443
+ }
1444
+ if (updates.timeoutMs !== void 0) {
1445
+ updateFields.push(`timeout_ms = $${paramIdx++}`);
1446
+ params.push(updates.timeoutMs ?? null);
1447
+ }
1448
+ if (updates.tags !== void 0) {
1449
+ updateFields.push(`tags = $${paramIdx++}`);
1450
+ params.push(updates.tags ?? null);
1451
+ }
1452
+ if (updateFields.length === 0) {
1453
+ log(`No fields to update for job ${jobId}`);
1454
+ return;
1455
+ }
1456
+ updateFields.push(`updated_at = NOW()`);
1457
+ params.push(jobId);
1458
+ const query = `
1459
+ UPDATE job_queue
1460
+ SET ${updateFields.join(", ")}
1461
+ WHERE id = $${paramIdx} AND status = 'pending'
1462
+ `;
1463
+ await client.query(query, params);
1464
+ const metadata = {};
1465
+ if (updates.payload !== void 0) metadata.payload = updates.payload;
1466
+ if (updates.maxAttempts !== void 0)
1467
+ metadata.maxAttempts = updates.maxAttempts;
1468
+ if (updates.priority !== void 0) metadata.priority = updates.priority;
1469
+ if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
1470
+ if (updates.timeoutMs !== void 0)
1471
+ metadata.timeoutMs = updates.timeoutMs;
1472
+ if (updates.tags !== void 0) metadata.tags = updates.tags;
1473
+ await this.recordJobEvent(jobId, "edited" /* Edited */, metadata);
1474
+ log(`Edited job ${jobId}: ${JSON.stringify(metadata)}`);
1144
1475
  } catch (error) {
1145
- log(`Error pausing cron schedule ${id}: ${error}`);
1476
+ log(`Error editing job ${jobId}: ${error}`);
1146
1477
  throw error;
1147
1478
  } finally {
1148
1479
  client.release();
1149
1480
  }
1150
1481
  }
1151
- /** Resume a paused cron schedule. */
1152
- async resumeCronSchedule(id) {
1482
+ async editAllPendingJobs(filters = void 0, updates) {
1153
1483
  const client = await this.pool.connect();
1154
1484
  try {
1155
- await client.query(
1156
- `UPDATE cron_schedules SET status = 'active', updated_at = NOW() WHERE id = $1`,
1157
- [id]
1158
- );
1159
- log(`Resumed cron schedule ${id}`);
1485
+ const updateFields = [];
1486
+ const params = [];
1487
+ let paramIdx = 1;
1488
+ if (updates.payload !== void 0) {
1489
+ updateFields.push(`payload = $${paramIdx++}`);
1490
+ params.push(updates.payload);
1491
+ }
1492
+ if (updates.maxAttempts !== void 0) {
1493
+ updateFields.push(`max_attempts = $${paramIdx++}`);
1494
+ params.push(updates.maxAttempts);
1495
+ }
1496
+ if (updates.priority !== void 0) {
1497
+ updateFields.push(`priority = $${paramIdx++}`);
1498
+ params.push(updates.priority);
1499
+ }
1500
+ if (updates.runAt !== void 0) {
1501
+ if (updates.runAt === null) {
1502
+ updateFields.push(`run_at = NOW()`);
1503
+ } else {
1504
+ updateFields.push(`run_at = $${paramIdx++}`);
1505
+ params.push(updates.runAt);
1506
+ }
1507
+ }
1508
+ if (updates.timeoutMs !== void 0) {
1509
+ updateFields.push(`timeout_ms = $${paramIdx++}`);
1510
+ params.push(updates.timeoutMs ?? null);
1511
+ }
1512
+ if (updates.tags !== void 0) {
1513
+ updateFields.push(`tags = $${paramIdx++}`);
1514
+ params.push(updates.tags ?? null);
1515
+ }
1516
+ if (updateFields.length === 0) {
1517
+ log(`No fields to update for batch edit`);
1518
+ return 0;
1519
+ }
1520
+ updateFields.push(`updated_at = NOW()`);
1521
+ let query = `
1522
+ UPDATE job_queue
1523
+ SET ${updateFields.join(", ")}
1524
+ WHERE status = 'pending'`;
1525
+ if (filters) {
1526
+ if (filters.jobType) {
1527
+ query += ` AND job_type = $${paramIdx++}`;
1528
+ params.push(filters.jobType);
1529
+ }
1530
+ if (filters.priority !== void 0) {
1531
+ query += ` AND priority = $${paramIdx++}`;
1532
+ params.push(filters.priority);
1533
+ }
1534
+ if (filters.runAt) {
1535
+ if (filters.runAt instanceof Date) {
1536
+ query += ` AND run_at = $${paramIdx++}`;
1537
+ params.push(filters.runAt);
1538
+ } else if (typeof filters.runAt === "object") {
1539
+ const ops = filters.runAt;
1540
+ if (ops.gt) {
1541
+ query += ` AND run_at > $${paramIdx++}`;
1542
+ params.push(ops.gt);
1543
+ }
1544
+ if (ops.gte) {
1545
+ query += ` AND run_at >= $${paramIdx++}`;
1546
+ params.push(ops.gte);
1547
+ }
1548
+ if (ops.lt) {
1549
+ query += ` AND run_at < $${paramIdx++}`;
1550
+ params.push(ops.lt);
1551
+ }
1552
+ if (ops.lte) {
1553
+ query += ` AND run_at <= $${paramIdx++}`;
1554
+ params.push(ops.lte);
1555
+ }
1556
+ if (ops.eq) {
1557
+ query += ` AND run_at = $${paramIdx++}`;
1558
+ params.push(ops.eq);
1559
+ }
1560
+ }
1561
+ }
1562
+ if (filters.tags && filters.tags.values && filters.tags.values.length > 0) {
1563
+ const mode = filters.tags.mode || "all";
1564
+ const tagValues = filters.tags.values;
1565
+ switch (mode) {
1566
+ case "exact":
1567
+ query += ` AND tags = $${paramIdx++}`;
1568
+ params.push(tagValues);
1569
+ break;
1570
+ case "all":
1571
+ query += ` AND tags @> $${paramIdx++}`;
1572
+ params.push(tagValues);
1573
+ break;
1574
+ case "any":
1575
+ query += ` AND tags && $${paramIdx++}`;
1576
+ params.push(tagValues);
1577
+ break;
1578
+ case "none":
1579
+ query += ` AND NOT (tags && $${paramIdx++})`;
1580
+ params.push(tagValues);
1581
+ break;
1582
+ default:
1583
+ query += ` AND tags @> $${paramIdx++}`;
1584
+ params.push(tagValues);
1585
+ }
1586
+ }
1587
+ }
1588
+ query += "\nRETURNING id";
1589
+ const result = await client.query(query, params);
1590
+ const editedCount = result.rowCount || 0;
1591
+ const metadata = {};
1592
+ if (updates.payload !== void 0) metadata.payload = updates.payload;
1593
+ if (updates.maxAttempts !== void 0)
1594
+ metadata.maxAttempts = updates.maxAttempts;
1595
+ if (updates.priority !== void 0) metadata.priority = updates.priority;
1596
+ if (updates.runAt !== void 0) metadata.runAt = updates.runAt;
1597
+ if (updates.timeoutMs !== void 0)
1598
+ metadata.timeoutMs = updates.timeoutMs;
1599
+ if (updates.tags !== void 0) metadata.tags = updates.tags;
1600
+ for (const row of result.rows) {
1601
+ await this.recordJobEvent(row.id, "edited" /* Edited */, metadata);
1602
+ }
1603
+ log(`Edited ${editedCount} pending jobs: ${JSON.stringify(metadata)}`);
1604
+ return editedCount;
1160
1605
  } catch (error) {
1161
- log(`Error resuming cron schedule ${id}: ${error}`);
1606
+ log(`Error editing pending jobs: ${error}`);
1162
1607
  throw error;
1163
1608
  } finally {
1164
1609
  client.release();
1165
1610
  }
1166
1611
  }
1167
- /** Edit a cron schedule. */
1168
- async editCronSchedule(id, updates, nextRunAt) {
1169
- const client = await this.pool.connect();
1612
+ /**
1613
+ * Delete completed jobs older than the given number of days.
1614
+ * Deletes in batches of 1000 to avoid long-running transactions
1615
+ * and excessive WAL bloat at scale.
1616
+ *
1617
+ * @param daysToKeep - Number of days to retain completed jobs (default 30).
1618
+ * @param batchSize - Number of rows to delete per batch (default 1000).
1619
+ * @returns Total number of deleted jobs.
1620
+ */
1621
+ async cleanupOldJobs(daysToKeep = 30, batchSize = 1e3) {
1622
+ let totalDeleted = 0;
1170
1623
  try {
1171
- const updateFields = [];
1172
- const params = [];
1173
- let paramIdx = 1;
1174
- if (updates.cronExpression !== void 0) {
1175
- updateFields.push(`cron_expression = $${paramIdx++}`);
1176
- params.push(updates.cronExpression);
1177
- }
1178
- if (updates.payload !== void 0) {
1179
- updateFields.push(`payload = $${paramIdx++}`);
1180
- params.push(updates.payload);
1181
- }
1182
- if (updates.maxAttempts !== void 0) {
1183
- updateFields.push(`max_attempts = $${paramIdx++}`);
1184
- params.push(updates.maxAttempts);
1185
- }
1186
- if (updates.priority !== void 0) {
1187
- updateFields.push(`priority = $${paramIdx++}`);
1188
- params.push(updates.priority);
1189
- }
1190
- if (updates.timeoutMs !== void 0) {
1191
- updateFields.push(`timeout_ms = $${paramIdx++}`);
1192
- params.push(updates.timeoutMs);
1193
- }
1194
- if (updates.forceKillOnTimeout !== void 0) {
1195
- updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
1196
- params.push(updates.forceKillOnTimeout);
1197
- }
1198
- if (updates.tags !== void 0) {
1199
- updateFields.push(`tags = $${paramIdx++}`);
1200
- params.push(updates.tags);
1201
- }
1202
- if (updates.timezone !== void 0) {
1203
- updateFields.push(`timezone = $${paramIdx++}`);
1204
- params.push(updates.timezone);
1205
- }
1206
- if (updates.allowOverlap !== void 0) {
1207
- updateFields.push(`allow_overlap = $${paramIdx++}`);
1208
- params.push(updates.allowOverlap);
1209
- }
1210
- if (nextRunAt !== void 0) {
1211
- updateFields.push(`next_run_at = $${paramIdx++}`);
1212
- params.push(nextRunAt);
1213
- }
1214
- if (updateFields.length === 0) {
1215
- log(`No fields to update for cron schedule ${id}`);
1216
- return;
1217
- }
1218
- updateFields.push(`updated_at = NOW()`);
1219
- params.push(id);
1220
- const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
1221
- await client.query(query, params);
1222
- log(`Edited cron schedule ${id}`);
1624
+ let deletedInBatch;
1625
+ do {
1626
+ const client = await this.pool.connect();
1627
+ try {
1628
+ const result = await client.query(
1629
+ `
1630
+ DELETE FROM job_queue
1631
+ WHERE id IN (
1632
+ SELECT id FROM job_queue
1633
+ WHERE status = 'completed'
1634
+ AND updated_at < NOW() - INTERVAL '1 day' * $1::int
1635
+ LIMIT $2
1636
+ )
1637
+ `,
1638
+ [daysToKeep, batchSize]
1639
+ );
1640
+ deletedInBatch = result.rowCount || 0;
1641
+ totalDeleted += deletedInBatch;
1642
+ } finally {
1643
+ client.release();
1644
+ }
1645
+ } while (deletedInBatch === batchSize);
1646
+ log(`Deleted ${totalDeleted} old jobs`);
1647
+ return totalDeleted;
1223
1648
  } catch (error) {
1224
- log(`Error editing cron schedule ${id}: ${error}`);
1649
+ log(`Error cleaning up old jobs: ${error}`);
1225
1650
  throw error;
1226
- } finally {
1227
- client.release();
1228
1651
  }
1229
1652
  }
1230
1653
  /**
1231
- * Atomically fetch all active cron schedules whose nextRunAt <= NOW().
1232
- * Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
1654
+ * Delete job events older than the given number of days.
1655
+ * Deletes in batches of 1000 to avoid long-running transactions
1656
+ * and excessive WAL bloat at scale.
1657
+ *
1658
+ * @param daysToKeep - Number of days to retain events (default 30).
1659
+ * @param batchSize - Number of rows to delete per batch (default 1000).
1660
+ * @returns Total number of deleted events.
1233
1661
  */
1234
- async getDueCronSchedules() {
1235
- const client = await this.pool.connect();
1662
+ async cleanupOldJobEvents(daysToKeep = 30, batchSize = 1e3) {
1663
+ let totalDeleted = 0;
1236
1664
  try {
1237
- const result = await client.query(
1238
- `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1239
- job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1240
- priority, timeout_ms AS "timeoutMs",
1241
- force_kill_on_timeout AS "forceKillOnTimeout", tags,
1242
- timezone, allow_overlap AS "allowOverlap", status,
1243
- last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1244
- next_run_at AS "nextRunAt",
1245
- created_at AS "createdAt", updated_at AS "updatedAt"
1246
- FROM cron_schedules
1247
- WHERE status = 'active'
1248
- AND next_run_at IS NOT NULL
1249
- AND next_run_at <= NOW()
1250
- ORDER BY next_run_at ASC
1251
- FOR UPDATE SKIP LOCKED`
1252
- );
1253
- log(`Found ${result.rows.length} due cron schedules`);
1254
- return result.rows;
1665
+ let deletedInBatch;
1666
+ do {
1667
+ const client = await this.pool.connect();
1668
+ try {
1669
+ const result = await client.query(
1670
+ `
1671
+ DELETE FROM job_events
1672
+ WHERE id IN (
1673
+ SELECT id FROM job_events
1674
+ WHERE created_at < NOW() - INTERVAL '1 day' * $1::int
1675
+ LIMIT $2
1676
+ )
1677
+ `,
1678
+ [daysToKeep, batchSize]
1679
+ );
1680
+ deletedInBatch = result.rowCount || 0;
1681
+ totalDeleted += deletedInBatch;
1682
+ } finally {
1683
+ client.release();
1684
+ }
1685
+ } while (deletedInBatch === batchSize);
1686
+ log(`Deleted ${totalDeleted} old job events`);
1687
+ return totalDeleted;
1255
1688
  } catch (error) {
1256
- if (error?.code === "42P01") {
1257
- log("cron_schedules table does not exist, skipping cron enqueue");
1258
- return [];
1259
- }
1260
- log(`Error getting due cron schedules: ${error}`);
1689
+ log(`Error cleaning up old job events: ${error}`);
1261
1690
  throw error;
1262
- } finally {
1263
- client.release();
1264
1691
  }
1265
1692
  }
1266
- /**
1267
- * Update a cron schedule after a job has been enqueued.
1268
- * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
1269
- */
1270
- async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
1693
+ async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
1271
1694
  const client = await this.pool.connect();
1272
1695
  try {
1273
- await client.query(
1274
- `UPDATE cron_schedules
1275
- SET last_enqueued_at = $2,
1276
- last_job_id = $3,
1277
- next_run_at = $4,
1278
- updated_at = NOW()
1279
- WHERE id = $1`,
1280
- [id, lastEnqueuedAt, lastJobId, nextRunAt]
1281
- );
1282
- log(
1283
- `Updated cron schedule ${id}: lastJobId=${lastJobId}, nextRunAt=${nextRunAt?.toISOString() ?? "null"}`
1696
+ const result = await client.query(
1697
+ `
1698
+ UPDATE job_queue
1699
+ SET status = 'pending', locked_at = NULL, locked_by = NULL, updated_at = NOW()
1700
+ WHERE status = 'processing'
1701
+ AND locked_at < NOW() - GREATEST(
1702
+ INTERVAL '1 minute' * $1::int,
1703
+ INTERVAL '1 millisecond' * COALESCE(timeout_ms, 0)
1704
+ )
1705
+ RETURNING id
1706
+ `,
1707
+ [maxProcessingTimeMinutes]
1284
1708
  );
1709
+ log(`Reclaimed ${result.rowCount} stuck jobs`);
1710
+ return result.rowCount || 0;
1285
1711
  } catch (error) {
1286
- log(`Error updating cron schedule ${id} after enqueue: ${error}`);
1712
+ log(`Error reclaiming stuck jobs: ${error}`);
1287
1713
  throw error;
1288
1714
  } finally {
1289
1715
  client.release();
1290
1716
  }
1291
1717
  }
1292
1718
  // ── Internal helpers ──────────────────────────────────────────────────
1293
- async setPendingReasonForUnpickedJobs(reason, jobType) {
1719
+ /**
1720
+ * Batch-insert multiple job events in a single query.
1721
+ * More efficient than individual recordJobEvent calls.
1722
+ */
1723
+ async recordJobEventsBatch(events) {
1724
+ if (events.length === 0) return;
1294
1725
  const client = await this.pool.connect();
1295
1726
  try {
1296
- let jobTypeFilter = "";
1297
- const params = [reason];
1298
- if (jobType) {
1299
- if (Array.isArray(jobType)) {
1300
- jobTypeFilter = ` AND job_type = ANY($2)`;
1301
- params.push(jobType);
1302
- } else {
1303
- jobTypeFilter = ` AND job_type = $2`;
1304
- params.push(jobType);
1305
- }
1306
- }
1307
- await client.query(
1308
- `UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
1309
- params
1310
- );
1311
- } finally {
1312
- client.release();
1313
- }
1314
- }
1315
- };
1316
- var recordJobEvent = async (pool, jobId, eventType, metadata) => new PostgresBackend(pool).recordJobEvent(jobId, eventType, metadata);
1317
- var waitJob = async (pool, jobId, options) => {
1318
- const client = await pool.connect();
1319
- try {
1320
- const result = await client.query(
1321
- `
1322
- UPDATE job_queue
1323
- SET status = 'waiting',
1324
- wait_until = $2,
1325
- wait_token_id = $3,
1326
- step_data = $4,
1327
- locked_at = NULL,
1328
- locked_by = NULL,
1329
- updated_at = NOW()
1330
- WHERE id = $1 AND status = 'processing'
1331
- `,
1332
- [
1333
- jobId,
1334
- options.waitUntil ?? null,
1335
- options.waitTokenId ?? null,
1336
- JSON.stringify(options.stepData)
1337
- ]
1338
- );
1339
- if (result.rowCount === 0) {
1340
- log(
1341
- `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
1342
- );
1343
- return;
1344
- }
1345
- await recordJobEvent(pool, jobId, "waiting" /* Waiting */, {
1346
- waitUntil: options.waitUntil?.toISOString() ?? null,
1347
- waitTokenId: options.waitTokenId ?? null
1348
- });
1349
- log(`Job ${jobId} set to waiting`);
1350
- } catch (error) {
1351
- log(`Error setting job ${jobId} to waiting: ${error}`);
1352
- throw error;
1353
- } finally {
1354
- client.release();
1355
- }
1356
- };
1357
- var updateStepData = async (pool, jobId, stepData) => {
1358
- const client = await pool.connect();
1359
- try {
1360
- await client.query(
1361
- `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
1362
- [jobId, JSON.stringify(stepData)]
1363
- );
1364
- } catch (error) {
1365
- log(`Error updating step_data for job ${jobId}: ${error}`);
1366
- } finally {
1367
- client.release();
1368
- }
1369
- };
1370
- var MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1e3;
1371
- function parseTimeoutString(timeout) {
1372
- const match = timeout.match(/^(\d+)(s|m|h|d)$/);
1373
- if (!match) {
1374
- throw new Error(
1375
- `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
1376
- );
1377
- }
1378
- const value = parseInt(match[1], 10);
1379
- const unit = match[2];
1380
- let ms;
1381
- switch (unit) {
1382
- case "s":
1383
- ms = value * 1e3;
1384
- break;
1385
- case "m":
1386
- ms = value * 60 * 1e3;
1387
- break;
1388
- case "h":
1389
- ms = value * 60 * 60 * 1e3;
1390
- break;
1391
- case "d":
1392
- ms = value * 24 * 60 * 60 * 1e3;
1393
- break;
1394
- default:
1395
- throw new Error(`Unknown timeout unit: "${unit}"`);
1396
- }
1397
- if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS) {
1398
- throw new Error(
1399
- `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
1400
- );
1401
- }
1402
- return ms;
1403
- }
1404
- var createWaitpoint = async (pool, jobId, options) => {
1405
- const client = await pool.connect();
1406
- try {
1407
- const id = `wp_${randomUUID()}`;
1408
- let timeoutAt = null;
1409
- if (options?.timeout) {
1410
- const ms = parseTimeoutString(options.timeout);
1411
- timeoutAt = new Date(Date.now() + ms);
1412
- }
1413
- await client.query(
1414
- `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
1415
- [id, jobId, timeoutAt, options?.tags ?? null]
1416
- );
1417
- log(`Created waitpoint ${id} for job ${jobId}`);
1418
- return { id };
1419
- } catch (error) {
1420
- log(`Error creating waitpoint: ${error}`);
1421
- throw error;
1422
- } finally {
1423
- client.release();
1424
- }
1425
- };
1426
- var completeWaitpoint = async (pool, tokenId, data) => {
1427
- const client = await pool.connect();
1428
- try {
1429
- await client.query("BEGIN");
1430
- const wpResult = await client.query(
1431
- `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
1432
- WHERE id = $1 AND status = 'waiting'
1433
- RETURNING job_id`,
1434
- [tokenId, data != null ? JSON.stringify(data) : null]
1435
- );
1436
- if (wpResult.rows.length === 0) {
1437
- await client.query("ROLLBACK");
1438
- log(`Waitpoint ${tokenId} not found or already completed`);
1439
- return;
1440
- }
1441
- const jobId = wpResult.rows[0].job_id;
1442
- if (jobId != null) {
1727
+ const values = [];
1728
+ const params = [];
1729
+ let paramIdx = 1;
1730
+ for (const event of events) {
1731
+ values.push(`($${paramIdx++}, $${paramIdx++}, $${paramIdx++})`);
1732
+ params.push(
1733
+ event.jobId,
1734
+ event.eventType,
1735
+ event.metadata ? JSON.stringify(event.metadata) : null
1736
+ );
1737
+ }
1443
1738
  await client.query(
1444
- `UPDATE job_queue
1445
- SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
1446
- WHERE id = $1 AND status = 'waiting'`,
1447
- [jobId]
1739
+ `INSERT INTO job_events (job_id, event_type, metadata) VALUES ${values.join(", ")}`,
1740
+ params
1448
1741
  );
1742
+ } catch (error) {
1743
+ log(`Error recording batch job events: ${error}`);
1744
+ } finally {
1745
+ client.release();
1449
1746
  }
1450
- await client.query("COMMIT");
1451
- log(`Completed waitpoint ${tokenId} for job ${jobId}`);
1452
- } catch (error) {
1453
- await client.query("ROLLBACK");
1454
- log(`Error completing waitpoint ${tokenId}: ${error}`);
1455
- throw error;
1456
- } finally {
1457
- client.release();
1458
- }
1459
- };
1460
- var getWaitpoint = async (pool, tokenId) => {
1461
- const client = await pool.connect();
1462
- try {
1463
- const result = await client.query(
1464
- `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
1465
- [tokenId]
1466
- );
1467
- if (result.rows.length === 0) return null;
1468
- return result.rows[0];
1469
- } catch (error) {
1470
- log(`Error getting waitpoint ${tokenId}: ${error}`);
1471
- throw error;
1472
- } finally {
1473
- client.release();
1474
1747
  }
1475
- };
1476
- var expireTimedOutWaitpoints = async (pool) => {
1477
- const client = await pool.connect();
1478
- try {
1479
- await client.query("BEGIN");
1480
- const result = await client.query(
1481
- `UPDATE waitpoints
1482
- SET status = 'timed_out'
1483
- WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
1484
- RETURNING id, job_id`
1485
- );
1486
- for (const row of result.rows) {
1487
- if (row.job_id != null) {
1488
- await client.query(
1489
- `UPDATE job_queue
1490
- SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
1491
- WHERE id = $1 AND status = 'waiting'`,
1492
- [row.job_id]
1748
+ // ── Cron schedules ──────────────────────────────────────────────────
1749
+ /** Create a cron schedule and return its ID. */
1750
+ async addCronSchedule(input) {
1751
+ const client = await this.pool.connect();
1752
+ try {
1753
+ const result = await client.query(
1754
+ `INSERT INTO cron_schedules
1755
+ (schedule_name, cron_expression, job_type, payload, max_attempts,
1756
+ priority, timeout_ms, force_kill_on_timeout, tags, timezone,
1757
+ allow_overlap, next_run_at)
1758
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
1759
+ RETURNING id`,
1760
+ [
1761
+ input.scheduleName,
1762
+ input.cronExpression,
1763
+ input.jobType,
1764
+ input.payload,
1765
+ input.maxAttempts,
1766
+ input.priority,
1767
+ input.timeoutMs,
1768
+ input.forceKillOnTimeout,
1769
+ input.tags ?? null,
1770
+ input.timezone,
1771
+ input.allowOverlap,
1772
+ input.nextRunAt
1773
+ ]
1774
+ );
1775
+ const id = result.rows[0].id;
1776
+ log(`Added cron schedule ${id}: "${input.scheduleName}"`);
1777
+ return id;
1778
+ } catch (error) {
1779
+ if (error?.code === "23505") {
1780
+ throw new Error(
1781
+ `Cron schedule with name "${input.scheduleName}" already exists`
1493
1782
  );
1494
1783
  }
1784
+ log(`Error adding cron schedule: ${error}`);
1785
+ throw error;
1786
+ } finally {
1787
+ client.release();
1495
1788
  }
1496
- await client.query("COMMIT");
1497
- const count = result.rowCount || 0;
1498
- if (count > 0) {
1499
- log(`Expired ${count} timed-out waitpoints`);
1500
- }
1501
- return count;
1502
- } catch (error) {
1503
- await client.query("ROLLBACK");
1504
- log(`Error expiring timed-out waitpoints: ${error}`);
1505
- throw error;
1506
- } finally {
1507
- client.release();
1508
- }
1509
- };
1510
- function tryExtractPool(backend) {
1511
- if (backend instanceof PostgresBackend) {
1512
- return backend.getPool();
1513
1789
  }
1514
- return null;
1515
- }
1516
- function buildBasicContext(backend, jobId, baseCtx) {
1517
- const waitError = () => new Error(
1518
- "Wait features (waitFor, waitUntil, createToken, waitForToken, ctx.run) are currently only supported with the PostgreSQL backend."
1519
- );
1520
- return {
1521
- prolong: baseCtx.prolong,
1522
- onTimeout: baseCtx.onTimeout,
1523
- run: async (_stepName, fn) => {
1524
- return fn();
1525
- },
1526
- waitFor: async () => {
1527
- throw waitError();
1528
- },
1529
- waitUntil: async () => {
1530
- throw waitError();
1531
- },
1532
- createToken: async () => {
1533
- throw waitError();
1534
- },
1535
- waitForToken: async () => {
1536
- throw waitError();
1537
- },
1538
- setProgress: async (percent) => {
1539
- if (percent < 0 || percent > 100)
1540
- throw new Error("Progress must be between 0 and 100");
1541
- await backend.updateProgress(jobId, Math.round(percent));
1542
- }
1543
- };
1544
- }
1545
- function validateHandlerSerializable(handler, jobType) {
1546
- try {
1547
- const handlerString = handler.toString();
1548
- if (handlerString.includes("this.") && !handlerString.match(/\([^)]*this[^)]*\)/)) {
1549
- throw new Error(
1550
- `Handler for job type "${jobType}" uses 'this' context which cannot be serialized. Use a regular function or avoid 'this' references when forceKillOnTimeout is enabled.`
1551
- );
1552
- }
1553
- if (handlerString.includes("[native code]")) {
1554
- throw new Error(
1555
- `Handler for job type "${jobType}" contains native code which cannot be serialized. Ensure your handler is a plain function when forceKillOnTimeout is enabled.`
1790
+ /** Get a cron schedule by ID. */
1791
+ async getCronSchedule(id) {
1792
+ const client = await this.pool.connect();
1793
+ try {
1794
+ const result = await client.query(
1795
+ `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1796
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1797
+ priority, timeout_ms AS "timeoutMs",
1798
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1799
+ timezone, allow_overlap AS "allowOverlap", status,
1800
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1801
+ next_run_at AS "nextRunAt",
1802
+ created_at AS "createdAt", updated_at AS "updatedAt"
1803
+ FROM cron_schedules WHERE id = $1`,
1804
+ [id]
1556
1805
  );
1806
+ if (result.rows.length === 0) return null;
1807
+ return result.rows[0];
1808
+ } catch (error) {
1809
+ log(`Error getting cron schedule ${id}: ${error}`);
1810
+ throw error;
1811
+ } finally {
1812
+ client.release();
1557
1813
  }
1814
+ }
1815
+ /** Get a cron schedule by its unique name. */
1816
+ async getCronScheduleByName(name) {
1817
+ const client = await this.pool.connect();
1558
1818
  try {
1559
- new Function("return " + handlerString);
1560
- } catch (parseError) {
1561
- throw new Error(
1562
- `Handler for job type "${jobType}" cannot be serialized: ${parseError instanceof Error ? parseError.message : String(parseError)}. When using forceKillOnTimeout, handlers must be serializable functions without closures over external variables.`
1819
+ const result = await client.query(
1820
+ `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1821
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1822
+ priority, timeout_ms AS "timeoutMs",
1823
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1824
+ timezone, allow_overlap AS "allowOverlap", status,
1825
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1826
+ next_run_at AS "nextRunAt",
1827
+ created_at AS "createdAt", updated_at AS "updatedAt"
1828
+ FROM cron_schedules WHERE schedule_name = $1`,
1829
+ [name]
1563
1830
  );
1564
- }
1565
- } catch (error) {
1566
- if (error instanceof Error) {
1831
+ if (result.rows.length === 0) return null;
1832
+ return result.rows[0];
1833
+ } catch (error) {
1834
+ log(`Error getting cron schedule by name "${name}": ${error}`);
1567
1835
  throw error;
1836
+ } finally {
1837
+ client.release();
1568
1838
  }
1569
- throw new Error(
1570
- `Failed to validate handler serialization for job type "${jobType}": ${String(error)}`
1571
- );
1572
1839
  }
1573
- }
1574
- async function runHandlerInWorker(handler, payload, timeoutMs, jobType) {
1575
- validateHandlerSerializable(handler, jobType);
1576
- return new Promise((resolve, reject) => {
1577
- const workerCode = `
1578
- (function() {
1579
- const { parentPort, workerData } = require('worker_threads');
1580
- const { handlerCode, payload, timeoutMs } = workerData;
1581
-
1582
- // Create an AbortController for the handler
1583
- const controller = new AbortController();
1584
- const signal = controller.signal;
1585
-
1586
- // Set up timeout
1587
- const timeoutId = setTimeout(() => {
1588
- controller.abort();
1589
- parentPort.postMessage({ type: 'timeout' });
1590
- }, timeoutMs);
1591
-
1592
- try {
1593
- // Execute the handler
1594
- // Note: This uses Function constructor which requires the handler to be serializable.
1595
- // The handler should be validated before reaching this point.
1596
- let handlerFn;
1597
- try {
1598
- // Wrap handlerCode in parentheses to ensure it's treated as an expression
1599
- // This handles both arrow functions and regular functions
1600
- const wrappedCode = handlerCode.trim().startsWith('async') || handlerCode.trim().startsWith('function')
1601
- ? handlerCode
1602
- : '(' + handlerCode + ')';
1603
- handlerFn = new Function('return ' + wrappedCode)();
1604
- } catch (parseError) {
1605
- clearTimeout(timeoutId);
1606
- parentPort.postMessage({
1607
- type: 'error',
1608
- error: {
1609
- message: 'Handler cannot be deserialized in worker thread. ' +
1610
- 'Ensure your handler is a standalone function without closures over external variables. ' +
1611
- 'Original error: ' + (parseError instanceof Error ? parseError.message : String(parseError)),
1612
- stack: parseError instanceof Error ? parseError.stack : undefined,
1613
- name: 'SerializationError',
1614
- },
1615
- });
1616
- return;
1617
- }
1618
-
1619
- // Ensure handlerFn is actually a function
1620
- if (typeof handlerFn !== 'function') {
1621
- clearTimeout(timeoutId);
1622
- parentPort.postMessage({
1623
- type: 'error',
1624
- error: {
1625
- message: 'Handler deserialization did not produce a function. ' +
1626
- 'Ensure your handler is a valid function when forceKillOnTimeout is enabled.',
1627
- name: 'SerializationError',
1628
- },
1629
- });
1630
- return;
1631
- }
1632
-
1633
- handlerFn(payload, signal)
1634
- .then(() => {
1635
- clearTimeout(timeoutId);
1636
- parentPort.postMessage({ type: 'success' });
1637
- })
1638
- .catch((error) => {
1639
- clearTimeout(timeoutId);
1640
- parentPort.postMessage({
1641
- type: 'error',
1642
- error: {
1643
- message: error.message,
1644
- stack: error.stack,
1645
- name: error.name,
1646
- },
1647
- });
1648
- });
1649
- } catch (error) {
1650
- clearTimeout(timeoutId);
1651
- parentPort.postMessage({
1652
- type: 'error',
1653
- error: {
1654
- message: error.message,
1655
- stack: error.stack,
1656
- name: error.name,
1657
- },
1658
- });
1659
- }
1660
- })();
1661
- `;
1662
- const worker = new Worker(workerCode, {
1663
- eval: true,
1664
- workerData: {
1665
- handlerCode: handler.toString(),
1666
- payload,
1667
- timeoutMs
1840
+ /** List cron schedules, optionally filtered by status. */
1841
+ async listCronSchedules(status) {
1842
+ const client = await this.pool.connect();
1843
+ try {
1844
+ let query = `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1845
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1846
+ priority, timeout_ms AS "timeoutMs",
1847
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1848
+ timezone, allow_overlap AS "allowOverlap", status,
1849
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1850
+ next_run_at AS "nextRunAt",
1851
+ created_at AS "createdAt", updated_at AS "updatedAt"
1852
+ FROM cron_schedules`;
1853
+ const params = [];
1854
+ if (status) {
1855
+ query += ` WHERE status = $1`;
1856
+ params.push(status);
1668
1857
  }
1669
- });
1670
- let resolved = false;
1671
- worker.on("message", (message) => {
1672
- if (resolved) return;
1673
- resolved = true;
1674
- if (message.type === "success") {
1675
- resolve();
1676
- } else if (message.type === "timeout") {
1677
- const timeoutError = new Error(
1678
- `Job timed out after ${timeoutMs} ms and was forcefully terminated`
1679
- );
1680
- timeoutError.failureReason = "timeout" /* Timeout */;
1681
- reject(timeoutError);
1682
- } else if (message.type === "error") {
1683
- const error = new Error(message.error.message);
1684
- error.stack = message.error.stack;
1685
- error.name = message.error.name;
1686
- reject(error);
1858
+ query += ` ORDER BY created_at ASC`;
1859
+ const result = await client.query(query, params);
1860
+ return result.rows;
1861
+ } catch (error) {
1862
+ log(`Error listing cron schedules: ${error}`);
1863
+ throw error;
1864
+ } finally {
1865
+ client.release();
1866
+ }
1867
+ }
1868
+ /** Delete a cron schedule by ID. */
1869
+ async removeCronSchedule(id) {
1870
+ const client = await this.pool.connect();
1871
+ try {
1872
+ await client.query(`DELETE FROM cron_schedules WHERE id = $1`, [id]);
1873
+ log(`Removed cron schedule ${id}`);
1874
+ } catch (error) {
1875
+ log(`Error removing cron schedule ${id}: ${error}`);
1876
+ throw error;
1877
+ } finally {
1878
+ client.release();
1879
+ }
1880
+ }
1881
+ /** Pause a cron schedule. */
1882
+ async pauseCronSchedule(id) {
1883
+ const client = await this.pool.connect();
1884
+ try {
1885
+ await client.query(
1886
+ `UPDATE cron_schedules SET status = 'paused', updated_at = NOW() WHERE id = $1`,
1887
+ [id]
1888
+ );
1889
+ log(`Paused cron schedule ${id}`);
1890
+ } catch (error) {
1891
+ log(`Error pausing cron schedule ${id}: ${error}`);
1892
+ throw error;
1893
+ } finally {
1894
+ client.release();
1895
+ }
1896
+ }
1897
+ /** Resume a paused cron schedule. */
1898
+ async resumeCronSchedule(id) {
1899
+ const client = await this.pool.connect();
1900
+ try {
1901
+ await client.query(
1902
+ `UPDATE cron_schedules SET status = 'active', updated_at = NOW() WHERE id = $1`,
1903
+ [id]
1904
+ );
1905
+ log(`Resumed cron schedule ${id}`);
1906
+ } catch (error) {
1907
+ log(`Error resuming cron schedule ${id}: ${error}`);
1908
+ throw error;
1909
+ } finally {
1910
+ client.release();
1911
+ }
1912
+ }
1913
+ /** Edit a cron schedule. */
1914
+ async editCronSchedule(id, updates, nextRunAt) {
1915
+ const client = await this.pool.connect();
1916
+ try {
1917
+ const updateFields = [];
1918
+ const params = [];
1919
+ let paramIdx = 1;
1920
+ if (updates.cronExpression !== void 0) {
1921
+ updateFields.push(`cron_expression = $${paramIdx++}`);
1922
+ params.push(updates.cronExpression);
1687
1923
  }
1688
- });
1689
- worker.on("error", (error) => {
1690
- if (resolved) return;
1691
- resolved = true;
1692
- reject(error);
1693
- });
1694
- worker.on("exit", (code) => {
1695
- if (resolved) return;
1696
- if (code !== 0) {
1697
- resolved = true;
1698
- reject(new Error(`Worker stopped with exit code ${code}`));
1924
+ if (updates.payload !== void 0) {
1925
+ updateFields.push(`payload = $${paramIdx++}`);
1926
+ params.push(updates.payload);
1699
1927
  }
1700
- });
1701
- setTimeout(() => {
1702
- if (!resolved) {
1703
- resolved = true;
1704
- worker.terminate().then(() => {
1705
- const timeoutError = new Error(
1706
- `Job timed out after ${timeoutMs} ms and was forcefully terminated`
1707
- );
1708
- timeoutError.failureReason = "timeout" /* Timeout */;
1709
- reject(timeoutError);
1710
- }).catch((err) => {
1711
- reject(err);
1712
- });
1928
+ if (updates.maxAttempts !== void 0) {
1929
+ updateFields.push(`max_attempts = $${paramIdx++}`);
1930
+ params.push(updates.maxAttempts);
1713
1931
  }
1714
- }, timeoutMs + 100);
1715
- });
1716
- }
1717
- function calculateWaitUntil(duration) {
1718
- const now = Date.now();
1719
- let ms = 0;
1720
- if (duration.seconds) ms += duration.seconds * 1e3;
1721
- if (duration.minutes) ms += duration.minutes * 60 * 1e3;
1722
- if (duration.hours) ms += duration.hours * 60 * 60 * 1e3;
1723
- if (duration.days) ms += duration.days * 24 * 60 * 60 * 1e3;
1724
- if (duration.weeks) ms += duration.weeks * 7 * 24 * 60 * 60 * 1e3;
1725
- if (duration.months) ms += duration.months * 30 * 24 * 60 * 60 * 1e3;
1726
- if (duration.years) ms += duration.years * 365 * 24 * 60 * 60 * 1e3;
1727
- if (ms <= 0) {
1728
- throw new Error(
1729
- "waitFor duration must be positive. Provide at least one positive duration field."
1730
- );
1731
- }
1732
- return new Date(now + ms);
1733
- }
1734
- async function resolveCompletedWaits(pool, stepData) {
1735
- for (const key of Object.keys(stepData)) {
1736
- if (!key.startsWith("__wait_")) continue;
1737
- const entry = stepData[key];
1738
- if (!entry || typeof entry !== "object" || entry.completed) continue;
1739
- if (entry.type === "duration" || entry.type === "date") {
1740
- stepData[key] = { ...entry, completed: true };
1741
- } else if (entry.type === "token" && entry.tokenId) {
1742
- const wp = await getWaitpoint(pool, entry.tokenId);
1743
- if (wp && wp.status === "completed") {
1744
- stepData[key] = {
1745
- ...entry,
1746
- completed: true,
1747
- result: { ok: true, output: wp.output }
1748
- };
1749
- } else if (wp && wp.status === "timed_out") {
1750
- stepData[key] = {
1751
- ...entry,
1752
- completed: true,
1753
- result: { ok: false, error: "Token timed out" }
1754
- };
1932
+ if (updates.priority !== void 0) {
1933
+ updateFields.push(`priority = $${paramIdx++}`);
1934
+ params.push(updates.priority);
1755
1935
  }
1756
- }
1757
- }
1758
- }
1759
- function buildWaitContext(backend, pool, jobId, stepData, baseCtx) {
1760
- let waitCounter = 0;
1761
- const ctx = {
1762
- prolong: baseCtx.prolong,
1763
- onTimeout: baseCtx.onTimeout,
1764
- run: async (stepName, fn) => {
1765
- const cached = stepData[stepName];
1766
- if (cached && typeof cached === "object" && cached.__completed) {
1767
- log(`Step "${stepName}" replayed from cache for job ${jobId}`);
1768
- return cached.result;
1936
+ if (updates.timeoutMs !== void 0) {
1937
+ updateFields.push(`timeout_ms = $${paramIdx++}`);
1938
+ params.push(updates.timeoutMs);
1769
1939
  }
1770
- const result = await fn();
1771
- stepData[stepName] = { __completed: true, result };
1772
- await updateStepData(pool, jobId, stepData);
1773
- return result;
1774
- },
1775
- waitFor: async (duration) => {
1776
- const waitKey = `__wait_${waitCounter++}`;
1777
- const cached = stepData[waitKey];
1778
- if (cached && typeof cached === "object" && cached.completed) {
1779
- log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
1780
- return;
1940
+ if (updates.forceKillOnTimeout !== void 0) {
1941
+ updateFields.push(`force_kill_on_timeout = $${paramIdx++}`);
1942
+ params.push(updates.forceKillOnTimeout);
1781
1943
  }
1782
- const waitUntilDate = calculateWaitUntil(duration);
1783
- stepData[waitKey] = { type: "duration", completed: false };
1784
- throw new WaitSignal("duration", waitUntilDate, void 0, stepData);
1785
- },
1786
- waitUntil: async (date) => {
1787
- const waitKey = `__wait_${waitCounter++}`;
1788
- const cached = stepData[waitKey];
1789
- if (cached && typeof cached === "object" && cached.completed) {
1790
- log(`Wait "${waitKey}" already completed for job ${jobId}, skipping`);
1791
- return;
1944
+ if (updates.tags !== void 0) {
1945
+ updateFields.push(`tags = $${paramIdx++}`);
1946
+ params.push(updates.tags);
1792
1947
  }
1793
- stepData[waitKey] = { type: "date", completed: false };
1794
- throw new WaitSignal("date", date, void 0, stepData);
1795
- },
1796
- createToken: async (options) => {
1797
- const token = await createWaitpoint(pool, jobId, options);
1798
- return token;
1799
- },
1800
- waitForToken: async (tokenId) => {
1801
- const waitKey = `__wait_${waitCounter++}`;
1802
- const cached = stepData[waitKey];
1803
- if (cached && typeof cached === "object" && cached.completed) {
1804
- log(
1805
- `Token wait "${waitKey}" already completed for job ${jobId}, returning cached result`
1806
- );
1807
- return cached.result;
1948
+ if (updates.timezone !== void 0) {
1949
+ updateFields.push(`timezone = $${paramIdx++}`);
1950
+ params.push(updates.timezone);
1808
1951
  }
1809
- const wp = await getWaitpoint(pool, tokenId);
1810
- if (wp && wp.status === "completed") {
1811
- const result = {
1812
- ok: true,
1813
- output: wp.output
1814
- };
1815
- stepData[waitKey] = {
1816
- type: "token",
1817
- tokenId,
1818
- completed: true,
1819
- result
1820
- };
1821
- await updateStepData(pool, jobId, stepData);
1822
- return result;
1952
+ if (updates.allowOverlap !== void 0) {
1953
+ updateFields.push(`allow_overlap = $${paramIdx++}`);
1954
+ params.push(updates.allowOverlap);
1823
1955
  }
1824
- if (wp && wp.status === "timed_out") {
1825
- const result = {
1826
- ok: false,
1827
- error: "Token timed out"
1828
- };
1829
- stepData[waitKey] = {
1830
- type: "token",
1831
- tokenId,
1832
- completed: true,
1833
- result
1834
- };
1835
- await updateStepData(pool, jobId, stepData);
1836
- return result;
1956
+ if (nextRunAt !== void 0) {
1957
+ updateFields.push(`next_run_at = $${paramIdx++}`);
1958
+ params.push(nextRunAt);
1959
+ }
1960
+ if (updateFields.length === 0) {
1961
+ log(`No fields to update for cron schedule ${id}`);
1962
+ return;
1837
1963
  }
1838
- stepData[waitKey] = { type: "token", tokenId, completed: false };
1839
- throw new WaitSignal("token", void 0, tokenId, stepData);
1840
- },
1841
- setProgress: async (percent) => {
1842
- if (percent < 0 || percent > 100)
1843
- throw new Error("Progress must be between 0 and 100");
1844
- await backend.updateProgress(jobId, Math.round(percent));
1964
+ updateFields.push(`updated_at = NOW()`);
1965
+ params.push(id);
1966
+ const query = `UPDATE cron_schedules SET ${updateFields.join(", ")} WHERE id = $${paramIdx}`;
1967
+ await client.query(query, params);
1968
+ log(`Edited cron schedule ${id}`);
1969
+ } catch (error) {
1970
+ log(`Error editing cron schedule ${id}: ${error}`);
1971
+ throw error;
1972
+ } finally {
1973
+ client.release();
1845
1974
  }
1846
- };
1847
- return ctx;
1848
- }
1849
- async function processJobWithHandlers(backend, job, jobHandlers) {
1850
- const handler = jobHandlers[job.jobType];
1851
- if (!handler) {
1852
- await backend.setPendingReasonForUnpickedJobs(
1853
- `No handler registered for job type: ${job.jobType}`,
1854
- job.jobType
1855
- );
1856
- await backend.failJob(
1857
- job.id,
1858
- new Error(`No handler registered for job type: ${job.jobType}`),
1859
- "no_handler" /* NoHandler */
1860
- );
1861
- return;
1862
- }
1863
- const stepData = { ...job.stepData || {} };
1864
- const pool = tryExtractPool(backend);
1865
- const hasStepHistory = Object.keys(stepData).some(
1866
- (k) => k.startsWith("__wait_")
1867
- );
1868
- if (hasStepHistory && pool) {
1869
- await resolveCompletedWaits(pool, stepData);
1870
- await updateStepData(pool, job.id, stepData);
1871
1975
  }
1872
- const timeoutMs = job.timeoutMs ?? void 0;
1873
- const forceKillOnTimeout = job.forceKillOnTimeout ?? false;
1874
- let timeoutId;
1875
- const controller = new AbortController();
1876
- try {
1877
- if (forceKillOnTimeout && timeoutMs && timeoutMs > 0) {
1878
- await runHandlerInWorker(handler, job.payload, timeoutMs, job.jobType);
1879
- } else {
1880
- let onTimeoutCallback;
1881
- let timeoutReject;
1882
- const armTimeout = (ms) => {
1883
- if (timeoutId) clearTimeout(timeoutId);
1884
- timeoutId = setTimeout(() => {
1885
- if (onTimeoutCallback) {
1886
- try {
1887
- const extension = onTimeoutCallback();
1888
- if (typeof extension === "number" && extension > 0) {
1889
- backend.prolongJob(job.id).catch(() => {
1890
- });
1891
- armTimeout(extension);
1892
- return;
1893
- }
1894
- } catch (callbackError) {
1895
- log(
1896
- `onTimeout callback threw for job ${job.id}: ${callbackError}`
1897
- );
1898
- }
1899
- }
1900
- controller.abort();
1901
- const timeoutError = new Error(`Job timed out after ${ms} ms`);
1902
- timeoutError.failureReason = "timeout" /* Timeout */;
1903
- if (timeoutReject) {
1904
- timeoutReject(timeoutError);
1905
- }
1906
- }, ms);
1907
- };
1908
- const hasTimeout = timeoutMs != null && timeoutMs > 0;
1909
- const baseCtx = hasTimeout ? {
1910
- prolong: (ms) => {
1911
- const duration = ms ?? timeoutMs;
1912
- if (duration != null && duration > 0) {
1913
- armTimeout(duration);
1914
- backend.prolongJob(job.id).catch(() => {
1915
- });
1916
- }
1917
- },
1918
- onTimeout: (callback) => {
1919
- onTimeoutCallback = callback;
1920
- }
1921
- } : {
1922
- prolong: () => {
1923
- log("prolong() called but ignored: job has no timeout set");
1924
- },
1925
- onTimeout: () => {
1926
- log("onTimeout() called but ignored: job has no timeout set");
1927
- }
1928
- };
1929
- const ctx = pool ? buildWaitContext(backend, pool, job.id, stepData, baseCtx) : buildBasicContext(backend, job.id, baseCtx);
1930
- if (forceKillOnTimeout && !hasTimeout) {
1931
- log(
1932
- `forceKillOnTimeout is set but no timeoutMs for job ${job.id}, running without force kill`
1933
- );
1934
- }
1935
- const jobPromise = handler(job.payload, controller.signal, ctx);
1936
- if (hasTimeout) {
1937
- await Promise.race([
1938
- jobPromise,
1939
- new Promise((_, reject) => {
1940
- timeoutReject = reject;
1941
- armTimeout(timeoutMs);
1942
- })
1943
- ]);
1944
- } else {
1945
- await jobPromise;
1976
+ /**
1977
+ * Atomically fetch all active cron schedules whose nextRunAt <= NOW().
1978
+ * Uses FOR UPDATE SKIP LOCKED to prevent duplicate enqueuing across workers.
1979
+ */
1980
+ async getDueCronSchedules() {
1981
+ const client = await this.pool.connect();
1982
+ try {
1983
+ const result = await client.query(
1984
+ `SELECT id, schedule_name AS "scheduleName", cron_expression AS "cronExpression",
1985
+ job_type AS "jobType", payload, max_attempts AS "maxAttempts",
1986
+ priority, timeout_ms AS "timeoutMs",
1987
+ force_kill_on_timeout AS "forceKillOnTimeout", tags,
1988
+ timezone, allow_overlap AS "allowOverlap", status,
1989
+ last_enqueued_at AS "lastEnqueuedAt", last_job_id AS "lastJobId",
1990
+ next_run_at AS "nextRunAt",
1991
+ created_at AS "createdAt", updated_at AS "updatedAt"
1992
+ FROM cron_schedules
1993
+ WHERE status = 'active'
1994
+ AND next_run_at IS NOT NULL
1995
+ AND next_run_at <= NOW()
1996
+ ORDER BY next_run_at ASC
1997
+ FOR UPDATE SKIP LOCKED`
1998
+ );
1999
+ log(`Found ${result.rows.length} due cron schedules`);
2000
+ return result.rows;
2001
+ } catch (error) {
2002
+ if (error?.code === "42P01") {
2003
+ log("cron_schedules table does not exist, skipping cron enqueue");
2004
+ return [];
1946
2005
  }
2006
+ log(`Error getting due cron schedules: ${error}`);
2007
+ throw error;
2008
+ } finally {
2009
+ client.release();
1947
2010
  }
1948
- if (timeoutId) clearTimeout(timeoutId);
1949
- await backend.completeJob(job.id);
1950
- } catch (error) {
1951
- if (timeoutId) clearTimeout(timeoutId);
1952
- if (error instanceof WaitSignal) {
1953
- if (!pool) {
1954
- await backend.failJob(
1955
- job.id,
1956
- new Error(
1957
- "WaitSignal received but wait features require the PostgreSQL backend."
1958
- ),
1959
- "handler_error" /* HandlerError */
1960
- );
1961
- return;
1962
- }
2011
+ }
2012
+ /**
2013
+ * Update a cron schedule after a job has been enqueued.
2014
+ * Sets lastEnqueuedAt, lastJobId, and advances nextRunAt.
2015
+ */
2016
+ async updateCronScheduleAfterEnqueue(id, lastEnqueuedAt, lastJobId, nextRunAt) {
2017
+ const client = await this.pool.connect();
2018
+ try {
2019
+ await client.query(
2020
+ `UPDATE cron_schedules
2021
+ SET last_enqueued_at = $2,
2022
+ last_job_id = $3,
2023
+ next_run_at = $4,
2024
+ updated_at = NOW()
2025
+ WHERE id = $1`,
2026
+ [id, lastEnqueuedAt, lastJobId, nextRunAt]
2027
+ );
1963
2028
  log(
1964
- `Job ${job.id} entering wait: type=${error.type}, waitUntil=${error.waitUntil?.toISOString() ?? "none"}, tokenId=${error.tokenId ?? "none"}`
2029
+ `Updated cron schedule ${id}: lastJobId=${lastJobId}, nextRunAt=${nextRunAt?.toISOString() ?? "null"}`
1965
2030
  );
1966
- await waitJob(pool, job.id, {
1967
- waitUntil: error.waitUntil,
1968
- waitTokenId: error.tokenId,
1969
- stepData: error.stepData
1970
- });
1971
- return;
1972
- }
1973
- console.error(`Error processing job ${job.id}:`, error);
1974
- let failureReason = "handler_error" /* HandlerError */;
1975
- if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
1976
- failureReason = "timeout" /* Timeout */;
2031
+ } catch (error) {
2032
+ log(`Error updating cron schedule ${id} after enqueue: ${error}`);
2033
+ throw error;
2034
+ } finally {
2035
+ client.release();
1977
2036
  }
1978
- await backend.failJob(
1979
- job.id,
1980
- error instanceof Error ? error : new Error(String(error)),
1981
- failureReason
1982
- );
1983
- }
1984
- }
1985
- async function processBatchWithHandlers(backend, workerId, batchSize, jobType, jobHandlers, concurrency, onError) {
1986
- const jobs = await backend.getNextBatch(
1987
- workerId,
1988
- batchSize,
1989
- jobType
1990
- );
1991
- if (!concurrency || concurrency >= jobs.length) {
1992
- await Promise.all(
1993
- jobs.map((job) => processJobWithHandlers(backend, job, jobHandlers))
1994
- );
1995
- return jobs.length;
1996
- }
1997
- let idx = 0;
1998
- let running = 0;
1999
- let finished = 0;
2000
- return new Promise((resolve, reject) => {
2001
- const next = () => {
2002
- if (finished === jobs.length) return resolve(jobs.length);
2003
- while (running < concurrency && idx < jobs.length) {
2004
- const job = jobs[idx++];
2005
- running++;
2006
- processJobWithHandlers(backend, job, jobHandlers).then(() => {
2007
- running--;
2008
- finished++;
2009
- next();
2010
- }).catch((err) => {
2011
- running--;
2012
- finished++;
2013
- if (onError) {
2014
- onError(err instanceof Error ? err : new Error(String(err)));
2015
- }
2016
- next();
2017
- });
2018
- }
2019
- };
2020
- next();
2021
- });
2022
- }
2023
- var createProcessor = (backend, handlers, options = {}, onBeforeBatch) => {
2024
- const {
2025
- workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
2026
- batchSize = 10,
2027
- pollInterval = 5e3,
2028
- onError = (error) => console.error("Job processor error:", error),
2029
- jobType,
2030
- concurrency = 3
2031
- } = options;
2032
- let running = false;
2033
- let intervalId = null;
2034
- let currentBatchPromise = null;
2035
- setLogContext(options.verbose ?? false);
2036
- const processJobs = async () => {
2037
- if (!running) return 0;
2038
- if (onBeforeBatch) {
2039
- try {
2040
- await onBeforeBatch();
2041
- } catch (hookError) {
2042
- log(`onBeforeBatch hook error: ${hookError}`);
2043
- if (onError) {
2044
- onError(
2045
- hookError instanceof Error ? hookError : new Error(String(hookError))
2046
- );
2047
- }
2037
+ }
2038
+ // ── Wait / step-data support ────────────────────────────────────────
2039
+ /**
2040
+ * Transition a job from 'processing' to 'waiting' status.
2041
+ * Persists step data so the handler can resume from where it left off.
2042
+ *
2043
+ * @param jobId - The job to pause.
2044
+ * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
2045
+ */
2046
+ async waitJob(jobId, options) {
2047
+ const client = await this.pool.connect();
2048
+ try {
2049
+ const result = await client.query(
2050
+ `
2051
+ UPDATE job_queue
2052
+ SET status = 'waiting',
2053
+ wait_until = $2,
2054
+ wait_token_id = $3,
2055
+ step_data = $4,
2056
+ locked_at = NULL,
2057
+ locked_by = NULL,
2058
+ updated_at = NOW()
2059
+ WHERE id = $1 AND status = 'processing'
2060
+ `,
2061
+ [
2062
+ jobId,
2063
+ options.waitUntil ?? null,
2064
+ options.waitTokenId ?? null,
2065
+ JSON.stringify(options.stepData)
2066
+ ]
2067
+ );
2068
+ if (result.rowCount === 0) {
2069
+ log(
2070
+ `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
2071
+ );
2072
+ return;
2048
2073
  }
2074
+ await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
2075
+ waitUntil: options.waitUntil?.toISOString() ?? null,
2076
+ waitTokenId: options.waitTokenId ?? null
2077
+ });
2078
+ log(`Job ${jobId} set to waiting`);
2079
+ } catch (error) {
2080
+ log(`Error setting job ${jobId} to waiting: ${error}`);
2081
+ throw error;
2082
+ } finally {
2083
+ client.release();
2049
2084
  }
2050
- log(
2051
- `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(",") : jobType}` : ""}`
2052
- );
2085
+ }
2086
+ /**
2087
+ * Persist step data for a job. Called after each ctx.run() step completes.
2088
+ * Best-effort: does not throw to avoid killing the running handler.
2089
+ *
2090
+ * @param jobId - The job to update.
2091
+ * @param stepData - The step data to persist.
2092
+ */
2093
+ async updateStepData(jobId, stepData) {
2094
+ const client = await this.pool.connect();
2053
2095
  try {
2054
- const processed = await processBatchWithHandlers(
2055
- backend,
2056
- workerId,
2057
- batchSize,
2058
- jobType,
2059
- handlers,
2060
- concurrency,
2061
- onError
2096
+ await client.query(
2097
+ `UPDATE job_queue SET step_data = $2, updated_at = NOW() WHERE id = $1`,
2098
+ [jobId, JSON.stringify(stepData)]
2062
2099
  );
2063
- return processed;
2064
2100
  } catch (error) {
2065
- onError(error instanceof Error ? error : new Error(String(error)));
2101
+ log(`Error updating step_data for job ${jobId}: ${error}`);
2102
+ } finally {
2103
+ client.release();
2066
2104
  }
2067
- return 0;
2068
- };
2069
- return {
2070
- /**
2071
- * Start the job processor in the background.
2072
- * - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
2073
- * - You have to call the stop method to stop the processor.
2074
- */
2075
- startInBackground: () => {
2076
- if (running) return;
2077
- log(`Starting job processor with workerId: ${workerId}`);
2078
- running = true;
2079
- const scheduleNext = (immediate) => {
2080
- if (!running) return;
2081
- if (immediate) {
2082
- intervalId = setTimeout(loop, 0);
2083
- } else {
2084
- intervalId = setTimeout(loop, pollInterval);
2085
- }
2086
- };
2087
- const loop = async () => {
2088
- if (!running) return;
2089
- currentBatchPromise = processJobs();
2090
- const processed = await currentBatchPromise;
2091
- currentBatchPromise = null;
2092
- scheduleNext(processed === batchSize);
2093
- };
2094
- loop();
2095
- },
2096
- /**
2097
- * Stop the job processor that runs in the background.
2098
- * Does not wait for in-flight jobs.
2099
- */
2100
- stop: () => {
2101
- log(`Stopping job processor with workerId: ${workerId}`);
2102
- running = false;
2103
- if (intervalId) {
2104
- clearTimeout(intervalId);
2105
- intervalId = null;
2106
- }
2107
- },
2108
- /**
2109
- * Stop the job processor and wait for all in-flight jobs to complete.
2110
- * Useful for graceful shutdown (e.g., SIGTERM handling).
2111
- */
2112
- stopAndDrain: async (drainTimeoutMs = 3e4) => {
2113
- log(`Stopping and draining job processor with workerId: ${workerId}`);
2114
- running = false;
2115
- if (intervalId) {
2116
- clearTimeout(intervalId);
2117
- intervalId = null;
2118
- }
2119
- if (currentBatchPromise) {
2120
- await Promise.race([
2121
- currentBatchPromise.catch(() => {
2122
- }),
2123
- new Promise((resolve) => setTimeout(resolve, drainTimeoutMs))
2124
- ]);
2125
- currentBatchPromise = null;
2126
- }
2127
- log(`Job processor ${workerId} drained`);
2128
- },
2129
- /**
2130
- * Start the job processor synchronously.
2131
- * - This will process all jobs immediately and then stop.
2132
- * - The pollInterval is ignored.
2133
- */
2134
- start: async () => {
2135
- log(`Starting job processor with workerId: ${workerId}`);
2136
- running = true;
2137
- const processed = await processJobs();
2138
- running = false;
2139
- return processed;
2140
- },
2141
- isRunning: () => running
2142
- };
2143
- };
2144
- function loadPemOrFile(value) {
2145
- if (!value) return void 0;
2146
- if (value.startsWith("file://")) {
2147
- const filePath = value.slice(7);
2148
- return fs.readFileSync(filePath, "utf8");
2149
2105
  }
2150
- return value;
2151
- }
2152
- var createPool = (config) => {
2153
- let searchPath;
2154
- let ssl = void 0;
2155
- let customCA;
2156
- let sslmode;
2157
- if (config.connectionString) {
2106
+ /**
2107
+ * Create a waitpoint token in the database.
2108
+ *
2109
+ * @param jobId - The job ID to associate with the token (null if created outside a handler).
2110
+ * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
2111
+ * @returns The created waitpoint with its unique ID.
2112
+ */
2113
+ async createWaitpoint(jobId, options) {
2114
+ const client = await this.pool.connect();
2158
2115
  try {
2159
- const url = new URL(config.connectionString);
2160
- searchPath = url.searchParams.get("search_path") || void 0;
2161
- sslmode = url.searchParams.get("sslmode") || void 0;
2162
- if (sslmode === "no-verify") {
2163
- ssl = { rejectUnauthorized: false };
2116
+ const id = `wp_${randomUUID()}`;
2117
+ let timeoutAt = null;
2118
+ if (options?.timeout) {
2119
+ const ms = parseTimeoutString(options.timeout);
2120
+ timeoutAt = new Date(Date.now() + ms);
2164
2121
  }
2165
- } catch (e) {
2166
- const parsed = parse(config.connectionString);
2167
- if (parsed.options) {
2168
- const match = parsed.options.match(/search_path=([^\s]+)/);
2169
- if (match) {
2170
- searchPath = match[1];
2171
- }
2122
+ await client.query(
2123
+ `INSERT INTO waitpoints (id, job_id, status, timeout_at, tags) VALUES ($1, $2, 'waiting', $3, $4)`,
2124
+ [id, jobId, timeoutAt, options?.tags ?? null]
2125
+ );
2126
+ log(`Created waitpoint ${id} for job ${jobId}`);
2127
+ return { id };
2128
+ } catch (error) {
2129
+ log(`Error creating waitpoint: ${error}`);
2130
+ throw error;
2131
+ } finally {
2132
+ client.release();
2133
+ }
2134
+ }
2135
+ /**
2136
+ * Complete a waitpoint token and move the associated job back to 'pending'.
2137
+ *
2138
+ * @param tokenId - The waitpoint token ID to complete.
2139
+ * @param data - Optional data to pass to the waiting handler.
2140
+ */
2141
+ async completeWaitpoint(tokenId, data) {
2142
+ const client = await this.pool.connect();
2143
+ try {
2144
+ await client.query("BEGIN");
2145
+ const wpResult = await client.query(
2146
+ `UPDATE waitpoints SET status = 'completed', output = $2, completed_at = NOW()
2147
+ WHERE id = $1 AND status = 'waiting'
2148
+ RETURNING job_id`,
2149
+ [tokenId, data != null ? JSON.stringify(data) : null]
2150
+ );
2151
+ if (wpResult.rows.length === 0) {
2152
+ await client.query("ROLLBACK");
2153
+ log(`Waitpoint ${tokenId} not found or already completed`);
2154
+ return;
2172
2155
  }
2173
- sslmode = typeof parsed.sslmode === "string" ? parsed.sslmode : void 0;
2174
- if (sslmode === "no-verify") {
2175
- ssl = { rejectUnauthorized: false };
2156
+ const jobId = wpResult.rows[0].job_id;
2157
+ if (jobId != null) {
2158
+ await client.query(
2159
+ `UPDATE job_queue
2160
+ SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
2161
+ WHERE id = $1 AND status = 'waiting'`,
2162
+ [jobId]
2163
+ );
2176
2164
  }
2165
+ await client.query("COMMIT");
2166
+ log(`Completed waitpoint ${tokenId} for job ${jobId}`);
2167
+ } catch (error) {
2168
+ await client.query("ROLLBACK");
2169
+ log(`Error completing waitpoint ${tokenId}: ${error}`);
2170
+ throw error;
2171
+ } finally {
2172
+ client.release();
2177
2173
  }
2178
2174
  }
2179
- if (config.ssl) {
2180
- if (typeof config.ssl.ca === "string") {
2181
- customCA = config.ssl.ca;
2182
- } else if (typeof process.env.PGSSLROOTCERT === "string") {
2183
- customCA = process.env.PGSSLROOTCERT;
2184
- } else {
2185
- customCA = void 0;
2175
+ /**
2176
+ * Retrieve a waitpoint token by its ID.
2177
+ *
2178
+ * @param tokenId - The waitpoint token ID to look up.
2179
+ * @returns The waitpoint record, or null if not found.
2180
+ */
2181
+ async getWaitpoint(tokenId) {
2182
+ const client = await this.pool.connect();
2183
+ try {
2184
+ const result = await client.query(
2185
+ `SELECT id, job_id AS "jobId", status, output, timeout_at AS "timeoutAt", created_at AS "createdAt", completed_at AS "completedAt", tags FROM waitpoints WHERE id = $1`,
2186
+ [tokenId]
2187
+ );
2188
+ if (result.rows.length === 0) return null;
2189
+ return result.rows[0];
2190
+ } catch (error) {
2191
+ log(`Error getting waitpoint ${tokenId}: ${error}`);
2192
+ throw error;
2193
+ } finally {
2194
+ client.release();
2186
2195
  }
2187
- const caValue = typeof customCA === "string" ? loadPemOrFile(customCA) : void 0;
2188
- ssl = {
2189
- ...ssl,
2190
- ...caValue ? { ca: caValue } : {},
2191
- cert: loadPemOrFile(
2192
- typeof config.ssl.cert === "string" ? config.ssl.cert : process.env.PGSSLCERT
2193
- ),
2194
- key: loadPemOrFile(
2195
- typeof config.ssl.key === "string" ? config.ssl.key : process.env.PGSSLKEY
2196
- ),
2197
- rejectUnauthorized: config.ssl.rejectUnauthorized !== void 0 ? config.ssl.rejectUnauthorized : true
2198
- };
2199
2196
  }
2200
- if (sslmode && customCA) {
2201
- const warning = `
2202
-
2203
- \x1B[33m**************************************************
2204
- \u26A0\uFE0F WARNING: SSL CONFIGURATION ISSUE
2205
- **************************************************
2206
- Both sslmode ('${sslmode}') is set in the connection string
2207
- and a custom CA is provided (via config.ssl.ca or PGSSLROOTCERT).
2208
- This combination may cause connection failures or unexpected behavior.
2209
-
2210
- Recommended: Remove sslmode from the connection string when using a custom CA.
2211
- **************************************************\x1B[0m
2212
- `;
2213
- console.warn(warning);
2197
+ /**
2198
+ * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
2199
+ *
2200
+ * @returns The number of tokens that were expired.
2201
+ */
2202
+ async expireTimedOutWaitpoints() {
2203
+ const client = await this.pool.connect();
2204
+ try {
2205
+ await client.query("BEGIN");
2206
+ const result = await client.query(
2207
+ `UPDATE waitpoints
2208
+ SET status = 'timed_out'
2209
+ WHERE status = 'waiting' AND timeout_at IS NOT NULL AND timeout_at <= NOW()
2210
+ RETURNING id, job_id`
2211
+ );
2212
+ for (const row of result.rows) {
2213
+ if (row.job_id != null) {
2214
+ await client.query(
2215
+ `UPDATE job_queue
2216
+ SET status = 'pending', wait_token_id = NULL, wait_until = NULL, updated_at = NOW()
2217
+ WHERE id = $1 AND status = 'waiting'`,
2218
+ [row.job_id]
2219
+ );
2220
+ }
2221
+ }
2222
+ await client.query("COMMIT");
2223
+ const count = result.rowCount || 0;
2224
+ if (count > 0) {
2225
+ log(`Expired ${count} timed-out waitpoints`);
2226
+ }
2227
+ return count;
2228
+ } catch (error) {
2229
+ await client.query("ROLLBACK");
2230
+ log(`Error expiring timed-out waitpoints: ${error}`);
2231
+ throw error;
2232
+ } finally {
2233
+ client.release();
2234
+ }
2214
2235
  }
2215
- const pool = new Pool({
2216
- ...config,
2217
- ...ssl ? { ssl } : {}
2218
- });
2219
- if (searchPath) {
2220
- pool.on("connect", (client) => {
2221
- client.query(`SET search_path TO ${searchPath}`);
2222
- });
2236
+ // ── Internal helpers ──────────────────────────────────────────────────
2237
+ async setPendingReasonForUnpickedJobs(reason, jobType) {
2238
+ const client = await this.pool.connect();
2239
+ try {
2240
+ let jobTypeFilter = "";
2241
+ const params = [reason];
2242
+ if (jobType) {
2243
+ if (Array.isArray(jobType)) {
2244
+ jobTypeFilter = ` AND job_type = ANY($2)`;
2245
+ params.push(jobType);
2246
+ } else {
2247
+ jobTypeFilter = ` AND job_type = $2`;
2248
+ params.push(jobType);
2249
+ }
2250
+ }
2251
+ await client.query(
2252
+ `UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
2253
+ params
2254
+ );
2255
+ } finally {
2256
+ client.release();
2257
+ }
2223
2258
  }
2224
- return pool;
2225
2259
  };
2226
2260
 
2227
2261
  // src/backends/redis-scripts.ts
@@ -2278,7 +2312,10 @@ redis.call('HMSET', jobKey,
2278
2312
  'lastFailedAt', 'null',
2279
2313
  'lastCancelledAt', 'null',
2280
2314
  'tags', tagsJson,
2281
- 'idempotencyKey', idempotencyKey
2315
+ 'idempotencyKey', idempotencyKey,
2316
+ 'waitUntil', 'null',
2317
+ 'waitTokenId', 'null',
2318
+ 'stepData', 'null'
2282
2319
  )
2283
2320
 
2284
2321
  -- Status index
@@ -2361,7 +2398,25 @@ for _, jobId in ipairs(retries) do
2361
2398
  redis.call('ZREM', prefix .. 'retry', jobId)
2362
2399
  end
2363
2400
 
2364
- -- 3. Parse job type filter
2401
+ -- 3. Move ready waiting jobs (time-based, no token) into queue
2402
+ local waitingJobs = redis.call('ZRANGEBYSCORE', prefix .. 'waiting', '-inf', nowMs, 'LIMIT', 0, 200)
2403
+ for _, jobId in ipairs(waitingJobs) do
2404
+ local jk = prefix .. 'job:' .. jobId
2405
+ local status = redis.call('HGET', jk, 'status')
2406
+ local waitTokenId = redis.call('HGET', jk, 'waitTokenId')
2407
+ if status == 'waiting' and (waitTokenId == false or waitTokenId == 'null') then
2408
+ local pri = tonumber(redis.call('HGET', jk, 'priority') or '0')
2409
+ local ca = tonumber(redis.call('HGET', jk, 'createdAt'))
2410
+ local score = pri * ${SCORE_RANGE} + (${SCORE_RANGE} - ca)
2411
+ redis.call('ZADD', prefix .. 'queue', score, jobId)
2412
+ redis.call('SREM', prefix .. 'status:waiting', jobId)
2413
+ redis.call('SADD', prefix .. 'status:pending', jobId)
2414
+ redis.call('HMSET', jk, 'status', 'pending', 'waitUntil', 'null')
2415
+ end
2416
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2417
+ end
2418
+
2419
+ -- 4. Parse job type filter
2365
2420
  local filterTypes = nil
2366
2421
  if jobTypeFilter ~= "null" then
2367
2422
  -- Could be a JSON array or a plain string
@@ -2374,7 +2429,7 @@ if jobTypeFilter ~= "null" then
2374
2429
  end
2375
2430
  end
2376
2431
 
2377
- -- 4. Pop candidates from queue (highest score first)
2432
+ -- 5. Pop candidates from queue (highest score first)
2378
2433
  -- We pop more than batchSize because some may be filtered out
2379
2434
  local popCount = batchSize * 3
2380
2435
  local candidates = redis.call('ZPOPMAX', prefix .. 'queue', popCount)
@@ -2458,7 +2513,10 @@ local jk = prefix .. 'job:' .. jobId
2458
2513
  redis.call('HMSET', jk,
2459
2514
  'status', 'completed',
2460
2515
  'updatedAt', nowMs,
2461
- 'completedAt', nowMs
2516
+ 'completedAt', nowMs,
2517
+ 'stepData', 'null',
2518
+ 'waitUntil', 'null',
2519
+ 'waitTokenId', 'null'
2462
2520
  )
2463
2521
  redis.call('SREM', prefix .. 'status:processing', jobId)
2464
2522
  redis.call('SADD', prefix .. 'status:completed', jobId)
@@ -2517,6 +2575,7 @@ local nowMs = tonumber(ARGV[2])
2517
2575
  local jk = prefix .. 'job:' .. jobId
2518
2576
 
2519
2577
  local oldStatus = redis.call('HGET', jk, 'status')
2578
+ if oldStatus ~= 'failed' and oldStatus ~= 'processing' then return 0 end
2520
2579
 
2521
2580
  redis.call('HMSET', jk,
2522
2581
  'status', 'pending',
@@ -2528,9 +2587,7 @@ redis.call('HMSET', jk,
2528
2587
  )
2529
2588
 
2530
2589
  -- Remove from old status, add to pending
2531
- if oldStatus then
2532
- redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
2533
- end
2590
+ redis.call('SREM', prefix .. 'status:' .. oldStatus, jobId)
2534
2591
  redis.call('SADD', prefix .. 'status:pending', jobId)
2535
2592
 
2536
2593
  -- Remove from retry sorted set if present
@@ -2551,18 +2608,21 @@ local nowMs = ARGV[2]
2551
2608
  local jk = prefix .. 'job:' .. jobId
2552
2609
 
2553
2610
  local status = redis.call('HGET', jk, 'status')
2554
- if status ~= 'pending' then return 0 end
2611
+ if status ~= 'pending' and status ~= 'waiting' then return 0 end
2555
2612
 
2556
2613
  redis.call('HMSET', jk,
2557
2614
  'status', 'cancelled',
2558
2615
  'updatedAt', nowMs,
2559
- 'lastCancelledAt', nowMs
2616
+ 'lastCancelledAt', nowMs,
2617
+ 'waitUntil', 'null',
2618
+ 'waitTokenId', 'null'
2560
2619
  )
2561
- redis.call('SREM', prefix .. 'status:pending', jobId)
2620
+ redis.call('SREM', prefix .. 'status:' .. status, jobId)
2562
2621
  redis.call('SADD', prefix .. 'status:cancelled', jobId)
2563
- -- Remove from queue / delayed
2622
+ -- Remove from queue / delayed / waiting
2564
2623
  redis.call('ZREM', prefix .. 'queue', jobId)
2565
2624
  redis.call('ZREM', prefix .. 'delayed', jobId)
2625
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2566
2626
 
2567
2627
  return 1
2568
2628
  `;
@@ -2630,18 +2690,16 @@ end
2630
2690
 
2631
2691
  return count
2632
2692
  `;
2633
- var CLEANUP_OLD_JOBS_SCRIPT = `
2693
+ var CLEANUP_OLD_JOBS_BATCH_SCRIPT = `
2634
2694
  local prefix = KEYS[1]
2635
2695
  local cutoffMs = tonumber(ARGV[1])
2636
-
2637
- local completed = redis.call('SMEMBERS', prefix .. 'status:completed')
2638
2696
  local count = 0
2639
2697
 
2640
- for _, jobId in ipairs(completed) do
2698
+ for i = 2, #ARGV do
2699
+ local jobId = ARGV[i]
2641
2700
  local jk = prefix .. 'job:' .. jobId
2642
2701
  local updatedAt = tonumber(redis.call('HGET', jk, 'updatedAt'))
2643
2702
  if updatedAt and updatedAt < cutoffMs then
2644
- -- Remove all indexes
2645
2703
  local jobType = redis.call('HGET', jk, 'jobType')
2646
2704
  local tagsJson = redis.call('HGET', jk, 'tags')
2647
2705
  local idempotencyKey = redis.call('HGET', jk, 'idempotencyKey')
@@ -2664,7 +2722,6 @@ for _, jobId in ipairs(completed) do
2664
2722
  if idempotencyKey and idempotencyKey ~= 'null' then
2665
2723
  redis.call('DEL', prefix .. 'idempotency:' .. idempotencyKey)
2666
2724
  end
2667
- -- Delete events
2668
2725
  redis.call('DEL', prefix .. 'events:' .. jobId)
2669
2726
 
2670
2727
  count = count + 1
@@ -2673,8 +2730,158 @@ end
2673
2730
 
2674
2731
  return count
2675
2732
  `;
2733
+ var WAIT_JOB_SCRIPT = `
2734
+ local prefix = KEYS[1]
2735
+ local jobId = ARGV[1]
2736
+ local waitUntilMs = ARGV[2]
2737
+ local waitTokenId = ARGV[3]
2738
+ local stepDataJson = ARGV[4]
2739
+ local nowMs = ARGV[5]
2740
+ local jk = prefix .. 'job:' .. jobId
2741
+
2742
+ local status = redis.call('HGET', jk, 'status')
2743
+ if status ~= 'processing' then return 0 end
2744
+
2745
+ redis.call('HMSET', jk,
2746
+ 'status', 'waiting',
2747
+ 'waitUntil', waitUntilMs,
2748
+ 'waitTokenId', waitTokenId,
2749
+ 'stepData', stepDataJson,
2750
+ 'lockedAt', 'null',
2751
+ 'lockedBy', 'null',
2752
+ 'updatedAt', nowMs
2753
+ )
2754
+ redis.call('SREM', prefix .. 'status:processing', jobId)
2755
+ redis.call('SADD', prefix .. 'status:waiting', jobId)
2756
+
2757
+ -- Add to waiting sorted set if time-based wait
2758
+ if waitUntilMs ~= 'null' then
2759
+ redis.call('ZADD', prefix .. 'waiting', tonumber(waitUntilMs), jobId)
2760
+ end
2761
+
2762
+ return 1
2763
+ `;
2764
+ var COMPLETE_WAITPOINT_SCRIPT = `
2765
+ local prefix = KEYS[1]
2766
+ local tokenId = ARGV[1]
2767
+ local outputJson = ARGV[2]
2768
+ local nowMs = ARGV[3]
2769
+ local wpk = prefix .. 'waitpoint:' .. tokenId
2770
+
2771
+ local wpStatus = redis.call('HGET', wpk, 'status')
2772
+ if not wpStatus or wpStatus ~= 'waiting' then return 0 end
2773
+
2774
+ redis.call('HMSET', wpk,
2775
+ 'status', 'completed',
2776
+ 'output', outputJson,
2777
+ 'completedAt', nowMs
2778
+ )
2779
+
2780
+ -- Move associated job back to pending
2781
+ local jobId = redis.call('HGET', wpk, 'jobId')
2782
+ if jobId and jobId ~= 'null' then
2783
+ local jk = prefix .. 'job:' .. jobId
2784
+ local jobStatus = redis.call('HGET', jk, 'status')
2785
+ if jobStatus == 'waiting' then
2786
+ redis.call('HMSET', jk,
2787
+ 'status', 'pending',
2788
+ 'waitTokenId', 'null',
2789
+ 'waitUntil', 'null',
2790
+ 'updatedAt', nowMs
2791
+ )
2792
+ redis.call('SREM', prefix .. 'status:waiting', jobId)
2793
+ redis.call('SADD', prefix .. 'status:pending', jobId)
2794
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2795
+
2796
+ -- Re-add to queue
2797
+ local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
2798
+ local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
2799
+ local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
2800
+ redis.call('ZADD', prefix .. 'queue', score, jobId)
2801
+ end
2802
+ end
2803
+
2804
+ return 1
2805
+ `;
2806
+ var EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT = `
2807
+ local prefix = KEYS[1]
2808
+ local nowMs = tonumber(ARGV[1])
2809
+
2810
+ local expiredIds = redis.call('ZRANGEBYSCORE', prefix .. 'waitpoint_timeout', '-inf', nowMs)
2811
+ local count = 0
2812
+
2813
+ for _, tokenId in ipairs(expiredIds) do
2814
+ local wpk = prefix .. 'waitpoint:' .. tokenId
2815
+ local wpStatus = redis.call('HGET', wpk, 'status')
2816
+ if wpStatus == 'waiting' then
2817
+ redis.call('HMSET', wpk,
2818
+ 'status', 'timed_out'
2819
+ )
2820
+
2821
+ -- Move associated job back to pending
2822
+ local jobId = redis.call('HGET', wpk, 'jobId')
2823
+ if jobId and jobId ~= 'null' then
2824
+ local jk = prefix .. 'job:' .. jobId
2825
+ local jobStatus = redis.call('HGET', jk, 'status')
2826
+ if jobStatus == 'waiting' then
2827
+ redis.call('HMSET', jk,
2828
+ 'status', 'pending',
2829
+ 'waitTokenId', 'null',
2830
+ 'waitUntil', 'null',
2831
+ 'updatedAt', nowMs
2832
+ )
2833
+ redis.call('SREM', prefix .. 'status:waiting', jobId)
2834
+ redis.call('SADD', prefix .. 'status:pending', jobId)
2835
+ redis.call('ZREM', prefix .. 'waiting', jobId)
2836
+
2837
+ local priority = tonumber(redis.call('HGET', jk, 'priority') or '0')
2838
+ local createdAt = tonumber(redis.call('HGET', jk, 'createdAt'))
2839
+ local score = priority * ${SCORE_RANGE} + (${SCORE_RANGE} - createdAt)
2840
+ redis.call('ZADD', prefix .. 'queue', score, jobId)
2841
+ end
2842
+ end
2843
+
2844
+ count = count + 1
2845
+ end
2846
+ redis.call('ZREM', prefix .. 'waitpoint_timeout', tokenId)
2847
+ end
2676
2848
 
2677
- // src/backends/redis.ts
2849
+ return count
2850
+ `;
2851
+ var MAX_TIMEOUT_MS2 = 365 * 24 * 60 * 60 * 1e3;
2852
+ function parseTimeoutString2(timeout) {
2853
+ const match = timeout.match(/^(\d+)(s|m|h|d)$/);
2854
+ if (!match) {
2855
+ throw new Error(
2856
+ `Invalid timeout format: "${timeout}". Expected format like "10m", "1h", "24h", "7d".`
2857
+ );
2858
+ }
2859
+ const value = parseInt(match[1], 10);
2860
+ const unit = match[2];
2861
+ let ms;
2862
+ switch (unit) {
2863
+ case "s":
2864
+ ms = value * 1e3;
2865
+ break;
2866
+ case "m":
2867
+ ms = value * 60 * 1e3;
2868
+ break;
2869
+ case "h":
2870
+ ms = value * 60 * 60 * 1e3;
2871
+ break;
2872
+ case "d":
2873
+ ms = value * 24 * 60 * 60 * 1e3;
2874
+ break;
2875
+ default:
2876
+ throw new Error(`Unknown timeout unit: "${unit}"`);
2877
+ }
2878
+ if (!Number.isFinite(ms) || ms > MAX_TIMEOUT_MS2) {
2879
+ throw new Error(
2880
+ `Timeout value "${timeout}" is too large. Maximum allowed is 365 days.`
2881
+ );
2882
+ }
2883
+ return ms;
2884
+ }
2678
2885
  function hashToObject(arr) {
2679
2886
  const obj = {};
2680
2887
  for (let i = 0; i < arr.length; i += 2) {
@@ -2740,9 +2947,20 @@ function deserializeJob(h) {
2740
2947
  lastCancelledAt: dateOrNull(h.lastCancelledAt),
2741
2948
  tags,
2742
2949
  idempotencyKey: nullish(h.idempotencyKey),
2743
- progress: numOrNull(h.progress)
2950
+ progress: numOrNull(h.progress),
2951
+ waitUntil: dateOrNull(h.waitUntil),
2952
+ waitTokenId: nullish(h.waitTokenId),
2953
+ stepData: parseStepData(h.stepData)
2744
2954
  };
2745
2955
  }
2956
+ function parseStepData(raw) {
2957
+ if (!raw || raw === "null") return void 0;
2958
+ try {
2959
+ return JSON.parse(raw);
2960
+ } catch {
2961
+ return void 0;
2962
+ }
2963
+ }
2746
2964
  var RedisBackend = class {
2747
2965
  constructor(redisConfig) {
2748
2966
  let IORedis;
@@ -2898,8 +3116,14 @@ var RedisBackend = class {
2898
3116
  if (filters.runAt) {
2899
3117
  jobs = this.filterByRunAt(jobs, filters.runAt);
2900
3118
  }
3119
+ if (filters.cursor !== void 0) {
3120
+ jobs = jobs.filter((j) => j.id < filters.cursor);
3121
+ }
3122
+ }
3123
+ jobs.sort((a, b) => b.id - a.id);
3124
+ if (filters?.cursor !== void 0) {
3125
+ return jobs.slice(0, limit);
2901
3126
  }
2902
- jobs.sort((a, b) => b.createdAt.getTime() - a.createdAt.getTime());
2903
3127
  return jobs.slice(offset, offset + limit);
2904
3128
  }
2905
3129
  async getJobsByTags(tags, mode = "all", limit = 100, offset = 0) {
@@ -3131,35 +3355,302 @@ var RedisBackend = class {
3131
3355
  log(`Edited ${count} pending jobs`);
3132
3356
  return count;
3133
3357
  }
3134
- async cleanupOldJobs(daysToKeep = 30) {
3358
+ /**
3359
+ * Delete completed jobs older than the given number of days.
3360
+ * Uses SSCAN to iterate the completed set in batches, avoiding
3361
+ * loading all IDs into memory and preventing long Redis blocks.
3362
+ *
3363
+ * @param daysToKeep - Number of days to retain completed jobs (default 30).
3364
+ * @param batchSize - Number of IDs to scan per SSCAN iteration (default 200).
3365
+ * @returns Total number of deleted jobs.
3366
+ */
3367
+ async cleanupOldJobs(daysToKeep = 30, batchSize = 200) {
3368
+ const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
3369
+ const setKey = `${this.prefix}status:completed`;
3370
+ let totalDeleted = 0;
3371
+ let cursor = "0";
3372
+ do {
3373
+ const [nextCursor, ids] = await this.client.sscan(
3374
+ setKey,
3375
+ cursor,
3376
+ "COUNT",
3377
+ batchSize
3378
+ );
3379
+ cursor = nextCursor;
3380
+ if (ids.length > 0) {
3381
+ const result = await this.client.eval(
3382
+ CLEANUP_OLD_JOBS_BATCH_SCRIPT,
3383
+ 1,
3384
+ this.prefix,
3385
+ cutoffMs,
3386
+ ...ids
3387
+ );
3388
+ totalDeleted += Number(result);
3389
+ }
3390
+ } while (cursor !== "0");
3391
+ log(`Deleted ${totalDeleted} old jobs`);
3392
+ return totalDeleted;
3393
+ }
3394
+ /**
3395
+ * Delete job events older than the given number of days.
3396
+ * Iterates all event lists and removes events whose createdAt is before the cutoff.
3397
+ * Also removes orphaned event lists (where the job no longer exists).
3398
+ *
3399
+ * @param daysToKeep - Number of days to retain events (default 30).
3400
+ * @param batchSize - Number of event keys to scan per SCAN iteration (default 200).
3401
+ * @returns Total number of deleted events.
3402
+ */
3403
+ async cleanupOldJobEvents(daysToKeep = 30, batchSize = 200) {
3135
3404
  const cutoffMs = this.nowMs() - daysToKeep * 24 * 60 * 60 * 1e3;
3405
+ const pattern = `${this.prefix}events:*`;
3406
+ let totalDeleted = 0;
3407
+ let cursor = "0";
3408
+ do {
3409
+ const [nextCursor, keys] = await this.client.scan(
3410
+ cursor,
3411
+ "MATCH",
3412
+ pattern,
3413
+ "COUNT",
3414
+ batchSize
3415
+ );
3416
+ cursor = nextCursor;
3417
+ for (const key of keys) {
3418
+ const jobIdStr = key.slice(`${this.prefix}events:`.length);
3419
+ const jobExists = await this.client.exists(
3420
+ `${this.prefix}job:${jobIdStr}`
3421
+ );
3422
+ if (!jobExists) {
3423
+ const len = await this.client.llen(key);
3424
+ await this.client.del(key);
3425
+ totalDeleted += len;
3426
+ continue;
3427
+ }
3428
+ const events = await this.client.lrange(key, 0, -1);
3429
+ const kept = [];
3430
+ for (const raw of events) {
3431
+ try {
3432
+ const e = JSON.parse(raw);
3433
+ if (e.createdAt >= cutoffMs) {
3434
+ kept.push(raw);
3435
+ } else {
3436
+ totalDeleted++;
3437
+ }
3438
+ } catch {
3439
+ totalDeleted++;
3440
+ }
3441
+ }
3442
+ if (kept.length === 0) {
3443
+ await this.client.del(key);
3444
+ } else if (kept.length < events.length) {
3445
+ const pipeline = this.client.pipeline();
3446
+ pipeline.del(key);
3447
+ for (const raw of kept) {
3448
+ pipeline.rpush(key, raw);
3449
+ }
3450
+ await pipeline.exec();
3451
+ }
3452
+ }
3453
+ } while (cursor !== "0");
3454
+ log(`Deleted ${totalDeleted} old job events`);
3455
+ return totalDeleted;
3456
+ }
3457
+ async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
3458
+ const maxAgeMs = maxProcessingTimeMinutes * 60 * 1e3;
3459
+ const now = this.nowMs();
3136
3460
  const result = await this.client.eval(
3137
- CLEANUP_OLD_JOBS_SCRIPT,
3461
+ RECLAIM_STUCK_JOBS_SCRIPT,
3138
3462
  1,
3139
3463
  this.prefix,
3140
- cutoffMs
3464
+ maxAgeMs,
3465
+ now
3141
3466
  );
3142
- log(`Deleted ${result} old jobs`);
3467
+ log(`Reclaimed ${result} stuck jobs`);
3143
3468
  return Number(result);
3144
3469
  }
3145
- async cleanupOldJobEvents(daysToKeep = 30) {
3146
- log(
3147
- `cleanupOldJobEvents is a no-op for Redis backend (events are cleaned up with their jobs)`
3470
+ // ── Wait / step-data support ────────────────────────────────────────
3471
+ /**
3472
+ * Transition a job from 'processing' to 'waiting' status.
3473
+ * Persists step data so the handler can resume from where it left off.
3474
+ *
3475
+ * @param jobId - The job to pause.
3476
+ * @param options - Wait configuration including optional waitUntil date, token ID, and step data.
3477
+ */
3478
+ async waitJob(jobId, options) {
3479
+ const now = this.nowMs();
3480
+ const waitUntilMs = options.waitUntil ? options.waitUntil.getTime().toString() : "null";
3481
+ const waitTokenId = options.waitTokenId ?? "null";
3482
+ const stepDataJson = JSON.stringify(options.stepData);
3483
+ const result = await this.client.eval(
3484
+ WAIT_JOB_SCRIPT,
3485
+ 1,
3486
+ this.prefix,
3487
+ jobId,
3488
+ waitUntilMs,
3489
+ waitTokenId,
3490
+ stepDataJson,
3491
+ now
3148
3492
  );
3149
- return 0;
3493
+ if (Number(result) === 0) {
3494
+ log(
3495
+ `Job ${jobId} could not be set to waiting (may have been reclaimed or is no longer processing)`
3496
+ );
3497
+ return;
3498
+ }
3499
+ await this.recordJobEvent(jobId, "waiting" /* Waiting */, {
3500
+ waitUntil: options.waitUntil?.toISOString() ?? null,
3501
+ waitTokenId: options.waitTokenId ?? null
3502
+ });
3503
+ log(`Job ${jobId} set to waiting`);
3150
3504
  }
3151
- async reclaimStuckJobs(maxProcessingTimeMinutes = 10) {
3152
- const maxAgeMs = maxProcessingTimeMinutes * 60 * 1e3;
3505
+ /**
3506
+ * Persist step data for a job. Called after each ctx.run() step completes.
3507
+ * Best-effort: does not throw to avoid killing the running handler.
3508
+ *
3509
+ * @param jobId - The job to update.
3510
+ * @param stepData - The step data to persist.
3511
+ */
3512
+ async updateStepData(jobId, stepData) {
3513
+ try {
3514
+ const now = this.nowMs();
3515
+ await this.client.hset(
3516
+ `${this.prefix}job:${jobId}`,
3517
+ "stepData",
3518
+ JSON.stringify(stepData),
3519
+ "updatedAt",
3520
+ now.toString()
3521
+ );
3522
+ } catch (error) {
3523
+ log(`Error updating stepData for job ${jobId}: ${error}`);
3524
+ }
3525
+ }
3526
+ /**
3527
+ * Create a waitpoint token.
3528
+ *
3529
+ * @param jobId - The job ID to associate with the token (null if created outside a handler).
3530
+ * @param options - Optional timeout string (e.g. '10m', '1h') and tags.
3531
+ * @returns The created waitpoint with its unique ID.
3532
+ */
3533
+ async createWaitpoint(jobId, options) {
3534
+ const id = `wp_${randomUUID()}`;
3535
+ const now = this.nowMs();
3536
+ let timeoutAt = null;
3537
+ if (options?.timeout) {
3538
+ const ms = parseTimeoutString2(options.timeout);
3539
+ timeoutAt = now + ms;
3540
+ }
3541
+ const key = `${this.prefix}waitpoint:${id}`;
3542
+ const fields = [
3543
+ "id",
3544
+ id,
3545
+ "jobId",
3546
+ jobId !== null ? jobId.toString() : "null",
3547
+ "status",
3548
+ "waiting",
3549
+ "output",
3550
+ "null",
3551
+ "timeoutAt",
3552
+ timeoutAt !== null ? timeoutAt.toString() : "null",
3553
+ "createdAt",
3554
+ now.toString(),
3555
+ "completedAt",
3556
+ "null",
3557
+ "tags",
3558
+ options?.tags ? JSON.stringify(options.tags) : "null"
3559
+ ];
3560
+ await this.client.hmset(key, ...fields);
3561
+ if (timeoutAt !== null) {
3562
+ await this.client.zadd(`${this.prefix}waitpoint_timeout`, timeoutAt, id);
3563
+ }
3564
+ log(`Created waitpoint ${id} for job ${jobId}`);
3565
+ return { id };
3566
+ }
3567
+ /**
3568
+ * Complete a waitpoint token and move the associated job back to 'pending'.
3569
+ *
3570
+ * @param tokenId - The waitpoint token ID to complete.
3571
+ * @param data - Optional data to pass to the waiting handler.
3572
+ */
3573
+ async completeWaitpoint(tokenId, data) {
3153
3574
  const now = this.nowMs();
3575
+ const outputJson = data != null ? JSON.stringify(data) : "null";
3154
3576
  const result = await this.client.eval(
3155
- RECLAIM_STUCK_JOBS_SCRIPT,
3577
+ COMPLETE_WAITPOINT_SCRIPT,
3156
3578
  1,
3157
3579
  this.prefix,
3158
- maxAgeMs,
3580
+ tokenId,
3581
+ outputJson,
3159
3582
  now
3160
3583
  );
3161
- log(`Reclaimed ${result} stuck jobs`);
3162
- return Number(result);
3584
+ if (Number(result) === 0) {
3585
+ log(`Waitpoint ${tokenId} not found or already completed`);
3586
+ return;
3587
+ }
3588
+ log(`Completed waitpoint ${tokenId}`);
3589
+ }
3590
+ /**
3591
+ * Retrieve a waitpoint token by its ID.
3592
+ *
3593
+ * @param tokenId - The waitpoint token ID to look up.
3594
+ * @returns The waitpoint record, or null if not found.
3595
+ */
3596
+ async getWaitpoint(tokenId) {
3597
+ const data = await this.client.hgetall(
3598
+ `${this.prefix}waitpoint:${tokenId}`
3599
+ );
3600
+ if (!data || Object.keys(data).length === 0) return null;
3601
+ const nullish = (v) => v === void 0 || v === "null" || v === "" ? null : v;
3602
+ const numOrNull = (v) => {
3603
+ const n = nullish(v);
3604
+ return n === null ? null : Number(n);
3605
+ };
3606
+ const dateOrNull = (v) => {
3607
+ const n = numOrNull(v);
3608
+ return n === null ? null : new Date(n);
3609
+ };
3610
+ let output = null;
3611
+ if (data.output && data.output !== "null") {
3612
+ try {
3613
+ output = JSON.parse(data.output);
3614
+ } catch {
3615
+ output = data.output;
3616
+ }
3617
+ }
3618
+ let tags = null;
3619
+ if (data.tags && data.tags !== "null") {
3620
+ try {
3621
+ tags = JSON.parse(data.tags);
3622
+ } catch {
3623
+ }
3624
+ }
3625
+ return {
3626
+ id: data.id,
3627
+ jobId: numOrNull(data.jobId),
3628
+ status: data.status,
3629
+ output,
3630
+ timeoutAt: dateOrNull(data.timeoutAt),
3631
+ createdAt: new Date(Number(data.createdAt)),
3632
+ completedAt: dateOrNull(data.completedAt),
3633
+ tags
3634
+ };
3635
+ }
3636
+ /**
3637
+ * Expire timed-out waitpoint tokens and move their associated jobs back to 'pending'.
3638
+ *
3639
+ * @returns The number of tokens that were expired.
3640
+ */
3641
+ async expireTimedOutWaitpoints() {
3642
+ const now = this.nowMs();
3643
+ const result = await this.client.eval(
3644
+ EXPIRE_TIMED_OUT_WAITPOINTS_SCRIPT,
3645
+ 1,
3646
+ this.prefix,
3647
+ now
3648
+ );
3649
+ const count = Number(result);
3650
+ if (count > 0) {
3651
+ log(`Expired ${count} timed-out waitpoints`);
3652
+ }
3653
+ return count;
3163
3654
  }
3164
3655
  // ── Internal helpers ──────────────────────────────────────────────────
3165
3656
  async setPendingReasonForUnpickedJobs(reason, jobType) {
@@ -3708,10 +4199,9 @@ var initJobQueue = (config) => {
3708
4199
  const backendType = config.backend ?? "postgres";
3709
4200
  setLogContext(config.verbose ?? false);
3710
4201
  let backend;
3711
- let pool;
3712
4202
  if (backendType === "postgres") {
3713
4203
  const pgConfig = config;
3714
- pool = createPool(pgConfig.databaseConfig);
4204
+ const pool = createPool(pgConfig.databaseConfig);
3715
4205
  backend = new PostgresBackend(pool);
3716
4206
  } else if (backendType === "redis") {
3717
4207
  const redisConfig = config.redisConfig;
@@ -3719,14 +4209,6 @@ var initJobQueue = (config) => {
3719
4209
  } else {
3720
4210
  throw new Error(`Unknown backend: ${backendType}`);
3721
4211
  }
3722
- const requirePool = () => {
3723
- if (!pool) {
3724
- throw new Error(
3725
- 'Wait/Token features require the PostgreSQL backend. Configure with backend: "postgres" to use these features.'
3726
- );
3727
- }
3728
- return pool;
3729
- };
3730
4212
  const enqueueDueCronJobsImpl = async () => {
3731
4213
  const dueSchedules = await backend.getDueCronSchedules();
3732
4214
  let count = 0;
@@ -3793,8 +4275,8 @@ var initJobQueue = (config) => {
3793
4275
  config.verbose ?? false
3794
4276
  ),
3795
4277
  retryJob: (jobId) => backend.retryJob(jobId),
3796
- cleanupOldJobs: (daysToKeep) => backend.cleanupOldJobs(daysToKeep),
3797
- cleanupOldJobEvents: (daysToKeep) => backend.cleanupOldJobEvents(daysToKeep),
4278
+ cleanupOldJobs: (daysToKeep, batchSize) => backend.cleanupOldJobs(daysToKeep, batchSize),
4279
+ cleanupOldJobEvents: (daysToKeep, batchSize) => backend.cleanupOldJobEvents(daysToKeep, batchSize),
3798
4280
  cancelJob: withLogContext(
3799
4281
  (jobId) => backend.cancelJob(jobId),
3800
4282
  config.verbose ?? false
@@ -3831,21 +4313,21 @@ var initJobQueue = (config) => {
3831
4313
  (jobId) => backend.getJobEvents(jobId),
3832
4314
  config.verbose ?? false
3833
4315
  ),
3834
- // Wait / Token support (PostgreSQL-only for now)
4316
+ // Wait / Token support (works with all backends)
3835
4317
  createToken: withLogContext(
3836
- (options) => createWaitpoint(requirePool(), null, options),
4318
+ (options) => backend.createWaitpoint(null, options),
3837
4319
  config.verbose ?? false
3838
4320
  ),
3839
4321
  completeToken: withLogContext(
3840
- (tokenId, data) => completeWaitpoint(requirePool(), tokenId, data),
4322
+ (tokenId, data) => backend.completeWaitpoint(tokenId, data),
3841
4323
  config.verbose ?? false
3842
4324
  ),
3843
4325
  getToken: withLogContext(
3844
- (tokenId) => getWaitpoint(requirePool(), tokenId),
4326
+ (tokenId) => backend.getWaitpoint(tokenId),
3845
4327
  config.verbose ?? false
3846
4328
  ),
3847
4329
  expireTimedOutTokens: withLogContext(
3848
- () => expireTimedOutWaitpoints(requirePool()),
4330
+ () => backend.expireTimedOutWaitpoints(),
3849
4331
  config.verbose ?? false
3850
4332
  ),
3851
4333
  // Cron schedule operations
@@ -3926,7 +4408,7 @@ var initJobQueue = (config) => {
3926
4408
  ),
3927
4409
  // Advanced access
3928
4410
  getPool: () => {
3929
- if (backendType !== "postgres") {
4411
+ if (!(backend instanceof PostgresBackend)) {
3930
4412
  throw new Error(
3931
4413
  "getPool() is only available with the PostgreSQL backend."
3932
4414
  );