@q32/core 0.1.20 → 0.1.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/jobs.ts CHANGED
@@ -3,11 +3,14 @@ import { parseJsonColumn, stringifyJsonColumn } from "./d1.js";
3
3
  import { createId } from "./ids.js";
4
4
  import { isFutureIso, nowIso } from "./time.js";
5
5
 
6
- export type JobStatus = "queued" | "running" | "succeeded" | "failed";
6
+ export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "stopping" | "stopped";
7
+ export type TerminalJobStatus = "succeeded" | "failed" | "stopped";
8
+ export type JobEnqueuePolicy = "enqueue" | "dedupe_active";
7
9
 
8
- export type JobHandlerResult<TResult = unknown> =
10
+ export type JobHandlerResult<TResult = unknown, TPayload = unknown> =
9
11
  | { kind: "done"; result?: TResult | null }
10
- | { kind: "requeue"; result?: TResult | null; availableAt?: string | null };
12
+ | { kind: "requeue"; payload?: TPayload; result?: TResult | null; availableAt?: string | null }
13
+ | { kind: "stopped"; result?: TResult | null };
11
14
 
12
15
  export type JobRecord<TPayload = unknown, TResult = unknown> = {
13
16
  jobId: string;
@@ -15,7 +18,14 @@ export type JobRecord<TPayload = unknown, TResult = unknown> = {
15
18
  status: JobStatus;
16
19
  payload: TPayload;
17
20
  result: TResult | null;
21
+ orgId: string | null;
22
+ siteId: string | null;
23
+ parentJobId: string | null;
18
24
  lockKey: string | null;
25
+ concurrencyKey: string | null;
26
+ concurrencyLimit: number | null;
27
+ enqueuePolicy: JobEnqueuePolicy;
28
+ metadata: Record<string, unknown>;
19
29
  attemptCount: number;
20
30
  maxAttempts: number;
21
31
  availableAt: string | null;
@@ -29,15 +39,350 @@ export type JobRecord<TPayload = unknown, TResult = unknown> = {
29
39
  export type EnqueueJobInput<TPayload = unknown> = {
30
40
  jobType: string;
31
41
  payload?: TPayload;
42
+ orgId?: string | null;
43
+ siteId?: string | null;
44
+ parentJobId?: string | null;
32
45
  lockKey?: string | null;
46
+ concurrencyKey?: string | null;
47
+ concurrencyLimit?: number | null;
48
+ enqueuePolicy?: JobEnqueuePolicy;
49
+ metadata?: Record<string, unknown> | null;
33
50
  maxAttempts?: number;
34
51
  availableAt?: string | null;
35
52
  };
36
53
 
37
- export class D1JobStore {
54
+ export type DurableJobEventInput = {
55
+ eventName: string;
56
+ status?: "started" | "ok" | "warning" | "error" | "skipped" | string;
57
+ severity?: "debug" | "info" | "warn" | "error";
58
+ message?: string | null;
59
+ error?: unknown;
60
+ metrics?: Record<string, unknown> | null;
61
+ metadata?: Record<string, unknown> | null;
62
+ };
63
+
64
+ export type DurableJobEventSink<TPayload = unknown, TResult = unknown> = (
65
+ job: JobRecord<TPayload, TResult>,
66
+ event: DurableJobEventInput,
67
+ ) => Promise<void> | void;
68
+
69
+ export interface DurableJobRepository {
70
+ enqueue<TPayload = unknown>(input: EnqueueJobInput<TPayload>): Promise<JobRecord<TPayload>>;
71
+ get<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null>;
72
+ claim<TPayload = unknown>(jobId: string): Promise<JobRecord<TPayload> | null>;
73
+ succeed<TResult = unknown>(jobId: string, result?: TResult | null): Promise<void>;
74
+ requeue<TPayload = unknown, TResult = unknown>(
75
+ jobId: string,
76
+ input?: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null },
77
+ ): Promise<void>;
78
+ fail(jobId: string, error: unknown): Promise<void>;
79
+ stop<TResult = unknown>(jobId: string, result?: TResult | null): Promise<void>;
80
+ requestStop(jobId: string): Promise<void>;
81
+ listChildren(parentJobId: string): Promise<JobRecord[]>;
82
+ summarizeChildren(parentJobId: string): Promise<JobChildrenSummary>;
83
+ markQueuedChildrenTerminal(parentJobId: string, status: "failed" | "stopped", error?: unknown): Promise<number>;
84
+ requestStopActiveChildren(parentJobId: string): Promise<number>;
85
+ }
86
+
87
+ export type JobStatusEventInput<TPayload = unknown, TResult = unknown> = {
88
+ job: JobRecord<TPayload, TResult>;
89
+ status: JobStatus;
90
+ message: string;
91
+ error?: unknown;
92
+ metadata?: Record<string, unknown> | null;
93
+ };
94
+
95
+ export type JobStatusEventSink = (event: JobStatusEventInput) => Promise<void> | void;
96
+
97
+ export type JobChildrenSummary = {
98
+ parentJobId: string;
99
+ total: number;
100
+ queued: number;
101
+ running: number;
102
+ stopping: number;
103
+ succeeded: number;
104
+ failed: number;
105
+ stopped: number;
106
+ active: number;
107
+ terminal: number;
108
+ allTerminal: boolean;
109
+ };
110
+
111
+ export type JobTreeRollupStatus = "queued" | "running" | "succeeded" | "failed" | "stopped";
112
+
113
+ export type JobEntityState = {
114
+ jobId: string;
115
+ jobType: string;
116
+ status: JobStatus;
117
+ entityKey: string;
118
+ entityType: string | null;
119
+ entityId: string | null;
120
+ label: string | null;
121
+ lastError: string | null;
122
+ result: unknown;
123
+ depth: number;
124
+ };
125
+
126
+ export type JobActivityEvent = {
127
+ opsEventId: string;
128
+ jobId: string | null;
129
+ level: "info" | "warn" | "error";
130
+ eventType: string;
131
+ message: string;
132
+ metadata: Record<string, unknown> | null;
133
+ createdAt: string;
134
+ };
135
+
136
+ export type JobTreeRecord<TPayload = unknown, TResult = unknown> = {
137
+ root: JobRecord<TPayload, TResult>;
138
+ children: Array<JobRecord & { depth: number }>;
139
+ summary: JobChildrenSummary;
140
+ rollupStatus: JobTreeRollupStatus;
141
+ rowStates: JobEntityState[];
142
+ activityEvents: JobActivityEvent[];
143
+ latestError: string | null;
144
+ };
145
+
146
+ export type DurableJobHandlerContext<TPayload = unknown, TResult = unknown, TServices = unknown> = {
147
+ job: JobRecord<TPayload, TResult>;
148
+ jobs: DurableJobRepository;
149
+ services: TServices;
150
+ events: {
151
+ info(input: Omit<DurableJobEventInput, "severity">): Promise<void>;
152
+ warn(input: Omit<DurableJobEventInput, "severity">): Promise<void>;
153
+ error(input: Omit<DurableJobEventInput, "severity">): Promise<void>;
154
+ };
155
+ shouldStop(): Promise<boolean>;
156
+ };
157
+
158
+ export class JobDeadline {
159
+ private readonly startedAt = Date.now();
160
+
161
+ constructor(private readonly budgetMs = 50_000) {}
162
+
163
+ shouldYield(): boolean {
164
+ return Date.now() - this.startedAt >= this.budgetMs;
165
+ }
166
+ }
167
+
168
+ export type JobOperationEntity = {
169
+ entityType: string;
170
+ entityId: string;
171
+ entityLabel?: string | null;
172
+ entityKey?: string | null;
173
+ };
174
+
175
+ export type EnqueueJobOperationInput<TPayload extends Record<string, unknown> = Record<string, unknown>> = {
176
+ jobType: string;
177
+ operationKey: string;
178
+ orgId?: string | null;
179
+ siteId?: string | null;
180
+ parentJobId?: string | null;
181
+ entity?: JobOperationEntity | null;
182
+ payload?: TPayload | null;
183
+ dedupeActive?: boolean;
184
+ concurrencyScope?: string | null;
185
+ concurrencyLimit?: number | null;
186
+ maxAttempts?: number | null;
187
+ availableAt?: string | null;
188
+ };
189
+
190
+ export type JobOperationKeys = {
191
+ lockKey: string | null;
192
+ concurrencyKey: string;
193
+ concurrencyLimit: number;
194
+ };
195
+
196
+ export type DurableJobHandler<TPayload = unknown, TResult = unknown, TServices = unknown> = (
197
+ context: DurableJobHandlerContext<TPayload, TResult, TServices>,
198
+ ) => Promise<JobHandlerResult<TResult, TPayload> | TResult | void>;
199
+
200
+ export type DurableJobRegistry<TServices = unknown> = Record<string, DurableJobHandler<unknown, unknown, TServices>>;
201
+
202
+ export type DurableJobDispatcherOptions<TServices = unknown> = {
203
+ jobs: DurableJobRepository;
204
+ handlers: DurableJobRegistry<TServices>;
205
+ services: TServices;
206
+ events?: DurableJobEventSink;
207
+ retryDelay?: (input: { job: JobRecord; error: unknown }) => string | null | undefined;
208
+ };
209
+
210
+ export class DurableJobDispatcher<TServices = unknown> {
211
+ constructor(private readonly options: DurableJobDispatcherOptions<TServices>) {}
212
+
213
+ async run<TResult = unknown>(jobId: string): Promise<JobHandlerResult<TResult>> {
214
+ const job = await this.options.jobs.claim(jobId);
215
+ if (!job) return { kind: "done", result: null };
216
+
217
+ const handler = this.options.handlers[job.jobType];
218
+ if (!handler) {
219
+ await this.options.jobs.fail(job.jobId, `No handler registered for job type: ${job.jobType}`);
220
+ throw new Error(`No handler registered for job type: ${job.jobType}`);
221
+ }
222
+
223
+ const events = new DurableJobEvents(job, this.options.events);
224
+ await events.info({ eventName: "job.started", status: "started", message: "Job started." });
225
+
226
+ try {
227
+ if (await this.shouldStop(job.jobId)) {
228
+ await this.options.jobs.stop(job.jobId);
229
+ await events.warn({ eventName: "job.stopped", status: "skipped", message: "Job stopped before handler ran." });
230
+ return { kind: "stopped", result: null };
231
+ }
232
+
233
+ const output = await handler({
234
+ job,
235
+ jobs: this.options.jobs,
236
+ services: this.options.services,
237
+ events,
238
+ shouldStop: () => this.shouldStop(job.jobId),
239
+ });
240
+ const normalized = normalizeJobResult<TResult>(output as JobHandlerResult<TResult> | TResult | void);
241
+ if (normalized.kind === "requeue") {
242
+ await this.options.jobs.requeue(job.jobId, {
243
+ payload: normalized.payload,
244
+ result: normalized.result ?? null,
245
+ availableAt: normalized.availableAt ?? null,
246
+ });
247
+ await events.info({ eventName: "job.requeued", status: "started", message: "Job requeued." });
248
+ } else if (normalized.kind === "stopped") {
249
+ await this.options.jobs.stop(job.jobId, normalized.result ?? null);
250
+ await events.warn({ eventName: "job.stopped", status: "skipped", message: "Job stopped." });
251
+ } else {
252
+ await this.options.jobs.succeed(job.jobId, normalized.result ?? null);
253
+ await events.info({ eventName: "job.succeeded", status: "ok", message: "Job succeeded." });
254
+ }
255
+ return normalized;
256
+ } catch (error) {
257
+ const latest = await this.options.jobs.get(job.jobId);
258
+ if (latest && latest.attemptCount < latest.maxAttempts) {
259
+ await this.options.jobs.requeue(job.jobId, {
260
+ result: { retryError: errorMessage(error) },
261
+ availableAt: this.options.retryDelay?.({ job: latest, error }) ?? null,
262
+ lastError: errorMessage(error),
263
+ });
264
+ await events.warn({ eventName: "job.retrying", status: "warning", message: "Job failed and will retry.", error });
265
+ return { kind: "requeue", result: null };
266
+ }
267
+ await this.options.jobs.fail(job.jobId, error);
268
+ await events.error({ eventName: "job.failed", status: "error", message: "Job failed.", error });
269
+ throw error;
270
+ }
271
+ }
272
+
273
+ private async shouldStop(jobId: string): Promise<boolean> {
274
+ const latest = await this.options.jobs.get(jobId);
275
+ return latest?.status === "stopping" || latest?.status === "stopped";
276
+ }
277
+ }
278
+
279
+ export type ParentJobOrchestrationInput<TPayload = unknown> = {
280
+ children: EnqueueJobInput<TPayload>[];
281
+ failureMode?: "fail_fast" | "continue";
282
+ queuedSiblingPolicy?: "fail" | "stop";
283
+ pollSeconds?: number;
284
+ now?: () => Date;
285
+ onChildQueued?: (job: JobRecord<TPayload>) => Promise<void> | void;
286
+ };
287
+
288
+ export type ParentJobOrchestrationResult = {
289
+ stage: "waiting_children" | "complete";
290
+ failureMode: "fail_fast" | "continue";
291
+ queuedSiblingPolicy: "fail" | "stop";
292
+ childJobIds: string[];
293
+ summary: JobChildrenSummary;
294
+ };
295
+
296
+ export async function runParentJobOrchestration<TPayload = unknown, TResult = unknown, TServices = unknown>(
297
+ context: DurableJobHandlerContext<TPayload, TResult, TServices>,
298
+ input: ParentJobOrchestrationInput<TPayload>,
299
+ ): Promise<JobHandlerResult> {
300
+ const existing = await context.jobs.listChildren(context.job.jobId);
301
+ const failureMode = input.failureMode ?? "fail_fast";
302
+ const queuedSiblingPolicy = input.queuedSiblingPolicy ?? "fail";
303
+ if (existing.length === 0) {
304
+ if (input.children.length === 0) {
305
+ return {
306
+ kind: "done",
307
+ result: {
308
+ stage: "complete",
309
+ failureMode,
310
+ queuedSiblingPolicy,
311
+ childJobIds: [],
312
+ summary: emptyChildrenSummary(context.job.jobId),
313
+ } satisfies ParentJobOrchestrationResult,
314
+ };
315
+ }
316
+ const children: JobRecord[] = [];
317
+ for (const child of input.children) {
318
+ const queued = await context.jobs.enqueue({
319
+ ...child,
320
+ parentJobId: child.parentJobId ?? context.job.jobId,
321
+ });
322
+ children.push(queued);
323
+ await input.onChildQueued?.(queued as JobRecord<TPayload>);
324
+ }
325
+ await context.events.info({
326
+ eventName: "job.children_queued",
327
+ status: "started",
328
+ message: "Child jobs queued.",
329
+ metrics: { queued: input.children.length },
330
+ });
331
+ return requeueParent<TPayload>(context.job.payload, children, summarizeJobs(context.job.jobId, children), input);
332
+ }
333
+
334
+ const summary = await context.jobs.summarizeChildren(context.job.jobId);
335
+ if (failureMode === "fail_fast" && summary.failed > 0) {
336
+ const failedReason =
337
+ (await context.jobs.listChildren(context.job.jobId)).find((child) => child.status === "failed" && child.lastError)
338
+ ?.lastError ?? `Child job failed for parent ${context.job.jobId}.`;
339
+ if (queuedSiblingPolicy === "stop") {
340
+ await context.jobs.markQueuedChildrenTerminal(context.job.jobId, "stopped", failedReason);
341
+ } else {
342
+ await context.jobs.markQueuedChildrenTerminal(context.job.jobId, "failed", failedReason);
343
+ }
344
+ await context.jobs.requestStopActiveChildren(context.job.jobId);
345
+ await context.events.warn({
346
+ eventName: "job.children_failed",
347
+ status: "warning",
348
+ message: "One or more child jobs failed.",
349
+ metrics: summary,
350
+ });
351
+ const updated = await context.jobs.summarizeChildren(context.job.jobId);
352
+ if (!updated.allTerminal) {
353
+ const children = await context.jobs.listChildren(context.job.jobId);
354
+ return requeueParent<TPayload>(context.job.payload, children, updated, input);
355
+ }
356
+ throw new Error(failedReason);
357
+ }
358
+
359
+ const children = await context.jobs.listChildren(context.job.jobId);
360
+ if (!summary.allTerminal) return requeueParent<TPayload>(context.job.payload, children, summary, input);
361
+ return {
362
+ kind: "done",
363
+ result: {
364
+ stage: "complete",
365
+ failureMode,
366
+ queuedSiblingPolicy,
367
+ childJobIds: children.map((child) => child.jobId),
368
+ summary,
369
+ } satisfies ParentJobOrchestrationResult,
370
+ };
371
+ }
372
+
373
+ export class D1JobStore implements DurableJobRepository {
38
374
  constructor(
39
375
  private readonly db: D1DatabaseLike,
40
- private readonly options: { tableName?: string; idPrefix?: string } = {},
376
+ private readonly options: {
377
+ tableName?: string;
378
+ idPrefix?: string;
379
+ staleRunningAfterSeconds?: number;
380
+ opsEventsTableName?: string;
381
+ defaultMaxAttempts?: number;
382
+ onStatusEvent?: JobStatusEventSink;
383
+ queuedStopStatus?: "stopping" | "stopped";
384
+ requestStopQueuedChildren?: boolean;
385
+ } = {},
41
386
  ) {}
42
387
 
43
388
  get tableName(): string {
@@ -47,19 +392,33 @@ export class D1JobStore {
47
392
  async enqueue<TPayload = unknown>(input: EnqueueJobInput<TPayload>): Promise<JobRecord<TPayload>> {
48
393
  const jobId = createId(this.options.idPrefix ?? "job");
49
394
  const now = nowIso();
395
+ const enqueuePolicy = input.enqueuePolicy ?? "enqueue";
396
+ if (enqueuePolicy === "dedupe_active" && input.lockKey) {
397
+ const existing = await this.getActiveForLockKey<TPayload>(input.lockKey);
398
+ if (existing) return existing;
399
+ }
400
+
50
401
  await this.db
51
402
  .prepare(
52
403
  `INSERT INTO ${this.tableName} (
53
- job_id, job_type, status, payload_json, result_json, lock_key,
404
+ job_id, job_type, status, org_id, site_id, payload_json, result_json, metadata_json,
405
+ parent_job_id, lock_key, concurrency_key, concurrency_limit, enqueue_policy,
54
406
  attempt_count, max_attempts, available_at, created_at, updated_at
55
- ) VALUES (?, ?, 'queued', ?, NULL, ?, 0, ?, ?, ?, ?)`,
407
+ ) VALUES (?, ?, 'queued', ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`,
56
408
  )
57
409
  .bind(
58
410
  jobId,
59
411
  input.jobType,
412
+ input.orgId ?? null,
413
+ input.siteId ?? null,
60
414
  stringifyJsonColumn(input.payload ?? {}),
415
+ stringifyJsonColumn(input.metadata ?? {}),
416
+ input.parentJobId ?? null,
61
417
  input.lockKey ?? null,
62
- Math.max(1, Math.floor(input.maxAttempts ?? 3)),
418
+ input.concurrencyKey ?? null,
419
+ normalizeConcurrencyLimit(input.concurrencyLimit),
420
+ enqueuePolicy,
421
+ Math.max(1, Math.floor(input.maxAttempts ?? this.options.defaultMaxAttempts ?? 3)),
63
422
  input.availableAt ?? null,
64
423
  now,
65
424
  now,
@@ -68,46 +427,43 @@ export class D1JobStore {
68
427
 
69
428
  const job = await this.get<TPayload>(jobId);
70
429
  if (!job) throw new Error(`Failed to load enqueued job: ${jobId}`);
430
+ await this.emitStatusEvent(job, "queued", "Job queued.");
71
431
  return job;
72
432
  }
73
433
 
74
434
  async get<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null> {
75
435
  const row = await this.db
76
- .prepare(
77
- `SELECT
78
- job_id AS jobId,
79
- job_type AS jobType,
80
- status,
81
- payload_json AS payloadJson,
82
- result_json AS resultJson,
83
- lock_key AS lockKey,
84
- attempt_count AS attemptCount,
85
- max_attempts AS maxAttempts,
86
- available_at AS availableAt,
87
- started_at AS startedAt,
88
- completed_at AS completedAt,
89
- last_error AS lastError,
90
- created_at AS createdAt,
91
- updated_at AS updatedAt
92
- FROM ${this.tableName}
93
- WHERE job_id = ?
94
- LIMIT 1`,
95
- )
436
+ .prepare(`${this.selectSql()} WHERE job_id = ? LIMIT 1`)
96
437
  .bind(jobId)
97
438
  .first<JobRow>();
98
439
  return row ? rowToJob<TPayload, TResult>(row) : null;
99
440
  }
100
441
 
442
+ async getJob<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null> {
443
+ return this.get(jobId);
444
+ }
445
+
446
+ async enqueueChild<TPayload = unknown>(
447
+ parentJobId: string,
448
+ input: Omit<EnqueueJobInput<TPayload>, "parentJobId">,
449
+ ): Promise<JobRecord<TPayload>> {
450
+ return this.enqueue({ ...input, parentJobId });
451
+ }
452
+
453
+ async enqueueChildren<TPayload = unknown>(
454
+ parentJobId: string,
455
+ inputs: Array<Omit<EnqueueJobInput<TPayload>, "parentJobId">>,
456
+ ): Promise<Array<JobRecord<TPayload>>> {
457
+ const children: Array<JobRecord<TPayload>> = [];
458
+ for (const input of inputs) children.push(await this.enqueueChild(parentJobId, input));
459
+ return children;
460
+ }
461
+
101
462
  async listQueued(limit = 25): Promise<JobRecord[]> {
102
463
  const now = nowIso();
103
464
  const rows = await this.db
104
465
  .prepare(
105
- `SELECT
106
- job_id AS jobId, job_type AS jobType, status, payload_json AS payloadJson,
107
- result_json AS resultJson, lock_key AS lockKey, attempt_count AS attemptCount,
108
- max_attempts AS maxAttempts, available_at AS availableAt, started_at AS startedAt,
109
- completed_at AS completedAt, last_error AS lastError, created_at AS createdAt, updated_at AS updatedAt
110
- FROM ${this.tableName}
466
+ `${this.selectSql()}
111
467
  WHERE status = 'queued' AND (available_at IS NULL OR available_at <= ?)
112
468
  ORDER BY created_at ASC
113
469
  LIMIT ?`,
@@ -117,14 +473,70 @@ export class D1JobStore {
117
473
  return (rows.results ?? []).map(rowToJob);
118
474
  }
119
475
 
476
+ async getLatestForSite(input: { siteId: string; jobType?: string | null }): Promise<JobRecord | null> {
477
+ const row = await this.db
478
+ .prepare(
479
+ `${this.selectSql()}
480
+ WHERE site_id = ?
481
+ ${input.jobType ? "AND job_type = ?" : ""}
482
+ ORDER BY created_at DESC
483
+ LIMIT 1`,
484
+ )
485
+ .bind(...([input.siteId, ...(input.jobType ? [input.jobType] : [])] as never[]))
486
+ .first<JobRow>();
487
+ return row ? rowToJob(row) : null;
488
+ }
489
+
490
+ async getActiveForSite(input: { siteId: string; jobType?: string | null }): Promise<JobRecord | null> {
491
+ const row = await this.db
492
+ .prepare(
493
+ `${this.selectSql()}
494
+ WHERE site_id = ?
495
+ AND status IN ('queued', 'running', 'stopping')
496
+ ${input.jobType ? "AND job_type = ?" : ""}
497
+ ORDER BY created_at DESC
498
+ LIMIT 1`,
499
+ )
500
+ .bind(...([input.siteId, ...(input.jobType ? [input.jobType] : [])] as never[]))
501
+ .first<JobRow>();
502
+ return row ? rowToJob(row) : null;
503
+ }
504
+
505
+ async getLatestForSiteByPayload(input: {
506
+ siteId: string;
507
+ jobType?: string | null;
508
+ key: string;
509
+ value: string | number | boolean | null;
510
+ }): Promise<JobRecord | null> {
511
+ return this.getForSiteByPayload({ ...input, activeOnly: false });
512
+ }
513
+
514
+ async getActiveForSiteByPayload(input: {
515
+ siteId: string;
516
+ jobType?: string | null;
517
+ key: string;
518
+ value: string | number | boolean | null;
519
+ }): Promise<JobRecord | null> {
520
+ return this.getForSiteByPayload({ ...input, activeOnly: true });
521
+ }
522
+
120
523
  async claim<TPayload = unknown>(jobId: string): Promise<JobRecord<TPayload> | null> {
121
524
  const current = await this.get<TPayload>(jobId);
122
- if (!current || current.status === "succeeded" || current.status === "failed") return null;
525
+ if (!current || isTerminalJobStatus(current.status) || current.status === "stopping") return null;
526
+ if (current.status === "running") {
527
+ if (!isStaleRunningJob(current, this.options.staleRunningAfterSeconds)) return null;
528
+ await this.markStaleRunningQueued(jobId);
529
+ }
123
530
  if (current.status === "queued" && isFutureIso(current.availableAt)) return null;
124
531
  if (current.attemptCount >= current.maxAttempts) {
125
532
  await this.fail(jobId, "Job exhausted all attempts.");
126
533
  return null;
127
534
  }
535
+ if (current.concurrencyKey && current.status === "queued") {
536
+ const limit = current.concurrencyLimit ?? 1;
537
+ const running = await this.countRunningForConcurrencyKey(current.concurrencyKey);
538
+ if (running >= limit) return null;
539
+ }
128
540
 
129
541
  const now = nowIso();
130
542
  const result = await this.db
@@ -132,85 +544,1005 @@ export class D1JobStore {
132
544
  `UPDATE ${this.tableName}
133
545
  SET status = 'running',
134
546
  attempt_count = attempt_count + 1,
135
- started_at = COALESCE(started_at, ?),
547
+ started_at = ?,
136
548
  updated_at = ?,
137
549
  last_error = NULL
138
550
  WHERE job_id = ?
139
- AND status IN ('queued', 'running')
551
+ AND status = 'queued'
140
552
  AND (available_at IS NULL OR available_at <= ?)`,
141
553
  )
142
554
  .bind(now, now, jobId, now)
143
555
  .run();
144
556
 
145
557
  if (!Number(result.meta.changes ?? 0)) return null;
146
- return this.get<TPayload>(jobId);
558
+ const job = await this.get<TPayload>(jobId);
559
+ if (job) await this.emitStatusEvent(job, "running", "Job started.");
560
+ return job;
561
+ }
562
+
563
+ async run<TPayload = unknown, TResult = unknown>(
564
+ jobId: string,
565
+ handler: (job: JobRecord<TPayload, TResult>) => Promise<JobHandlerResult<TResult, TPayload> | TResult | void>,
566
+ ): Promise<JobHandlerResult<TResult>> {
567
+ const job = await this.claim<TPayload>(jobId);
568
+ if (!job) return { kind: "done", result: null };
569
+
570
+ try {
571
+ const output = await handler(job as JobRecord<TPayload, TResult>);
572
+ const normalized = normalizeJobResult<TResult>(output);
573
+ if (normalized.kind === "requeue") {
574
+ await this.requeue(jobId, {
575
+ payload: normalized.payload as TPayload | undefined,
576
+ result: normalized.result ?? null,
577
+ availableAt: normalized.availableAt ?? null,
578
+ });
579
+ } else if (normalized.kind === "stopped") {
580
+ await this.stop(jobId, normalized.result ?? null);
581
+ } else {
582
+ await this.succeed(jobId, normalized.result ?? null);
583
+ }
584
+ return normalized;
585
+ } catch (error) {
586
+ const latest = await this.get(jobId);
587
+ if (latest && latest.attemptCount < latest.maxAttempts) {
588
+ await this.requeue(jobId, { result: { retryError: errorMessage(error) }, lastError: errorMessage(error) });
589
+ return { kind: "requeue", result: null };
590
+ }
591
+ await this.fail(jobId, error);
592
+ throw error;
593
+ }
147
594
  }
148
595
 
149
596
  async succeed<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
597
+ await this.complete(jobId, "succeeded", result, null);
598
+ }
599
+
600
+ async stop<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
601
+ await this.complete(jobId, "stopped", result, null);
602
+ }
603
+
604
+ async requeue<TResult = unknown>(jobId: string, result: TResult | null, availableAt?: string | null): Promise<void>;
605
+ async requeue<TPayload = unknown, TResult = unknown>(
606
+ jobId: string,
607
+ input?: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null },
608
+ ): Promise<void>;
609
+ async requeue<TPayload = unknown, TResult = unknown>(
610
+ jobId: string,
611
+ inputOrResult: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null } | TResult | null = {},
612
+ legacyAvailableAt?: string | null,
613
+ ): Promise<void> {
614
+ const input =
615
+ isRequeueInput(inputOrResult)
616
+ ? inputOrResult
617
+ : { result: inputOrResult as TResult | null, availableAt: legacyAvailableAt ?? null };
150
618
  const now = nowIso();
619
+ const payloadSql = input.payload === undefined ? "" : "payload_json = ?,";
620
+ const bindings = input.payload === undefined ? [] : [stringifyJsonColumn(input.payload)];
151
621
  await this.db
152
622
  .prepare(
153
623
  `UPDATE ${this.tableName}
154
- SET status = 'succeeded', result_json = ?, completed_at = ?, updated_at = ?
624
+ SET status = 'queued',
625
+ ${payloadSql}
626
+ result_json = ?,
627
+ available_at = ?,
628
+ last_error = ?,
629
+ updated_at = ?
155
630
  WHERE job_id = ?`,
156
631
  )
157
- .bind(stringifyJsonColumn(result), now, now, jobId)
632
+ .bind(
633
+ ...(bindings as never[]),
634
+ stringifyJsonColumn(input.result ?? null),
635
+ input.availableAt ?? null,
636
+ input.lastError ?? null,
637
+ now,
638
+ jobId,
639
+ )
158
640
  .run();
641
+ const job = await this.get(jobId);
642
+ if (job) {
643
+ await this.emitStatusEvent(job, "queued", "Job requeued.", {
644
+ availableAt: input.availableAt ?? null,
645
+ ...(input.lastError ? { lastError: input.lastError } : {}),
646
+ });
647
+ }
159
648
  }
160
649
 
161
- async requeue<TResult = unknown>(jobId: string, result: TResult | null = null, availableAt: string | null = null): Promise<void> {
650
+ async fail(jobId: string, error: unknown): Promise<void> {
651
+ await this.complete(jobId, "failed", null, errorMessage(error));
652
+ }
653
+
654
+ async requestStop(jobId: string): Promise<void> {
162
655
  const now = nowIso();
656
+ const queuedStopStatus = this.options.queuedStopStatus ?? "stopped";
163
657
  await this.db
164
658
  .prepare(
165
659
  `UPDATE ${this.tableName}
166
- SET status = 'queued', result_json = ?, available_at = ?, updated_at = ?
167
- WHERE job_id = ?`,
660
+ SET status = CASE WHEN status = 'queued' THEN ? ELSE 'stopping' END,
661
+ completed_at = CASE WHEN status = 'queued' AND ? = 'stopped' THEN ? ELSE completed_at END,
662
+ updated_at = ?
663
+ WHERE job_id = ? AND status IN ('queued', 'running')`,
168
664
  )
169
- .bind(stringifyJsonColumn(result), availableAt, now, jobId)
665
+ .bind(queuedStopStatus, queuedStopStatus, now, now, jobId)
170
666
  .run();
667
+ const job = await this.get(jobId);
668
+ if (job) await this.emitStatusEvent(job, job.status, "Job stop requested.");
171
669
  }
172
670
 
173
- async fail(jobId: string, error: unknown): Promise<void> {
671
+ async listChildren(parentJobId: string): Promise<JobRecord[]> {
672
+ const rows = await this.db
673
+ .prepare(`${this.selectSql()} WHERE parent_job_id = ? ORDER BY created_at ASC`)
674
+ .bind(parentJobId)
675
+ .all<JobRow>();
676
+ return (rows.results ?? []).map(rowToJob);
677
+ }
678
+
679
+ async getJobTree<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobTreeRecord<TPayload, TResult> | null> {
680
+ const root = await this.get<TPayload, TResult>(jobId);
681
+ if (!root) return null;
682
+ const rows = await this.db
683
+ .prepare(
684
+ `WITH RECURSIVE tree(job_id, depth) AS (
685
+ SELECT job_id, 0 FROM ${this.tableName} WHERE job_id = ?
686
+ UNION ALL
687
+ SELECT jobs.job_id, tree.depth + 1
688
+ FROM ${this.tableName} jobs
689
+ JOIN tree ON jobs.parent_job_id = tree.job_id
690
+ )
691
+ SELECT
692
+ jobs.job_id AS jobId,
693
+ jobs.job_type AS jobType,
694
+ jobs.status,
695
+ jobs.org_id AS orgId,
696
+ jobs.site_id AS siteId,
697
+ jobs.payload_json AS payloadJson,
698
+ jobs.result_json AS resultJson,
699
+ jobs.metadata_json AS metadataJson,
700
+ jobs.parent_job_id AS parentJobId,
701
+ jobs.lock_key AS lockKey,
702
+ jobs.concurrency_key AS concurrencyKey,
703
+ jobs.concurrency_limit AS concurrencyLimit,
704
+ jobs.enqueue_policy AS enqueuePolicy,
705
+ jobs.attempt_count AS attemptCount,
706
+ jobs.max_attempts AS maxAttempts,
707
+ jobs.available_at AS availableAt,
708
+ jobs.started_at AS startedAt,
709
+ jobs.completed_at AS completedAt,
710
+ jobs.last_error AS lastError,
711
+ jobs.created_at AS createdAt,
712
+ jobs.updated_at AS updatedAt,
713
+ tree.depth AS depth
714
+ FROM ${this.tableName} jobs
715
+ JOIN tree ON jobs.job_id = tree.job_id
716
+ WHERE jobs.job_id <> ?
717
+ ORDER BY tree.depth ASC, jobs.created_at ASC`,
718
+ )
719
+ .bind(jobId, jobId)
720
+ .all<JobTreeRow>();
721
+ const children = (rows.results ?? []).map((row) => ({ ...rowToJob(row), depth: Number(row.depth) }));
722
+ const allJobs = [root as JobRecord, ...children];
723
+ return {
724
+ root,
725
+ children,
726
+ summary: summarizeJobs(root.jobId, children),
727
+ rollupStatus: rollupJobTreeStatus(allJobs),
728
+ rowStates: [
729
+ jobEntityState(root as JobRecord, 0),
730
+ ...children.map((child) => jobEntityState(child, child.depth)),
731
+ ],
732
+ activityEvents: await this.listJobActivityEvents(allJobs.map((job) => job.jobId)),
733
+ latestError: latestJobError(allJobs),
734
+ };
735
+ }
736
+
737
+ async summarizeChildren(parentJobId: string): Promise<JobChildrenSummary> {
738
+ return summarizeJobs(parentJobId, await this.listChildren(parentJobId));
739
+ }
740
+
741
+ async markQueuedChildrenTerminal(parentJobId: string, status: "failed" | "stopped", error?: unknown): Promise<number> {
742
+ const now = nowIso();
743
+ const result = await this.db
744
+ .prepare(
745
+ `UPDATE ${this.tableName}
746
+ SET status = ?, last_error = ?, completed_at = ?, updated_at = ?
747
+ WHERE parent_job_id = ? AND status = 'queued'`,
748
+ )
749
+ .bind(status, status === "failed" && error ? errorMessage(error) : null, now, now, parentJobId)
750
+ .run();
751
+ const changed = Number(result.meta.changes ?? 0);
752
+ if (changed > 0 && this.options.onStatusEvent) {
753
+ for (const child of await this.listChildren(parentJobId)) {
754
+ if (child.status === status) {
755
+ await this.emitStatusEvent(
756
+ child,
757
+ status,
758
+ status === "failed" ? errorMessage(error ?? "Child job failed.") : "Job stopped.",
759
+ error ? { reason: errorMessage(error) } : null,
760
+ );
761
+ }
762
+ }
763
+ }
764
+ return changed;
765
+ }
766
+
767
+ async requestStopActiveChildren(parentJobId: string): Promise<number> {
768
+ const now = nowIso();
769
+ const statuses = this.options.requestStopQueuedChildren ? "'queued', 'running'" : "'running'";
770
+ const result = await this.db
771
+ .prepare(
772
+ `UPDATE ${this.tableName}
773
+ SET status = 'stopping', updated_at = ?
774
+ WHERE parent_job_id = ? AND status IN (${statuses})`,
775
+ )
776
+ .bind(now, parentJobId)
777
+ .run();
778
+ const changed = Number(result.meta.changes ?? 0);
779
+ if (changed > 0 && this.options.onStatusEvent) {
780
+ for (const child of await this.listChildren(parentJobId)) {
781
+ if (child.status === "stopping") await this.emitStatusEvent(child, "stopping", "Job stop requested.");
782
+ }
783
+ }
784
+ return changed;
785
+ }
786
+
787
+ async failQueuedChildren(input: { parentJobId: string; reason?: string | null }): Promise<number> {
788
+ return this.markQueuedChildrenTerminal(input.parentJobId, "failed", input.reason ?? "Child job failed.");
789
+ }
790
+
791
+ async stopQueuedChildren(input: { parentJobId: string; reason?: string | null }): Promise<number> {
792
+ return this.markQueuedChildrenTerminal(input.parentJobId, "stopped", input.reason ?? "Child job stopped.");
793
+ }
794
+
795
+ private async getActiveForLockKey<TPayload = unknown>(lockKey: string): Promise<JobRecord<TPayload> | null> {
796
+ const row = await this.db
797
+ .prepare(`${this.selectSql()} WHERE lock_key = ? AND status IN ('queued', 'running', 'stopping') ORDER BY created_at ASC LIMIT 1`)
798
+ .bind(lockKey)
799
+ .first<JobRow>();
800
+ return row ? rowToJob<TPayload>(row) : null;
801
+ }
802
+
803
+ private async countRunningForConcurrencyKey(concurrencyKey: string): Promise<number> {
804
+ const row = await this.db
805
+ .prepare(
806
+ `SELECT COUNT(*) AS count
807
+ FROM ${this.tableName}
808
+ WHERE concurrency_key = ? AND status IN ('running', 'stopping')`,
809
+ )
810
+ .bind(concurrencyKey)
811
+ .first<{ count: number }>();
812
+ return Number(row?.count ?? 0);
813
+ }
814
+
815
+ private async getForSiteByPayload(input: {
816
+ siteId: string;
817
+ jobType?: string | null;
818
+ key: string;
819
+ value: string | number | boolean | null;
820
+ activeOnly: boolean;
821
+ }): Promise<JobRecord | null> {
822
+ const row = await this.db
823
+ .prepare(
824
+ `${this.selectSql()}
825
+ WHERE site_id = ?
826
+ ${input.jobType ? "AND job_type = ?" : ""}
827
+ ${input.activeOnly ? "AND status IN ('queued', 'running', 'stopping')" : ""}
828
+ AND json_extract(payload_json, ?) = ?
829
+ ORDER BY created_at DESC
830
+ LIMIT 1`,
831
+ )
832
+ .bind(
833
+ ...([
834
+ input.siteId,
835
+ ...(input.jobType ? [input.jobType] : []),
836
+ `$.${input.key}`,
837
+ input.value,
838
+ ] as never[]),
839
+ )
840
+ .first<JobRow>();
841
+ return row ? rowToJob(row) : null;
842
+ }
843
+
844
+ private async listJobActivityEvents(jobIds: string[]): Promise<JobActivityEvent[]> {
845
+ if (jobIds.length === 0) return [];
846
+ const placeholders = jobIds.map(() => "?").join(", ");
847
+ try {
848
+ const rows = await this.db
849
+ .prepare(
850
+ `SELECT
851
+ ops_event_id AS opsEventId,
852
+ job_id AS jobId,
853
+ level,
854
+ event_type AS eventType,
855
+ message,
856
+ metadata_json AS metadataJson,
857
+ created_at AS createdAt
858
+ FROM ${this.options.opsEventsTableName ?? "ops_events"}
859
+ WHERE job_id IN (${placeholders})
860
+ ORDER BY created_at ASC
861
+ LIMIT 200`,
862
+ )
863
+ .bind(...(jobIds as never[]))
864
+ .all<JobActivityEventRow>();
865
+ return (rows.results ?? []).map(rowToActivityEvent);
866
+ } catch {
867
+ return [];
868
+ }
869
+ }
870
+
871
+ private async complete<TResult = unknown>(
872
+ jobId: string,
873
+ status: TerminalJobStatus,
874
+ result: TResult | null,
875
+ lastError: string | null,
876
+ ): Promise<void> {
174
877
  const now = nowIso();
175
- const message = error instanceof Error ? error.message : String(error);
176
878
  await this.db
177
879
  .prepare(
178
880
  `UPDATE ${this.tableName}
179
- SET status = 'failed', last_error = ?, completed_at = ?, updated_at = ?
881
+ SET status = ?, result_json = ?, last_error = ?, completed_at = ?, updated_at = ?
180
882
  WHERE job_id = ?`,
181
883
  )
182
- .bind(message, now, now, jobId)
884
+ .bind(status, stringifyJsonColumn(result), lastError, now, now, jobId)
183
885
  .run();
886
+ const job = await this.get(jobId);
887
+ if (job) {
888
+ await this.emitStatusEvent(
889
+ job,
890
+ status,
891
+ status === "succeeded" ? "Job succeeded." : status === "stopped" ? "Job stopped." : (lastError ?? "Job failed."),
892
+ );
893
+ }
184
894
  }
185
895
 
186
- async run<TPayload = unknown, TResult = unknown>(
896
+ private async markStaleRunningQueued(jobId: string): Promise<void> {
897
+ const now = nowIso();
898
+ await this.db
899
+ .prepare(
900
+ `UPDATE ${this.tableName}
901
+ SET status = 'queued',
902
+ available_at = ?,
903
+ last_error = ?,
904
+ updated_at = ?
905
+ WHERE job_id = ? AND status = 'running'`,
906
+ )
907
+ .bind(now, "Job reclaimed after stale running lease.", now, jobId)
908
+ .run();
909
+ const job = await this.get(jobId);
910
+ if (job) {
911
+ await this.emitStatusEvent(job, "queued", "Stale running job requeued.", {
912
+ previousStatus: "running",
913
+ });
914
+ }
915
+ }
916
+
917
+ private async emitStatusEvent(
918
+ job: JobRecord,
919
+ status: JobStatus,
920
+ message: string,
921
+ metadata?: Record<string, unknown> | null,
922
+ error?: unknown,
923
+ ): Promise<void> {
924
+ await this.options.onStatusEvent?.({ job, status, message, metadata: metadata ?? null, error });
925
+ }
926
+
927
+ private selectSql(): string {
928
+ return `SELECT
929
+ job_id AS jobId,
930
+ job_type AS jobType,
931
+ status,
932
+ org_id AS orgId,
933
+ site_id AS siteId,
934
+ payload_json AS payloadJson,
935
+ result_json AS resultJson,
936
+ metadata_json AS metadataJson,
937
+ parent_job_id AS parentJobId,
938
+ lock_key AS lockKey,
939
+ concurrency_key AS concurrencyKey,
940
+ concurrency_limit AS concurrencyLimit,
941
+ enqueue_policy AS enqueuePolicy,
942
+ attempt_count AS attemptCount,
943
+ max_attempts AS maxAttempts,
944
+ available_at AS availableAt,
945
+ started_at AS startedAt,
946
+ completed_at AS completedAt,
947
+ last_error AS lastError,
948
+ created_at AS createdAt,
949
+ updated_at AS updatedAt
950
+ FROM ${this.tableName}`;
951
+ }
952
+ }
953
+
954
+ export type PostgresSqlLike = {
955
+ unsafe<Row extends object = Record<string, unknown>>(query: string, values?: readonly unknown[]): Promise<Row[]>;
956
+ json?: (value: unknown) => unknown;
957
+ };
958
+
959
+ export type PostgresJobStoreOptions = {
960
+ tableName?: string;
961
+ idPrefix?: string;
962
+ queueName?: string;
963
+ statusMap?: Partial<Record<JobStatus, string>>;
964
+ reverseStatusMap?: Record<string, JobStatus>;
965
+ columns?: Partial<PostgresJobColumns>;
966
+ defaultMaxAttempts?: number;
967
+ initialResult?: unknown;
968
+ retryDelaySql?: string;
969
+ };
970
+
971
+ export type PostgresJobColumns = {
972
+ jobId: string;
973
+ orgId: string | null;
974
+ siteId: string | null;
975
+ queueName: string;
976
+ jobType: string;
977
+ status: string;
978
+ payloadJson: string;
979
+ resultJson: string;
980
+ metadataJson: string | null;
981
+ parentJobId: string | null;
982
+ lockKey: string | null;
983
+ concurrencyKey: string | null;
984
+ concurrencyLimit: string | null;
985
+ enqueuePolicy: string | null;
986
+ attemptCount: string;
987
+ maxAttempts: string;
988
+ availableAt: string;
989
+ startedAt: string | null;
990
+ completedAt: string | null;
991
+ lastError: string | null;
992
+ createdAt: string;
993
+ updatedAt: string;
994
+ };
995
+
996
+ const DEFAULT_POSTGRES_JOB_COLUMNS: PostgresJobColumns = {
997
+ jobId: "job_id",
998
+ orgId: "org_id",
999
+ siteId: "site_id",
1000
+ queueName: "queue_name",
1001
+ jobType: "job_type",
1002
+ status: "status",
1003
+ payloadJson: "payload_json",
1004
+ resultJson: "result_json",
1005
+ metadataJson: "metadata_json",
1006
+ parentJobId: "parent_job_id",
1007
+ lockKey: "lock_key",
1008
+ concurrencyKey: "concurrency_key",
1009
+ concurrencyLimit: "concurrency_limit",
1010
+ enqueuePolicy: "enqueue_policy",
1011
+ attemptCount: "attempt_count",
1012
+ maxAttempts: "max_attempts",
1013
+ availableAt: "available_at",
1014
+ startedAt: "started_at",
1015
+ completedAt: "completed_at",
1016
+ lastError: "last_error",
1017
+ createdAt: "created_at",
1018
+ updatedAt: "updated_at",
1019
+ };
1020
+
1021
+ type PostgresJobRow = Omit<
1022
+ JobRow,
1023
+ | "payloadJson"
1024
+ | "resultJson"
1025
+ | "metadataJson"
1026
+ | "availableAt"
1027
+ | "startedAt"
1028
+ | "completedAt"
1029
+ | "createdAt"
1030
+ | "updatedAt"
1031
+ > & {
1032
+ payloadJson: unknown;
1033
+ resultJson: unknown;
1034
+ metadataJson?: unknown;
1035
+ availableAt: unknown;
1036
+ startedAt: unknown;
1037
+ completedAt: unknown;
1038
+ createdAt: unknown;
1039
+ updatedAt: unknown;
1040
+ };
1041
+
1042
+ export class PostgresJobStore implements DurableJobRepository {
1043
+ private readonly tableName: string;
1044
+ private readonly columns: PostgresJobColumns;
1045
+
1046
+ constructor(
1047
+ private readonly sql: PostgresSqlLike,
1048
+ private readonly options: PostgresJobStoreOptions = {},
1049
+ ) {
1050
+ this.tableName = quotePgIdent(options.tableName ?? "jobs");
1051
+ this.columns = { ...DEFAULT_POSTGRES_JOB_COLUMNS, ...(options.columns ?? {}) };
1052
+ }
1053
+
1054
+ async enqueue<TPayload = unknown>(input: EnqueueJobInput<TPayload>): Promise<JobRecord<TPayload>> {
1055
+ if (input.enqueuePolicy === "dedupe_active" && input.lockKey) {
1056
+ const existing = await this.getActiveForLockKey<TPayload>(input.lockKey);
1057
+ if (existing) return existing;
1058
+ }
1059
+ const jobId = createId(this.options.idPrefix ?? "job");
1060
+ const queueName = stringValue(input.metadata?.queueName) ?? queueNameFromPayload(input.payload) ?? this.options.queueName ?? null;
1061
+ const insertColumnKeys: Array<keyof PostgresJobColumns> = [
1062
+ "jobId",
1063
+ "orgId",
1064
+ "siteId",
1065
+ "queueName",
1066
+ "jobType",
1067
+ "status",
1068
+ "payloadJson",
1069
+ "resultJson",
1070
+ "metadataJson",
1071
+ "parentJobId",
1072
+ "lockKey",
1073
+ "concurrencyKey",
1074
+ "concurrencyLimit",
1075
+ "enqueuePolicy",
1076
+ "maxAttempts",
1077
+ "availableAt",
1078
+ ];
1079
+ const insertColumns = insertColumnKeys.filter((key) => this.columns[key] !== null);
1080
+ const insertValues: Record<keyof PostgresJobColumns, unknown> = {
1081
+ jobId,
1082
+ orgId: input.orgId ?? null,
1083
+ siteId: input.siteId ?? null,
1084
+ queueName,
1085
+ jobType: input.jobType,
1086
+ status: this.toDbStatus("queued"),
1087
+ payloadJson: jsonParam(input.payload ?? {}),
1088
+ resultJson: jsonParam(this.options.initialResult ?? null),
1089
+ metadataJson: jsonParam(input.metadata ?? {}),
1090
+ parentJobId: input.parentJobId ?? null,
1091
+ lockKey: input.lockKey ?? null,
1092
+ concurrencyKey: input.concurrencyKey ?? null,
1093
+ concurrencyLimit: normalizeConcurrencyLimit(input.concurrencyLimit),
1094
+ enqueuePolicy: input.enqueuePolicy ?? "enqueue",
1095
+ attemptCount: 0,
1096
+ maxAttempts: Math.max(1, Math.floor(input.maxAttempts ?? this.options.defaultMaxAttempts ?? 3)),
1097
+ availableAt: input.availableAt ?? null,
1098
+ startedAt: null,
1099
+ completedAt: null,
1100
+ lastError: null,
1101
+ createdAt: null,
1102
+ updatedAt: null,
1103
+ };
1104
+ const values = insertColumns.map((key) => insertValues[key]);
1105
+ await this.sql.unsafe(
1106
+ `INSERT INTO ${this.tableName} (
1107
+ ${insertColumns.map((key) => this.col(key)).join(", ")}
1108
+ ) VALUES (${insertColumns.map((key, index) => this.insertPlaceholder(key, index + 1)).join(", ")})`,
1109
+ values,
1110
+ );
1111
+ const job = await this.get<TPayload>(jobId);
1112
+ if (!job) throw new Error(`Failed to load enqueued job: ${jobId}`);
1113
+ return job;
1114
+ }
1115
+
1116
+ async get<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null> {
1117
+ const rows = await this.sql.unsafe<PostgresJobRow>(`${this.selectSql()} WHERE ${this.col("jobId")} = $1 LIMIT 1`, [jobId]);
1118
+ return rows[0] ? this.rowToJob<TPayload, TResult>(rows[0]) : null;
1119
+ }
1120
+
1121
+ async claim<TPayload = unknown>(jobId: string): Promise<JobRecord<TPayload> | null> {
1122
+ const queueClause = this.options.queueName ? `AND ${this.col("queueName")} = $2` : "";
1123
+ const values = this.options.queueName ? [jobId, this.options.queueName] : [jobId];
1124
+ const rows = await this.sql.unsafe<PostgresJobRow>(
1125
+ `UPDATE ${this.tableName}
1126
+ SET ${this.col("status")} = '${this.toDbStatus("running")}',
1127
+ ${this.col("attemptCount")} = ${this.col("attemptCount")} + 1,
1128
+ ${this.setSql("startedAt", "now(),")}
1129
+ ${this.col("updatedAt")} = now(),
1130
+ ${this.setSql("lastError", "NULL")}
1131
+ WHERE ${this.col("jobId")} = $1
1132
+ ${queueClause}
1133
+ AND ${this.col("status")} = '${this.toDbStatus("queued")}'
1134
+ AND ${this.col("availableAt")} <= now()
1135
+ AND (
1136
+ ${this.nullableColSql("concurrencyKey", "NULL")} IS NULL
1137
+ OR (
1138
+ SELECT COUNT(*)
1139
+ FROM ${this.tableName} active
1140
+ WHERE active.${this.colRaw("concurrencyKey")} = ${this.tableName}.${this.colRaw("concurrencyKey")}
1141
+ AND active.${this.colRaw("status")} IN ('${this.toDbStatus("running")}', '${this.toDbStatus("stopping")}')
1142
+ AND active.${this.colRaw("jobId")} <> ${this.tableName}.${this.colRaw("jobId")}
1143
+ ) < COALESCE(${this.nullableColSql("concurrencyLimit", "1")}, 1)
1144
+ )
1145
+ RETURNING ${this.returningSql()}`,
1146
+ values,
1147
+ );
1148
+ return rows[0] ? this.rowToJob<TPayload>(rows[0]) : null;
1149
+ }
1150
+
1151
+ async succeed<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
1152
+ await this.complete(jobId, "succeeded", result, null);
1153
+ }
1154
+
1155
+ async requeue<TPayload = unknown, TResult = unknown>(
187
1156
  jobId: string,
188
- handler: (job: JobRecord<TPayload, TResult>) => Promise<JobHandlerResult<TResult> | TResult | void>,
189
- ): Promise<JobHandlerResult<TResult>> {
190
- const job = await this.claim<TPayload>(jobId);
191
- if (!job) return { kind: "done", result: null };
1157
+ input: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null } = {},
1158
+ ): Promise<void> {
1159
+ const payloadSet = input.payload === undefined ? "" : `${this.col("payloadJson")} = $5::jsonb,`;
1160
+ const values = [
1161
+ jsonParam(input.result ?? null),
1162
+ input.availableAt ?? null,
1163
+ input.lastError ?? null,
1164
+ jobId,
1165
+ ...(input.payload === undefined ? [] : [jsonParam(input.payload)]),
1166
+ ];
1167
+ await this.sql.unsafe(
1168
+ `UPDATE ${this.tableName}
1169
+ SET ${this.col("status")} = '${this.toDbStatus("queued")}',
1170
+ ${payloadSet}
1171
+ ${this.col("resultJson")} = $1::jsonb,
1172
+ ${this.col("availableAt")} = COALESCE($2::timestamptz, now()),
1173
+ ${this.setSql("lastError", "$3,")}
1174
+ ${this.col("updatedAt")} = now()
1175
+ WHERE ${this.col("jobId")} = $4`,
1176
+ values,
1177
+ );
1178
+ }
192
1179
 
193
- try {
194
- const output = await handler(job as JobRecord<TPayload, TResult>);
195
- const normalized = normalizeJobResult<TResult>(output);
196
- if (normalized.kind === "requeue") {
197
- await this.requeue(jobId, normalized.result ?? null, normalized.availableAt ?? null);
198
- } else {
199
- await this.succeed(jobId, normalized.result ?? null);
200
- }
201
- return normalized;
202
- } catch (error) {
203
- const latest = await this.get(jobId);
204
- if (latest && latest.attemptCount < latest.maxAttempts) {
205
- await this.requeue(jobId, { retryError: error instanceof Error ? error.message : String(error) });
206
- return { kind: "requeue", result: null };
207
- }
208
- await this.fail(jobId, error);
209
- throw error;
1180
+ async fail(jobId: string, error: unknown): Promise<void> {
1181
+ await this.complete(jobId, "failed", null, errorMessage(error));
1182
+ }
1183
+
1184
+ async stop<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
1185
+ await this.complete(jobId, "stopped", result, null);
1186
+ }
1187
+
1188
+ async requestStop(jobId: string): Promise<void> {
1189
+ await this.sql.unsafe(
1190
+ `UPDATE ${this.tableName}
1191
+ SET ${this.col("status")} = CASE
1192
+ WHEN ${this.col("status")} = '${this.toDbStatus("queued")}' THEN '${this.toDbStatus("stopped")}'
1193
+ ELSE '${this.toDbStatus("stopping")}'
1194
+ END,
1195
+ ${this.setSql("completedAt", `CASE
1196
+ WHEN ${this.col("status")} = '${this.toDbStatus("queued")}' THEN now()
1197
+ ELSE ${this.col("completedAt")}
1198
+ END,`)}
1199
+ ${this.col("updatedAt")} = now()
1200
+ WHERE ${this.col("jobId")} = $1
1201
+ AND ${this.col("status")} IN ('${this.toDbStatus("queued")}', '${this.toDbStatus("running")}')`,
1202
+ [jobId],
1203
+ );
1204
+ }
1205
+
1206
+ async listChildren(parentJobId: string): Promise<JobRecord[]> {
1207
+ const rows = await this.sql.unsafe<PostgresJobRow>(
1208
+ `${this.selectSql()} WHERE ${this.col("parentJobId")} = $1 ORDER BY ${this.col("createdAt")} ASC`,
1209
+ [parentJobId],
1210
+ );
1211
+ return rows.map((row) => this.rowToJob(row));
1212
+ }
1213
+
1214
+ async summarizeChildren(parentJobId: string): Promise<JobChildrenSummary> {
1215
+ return summarizeJobs(parentJobId, await this.listChildren(parentJobId));
1216
+ }
1217
+
1218
+ async markQueuedChildrenTerminal(parentJobId: string, status: "failed" | "stopped", error?: unknown): Promise<number> {
1219
+ const rows = await this.sql.unsafe<{ changed: number }>(
1220
+ `WITH updated AS (
1221
+ UPDATE ${this.tableName}
1222
+ SET ${this.col("status")} = $1,
1223
+ ${this.setSql("lastError", "$2,")}
1224
+ ${this.setSql("completedAt", "now(),")}
1225
+ ${this.col("updatedAt")} = now()
1226
+ WHERE ${this.col("parentJobId")} = $3
1227
+ AND ${this.col("status")} = '${this.toDbStatus("queued")}'
1228
+ RETURNING ${this.col("jobId")}
1229
+ )
1230
+ SELECT COUNT(*)::int AS changed FROM updated`,
1231
+ [this.toDbStatus(status), status === "failed" && error ? errorMessage(error) : null, parentJobId],
1232
+ );
1233
+ return Number(rows[0]?.changed ?? 0);
1234
+ }
1235
+
1236
+ async requestStopActiveChildren(parentJobId: string): Promise<number> {
1237
+ const rows = await this.sql.unsafe<{ changed: number }>(
1238
+ `WITH updated AS (
1239
+ UPDATE ${this.tableName}
1240
+ SET ${this.col("status")} = '${this.toDbStatus("stopping")}',
1241
+ ${this.col("updatedAt")} = now()
1242
+ WHERE ${this.col("parentJobId")} = $1
1243
+ AND ${this.col("status")} = '${this.toDbStatus("running")}'
1244
+ RETURNING ${this.col("jobId")}
1245
+ )
1246
+ SELECT COUNT(*)::int AS changed FROM updated`,
1247
+ [parentJobId],
1248
+ );
1249
+ return Number(rows[0]?.changed ?? 0);
1250
+ }
1251
+
1252
+ private async getActiveForLockKey<TPayload = unknown>(lockKey: string): Promise<JobRecord<TPayload> | null> {
1253
+ const rows = await this.sql.unsafe<PostgresJobRow>(
1254
+ `${this.selectSql()}
1255
+ WHERE ${this.col("lockKey")} = $1
1256
+ AND ${this.col("status")} IN ('${this.toDbStatus("queued")}', '${this.toDbStatus("running")}', '${this.toDbStatus("stopping")}')
1257
+ ORDER BY ${this.col("createdAt")} DESC
1258
+ LIMIT 1`,
1259
+ [lockKey],
1260
+ );
1261
+ return rows[0] ? this.rowToJob<TPayload>(rows[0]) : null;
1262
+ }
1263
+
1264
+ private async complete<TResult>(jobId: string, status: TerminalJobStatus, result: TResult | null, lastError: string | null): Promise<void> {
1265
+ await this.sql.unsafe(
1266
+ `UPDATE ${this.tableName}
1267
+ SET ${this.col("status")} = $1,
1268
+ ${this.col("resultJson")} = $2::jsonb,
1269
+ ${this.setSql("lastError", "$3,")}
1270
+ ${this.setSql("completedAt", "now(),")}
1271
+ ${this.col("updatedAt")} = now()
1272
+ WHERE ${this.col("jobId")} = $4`,
1273
+ [this.toDbStatus(status), jsonParam(result), lastError, jobId],
1274
+ );
1275
+ }
1276
+
1277
+ private selectSql(): string {
1278
+ return `SELECT ${this.returningSql()} FROM ${this.tableName}`;
1279
+ }
1280
+
1281
+ private returningSql(): string {
1282
+ return ([
1283
+ ["jobId", "jobId"],
1284
+ ["jobType", "jobType"],
1285
+ ["status", "status"],
1286
+ ["orgId", "orgId"],
1287
+ ["siteId", "siteId"],
1288
+ ["payloadJson", "payloadJson"],
1289
+ ["resultJson", "resultJson"],
1290
+ ["metadataJson", "metadataJson"],
1291
+ ["parentJobId", "parentJobId"],
1292
+ ["lockKey", "lockKey"],
1293
+ ["concurrencyKey", "concurrencyKey"],
1294
+ ["concurrencyLimit", "concurrencyLimit"],
1295
+ ["enqueuePolicy", "enqueuePolicy"],
1296
+ ["attemptCount", "attemptCount"],
1297
+ ["maxAttempts", "maxAttempts"],
1298
+ ["availableAt", "availableAt"],
1299
+ ["startedAt", "startedAt"],
1300
+ ["completedAt", "completedAt"],
1301
+ ["lastError", "lastError"],
1302
+ ["createdAt", "createdAt"],
1303
+ ["updatedAt", "updatedAt"],
1304
+ ] as Array<[keyof PostgresJobColumns, string]>)
1305
+ .map(([key, alias]) => `${this.selectExpr(key)} AS "${alias}"`)
1306
+ .join(", ");
1307
+ }
1308
+
1309
+ private rowToJob<TPayload = unknown, TResult = unknown>(row: PostgresJobRow): JobRecord<TPayload, TResult> {
1310
+ return rowToJob({
1311
+ ...row,
1312
+ status: this.fromDbStatus(String(row.status)),
1313
+ payloadJson: jsonText(row.payloadJson) ?? "{}",
1314
+ resultJson: jsonText(row.resultJson),
1315
+ metadataJson: jsonText(row.metadataJson),
1316
+ availableAt: dateText(row.availableAt),
1317
+ startedAt: dateText(row.startedAt),
1318
+ completedAt: dateText(row.completedAt),
1319
+ createdAt: dateText(row.createdAt) ?? "",
1320
+ updatedAt: dateText(row.updatedAt) ?? "",
1321
+ });
1322
+ }
1323
+
1324
+ private toDbStatus(status: JobStatus): string {
1325
+ return this.options.statusMap?.[status] ?? status;
1326
+ }
1327
+
1328
+ private fromDbStatus(status: string): JobStatus {
1329
+ return this.options.reverseStatusMap?.[status] ?? (status as JobStatus);
1330
+ }
1331
+
1332
+ private col(key: keyof PostgresJobColumns): string {
1333
+ const column = this.columns[key];
1334
+ if (!column) throw new Error(`Postgres job column is not configured: ${String(key)}`);
1335
+ return quotePgIdent(column);
1336
+ }
1337
+
1338
+ private colRaw(key: keyof PostgresJobColumns): string {
1339
+ const column = this.columns[key];
1340
+ if (!column) throw new Error(`Postgres job column is not configured: ${String(key)}`);
1341
+ return column;
1342
+ }
1343
+
1344
+ private insertPlaceholder(key: keyof PostgresJobColumns, index: number): string {
1345
+ if (key === "payloadJson" || key === "resultJson" || key === "metadataJson") return `$${index}::jsonb`;
1346
+ if (key === "availableAt" || key === "startedAt" || key === "completedAt") return `COALESCE($${index}::timestamptz, now())`;
1347
+ return `$${index}`;
1348
+ }
1349
+
1350
+ private nullableColSql(key: keyof PostgresJobColumns, fallback: string): string {
1351
+ return this.columns[key] ? this.col(key) : fallback;
1352
+ }
1353
+
1354
+ private selectExpr(key: keyof PostgresJobColumns): string {
1355
+ if (this.columns[key]) return this.col(key);
1356
+ if (key === "metadataJson") return `'{}'::jsonb`;
1357
+ if (key === "resultJson") return `'null'::jsonb`;
1358
+ if (key === "enqueuePolicy") return `'enqueue'`;
1359
+ if (key === "concurrencyLimit" || key === "attemptCount" || key === "maxAttempts") return "0";
1360
+ return "NULL";
1361
+ }
1362
+
1363
+ private setSql(key: keyof PostgresJobColumns, assignment: string): string {
1364
+ return this.columns[key] ? `${this.col(key)} = ${assignment}` : "";
1365
+ }
1366
+ }
1367
+
1368
+ export type PipelineJob<StepName extends string, State> = {
1369
+ pipeline: string;
1370
+ runId: string;
1371
+ step: StepName;
1372
+ continuationId?: string;
1373
+ state: State;
1374
+ };
1375
+
1376
+ export type PipelineStepHandlerInput<StepName extends string, State> = {
1377
+ job: PipelineJob<StepName, State>;
1378
+ stepIndex: number;
1379
+ isFinalStep: boolean;
1380
+ };
1381
+
1382
+ export type PipelineCompletion = {
1383
+ status: string;
1384
+ targetsTotal: number;
1385
+ targetsSucceeded: number;
1386
+ targetsFailed: number;
1387
+ message?: string | null;
1388
+ };
1389
+
1390
+ export type PipelineStepHandlerResult<StepName extends string, State> = {
1391
+ state?: State;
1392
+ nextStep?: { step: StepName; state?: State } | null;
1393
+ nextStepDelaySeconds?: number;
1394
+ completion?: PipelineCompletion | null;
1395
+ };
1396
+
1397
+ export type PipelineStepHandler<StepName extends string, State> = (
1398
+ input: PipelineStepHandlerInput<StepName, State>,
1399
+ ) => Promise<PipelineStepHandlerResult<StepName, State> | undefined> | Promise<void>;
1400
+
1401
+ export type PipelineManagerOptions<StepName extends string, State> = {
1402
+ pipeline: string;
1403
+ steps: readonly StepName[];
1404
+ handlers: Record<StepName, PipelineStepHandler<StepName, State>>;
1405
+ enqueue: (job: PipelineJob<StepName, State>, options?: { delaySeconds?: number }) => Promise<void>;
1406
+ makeContinuationId?: () => string;
1407
+ };
1408
+
1409
+ export class PipelineManager<StepName extends string, State> {
1410
+ constructor(private readonly options: PipelineManagerOptions<StepName, State>) {}
1411
+
1412
+ getNextStep(step: StepName): StepName | null {
1413
+ const index = this.options.steps.indexOf(step);
1414
+ if (index === -1) throw new Error(`Unknown pipeline step: ${step}`);
1415
+ return this.options.steps[index + 1] ?? null;
1416
+ }
1417
+
1418
+ async run(job: PipelineJob<StepName, State>): Promise<{
1419
+ nextStep: StepName | null;
1420
+ state: State;
1421
+ completion: PipelineCompletion | null;
1422
+ }> {
1423
+ if (job.pipeline !== this.options.pipeline) throw new Error(`Unexpected pipeline: ${job.pipeline}`);
1424
+ const stepIndex = this.options.steps.indexOf(job.step);
1425
+ if (stepIndex === -1) throw new Error(`Unknown pipeline step: ${job.step}`);
1426
+
1427
+ const defaultNextStep = this.options.steps[stepIndex + 1] ?? null;
1428
+ const result = await this.options.handlers[job.step]({
1429
+ job,
1430
+ stepIndex,
1431
+ isFinalStep: defaultNextStep === null,
1432
+ });
1433
+ const nextState = result?.state ?? job.state;
1434
+ const explicitNextStep = result?.nextStep;
1435
+ const nextStep =
1436
+ explicitNextStep === undefined
1437
+ ? defaultNextStep
1438
+ : explicitNextStep === null
1439
+ ? null
1440
+ : (explicitNextStep.step as StepName);
1441
+ const enqueuedState = explicitNextStep && explicitNextStep !== null ? (explicitNextStep.state ?? nextState) : nextState;
1442
+
1443
+ if (nextStep) {
1444
+ const continuationId = this.options.makeContinuationId?.();
1445
+ const nextJob = {
1446
+ pipeline: job.pipeline,
1447
+ runId: job.runId,
1448
+ step: nextStep,
1449
+ ...(continuationId ? { continuationId } : {}),
1450
+ state: enqueuedState,
1451
+ };
1452
+ if (result?.nextStepDelaySeconds === undefined) await this.options.enqueue(nextJob);
1453
+ else await this.options.enqueue(nextJob, { delaySeconds: result.nextStepDelaySeconds });
210
1454
  }
1455
+
1456
+ const completion =
1457
+ result?.completion ??
1458
+ (nextStep === null && defaultNextStep === null
1459
+ ? {
1460
+ status: "completed",
1461
+ targetsTotal: 0,
1462
+ targetsSucceeded: 0,
1463
+ targetsFailed: 0,
1464
+ message: `${job.pipeline} pipeline completed`,
1465
+ }
1466
+ : null);
1467
+
1468
+ return { nextStep, state: enqueuedState, completion };
211
1469
  }
212
1470
  }
213
1471
 
1472
+ export type QueueSendOptions = { delaySeconds?: number };
1473
+ export type QueueBindingLike<TMessage> = {
1474
+ send(message: TMessage, options?: QueueSendOptions): Promise<void>;
1475
+ };
1476
+
1477
+ export type JobPublisher<TMessage> = {
1478
+ put(message: TMessage, options?: QueueSendOptions): Promise<void>;
1479
+ };
1480
+
1481
+ export type JobPublisherOptions<TMessage> = {
1482
+ queue?: QueueBindingLike<TMessage>;
1483
+ inline?: boolean;
1484
+ handleInline?: (message: TMessage) => Promise<void>;
1485
+ onQueued?: (message: TMessage, options: QueueSendOptions & { mode: "inline" | "queue" }) => Promise<void> | void;
1486
+ };
1487
+
1488
+ export function createJobPublisher<TMessage>(options: JobPublisherOptions<TMessage>): JobPublisher<TMessage> {
1489
+ return {
1490
+ async put(message, sendOptions = {}) {
1491
+ if (options.inline) {
1492
+ if (!options.handleInline) throw new Error("Inline job publisher requires handleInline.");
1493
+ await options.onQueued?.(message, { ...sendOptions, mode: "inline" });
1494
+ await options.handleInline(message);
1495
+ return;
1496
+ }
1497
+ if (!options.queue) throw new Error("Job queue is not configured.");
1498
+ await options.onQueued?.(message, { ...sendOptions, mode: "queue" });
1499
+ await options.queue.send(message, sendOptions);
1500
+ },
1501
+ };
1502
+ }
1503
+
1504
+ export async function enqueueJobOperation<TPayload extends Record<string, unknown> = Record<string, unknown>>(
1505
+ jobs: Pick<DurableJobRepository, "enqueue">,
1506
+ input: EnqueueJobOperationInput<TPayload>,
1507
+ ): Promise<JobRecord> {
1508
+ const keys = jobOperationKeys(input);
1509
+ return jobs.enqueue({
1510
+ jobType: input.jobType,
1511
+ orgId: input.orgId ?? null,
1512
+ siteId: input.siteId ?? null,
1513
+ parentJobId: input.parentJobId ?? null,
1514
+ payload: jobOperationPayload(input),
1515
+ lockKey: keys.lockKey,
1516
+ concurrencyKey: keys.concurrencyKey,
1517
+ concurrencyLimit: keys.concurrencyLimit,
1518
+ enqueuePolicy: input.dedupeActive === false ? "enqueue" : "dedupe_active",
1519
+ maxAttempts: input.maxAttempts ?? undefined,
1520
+ availableAt: input.availableAt ?? null,
1521
+ });
1522
+ }
1523
+
1524
+ export function jobOperationKeys(input: {
1525
+ operationKey: string;
1526
+ siteId?: string | null;
1527
+ entity?: JobOperationEntity | null;
1528
+ dedupeActive?: boolean;
1529
+ concurrencyScope?: string | null;
1530
+ concurrencyLimit?: number | null;
1531
+ }): JobOperationKeys {
1532
+ const scope = keyPart(input.siteId ? `site:${input.siteId}` : "global");
1533
+ const operation = keyPart(input.operationKey);
1534
+ const entityKey = input.entity ? jobOperationEntityKey(input.entity) : null;
1535
+ const concurrencyScope = keyPart(input.concurrencyScope ?? input.operationKey);
1536
+ return {
1537
+ lockKey:
1538
+ input.dedupeActive === false
1539
+ ? null
1540
+ : ["jobop", scope, operation, entityKey ? keyPart(entityKey) : "root"].join(":"),
1541
+ concurrencyKey: ["jobop", scope, concurrencyScope].join(":"),
1542
+ concurrencyLimit: normalizeOperationConcurrencyLimit(input.concurrencyLimit),
1543
+ };
1544
+ }
1545
+
214
1546
  export const D1_JOBS_SCHEMA = `
215
1547
  CREATE TABLE IF NOT EXISTS jobs (
216
1548
  job_id TEXT PRIMARY KEY,
@@ -218,7 +1550,14 @@ CREATE TABLE IF NOT EXISTS jobs (
218
1550
  status TEXT NOT NULL,
219
1551
  payload_json TEXT NOT NULL DEFAULT '{}',
220
1552
  result_json TEXT,
1553
+ org_id TEXT,
1554
+ site_id TEXT,
1555
+ metadata_json TEXT NOT NULL DEFAULT '{}',
1556
+ parent_job_id TEXT,
221
1557
  lock_key TEXT,
1558
+ concurrency_key TEXT,
1559
+ concurrency_limit INTEGER,
1560
+ enqueue_policy TEXT NOT NULL DEFAULT 'enqueue',
222
1561
  attempt_count INTEGER NOT NULL DEFAULT 0,
223
1562
  max_attempts INTEGER NOT NULL DEFAULT 3,
224
1563
  available_at TEXT,
@@ -230,17 +1569,29 @@ CREATE TABLE IF NOT EXISTS jobs (
230
1569
  );
231
1570
  CREATE INDEX IF NOT EXISTS jobs_status_available_idx ON jobs(status, available_at, created_at);
232
1571
  CREATE INDEX IF NOT EXISTS jobs_type_status_idx ON jobs(job_type, status, created_at);
1572
+ CREATE INDEX IF NOT EXISTS jobs_org_status_idx ON jobs(org_id, status, created_at);
1573
+ CREATE INDEX IF NOT EXISTS jobs_site_status_idx ON jobs(site_id, status, created_at);
1574
+ CREATE INDEX IF NOT EXISTS jobs_parent_status_idx ON jobs(parent_job_id, status, created_at);
1575
+ CREATE INDEX IF NOT EXISTS jobs_concurrency_active_idx ON jobs(concurrency_key, status)
1576
+ WHERE concurrency_key IS NOT NULL AND status IN ('running', 'stopping');
233
1577
  CREATE UNIQUE INDEX IF NOT EXISTS jobs_active_lock_key_idx ON jobs(lock_key)
234
- WHERE lock_key IS NOT NULL AND status IN ('queued', 'running');
1578
+ WHERE lock_key IS NOT NULL AND status IN ('queued', 'running', 'stopping');
235
1579
  `;
236
1580
 
237
1581
  type JobRow = {
238
1582
  jobId: string;
239
1583
  jobType: string;
240
1584
  status: JobStatus;
1585
+ orgId?: string | null;
1586
+ siteId?: string | null;
241
1587
  payloadJson: string;
242
1588
  resultJson: string | null;
1589
+ metadataJson?: string | null;
1590
+ parentJobId?: string | null;
243
1591
  lockKey: string | null;
1592
+ concurrencyKey?: string | null;
1593
+ concurrencyLimit?: number | null;
1594
+ enqueuePolicy?: JobEnqueuePolicy | null;
244
1595
  attemptCount: number;
245
1596
  maxAttempts: number;
246
1597
  availableAt: string | null;
@@ -251,6 +1602,20 @@ type JobRow = {
251
1602
  updatedAt: string;
252
1603
  };
253
1604
 
1605
+ type JobTreeRow = JobRow & {
1606
+ depth: number | string;
1607
+ };
1608
+
1609
+ type JobActivityEventRow = {
1610
+ opsEventId: string;
1611
+ jobId: string | null;
1612
+ level: string;
1613
+ eventType: string;
1614
+ message: string;
1615
+ metadataJson: string | null;
1616
+ createdAt: string;
1617
+ };
1618
+
254
1619
  function rowToJob<TPayload = unknown, TResult = unknown>(row: JobRow): JobRecord<TPayload, TResult> {
255
1620
  return {
256
1621
  jobId: row.jobId,
@@ -258,7 +1623,14 @@ function rowToJob<TPayload = unknown, TResult = unknown>(row: JobRow): JobRecord
258
1623
  status: row.status,
259
1624
  payload: parseJsonColumn<TPayload>(row.payloadJson, {} as TPayload),
260
1625
  result: parseJsonColumn<TResult | null>(row.resultJson, null),
1626
+ orgId: row.orgId ?? null,
1627
+ siteId: row.siteId ?? null,
1628
+ parentJobId: row.parentJobId ?? null,
261
1629
  lockKey: row.lockKey,
1630
+ concurrencyKey: row.concurrencyKey ?? null,
1631
+ concurrencyLimit: row.concurrencyLimit === undefined || row.concurrencyLimit === null ? null : Number(row.concurrencyLimit),
1632
+ enqueuePolicy: row.enqueuePolicy ?? "enqueue",
1633
+ metadata: parseJsonColumn<Record<string, unknown>>(row.metadataJson, {}),
262
1634
  attemptCount: Number(row.attemptCount),
263
1635
  maxAttempts: Number(row.maxAttempts),
264
1636
  availableAt: row.availableAt,
@@ -274,3 +1646,225 @@ function normalizeJobResult<TResult>(output: JobHandlerResult<TResult> | TResult
274
1646
  if (output && typeof output === "object" && "kind" in output) return output as JobHandlerResult<TResult>;
275
1647
  return { kind: "done", result: (output ?? null) as TResult | null };
276
1648
  }
1649
+
1650
+ function quotePgIdent(value: string): string {
1651
+ if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) throw new Error(`Invalid Postgres identifier: ${value}`);
1652
+ return `"${value}"`;
1653
+ }
1654
+
1655
+ function jsonText(value: unknown): string | null {
1656
+ if (value === null || value === undefined) return null;
1657
+ return typeof value === "string" ? value : JSON.stringify(value);
1658
+ }
1659
+
1660
+ function jsonParam(value: unknown): string {
1661
+ return JSON.stringify(value ?? null);
1662
+ }
1663
+
1664
+ function queueNameFromPayload(value: unknown): string | null {
1665
+ if (!value || typeof value !== "object" || Array.isArray(value)) return null;
1666
+ return stringValue((value as Record<string, unknown>).queueName);
1667
+ }
1668
+
1669
+ function dateText(value: unknown): string | null {
1670
+ if (value === null || value === undefined) return null;
1671
+ if (value instanceof Date) return value.toISOString();
1672
+ if (typeof value === "string") return value;
1673
+ return String(value);
1674
+ }
1675
+
1676
+ function rowToActivityEvent(row: JobActivityEventRow): JobActivityEvent {
1677
+ const level = row.level === "warn" || row.level === "error" ? row.level : "info";
1678
+ return {
1679
+ opsEventId: row.opsEventId,
1680
+ jobId: row.jobId,
1681
+ level,
1682
+ eventType: row.eventType,
1683
+ message: row.message,
1684
+ metadata: parseJsonColumn<Record<string, unknown> | null>(row.metadataJson, null),
1685
+ createdAt: row.createdAt,
1686
+ };
1687
+ }
1688
+
1689
+ function summarizeJobs(parentJobId: string, jobs: readonly JobRecord[]): JobChildrenSummary {
1690
+ const summary: JobChildrenSummary = {
1691
+ parentJobId,
1692
+ total: jobs.length,
1693
+ queued: 0,
1694
+ running: 0,
1695
+ stopping: 0,
1696
+ succeeded: 0,
1697
+ failed: 0,
1698
+ stopped: 0,
1699
+ active: 0,
1700
+ terminal: 0,
1701
+ allTerminal: jobs.length > 0,
1702
+ };
1703
+ for (const job of jobs) {
1704
+ summary[job.status] += 1;
1705
+ if (isTerminalJobStatus(job.status)) summary.terminal += 1;
1706
+ else summary.active += 1;
1707
+ }
1708
+ summary.allTerminal = summary.total > 0 && summary.terminal === summary.total;
1709
+ return summary;
1710
+ }
1711
+
1712
+ function rollupJobTreeStatus(jobs: readonly JobRecord[]): JobTreeRollupStatus {
1713
+ if (jobs.some((job) => job.status === "failed")) return "failed";
1714
+ if (jobs.some((job) => job.status === "stopped")) return "stopped";
1715
+ if (jobs.some((job) => job.status === "running" || job.status === "stopping")) return "running";
1716
+ if (jobs.some((job) => job.status === "queued")) return "queued";
1717
+ return "succeeded";
1718
+ }
1719
+
1720
+ function latestJobError(jobs: readonly JobRecord[]): string | null {
1721
+ return (
1722
+ [...jobs]
1723
+ .sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))
1724
+ .find((job) => job.lastError && job.lastError.trim().length > 0)?.lastError ?? null
1725
+ );
1726
+ }
1727
+
1728
+ function jobEntityState(job: JobRecord, depth: number): JobEntityState {
1729
+ const payload = isRecord(job.payload) ? job.payload : {};
1730
+ const entityType = stringValue(payload.entityType);
1731
+ const entityId = stringValue(payload.entityId);
1732
+ const entityKey = stringValue(payload.entityKey) ?? (entityType && entityId ? `${entityType}:${entityId}` : job.jobId);
1733
+ return {
1734
+ jobId: job.jobId,
1735
+ jobType: job.jobType,
1736
+ status: job.status,
1737
+ entityKey,
1738
+ entityType,
1739
+ entityId,
1740
+ label: stringValue(payload.entityLabel),
1741
+ lastError: job.lastError,
1742
+ result: job.result,
1743
+ depth,
1744
+ };
1745
+ }
1746
+
1747
+ function emptyChildrenSummary(parentJobId: string): JobChildrenSummary {
1748
+ return summarizeJobs(parentJobId, []);
1749
+ }
1750
+
1751
+ function requeueParent<TPayload>(
1752
+ payload: TPayload,
1753
+ children: readonly JobRecord[],
1754
+ summary: JobChildrenSummary,
1755
+ input: ParentJobOrchestrationInput<TPayload>,
1756
+ ): JobHandlerResult {
1757
+ const pollSeconds = Math.max(1, Math.floor(input.pollSeconds ?? 10));
1758
+ const now = input.now?.() ?? new Date();
1759
+ return {
1760
+ kind: "requeue",
1761
+ payload,
1762
+ result: {
1763
+ stage: "waiting_children",
1764
+ failureMode: input.failureMode ?? "fail_fast",
1765
+ queuedSiblingPolicy: input.queuedSiblingPolicy ?? "fail",
1766
+ childJobIds: children.map((child) => child.jobId),
1767
+ summary,
1768
+ } satisfies ParentJobOrchestrationResult,
1769
+ availableAt: new Date(now.getTime() + pollSeconds * 1000).toISOString(),
1770
+ };
1771
+ }
1772
+
1773
+ function isTerminalJobStatus(status: JobStatus): status is TerminalJobStatus {
1774
+ return status === "succeeded" || status === "failed" || status === "stopped";
1775
+ }
1776
+
1777
+ function normalizeConcurrencyLimit(value: number | null | undefined): number | null {
1778
+ if (value === null || value === undefined) return null;
1779
+ return Math.max(1, Math.floor(value));
1780
+ }
1781
+
1782
+ function jobOperationPayload<TPayload extends Record<string, unknown>>(
1783
+ input: EnqueueJobOperationInput<TPayload>,
1784
+ ): Record<string, unknown> {
1785
+ return {
1786
+ ...(input.payload ?? {}),
1787
+ operationKey: input.operationKey,
1788
+ ...(input.entity
1789
+ ? {
1790
+ entityType: input.entity.entityType,
1791
+ entityId: input.entity.entityId,
1792
+ entityKey: jobOperationEntityKey(input.entity),
1793
+ ...(input.entity.entityLabel ? { entityLabel: input.entity.entityLabel } : {}),
1794
+ }
1795
+ : {}),
1796
+ };
1797
+ }
1798
+
1799
+ function jobOperationEntityKey(entity: JobOperationEntity): string {
1800
+ return entity.entityKey?.trim() || `${entity.entityType}:${entity.entityId}`;
1801
+ }
1802
+
1803
+ function keyPart(value: string): string {
1804
+ return value
1805
+ .trim()
1806
+ .toLowerCase()
1807
+ .replace(/[^a-z0-9:_-]+/g, "-")
1808
+ .replace(/-+/g, "-")
1809
+ .replace(/^-|-$/g, "");
1810
+ }
1811
+
1812
+ function normalizeOperationConcurrencyLimit(value: number | null | undefined): number {
1813
+ if (!Number.isFinite(value) || !value) return 1;
1814
+ return Math.max(1, Math.floor(value));
1815
+ }
1816
+
1817
+ function isRecord(value: unknown): value is Record<string, unknown> {
1818
+ return !!value && typeof value === "object" && !Array.isArray(value);
1819
+ }
1820
+
1821
+ function stringValue(value: unknown): string | null {
1822
+ return typeof value === "string" && value.trim().length > 0 ? value : null;
1823
+ }
1824
+
1825
+ function isRequeueInput(value: unknown): value is {
1826
+ payload?: unknown;
1827
+ result?: unknown;
1828
+ availableAt?: string | null;
1829
+ lastError?: string | null;
1830
+ } {
1831
+ return (
1832
+ !!value &&
1833
+ typeof value === "object" &&
1834
+ ("payload" in value || "result" in value || "availableAt" in value || "lastError" in value)
1835
+ );
1836
+ }
1837
+
1838
+ function isStaleRunningJob(job: JobRecord, staleRunningAfterSeconds: number | undefined): boolean {
1839
+ if (!staleRunningAfterSeconds || staleRunningAfterSeconds <= 0 || !job.startedAt) return false;
1840
+ const startedAt = Date.parse(job.startedAt);
1841
+ if (!Number.isFinite(startedAt)) return false;
1842
+ return Date.now() - startedAt >= staleRunningAfterSeconds * 1000;
1843
+ }
1844
+
1845
+ function errorMessage(error: unknown): string {
1846
+ return error instanceof Error ? error.message : String(error);
1847
+ }
1848
+
1849
+ class DurableJobEvents<TPayload = unknown, TResult = unknown> {
1850
+ constructor(
1851
+ private readonly job: JobRecord<TPayload, TResult>,
1852
+ private readonly sink: DurableJobEventSink<TPayload, TResult> | undefined,
1853
+ ) {}
1854
+
1855
+ async info(input: Omit<DurableJobEventInput, "severity">): Promise<void> {
1856
+ await this.emit({ ...input, severity: "info" });
1857
+ }
1858
+
1859
+ async warn(input: Omit<DurableJobEventInput, "severity">): Promise<void> {
1860
+ await this.emit({ ...input, severity: "warn" });
1861
+ }
1862
+
1863
+ async error(input: Omit<DurableJobEventInput, "severity">): Promise<void> {
1864
+ await this.emit({ ...input, severity: "error" });
1865
+ }
1866
+
1867
+ private async emit(input: DurableJobEventInput): Promise<void> {
1868
+ await this.sink?.(this.job, input);
1869
+ }
1870
+ }