@q32/core 0.1.20 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -2
- package/dist/auth.d.ts +98 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +230 -0
- package/dist/auth.js.map +1 -0
- package/dist/hono.d.ts +22 -0
- package/dist/hono.d.ts.map +1 -0
- package/dist/hono.js +26 -0
- package/dist/hono.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/jobs.d.ts +394 -6
- package/dist/jobs.d.ts.map +1 -1
- package/dist/jobs.js +1134 -62
- package/dist/jobs.js.map +1 -1
- package/dist/react-router.d.ts +14 -0
- package/dist/react-router.d.ts.map +1 -0
- package/dist/react-router.js +30 -0
- package/dist/react-router.js.map +1 -0
- package/docs/evaluation.md +5 -4
- package/docs/jobs.md +49 -0
- package/package.json +13 -1
- package/src/auth.ts +338 -0
- package/src/hono.ts +54 -0
- package/src/index.ts +3 -0
- package/src/jobs.ts +1665 -71
- package/src/react-router.ts +53 -0
package/src/jobs.ts
CHANGED
|
@@ -3,11 +3,14 @@ import { parseJsonColumn, stringifyJsonColumn } from "./d1.js";
|
|
|
3
3
|
import { createId } from "./ids.js";
|
|
4
4
|
import { isFutureIso, nowIso } from "./time.js";
|
|
5
5
|
|
|
6
|
-
export type JobStatus = "queued" | "running" | "succeeded" | "failed";
|
|
6
|
+
export type JobStatus = "queued" | "running" | "succeeded" | "failed" | "stopping" | "stopped";
|
|
7
|
+
export type TerminalJobStatus = "succeeded" | "failed" | "stopped";
|
|
8
|
+
export type JobEnqueuePolicy = "enqueue" | "dedupe_active";
|
|
7
9
|
|
|
8
|
-
export type JobHandlerResult<TResult = unknown> =
|
|
10
|
+
export type JobHandlerResult<TResult = unknown, TPayload = unknown> =
|
|
9
11
|
| { kind: "done"; result?: TResult | null }
|
|
10
|
-
| { kind: "requeue"; result?: TResult | null; availableAt?: string | null }
|
|
12
|
+
| { kind: "requeue"; payload?: TPayload; result?: TResult | null; availableAt?: string | null }
|
|
13
|
+
| { kind: "stopped"; result?: TResult | null };
|
|
11
14
|
|
|
12
15
|
export type JobRecord<TPayload = unknown, TResult = unknown> = {
|
|
13
16
|
jobId: string;
|
|
@@ -15,7 +18,14 @@ export type JobRecord<TPayload = unknown, TResult = unknown> = {
|
|
|
15
18
|
status: JobStatus;
|
|
16
19
|
payload: TPayload;
|
|
17
20
|
result: TResult | null;
|
|
21
|
+
orgId: string | null;
|
|
22
|
+
siteId: string | null;
|
|
23
|
+
parentJobId: string | null;
|
|
18
24
|
lockKey: string | null;
|
|
25
|
+
concurrencyKey: string | null;
|
|
26
|
+
concurrencyLimit: number | null;
|
|
27
|
+
enqueuePolicy: JobEnqueuePolicy;
|
|
28
|
+
metadata: Record<string, unknown>;
|
|
19
29
|
attemptCount: number;
|
|
20
30
|
maxAttempts: number;
|
|
21
31
|
availableAt: string | null;
|
|
@@ -29,15 +39,350 @@ export type JobRecord<TPayload = unknown, TResult = unknown> = {
|
|
|
29
39
|
export type EnqueueJobInput<TPayload = unknown> = {
|
|
30
40
|
jobType: string;
|
|
31
41
|
payload?: TPayload;
|
|
42
|
+
orgId?: string | null;
|
|
43
|
+
siteId?: string | null;
|
|
44
|
+
parentJobId?: string | null;
|
|
32
45
|
lockKey?: string | null;
|
|
46
|
+
concurrencyKey?: string | null;
|
|
47
|
+
concurrencyLimit?: number | null;
|
|
48
|
+
enqueuePolicy?: JobEnqueuePolicy;
|
|
49
|
+
metadata?: Record<string, unknown> | null;
|
|
33
50
|
maxAttempts?: number;
|
|
34
51
|
availableAt?: string | null;
|
|
35
52
|
};
|
|
36
53
|
|
|
37
|
-
export
|
|
54
|
+
export type DurableJobEventInput = {
|
|
55
|
+
eventName: string;
|
|
56
|
+
status?: "started" | "ok" | "warning" | "error" | "skipped" | string;
|
|
57
|
+
severity?: "debug" | "info" | "warn" | "error";
|
|
58
|
+
message?: string | null;
|
|
59
|
+
error?: unknown;
|
|
60
|
+
metrics?: Record<string, unknown> | null;
|
|
61
|
+
metadata?: Record<string, unknown> | null;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
export type DurableJobEventSink<TPayload = unknown, TResult = unknown> = (
|
|
65
|
+
job: JobRecord<TPayload, TResult>,
|
|
66
|
+
event: DurableJobEventInput,
|
|
67
|
+
) => Promise<void> | void;
|
|
68
|
+
|
|
69
|
+
export interface DurableJobRepository {
|
|
70
|
+
enqueue<TPayload = unknown>(input: EnqueueJobInput<TPayload>): Promise<JobRecord<TPayload>>;
|
|
71
|
+
get<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null>;
|
|
72
|
+
claim<TPayload = unknown>(jobId: string): Promise<JobRecord<TPayload> | null>;
|
|
73
|
+
succeed<TResult = unknown>(jobId: string, result?: TResult | null): Promise<void>;
|
|
74
|
+
requeue<TPayload = unknown, TResult = unknown>(
|
|
75
|
+
jobId: string,
|
|
76
|
+
input?: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null },
|
|
77
|
+
): Promise<void>;
|
|
78
|
+
fail(jobId: string, error: unknown): Promise<void>;
|
|
79
|
+
stop<TResult = unknown>(jobId: string, result?: TResult | null): Promise<void>;
|
|
80
|
+
requestStop(jobId: string): Promise<void>;
|
|
81
|
+
listChildren(parentJobId: string): Promise<JobRecord[]>;
|
|
82
|
+
summarizeChildren(parentJobId: string): Promise<JobChildrenSummary>;
|
|
83
|
+
markQueuedChildrenTerminal(parentJobId: string, status: "failed" | "stopped", error?: unknown): Promise<number>;
|
|
84
|
+
requestStopActiveChildren(parentJobId: string): Promise<number>;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export type JobStatusEventInput<TPayload = unknown, TResult = unknown> = {
|
|
88
|
+
job: JobRecord<TPayload, TResult>;
|
|
89
|
+
status: JobStatus;
|
|
90
|
+
message: string;
|
|
91
|
+
error?: unknown;
|
|
92
|
+
metadata?: Record<string, unknown> | null;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
export type JobStatusEventSink = (event: JobStatusEventInput) => Promise<void> | void;
|
|
96
|
+
|
|
97
|
+
export type JobChildrenSummary = {
|
|
98
|
+
parentJobId: string;
|
|
99
|
+
total: number;
|
|
100
|
+
queued: number;
|
|
101
|
+
running: number;
|
|
102
|
+
stopping: number;
|
|
103
|
+
succeeded: number;
|
|
104
|
+
failed: number;
|
|
105
|
+
stopped: number;
|
|
106
|
+
active: number;
|
|
107
|
+
terminal: number;
|
|
108
|
+
allTerminal: boolean;
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
export type JobTreeRollupStatus = "queued" | "running" | "succeeded" | "failed" | "stopped";
|
|
112
|
+
|
|
113
|
+
export type JobEntityState = {
|
|
114
|
+
jobId: string;
|
|
115
|
+
jobType: string;
|
|
116
|
+
status: JobStatus;
|
|
117
|
+
entityKey: string;
|
|
118
|
+
entityType: string | null;
|
|
119
|
+
entityId: string | null;
|
|
120
|
+
label: string | null;
|
|
121
|
+
lastError: string | null;
|
|
122
|
+
result: unknown;
|
|
123
|
+
depth: number;
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
export type JobActivityEvent = {
|
|
127
|
+
opsEventId: string;
|
|
128
|
+
jobId: string | null;
|
|
129
|
+
level: "info" | "warn" | "error";
|
|
130
|
+
eventType: string;
|
|
131
|
+
message: string;
|
|
132
|
+
metadata: Record<string, unknown> | null;
|
|
133
|
+
createdAt: string;
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
export type JobTreeRecord<TPayload = unknown, TResult = unknown> = {
|
|
137
|
+
root: JobRecord<TPayload, TResult>;
|
|
138
|
+
children: Array<JobRecord & { depth: number }>;
|
|
139
|
+
summary: JobChildrenSummary;
|
|
140
|
+
rollupStatus: JobTreeRollupStatus;
|
|
141
|
+
rowStates: JobEntityState[];
|
|
142
|
+
activityEvents: JobActivityEvent[];
|
|
143
|
+
latestError: string | null;
|
|
144
|
+
};
|
|
145
|
+
|
|
146
|
+
export type DurableJobHandlerContext<TPayload = unknown, TResult = unknown, TServices = unknown> = {
|
|
147
|
+
job: JobRecord<TPayload, TResult>;
|
|
148
|
+
jobs: DurableJobRepository;
|
|
149
|
+
services: TServices;
|
|
150
|
+
events: {
|
|
151
|
+
info(input: Omit<DurableJobEventInput, "severity">): Promise<void>;
|
|
152
|
+
warn(input: Omit<DurableJobEventInput, "severity">): Promise<void>;
|
|
153
|
+
error(input: Omit<DurableJobEventInput, "severity">): Promise<void>;
|
|
154
|
+
};
|
|
155
|
+
shouldStop(): Promise<boolean>;
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
export class JobDeadline {
|
|
159
|
+
private readonly startedAt = Date.now();
|
|
160
|
+
|
|
161
|
+
constructor(private readonly budgetMs = 50_000) {}
|
|
162
|
+
|
|
163
|
+
shouldYield(): boolean {
|
|
164
|
+
return Date.now() - this.startedAt >= this.budgetMs;
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
export type JobOperationEntity = {
|
|
169
|
+
entityType: string;
|
|
170
|
+
entityId: string;
|
|
171
|
+
entityLabel?: string | null;
|
|
172
|
+
entityKey?: string | null;
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
export type EnqueueJobOperationInput<TPayload extends Record<string, unknown> = Record<string, unknown>> = {
|
|
176
|
+
jobType: string;
|
|
177
|
+
operationKey: string;
|
|
178
|
+
orgId?: string | null;
|
|
179
|
+
siteId?: string | null;
|
|
180
|
+
parentJobId?: string | null;
|
|
181
|
+
entity?: JobOperationEntity | null;
|
|
182
|
+
payload?: TPayload | null;
|
|
183
|
+
dedupeActive?: boolean;
|
|
184
|
+
concurrencyScope?: string | null;
|
|
185
|
+
concurrencyLimit?: number | null;
|
|
186
|
+
maxAttempts?: number | null;
|
|
187
|
+
availableAt?: string | null;
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
export type JobOperationKeys = {
|
|
191
|
+
lockKey: string | null;
|
|
192
|
+
concurrencyKey: string;
|
|
193
|
+
concurrencyLimit: number;
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
export type DurableJobHandler<TPayload = unknown, TResult = unknown, TServices = unknown> = (
|
|
197
|
+
context: DurableJobHandlerContext<TPayload, TResult, TServices>,
|
|
198
|
+
) => Promise<JobHandlerResult<TResult, TPayload> | TResult | void>;
|
|
199
|
+
|
|
200
|
+
export type DurableJobRegistry<TServices = unknown> = Record<string, DurableJobHandler<unknown, unknown, TServices>>;
|
|
201
|
+
|
|
202
|
+
export type DurableJobDispatcherOptions<TServices = unknown> = {
|
|
203
|
+
jobs: DurableJobRepository;
|
|
204
|
+
handlers: DurableJobRegistry<TServices>;
|
|
205
|
+
services: TServices;
|
|
206
|
+
events?: DurableJobEventSink;
|
|
207
|
+
retryDelay?: (input: { job: JobRecord; error: unknown }) => string | null | undefined;
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
export class DurableJobDispatcher<TServices = unknown> {
|
|
211
|
+
constructor(private readonly options: DurableJobDispatcherOptions<TServices>) {}
|
|
212
|
+
|
|
213
|
+
async run<TResult = unknown>(jobId: string): Promise<JobHandlerResult<TResult>> {
|
|
214
|
+
const job = await this.options.jobs.claim(jobId);
|
|
215
|
+
if (!job) return { kind: "done", result: null };
|
|
216
|
+
|
|
217
|
+
const handler = this.options.handlers[job.jobType];
|
|
218
|
+
if (!handler) {
|
|
219
|
+
await this.options.jobs.fail(job.jobId, `No handler registered for job type: ${job.jobType}`);
|
|
220
|
+
throw new Error(`No handler registered for job type: ${job.jobType}`);
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const events = new DurableJobEvents(job, this.options.events);
|
|
224
|
+
await events.info({ eventName: "job.started", status: "started", message: "Job started." });
|
|
225
|
+
|
|
226
|
+
try {
|
|
227
|
+
if (await this.shouldStop(job.jobId)) {
|
|
228
|
+
await this.options.jobs.stop(job.jobId);
|
|
229
|
+
await events.warn({ eventName: "job.stopped", status: "skipped", message: "Job stopped before handler ran." });
|
|
230
|
+
return { kind: "stopped", result: null };
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
const output = await handler({
|
|
234
|
+
job,
|
|
235
|
+
jobs: this.options.jobs,
|
|
236
|
+
services: this.options.services,
|
|
237
|
+
events,
|
|
238
|
+
shouldStop: () => this.shouldStop(job.jobId),
|
|
239
|
+
});
|
|
240
|
+
const normalized = normalizeJobResult<TResult>(output as JobHandlerResult<TResult> | TResult | void);
|
|
241
|
+
if (normalized.kind === "requeue") {
|
|
242
|
+
await this.options.jobs.requeue(job.jobId, {
|
|
243
|
+
payload: normalized.payload,
|
|
244
|
+
result: normalized.result ?? null,
|
|
245
|
+
availableAt: normalized.availableAt ?? null,
|
|
246
|
+
});
|
|
247
|
+
await events.info({ eventName: "job.requeued", status: "started", message: "Job requeued." });
|
|
248
|
+
} else if (normalized.kind === "stopped") {
|
|
249
|
+
await this.options.jobs.stop(job.jobId, normalized.result ?? null);
|
|
250
|
+
await events.warn({ eventName: "job.stopped", status: "skipped", message: "Job stopped." });
|
|
251
|
+
} else {
|
|
252
|
+
await this.options.jobs.succeed(job.jobId, normalized.result ?? null);
|
|
253
|
+
await events.info({ eventName: "job.succeeded", status: "ok", message: "Job succeeded." });
|
|
254
|
+
}
|
|
255
|
+
return normalized;
|
|
256
|
+
} catch (error) {
|
|
257
|
+
const latest = await this.options.jobs.get(job.jobId);
|
|
258
|
+
if (latest && latest.attemptCount < latest.maxAttempts) {
|
|
259
|
+
await this.options.jobs.requeue(job.jobId, {
|
|
260
|
+
result: { retryError: errorMessage(error) },
|
|
261
|
+
availableAt: this.options.retryDelay?.({ job: latest, error }) ?? null,
|
|
262
|
+
lastError: errorMessage(error),
|
|
263
|
+
});
|
|
264
|
+
await events.warn({ eventName: "job.retrying", status: "warning", message: "Job failed and will retry.", error });
|
|
265
|
+
return { kind: "requeue", result: null };
|
|
266
|
+
}
|
|
267
|
+
await this.options.jobs.fail(job.jobId, error);
|
|
268
|
+
await events.error({ eventName: "job.failed", status: "error", message: "Job failed.", error });
|
|
269
|
+
throw error;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
private async shouldStop(jobId: string): Promise<boolean> {
|
|
274
|
+
const latest = await this.options.jobs.get(jobId);
|
|
275
|
+
return latest?.status === "stopping" || latest?.status === "stopped";
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
export type ParentJobOrchestrationInput<TPayload = unknown> = {
|
|
280
|
+
children: EnqueueJobInput<TPayload>[];
|
|
281
|
+
failureMode?: "fail_fast" | "continue";
|
|
282
|
+
queuedSiblingPolicy?: "fail" | "stop";
|
|
283
|
+
pollSeconds?: number;
|
|
284
|
+
now?: () => Date;
|
|
285
|
+
onChildQueued?: (job: JobRecord<TPayload>) => Promise<void> | void;
|
|
286
|
+
};
|
|
287
|
+
|
|
288
|
+
export type ParentJobOrchestrationResult = {
|
|
289
|
+
stage: "waiting_children" | "complete";
|
|
290
|
+
failureMode: "fail_fast" | "continue";
|
|
291
|
+
queuedSiblingPolicy: "fail" | "stop";
|
|
292
|
+
childJobIds: string[];
|
|
293
|
+
summary: JobChildrenSummary;
|
|
294
|
+
};
|
|
295
|
+
|
|
296
|
+
export async function runParentJobOrchestration<TPayload = unknown, TResult = unknown, TServices = unknown>(
|
|
297
|
+
context: DurableJobHandlerContext<TPayload, TResult, TServices>,
|
|
298
|
+
input: ParentJobOrchestrationInput<TPayload>,
|
|
299
|
+
): Promise<JobHandlerResult> {
|
|
300
|
+
const existing = await context.jobs.listChildren(context.job.jobId);
|
|
301
|
+
const failureMode = input.failureMode ?? "fail_fast";
|
|
302
|
+
const queuedSiblingPolicy = input.queuedSiblingPolicy ?? "fail";
|
|
303
|
+
if (existing.length === 0) {
|
|
304
|
+
if (input.children.length === 0) {
|
|
305
|
+
return {
|
|
306
|
+
kind: "done",
|
|
307
|
+
result: {
|
|
308
|
+
stage: "complete",
|
|
309
|
+
failureMode,
|
|
310
|
+
queuedSiblingPolicy,
|
|
311
|
+
childJobIds: [],
|
|
312
|
+
summary: emptyChildrenSummary(context.job.jobId),
|
|
313
|
+
} satisfies ParentJobOrchestrationResult,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
const children: JobRecord[] = [];
|
|
317
|
+
for (const child of input.children) {
|
|
318
|
+
const queued = await context.jobs.enqueue({
|
|
319
|
+
...child,
|
|
320
|
+
parentJobId: child.parentJobId ?? context.job.jobId,
|
|
321
|
+
});
|
|
322
|
+
children.push(queued);
|
|
323
|
+
await input.onChildQueued?.(queued as JobRecord<TPayload>);
|
|
324
|
+
}
|
|
325
|
+
await context.events.info({
|
|
326
|
+
eventName: "job.children_queued",
|
|
327
|
+
status: "started",
|
|
328
|
+
message: "Child jobs queued.",
|
|
329
|
+
metrics: { queued: input.children.length },
|
|
330
|
+
});
|
|
331
|
+
return requeueParent<TPayload>(context.job.payload, children, summarizeJobs(context.job.jobId, children), input);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const summary = await context.jobs.summarizeChildren(context.job.jobId);
|
|
335
|
+
if (failureMode === "fail_fast" && summary.failed > 0) {
|
|
336
|
+
const failedReason =
|
|
337
|
+
(await context.jobs.listChildren(context.job.jobId)).find((child) => child.status === "failed" && child.lastError)
|
|
338
|
+
?.lastError ?? `Child job failed for parent ${context.job.jobId}.`;
|
|
339
|
+
if (queuedSiblingPolicy === "stop") {
|
|
340
|
+
await context.jobs.markQueuedChildrenTerminal(context.job.jobId, "stopped", failedReason);
|
|
341
|
+
} else {
|
|
342
|
+
await context.jobs.markQueuedChildrenTerminal(context.job.jobId, "failed", failedReason);
|
|
343
|
+
}
|
|
344
|
+
await context.jobs.requestStopActiveChildren(context.job.jobId);
|
|
345
|
+
await context.events.warn({
|
|
346
|
+
eventName: "job.children_failed",
|
|
347
|
+
status: "warning",
|
|
348
|
+
message: "One or more child jobs failed.",
|
|
349
|
+
metrics: summary,
|
|
350
|
+
});
|
|
351
|
+
const updated = await context.jobs.summarizeChildren(context.job.jobId);
|
|
352
|
+
if (!updated.allTerminal) {
|
|
353
|
+
const children = await context.jobs.listChildren(context.job.jobId);
|
|
354
|
+
return requeueParent<TPayload>(context.job.payload, children, updated, input);
|
|
355
|
+
}
|
|
356
|
+
throw new Error(failedReason);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
const children = await context.jobs.listChildren(context.job.jobId);
|
|
360
|
+
if (!summary.allTerminal) return requeueParent<TPayload>(context.job.payload, children, summary, input);
|
|
361
|
+
return {
|
|
362
|
+
kind: "done",
|
|
363
|
+
result: {
|
|
364
|
+
stage: "complete",
|
|
365
|
+
failureMode,
|
|
366
|
+
queuedSiblingPolicy,
|
|
367
|
+
childJobIds: children.map((child) => child.jobId),
|
|
368
|
+
summary,
|
|
369
|
+
} satisfies ParentJobOrchestrationResult,
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
export class D1JobStore implements DurableJobRepository {
|
|
38
374
|
constructor(
|
|
39
375
|
private readonly db: D1DatabaseLike,
|
|
40
|
-
private readonly options: {
|
|
376
|
+
private readonly options: {
|
|
377
|
+
tableName?: string;
|
|
378
|
+
idPrefix?: string;
|
|
379
|
+
staleRunningAfterSeconds?: number;
|
|
380
|
+
opsEventsTableName?: string;
|
|
381
|
+
defaultMaxAttempts?: number;
|
|
382
|
+
onStatusEvent?: JobStatusEventSink;
|
|
383
|
+
queuedStopStatus?: "stopping" | "stopped";
|
|
384
|
+
requestStopQueuedChildren?: boolean;
|
|
385
|
+
} = {},
|
|
41
386
|
) {}
|
|
42
387
|
|
|
43
388
|
get tableName(): string {
|
|
@@ -47,19 +392,33 @@ export class D1JobStore {
|
|
|
47
392
|
async enqueue<TPayload = unknown>(input: EnqueueJobInput<TPayload>): Promise<JobRecord<TPayload>> {
|
|
48
393
|
const jobId = createId(this.options.idPrefix ?? "job");
|
|
49
394
|
const now = nowIso();
|
|
395
|
+
const enqueuePolicy = input.enqueuePolicy ?? "enqueue";
|
|
396
|
+
if (enqueuePolicy === "dedupe_active" && input.lockKey) {
|
|
397
|
+
const existing = await this.getActiveForLockKey<TPayload>(input.lockKey);
|
|
398
|
+
if (existing) return existing;
|
|
399
|
+
}
|
|
400
|
+
|
|
50
401
|
await this.db
|
|
51
402
|
.prepare(
|
|
52
403
|
`INSERT INTO ${this.tableName} (
|
|
53
|
-
job_id, job_type, status, payload_json, result_json,
|
|
404
|
+
job_id, job_type, status, org_id, site_id, payload_json, result_json, metadata_json,
|
|
405
|
+
parent_job_id, lock_key, concurrency_key, concurrency_limit, enqueue_policy,
|
|
54
406
|
attempt_count, max_attempts, available_at, created_at, updated_at
|
|
55
|
-
) VALUES (?, ?, 'queued', ?, NULL, ?, 0, ?, ?, ?, ?)`,
|
|
407
|
+
) VALUES (?, ?, 'queued', ?, ?, ?, NULL, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)`,
|
|
56
408
|
)
|
|
57
409
|
.bind(
|
|
58
410
|
jobId,
|
|
59
411
|
input.jobType,
|
|
412
|
+
input.orgId ?? null,
|
|
413
|
+
input.siteId ?? null,
|
|
60
414
|
stringifyJsonColumn(input.payload ?? {}),
|
|
415
|
+
stringifyJsonColumn(input.metadata ?? {}),
|
|
416
|
+
input.parentJobId ?? null,
|
|
61
417
|
input.lockKey ?? null,
|
|
62
|
-
|
|
418
|
+
input.concurrencyKey ?? null,
|
|
419
|
+
normalizeConcurrencyLimit(input.concurrencyLimit),
|
|
420
|
+
enqueuePolicy,
|
|
421
|
+
Math.max(1, Math.floor(input.maxAttempts ?? this.options.defaultMaxAttempts ?? 3)),
|
|
63
422
|
input.availableAt ?? null,
|
|
64
423
|
now,
|
|
65
424
|
now,
|
|
@@ -68,46 +427,43 @@ export class D1JobStore {
|
|
|
68
427
|
|
|
69
428
|
const job = await this.get<TPayload>(jobId);
|
|
70
429
|
if (!job) throw new Error(`Failed to load enqueued job: ${jobId}`);
|
|
430
|
+
await this.emitStatusEvent(job, "queued", "Job queued.");
|
|
71
431
|
return job;
|
|
72
432
|
}
|
|
73
433
|
|
|
74
434
|
async get<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null> {
|
|
75
435
|
const row = await this.db
|
|
76
|
-
.prepare(
|
|
77
|
-
`SELECT
|
|
78
|
-
job_id AS jobId,
|
|
79
|
-
job_type AS jobType,
|
|
80
|
-
status,
|
|
81
|
-
payload_json AS payloadJson,
|
|
82
|
-
result_json AS resultJson,
|
|
83
|
-
lock_key AS lockKey,
|
|
84
|
-
attempt_count AS attemptCount,
|
|
85
|
-
max_attempts AS maxAttempts,
|
|
86
|
-
available_at AS availableAt,
|
|
87
|
-
started_at AS startedAt,
|
|
88
|
-
completed_at AS completedAt,
|
|
89
|
-
last_error AS lastError,
|
|
90
|
-
created_at AS createdAt,
|
|
91
|
-
updated_at AS updatedAt
|
|
92
|
-
FROM ${this.tableName}
|
|
93
|
-
WHERE job_id = ?
|
|
94
|
-
LIMIT 1`,
|
|
95
|
-
)
|
|
436
|
+
.prepare(`${this.selectSql()} WHERE job_id = ? LIMIT 1`)
|
|
96
437
|
.bind(jobId)
|
|
97
438
|
.first<JobRow>();
|
|
98
439
|
return row ? rowToJob<TPayload, TResult>(row) : null;
|
|
99
440
|
}
|
|
100
441
|
|
|
442
|
+
async getJob<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null> {
|
|
443
|
+
return this.get(jobId);
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
async enqueueChild<TPayload = unknown>(
|
|
447
|
+
parentJobId: string,
|
|
448
|
+
input: Omit<EnqueueJobInput<TPayload>, "parentJobId">,
|
|
449
|
+
): Promise<JobRecord<TPayload>> {
|
|
450
|
+
return this.enqueue({ ...input, parentJobId });
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
async enqueueChildren<TPayload = unknown>(
|
|
454
|
+
parentJobId: string,
|
|
455
|
+
inputs: Array<Omit<EnqueueJobInput<TPayload>, "parentJobId">>,
|
|
456
|
+
): Promise<Array<JobRecord<TPayload>>> {
|
|
457
|
+
const children: Array<JobRecord<TPayload>> = [];
|
|
458
|
+
for (const input of inputs) children.push(await this.enqueueChild(parentJobId, input));
|
|
459
|
+
return children;
|
|
460
|
+
}
|
|
461
|
+
|
|
101
462
|
async listQueued(limit = 25): Promise<JobRecord[]> {
|
|
102
463
|
const now = nowIso();
|
|
103
464
|
const rows = await this.db
|
|
104
465
|
.prepare(
|
|
105
|
-
|
|
106
|
-
job_id AS jobId, job_type AS jobType, status, payload_json AS payloadJson,
|
|
107
|
-
result_json AS resultJson, lock_key AS lockKey, attempt_count AS attemptCount,
|
|
108
|
-
max_attempts AS maxAttempts, available_at AS availableAt, started_at AS startedAt,
|
|
109
|
-
completed_at AS completedAt, last_error AS lastError, created_at AS createdAt, updated_at AS updatedAt
|
|
110
|
-
FROM ${this.tableName}
|
|
466
|
+
`${this.selectSql()}
|
|
111
467
|
WHERE status = 'queued' AND (available_at IS NULL OR available_at <= ?)
|
|
112
468
|
ORDER BY created_at ASC
|
|
113
469
|
LIMIT ?`,
|
|
@@ -117,14 +473,70 @@ export class D1JobStore {
|
|
|
117
473
|
return (rows.results ?? []).map(rowToJob);
|
|
118
474
|
}
|
|
119
475
|
|
|
476
|
+
async getLatestForSite(input: { siteId: string; jobType?: string | null }): Promise<JobRecord | null> {
|
|
477
|
+
const row = await this.db
|
|
478
|
+
.prepare(
|
|
479
|
+
`${this.selectSql()}
|
|
480
|
+
WHERE site_id = ?
|
|
481
|
+
${input.jobType ? "AND job_type = ?" : ""}
|
|
482
|
+
ORDER BY created_at DESC
|
|
483
|
+
LIMIT 1`,
|
|
484
|
+
)
|
|
485
|
+
.bind(...([input.siteId, ...(input.jobType ? [input.jobType] : [])] as never[]))
|
|
486
|
+
.first<JobRow>();
|
|
487
|
+
return row ? rowToJob(row) : null;
|
|
488
|
+
}
|
|
489
|
+
|
|
490
|
+
async getActiveForSite(input: { siteId: string; jobType?: string | null }): Promise<JobRecord | null> {
|
|
491
|
+
const row = await this.db
|
|
492
|
+
.prepare(
|
|
493
|
+
`${this.selectSql()}
|
|
494
|
+
WHERE site_id = ?
|
|
495
|
+
AND status IN ('queued', 'running', 'stopping')
|
|
496
|
+
${input.jobType ? "AND job_type = ?" : ""}
|
|
497
|
+
ORDER BY created_at DESC
|
|
498
|
+
LIMIT 1`,
|
|
499
|
+
)
|
|
500
|
+
.bind(...([input.siteId, ...(input.jobType ? [input.jobType] : [])] as never[]))
|
|
501
|
+
.first<JobRow>();
|
|
502
|
+
return row ? rowToJob(row) : null;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
async getLatestForSiteByPayload(input: {
|
|
506
|
+
siteId: string;
|
|
507
|
+
jobType?: string | null;
|
|
508
|
+
key: string;
|
|
509
|
+
value: string | number | boolean | null;
|
|
510
|
+
}): Promise<JobRecord | null> {
|
|
511
|
+
return this.getForSiteByPayload({ ...input, activeOnly: false });
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
async getActiveForSiteByPayload(input: {
|
|
515
|
+
siteId: string;
|
|
516
|
+
jobType?: string | null;
|
|
517
|
+
key: string;
|
|
518
|
+
value: string | number | boolean | null;
|
|
519
|
+
}): Promise<JobRecord | null> {
|
|
520
|
+
return this.getForSiteByPayload({ ...input, activeOnly: true });
|
|
521
|
+
}
|
|
522
|
+
|
|
120
523
|
async claim<TPayload = unknown>(jobId: string): Promise<JobRecord<TPayload> | null> {
|
|
121
524
|
const current = await this.get<TPayload>(jobId);
|
|
122
|
-
if (!current || current.status
|
|
525
|
+
if (!current || isTerminalJobStatus(current.status) || current.status === "stopping") return null;
|
|
526
|
+
if (current.status === "running") {
|
|
527
|
+
if (!isStaleRunningJob(current, this.options.staleRunningAfterSeconds)) return null;
|
|
528
|
+
await this.markStaleRunningQueued(jobId);
|
|
529
|
+
}
|
|
123
530
|
if (current.status === "queued" && isFutureIso(current.availableAt)) return null;
|
|
124
531
|
if (current.attemptCount >= current.maxAttempts) {
|
|
125
532
|
await this.fail(jobId, "Job exhausted all attempts.");
|
|
126
533
|
return null;
|
|
127
534
|
}
|
|
535
|
+
if (current.concurrencyKey && current.status === "queued") {
|
|
536
|
+
const limit = current.concurrencyLimit ?? 1;
|
|
537
|
+
const running = await this.countRunningForConcurrencyKey(current.concurrencyKey);
|
|
538
|
+
if (running >= limit) return null;
|
|
539
|
+
}
|
|
128
540
|
|
|
129
541
|
const now = nowIso();
|
|
130
542
|
const result = await this.db
|
|
@@ -132,85 +544,1005 @@ export class D1JobStore {
|
|
|
132
544
|
`UPDATE ${this.tableName}
|
|
133
545
|
SET status = 'running',
|
|
134
546
|
attempt_count = attempt_count + 1,
|
|
135
|
-
started_at =
|
|
547
|
+
started_at = ?,
|
|
136
548
|
updated_at = ?,
|
|
137
549
|
last_error = NULL
|
|
138
550
|
WHERE job_id = ?
|
|
139
|
-
AND status
|
|
551
|
+
AND status = 'queued'
|
|
140
552
|
AND (available_at IS NULL OR available_at <= ?)`,
|
|
141
553
|
)
|
|
142
554
|
.bind(now, now, jobId, now)
|
|
143
555
|
.run();
|
|
144
556
|
|
|
145
557
|
if (!Number(result.meta.changes ?? 0)) return null;
|
|
146
|
-
|
|
558
|
+
const job = await this.get<TPayload>(jobId);
|
|
559
|
+
if (job) await this.emitStatusEvent(job, "running", "Job started.");
|
|
560
|
+
return job;
|
|
561
|
+
}
|
|
562
|
+
|
|
563
|
+
async run<TPayload = unknown, TResult = unknown>(
|
|
564
|
+
jobId: string,
|
|
565
|
+
handler: (job: JobRecord<TPayload, TResult>) => Promise<JobHandlerResult<TResult, TPayload> | TResult | void>,
|
|
566
|
+
): Promise<JobHandlerResult<TResult>> {
|
|
567
|
+
const job = await this.claim<TPayload>(jobId);
|
|
568
|
+
if (!job) return { kind: "done", result: null };
|
|
569
|
+
|
|
570
|
+
try {
|
|
571
|
+
const output = await handler(job as JobRecord<TPayload, TResult>);
|
|
572
|
+
const normalized = normalizeJobResult<TResult>(output);
|
|
573
|
+
if (normalized.kind === "requeue") {
|
|
574
|
+
await this.requeue(jobId, {
|
|
575
|
+
payload: normalized.payload as TPayload | undefined,
|
|
576
|
+
result: normalized.result ?? null,
|
|
577
|
+
availableAt: normalized.availableAt ?? null,
|
|
578
|
+
});
|
|
579
|
+
} else if (normalized.kind === "stopped") {
|
|
580
|
+
await this.stop(jobId, normalized.result ?? null);
|
|
581
|
+
} else {
|
|
582
|
+
await this.succeed(jobId, normalized.result ?? null);
|
|
583
|
+
}
|
|
584
|
+
return normalized;
|
|
585
|
+
} catch (error) {
|
|
586
|
+
const latest = await this.get(jobId);
|
|
587
|
+
if (latest && latest.attemptCount < latest.maxAttempts) {
|
|
588
|
+
await this.requeue(jobId, { result: { retryError: errorMessage(error) }, lastError: errorMessage(error) });
|
|
589
|
+
return { kind: "requeue", result: null };
|
|
590
|
+
}
|
|
591
|
+
await this.fail(jobId, error);
|
|
592
|
+
throw error;
|
|
593
|
+
}
|
|
147
594
|
}
|
|
148
595
|
|
|
149
596
|
async succeed<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
|
|
597
|
+
await this.complete(jobId, "succeeded", result, null);
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
async stop<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
|
|
601
|
+
await this.complete(jobId, "stopped", result, null);
|
|
602
|
+
}
|
|
603
|
+
|
|
604
|
+
async requeue<TResult = unknown>(jobId: string, result: TResult | null, availableAt?: string | null): Promise<void>;
|
|
605
|
+
async requeue<TPayload = unknown, TResult = unknown>(
|
|
606
|
+
jobId: string,
|
|
607
|
+
input?: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null },
|
|
608
|
+
): Promise<void>;
|
|
609
|
+
async requeue<TPayload = unknown, TResult = unknown>(
|
|
610
|
+
jobId: string,
|
|
611
|
+
inputOrResult: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null } | TResult | null = {},
|
|
612
|
+
legacyAvailableAt?: string | null,
|
|
613
|
+
): Promise<void> {
|
|
614
|
+
const input =
|
|
615
|
+
isRequeueInput(inputOrResult)
|
|
616
|
+
? inputOrResult
|
|
617
|
+
: { result: inputOrResult as TResult | null, availableAt: legacyAvailableAt ?? null };
|
|
150
618
|
const now = nowIso();
|
|
619
|
+
const payloadSql = input.payload === undefined ? "" : "payload_json = ?,";
|
|
620
|
+
const bindings = input.payload === undefined ? [] : [stringifyJsonColumn(input.payload)];
|
|
151
621
|
await this.db
|
|
152
622
|
.prepare(
|
|
153
623
|
`UPDATE ${this.tableName}
|
|
154
|
-
SET status = '
|
|
624
|
+
SET status = 'queued',
|
|
625
|
+
${payloadSql}
|
|
626
|
+
result_json = ?,
|
|
627
|
+
available_at = ?,
|
|
628
|
+
last_error = ?,
|
|
629
|
+
updated_at = ?
|
|
155
630
|
WHERE job_id = ?`,
|
|
156
631
|
)
|
|
157
|
-
.bind(
|
|
632
|
+
.bind(
|
|
633
|
+
...(bindings as never[]),
|
|
634
|
+
stringifyJsonColumn(input.result ?? null),
|
|
635
|
+
input.availableAt ?? null,
|
|
636
|
+
input.lastError ?? null,
|
|
637
|
+
now,
|
|
638
|
+
jobId,
|
|
639
|
+
)
|
|
158
640
|
.run();
|
|
641
|
+
const job = await this.get(jobId);
|
|
642
|
+
if (job) {
|
|
643
|
+
await this.emitStatusEvent(job, "queued", "Job requeued.", {
|
|
644
|
+
availableAt: input.availableAt ?? null,
|
|
645
|
+
...(input.lastError ? { lastError: input.lastError } : {}),
|
|
646
|
+
});
|
|
647
|
+
}
|
|
159
648
|
}
|
|
160
649
|
|
|
161
|
-
async
|
|
650
|
+
async fail(jobId: string, error: unknown): Promise<void> {
|
|
651
|
+
await this.complete(jobId, "failed", null, errorMessage(error));
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
async requestStop(jobId: string): Promise<void> {
|
|
162
655
|
const now = nowIso();
|
|
656
|
+
const queuedStopStatus = this.options.queuedStopStatus ?? "stopped";
|
|
163
657
|
await this.db
|
|
164
658
|
.prepare(
|
|
165
659
|
`UPDATE ${this.tableName}
|
|
166
|
-
SET status =
|
|
167
|
-
|
|
660
|
+
SET status = CASE WHEN status = 'queued' THEN ? ELSE 'stopping' END,
|
|
661
|
+
completed_at = CASE WHEN status = 'queued' AND ? = 'stopped' THEN ? ELSE completed_at END,
|
|
662
|
+
updated_at = ?
|
|
663
|
+
WHERE job_id = ? AND status IN ('queued', 'running')`,
|
|
168
664
|
)
|
|
169
|
-
.bind(
|
|
665
|
+
.bind(queuedStopStatus, queuedStopStatus, now, now, jobId)
|
|
170
666
|
.run();
|
|
667
|
+
const job = await this.get(jobId);
|
|
668
|
+
if (job) await this.emitStatusEvent(job, job.status, "Job stop requested.");
|
|
171
669
|
}
|
|
172
670
|
|
|
173
|
-
async
|
|
671
|
+
async listChildren(parentJobId: string): Promise<JobRecord[]> {
|
|
672
|
+
const rows = await this.db
|
|
673
|
+
.prepare(`${this.selectSql()} WHERE parent_job_id = ? ORDER BY created_at ASC`)
|
|
674
|
+
.bind(parentJobId)
|
|
675
|
+
.all<JobRow>();
|
|
676
|
+
return (rows.results ?? []).map(rowToJob);
|
|
677
|
+
}
|
|
678
|
+
|
|
679
|
+
async getJobTree<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobTreeRecord<TPayload, TResult> | null> {
|
|
680
|
+
const root = await this.get<TPayload, TResult>(jobId);
|
|
681
|
+
if (!root) return null;
|
|
682
|
+
const rows = await this.db
|
|
683
|
+
.prepare(
|
|
684
|
+
`WITH RECURSIVE tree(job_id, depth) AS (
|
|
685
|
+
SELECT job_id, 0 FROM ${this.tableName} WHERE job_id = ?
|
|
686
|
+
UNION ALL
|
|
687
|
+
SELECT jobs.job_id, tree.depth + 1
|
|
688
|
+
FROM ${this.tableName} jobs
|
|
689
|
+
JOIN tree ON jobs.parent_job_id = tree.job_id
|
|
690
|
+
)
|
|
691
|
+
SELECT
|
|
692
|
+
jobs.job_id AS jobId,
|
|
693
|
+
jobs.job_type AS jobType,
|
|
694
|
+
jobs.status,
|
|
695
|
+
jobs.org_id AS orgId,
|
|
696
|
+
jobs.site_id AS siteId,
|
|
697
|
+
jobs.payload_json AS payloadJson,
|
|
698
|
+
jobs.result_json AS resultJson,
|
|
699
|
+
jobs.metadata_json AS metadataJson,
|
|
700
|
+
jobs.parent_job_id AS parentJobId,
|
|
701
|
+
jobs.lock_key AS lockKey,
|
|
702
|
+
jobs.concurrency_key AS concurrencyKey,
|
|
703
|
+
jobs.concurrency_limit AS concurrencyLimit,
|
|
704
|
+
jobs.enqueue_policy AS enqueuePolicy,
|
|
705
|
+
jobs.attempt_count AS attemptCount,
|
|
706
|
+
jobs.max_attempts AS maxAttempts,
|
|
707
|
+
jobs.available_at AS availableAt,
|
|
708
|
+
jobs.started_at AS startedAt,
|
|
709
|
+
jobs.completed_at AS completedAt,
|
|
710
|
+
jobs.last_error AS lastError,
|
|
711
|
+
jobs.created_at AS createdAt,
|
|
712
|
+
jobs.updated_at AS updatedAt,
|
|
713
|
+
tree.depth AS depth
|
|
714
|
+
FROM ${this.tableName} jobs
|
|
715
|
+
JOIN tree ON jobs.job_id = tree.job_id
|
|
716
|
+
WHERE jobs.job_id <> ?
|
|
717
|
+
ORDER BY tree.depth ASC, jobs.created_at ASC`,
|
|
718
|
+
)
|
|
719
|
+
.bind(jobId, jobId)
|
|
720
|
+
.all<JobTreeRow>();
|
|
721
|
+
const children = (rows.results ?? []).map((row) => ({ ...rowToJob(row), depth: Number(row.depth) }));
|
|
722
|
+
const allJobs = [root as JobRecord, ...children];
|
|
723
|
+
return {
|
|
724
|
+
root,
|
|
725
|
+
children,
|
|
726
|
+
summary: summarizeJobs(root.jobId, children),
|
|
727
|
+
rollupStatus: rollupJobTreeStatus(allJobs),
|
|
728
|
+
rowStates: [
|
|
729
|
+
jobEntityState(root as JobRecord, 0),
|
|
730
|
+
...children.map((child) => jobEntityState(child, child.depth)),
|
|
731
|
+
],
|
|
732
|
+
activityEvents: await this.listJobActivityEvents(allJobs.map((job) => job.jobId)),
|
|
733
|
+
latestError: latestJobError(allJobs),
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
async summarizeChildren(parentJobId: string): Promise<JobChildrenSummary> {
|
|
738
|
+
return summarizeJobs(parentJobId, await this.listChildren(parentJobId));
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
async markQueuedChildrenTerminal(parentJobId: string, status: "failed" | "stopped", error?: unknown): Promise<number> {
|
|
742
|
+
const now = nowIso();
|
|
743
|
+
const result = await this.db
|
|
744
|
+
.prepare(
|
|
745
|
+
`UPDATE ${this.tableName}
|
|
746
|
+
SET status = ?, last_error = ?, completed_at = ?, updated_at = ?
|
|
747
|
+
WHERE parent_job_id = ? AND status = 'queued'`,
|
|
748
|
+
)
|
|
749
|
+
.bind(status, status === "failed" && error ? errorMessage(error) : null, now, now, parentJobId)
|
|
750
|
+
.run();
|
|
751
|
+
const changed = Number(result.meta.changes ?? 0);
|
|
752
|
+
if (changed > 0 && this.options.onStatusEvent) {
|
|
753
|
+
for (const child of await this.listChildren(parentJobId)) {
|
|
754
|
+
if (child.status === status) {
|
|
755
|
+
await this.emitStatusEvent(
|
|
756
|
+
child,
|
|
757
|
+
status,
|
|
758
|
+
status === "failed" ? errorMessage(error ?? "Child job failed.") : "Job stopped.",
|
|
759
|
+
error ? { reason: errorMessage(error) } : null,
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
}
|
|
764
|
+
return changed;
|
|
765
|
+
}
|
|
766
|
+
|
|
767
|
+
async requestStopActiveChildren(parentJobId: string): Promise<number> {
|
|
768
|
+
const now = nowIso();
|
|
769
|
+
const statuses = this.options.requestStopQueuedChildren ? "'queued', 'running'" : "'running'";
|
|
770
|
+
const result = await this.db
|
|
771
|
+
.prepare(
|
|
772
|
+
`UPDATE ${this.tableName}
|
|
773
|
+
SET status = 'stopping', updated_at = ?
|
|
774
|
+
WHERE parent_job_id = ? AND status IN (${statuses})`,
|
|
775
|
+
)
|
|
776
|
+
.bind(now, parentJobId)
|
|
777
|
+
.run();
|
|
778
|
+
const changed = Number(result.meta.changes ?? 0);
|
|
779
|
+
if (changed > 0 && this.options.onStatusEvent) {
|
|
780
|
+
for (const child of await this.listChildren(parentJobId)) {
|
|
781
|
+
if (child.status === "stopping") await this.emitStatusEvent(child, "stopping", "Job stop requested.");
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
return changed;
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
async failQueuedChildren(input: { parentJobId: string; reason?: string | null }): Promise<number> {
|
|
788
|
+
return this.markQueuedChildrenTerminal(input.parentJobId, "failed", input.reason ?? "Child job failed.");
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
async stopQueuedChildren(input: { parentJobId: string; reason?: string | null }): Promise<number> {
|
|
792
|
+
return this.markQueuedChildrenTerminal(input.parentJobId, "stopped", input.reason ?? "Child job stopped.");
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
private async getActiveForLockKey<TPayload = unknown>(lockKey: string): Promise<JobRecord<TPayload> | null> {
|
|
796
|
+
const row = await this.db
|
|
797
|
+
.prepare(`${this.selectSql()} WHERE lock_key = ? AND status IN ('queued', 'running', 'stopping') ORDER BY created_at ASC LIMIT 1`)
|
|
798
|
+
.bind(lockKey)
|
|
799
|
+
.first<JobRow>();
|
|
800
|
+
return row ? rowToJob<TPayload>(row) : null;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
private async countRunningForConcurrencyKey(concurrencyKey: string): Promise<number> {
|
|
804
|
+
const row = await this.db
|
|
805
|
+
.prepare(
|
|
806
|
+
`SELECT COUNT(*) AS count
|
|
807
|
+
FROM ${this.tableName}
|
|
808
|
+
WHERE concurrency_key = ? AND status IN ('running', 'stopping')`,
|
|
809
|
+
)
|
|
810
|
+
.bind(concurrencyKey)
|
|
811
|
+
.first<{ count: number }>();
|
|
812
|
+
return Number(row?.count ?? 0);
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
private async getForSiteByPayload(input: {
|
|
816
|
+
siteId: string;
|
|
817
|
+
jobType?: string | null;
|
|
818
|
+
key: string;
|
|
819
|
+
value: string | number | boolean | null;
|
|
820
|
+
activeOnly: boolean;
|
|
821
|
+
}): Promise<JobRecord | null> {
|
|
822
|
+
const row = await this.db
|
|
823
|
+
.prepare(
|
|
824
|
+
`${this.selectSql()}
|
|
825
|
+
WHERE site_id = ?
|
|
826
|
+
${input.jobType ? "AND job_type = ?" : ""}
|
|
827
|
+
${input.activeOnly ? "AND status IN ('queued', 'running', 'stopping')" : ""}
|
|
828
|
+
AND json_extract(payload_json, ?) = ?
|
|
829
|
+
ORDER BY created_at DESC
|
|
830
|
+
LIMIT 1`,
|
|
831
|
+
)
|
|
832
|
+
.bind(
|
|
833
|
+
...([
|
|
834
|
+
input.siteId,
|
|
835
|
+
...(input.jobType ? [input.jobType] : []),
|
|
836
|
+
`$.${input.key}`,
|
|
837
|
+
input.value,
|
|
838
|
+
] as never[]),
|
|
839
|
+
)
|
|
840
|
+
.first<JobRow>();
|
|
841
|
+
return row ? rowToJob(row) : null;
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
private async listJobActivityEvents(jobIds: string[]): Promise<JobActivityEvent[]> {
|
|
845
|
+
if (jobIds.length === 0) return [];
|
|
846
|
+
const placeholders = jobIds.map(() => "?").join(", ");
|
|
847
|
+
try {
|
|
848
|
+
const rows = await this.db
|
|
849
|
+
.prepare(
|
|
850
|
+
`SELECT
|
|
851
|
+
ops_event_id AS opsEventId,
|
|
852
|
+
job_id AS jobId,
|
|
853
|
+
level,
|
|
854
|
+
event_type AS eventType,
|
|
855
|
+
message,
|
|
856
|
+
metadata_json AS metadataJson,
|
|
857
|
+
created_at AS createdAt
|
|
858
|
+
FROM ${this.options.opsEventsTableName ?? "ops_events"}
|
|
859
|
+
WHERE job_id IN (${placeholders})
|
|
860
|
+
ORDER BY created_at ASC
|
|
861
|
+
LIMIT 200`,
|
|
862
|
+
)
|
|
863
|
+
.bind(...(jobIds as never[]))
|
|
864
|
+
.all<JobActivityEventRow>();
|
|
865
|
+
return (rows.results ?? []).map(rowToActivityEvent);
|
|
866
|
+
} catch {
|
|
867
|
+
return [];
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
private async complete<TResult = unknown>(
|
|
872
|
+
jobId: string,
|
|
873
|
+
status: TerminalJobStatus,
|
|
874
|
+
result: TResult | null,
|
|
875
|
+
lastError: string | null,
|
|
876
|
+
): Promise<void> {
|
|
174
877
|
const now = nowIso();
|
|
175
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
176
878
|
await this.db
|
|
177
879
|
.prepare(
|
|
178
880
|
`UPDATE ${this.tableName}
|
|
179
|
-
SET status =
|
|
881
|
+
SET status = ?, result_json = ?, last_error = ?, completed_at = ?, updated_at = ?
|
|
180
882
|
WHERE job_id = ?`,
|
|
181
883
|
)
|
|
182
|
-
.bind(
|
|
884
|
+
.bind(status, stringifyJsonColumn(result), lastError, now, now, jobId)
|
|
183
885
|
.run();
|
|
886
|
+
const job = await this.get(jobId);
|
|
887
|
+
if (job) {
|
|
888
|
+
await this.emitStatusEvent(
|
|
889
|
+
job,
|
|
890
|
+
status,
|
|
891
|
+
status === "succeeded" ? "Job succeeded." : status === "stopped" ? "Job stopped." : (lastError ?? "Job failed."),
|
|
892
|
+
);
|
|
893
|
+
}
|
|
184
894
|
}
|
|
185
895
|
|
|
186
|
-
async
|
|
896
|
+
private async markStaleRunningQueued(jobId: string): Promise<void> {
|
|
897
|
+
const now = nowIso();
|
|
898
|
+
await this.db
|
|
899
|
+
.prepare(
|
|
900
|
+
`UPDATE ${this.tableName}
|
|
901
|
+
SET status = 'queued',
|
|
902
|
+
available_at = ?,
|
|
903
|
+
last_error = ?,
|
|
904
|
+
updated_at = ?
|
|
905
|
+
WHERE job_id = ? AND status = 'running'`,
|
|
906
|
+
)
|
|
907
|
+
.bind(now, "Job reclaimed after stale running lease.", now, jobId)
|
|
908
|
+
.run();
|
|
909
|
+
const job = await this.get(jobId);
|
|
910
|
+
if (job) {
|
|
911
|
+
await this.emitStatusEvent(job, "queued", "Stale running job requeued.", {
|
|
912
|
+
previousStatus: "running",
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
}
|
|
916
|
+
|
|
917
|
+
private async emitStatusEvent(
|
|
918
|
+
job: JobRecord,
|
|
919
|
+
status: JobStatus,
|
|
920
|
+
message: string,
|
|
921
|
+
metadata?: Record<string, unknown> | null,
|
|
922
|
+
error?: unknown,
|
|
923
|
+
): Promise<void> {
|
|
924
|
+
await this.options.onStatusEvent?.({ job, status, message, metadata: metadata ?? null, error });
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
private selectSql(): string {
|
|
928
|
+
return `SELECT
|
|
929
|
+
job_id AS jobId,
|
|
930
|
+
job_type AS jobType,
|
|
931
|
+
status,
|
|
932
|
+
org_id AS orgId,
|
|
933
|
+
site_id AS siteId,
|
|
934
|
+
payload_json AS payloadJson,
|
|
935
|
+
result_json AS resultJson,
|
|
936
|
+
metadata_json AS metadataJson,
|
|
937
|
+
parent_job_id AS parentJobId,
|
|
938
|
+
lock_key AS lockKey,
|
|
939
|
+
concurrency_key AS concurrencyKey,
|
|
940
|
+
concurrency_limit AS concurrencyLimit,
|
|
941
|
+
enqueue_policy AS enqueuePolicy,
|
|
942
|
+
attempt_count AS attemptCount,
|
|
943
|
+
max_attempts AS maxAttempts,
|
|
944
|
+
available_at AS availableAt,
|
|
945
|
+
started_at AS startedAt,
|
|
946
|
+
completed_at AS completedAt,
|
|
947
|
+
last_error AS lastError,
|
|
948
|
+
created_at AS createdAt,
|
|
949
|
+
updated_at AS updatedAt
|
|
950
|
+
FROM ${this.tableName}`;
|
|
951
|
+
}
|
|
952
|
+
}
|
|
953
|
+
|
|
954
|
+
export type PostgresSqlLike = {
|
|
955
|
+
unsafe<Row extends object = Record<string, unknown>>(query: string, values?: readonly unknown[]): Promise<Row[]>;
|
|
956
|
+
json?: (value: unknown) => unknown;
|
|
957
|
+
};
|
|
958
|
+
|
|
959
|
+
export type PostgresJobStoreOptions = {
|
|
960
|
+
tableName?: string;
|
|
961
|
+
idPrefix?: string;
|
|
962
|
+
queueName?: string;
|
|
963
|
+
statusMap?: Partial<Record<JobStatus, string>>;
|
|
964
|
+
reverseStatusMap?: Record<string, JobStatus>;
|
|
965
|
+
columns?: Partial<PostgresJobColumns>;
|
|
966
|
+
defaultMaxAttempts?: number;
|
|
967
|
+
initialResult?: unknown;
|
|
968
|
+
retryDelaySql?: string;
|
|
969
|
+
};
|
|
970
|
+
|
|
971
|
+
export type PostgresJobColumns = {
|
|
972
|
+
jobId: string;
|
|
973
|
+
orgId: string | null;
|
|
974
|
+
siteId: string | null;
|
|
975
|
+
queueName: string;
|
|
976
|
+
jobType: string;
|
|
977
|
+
status: string;
|
|
978
|
+
payloadJson: string;
|
|
979
|
+
resultJson: string;
|
|
980
|
+
metadataJson: string | null;
|
|
981
|
+
parentJobId: string | null;
|
|
982
|
+
lockKey: string | null;
|
|
983
|
+
concurrencyKey: string | null;
|
|
984
|
+
concurrencyLimit: string | null;
|
|
985
|
+
enqueuePolicy: string | null;
|
|
986
|
+
attemptCount: string;
|
|
987
|
+
maxAttempts: string;
|
|
988
|
+
availableAt: string;
|
|
989
|
+
startedAt: string | null;
|
|
990
|
+
completedAt: string | null;
|
|
991
|
+
lastError: string | null;
|
|
992
|
+
createdAt: string;
|
|
993
|
+
updatedAt: string;
|
|
994
|
+
};
|
|
995
|
+
|
|
996
|
+
const DEFAULT_POSTGRES_JOB_COLUMNS: PostgresJobColumns = {
|
|
997
|
+
jobId: "job_id",
|
|
998
|
+
orgId: "org_id",
|
|
999
|
+
siteId: "site_id",
|
|
1000
|
+
queueName: "queue_name",
|
|
1001
|
+
jobType: "job_type",
|
|
1002
|
+
status: "status",
|
|
1003
|
+
payloadJson: "payload_json",
|
|
1004
|
+
resultJson: "result_json",
|
|
1005
|
+
metadataJson: "metadata_json",
|
|
1006
|
+
parentJobId: "parent_job_id",
|
|
1007
|
+
lockKey: "lock_key",
|
|
1008
|
+
concurrencyKey: "concurrency_key",
|
|
1009
|
+
concurrencyLimit: "concurrency_limit",
|
|
1010
|
+
enqueuePolicy: "enqueue_policy",
|
|
1011
|
+
attemptCount: "attempt_count",
|
|
1012
|
+
maxAttempts: "max_attempts",
|
|
1013
|
+
availableAt: "available_at",
|
|
1014
|
+
startedAt: "started_at",
|
|
1015
|
+
completedAt: "completed_at",
|
|
1016
|
+
lastError: "last_error",
|
|
1017
|
+
createdAt: "created_at",
|
|
1018
|
+
updatedAt: "updated_at",
|
|
1019
|
+
};
|
|
1020
|
+
|
|
1021
|
+
type PostgresJobRow = Omit<
|
|
1022
|
+
JobRow,
|
|
1023
|
+
| "payloadJson"
|
|
1024
|
+
| "resultJson"
|
|
1025
|
+
| "metadataJson"
|
|
1026
|
+
| "availableAt"
|
|
1027
|
+
| "startedAt"
|
|
1028
|
+
| "completedAt"
|
|
1029
|
+
| "createdAt"
|
|
1030
|
+
| "updatedAt"
|
|
1031
|
+
> & {
|
|
1032
|
+
payloadJson: unknown;
|
|
1033
|
+
resultJson: unknown;
|
|
1034
|
+
metadataJson?: unknown;
|
|
1035
|
+
availableAt: unknown;
|
|
1036
|
+
startedAt: unknown;
|
|
1037
|
+
completedAt: unknown;
|
|
1038
|
+
createdAt: unknown;
|
|
1039
|
+
updatedAt: unknown;
|
|
1040
|
+
};
|
|
1041
|
+
|
|
1042
|
+
export class PostgresJobStore implements DurableJobRepository {
|
|
1043
|
+
private readonly tableName: string;
|
|
1044
|
+
private readonly columns: PostgresJobColumns;
|
|
1045
|
+
|
|
1046
|
+
constructor(
|
|
1047
|
+
private readonly sql: PostgresSqlLike,
|
|
1048
|
+
private readonly options: PostgresJobStoreOptions = {},
|
|
1049
|
+
) {
|
|
1050
|
+
this.tableName = quotePgIdent(options.tableName ?? "jobs");
|
|
1051
|
+
this.columns = { ...DEFAULT_POSTGRES_JOB_COLUMNS, ...(options.columns ?? {}) };
|
|
1052
|
+
}
|
|
1053
|
+
|
|
1054
|
+
async enqueue<TPayload = unknown>(input: EnqueueJobInput<TPayload>): Promise<JobRecord<TPayload>> {
|
|
1055
|
+
if (input.enqueuePolicy === "dedupe_active" && input.lockKey) {
|
|
1056
|
+
const existing = await this.getActiveForLockKey<TPayload>(input.lockKey);
|
|
1057
|
+
if (existing) return existing;
|
|
1058
|
+
}
|
|
1059
|
+
const jobId = createId(this.options.idPrefix ?? "job");
|
|
1060
|
+
const queueName = stringValue(input.metadata?.queueName) ?? queueNameFromPayload(input.payload) ?? this.options.queueName ?? null;
|
|
1061
|
+
const insertColumnKeys: Array<keyof PostgresJobColumns> = [
|
|
1062
|
+
"jobId",
|
|
1063
|
+
"orgId",
|
|
1064
|
+
"siteId",
|
|
1065
|
+
"queueName",
|
|
1066
|
+
"jobType",
|
|
1067
|
+
"status",
|
|
1068
|
+
"payloadJson",
|
|
1069
|
+
"resultJson",
|
|
1070
|
+
"metadataJson",
|
|
1071
|
+
"parentJobId",
|
|
1072
|
+
"lockKey",
|
|
1073
|
+
"concurrencyKey",
|
|
1074
|
+
"concurrencyLimit",
|
|
1075
|
+
"enqueuePolicy",
|
|
1076
|
+
"maxAttempts",
|
|
1077
|
+
"availableAt",
|
|
1078
|
+
];
|
|
1079
|
+
const insertColumns = insertColumnKeys.filter((key) => this.columns[key] !== null);
|
|
1080
|
+
const insertValues: Record<keyof PostgresJobColumns, unknown> = {
|
|
1081
|
+
jobId,
|
|
1082
|
+
orgId: input.orgId ?? null,
|
|
1083
|
+
siteId: input.siteId ?? null,
|
|
1084
|
+
queueName,
|
|
1085
|
+
jobType: input.jobType,
|
|
1086
|
+
status: this.toDbStatus("queued"),
|
|
1087
|
+
payloadJson: jsonParam(input.payload ?? {}),
|
|
1088
|
+
resultJson: jsonParam(this.options.initialResult ?? null),
|
|
1089
|
+
metadataJson: jsonParam(input.metadata ?? {}),
|
|
1090
|
+
parentJobId: input.parentJobId ?? null,
|
|
1091
|
+
lockKey: input.lockKey ?? null,
|
|
1092
|
+
concurrencyKey: input.concurrencyKey ?? null,
|
|
1093
|
+
concurrencyLimit: normalizeConcurrencyLimit(input.concurrencyLimit),
|
|
1094
|
+
enqueuePolicy: input.enqueuePolicy ?? "enqueue",
|
|
1095
|
+
attemptCount: 0,
|
|
1096
|
+
maxAttempts: Math.max(1, Math.floor(input.maxAttempts ?? this.options.defaultMaxAttempts ?? 3)),
|
|
1097
|
+
availableAt: input.availableAt ?? null,
|
|
1098
|
+
startedAt: null,
|
|
1099
|
+
completedAt: null,
|
|
1100
|
+
lastError: null,
|
|
1101
|
+
createdAt: null,
|
|
1102
|
+
updatedAt: null,
|
|
1103
|
+
};
|
|
1104
|
+
const values = insertColumns.map((key) => insertValues[key]);
|
|
1105
|
+
await this.sql.unsafe(
|
|
1106
|
+
`INSERT INTO ${this.tableName} (
|
|
1107
|
+
${insertColumns.map((key) => this.col(key)).join(", ")}
|
|
1108
|
+
) VALUES (${insertColumns.map((key, index) => this.insertPlaceholder(key, index + 1)).join(", ")})`,
|
|
1109
|
+
values,
|
|
1110
|
+
);
|
|
1111
|
+
const job = await this.get<TPayload>(jobId);
|
|
1112
|
+
if (!job) throw new Error(`Failed to load enqueued job: ${jobId}`);
|
|
1113
|
+
return job;
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
async get<TPayload = unknown, TResult = unknown>(jobId: string): Promise<JobRecord<TPayload, TResult> | null> {
|
|
1117
|
+
const rows = await this.sql.unsafe<PostgresJobRow>(`${this.selectSql()} WHERE ${this.col("jobId")} = $1 LIMIT 1`, [jobId]);
|
|
1118
|
+
return rows[0] ? this.rowToJob<TPayload, TResult>(rows[0]) : null;
|
|
1119
|
+
}
|
|
1120
|
+
|
|
1121
|
+
async claim<TPayload = unknown>(jobId: string): Promise<JobRecord<TPayload> | null> {
|
|
1122
|
+
const queueClause = this.options.queueName ? `AND ${this.col("queueName")} = $2` : "";
|
|
1123
|
+
const values = this.options.queueName ? [jobId, this.options.queueName] : [jobId];
|
|
1124
|
+
const rows = await this.sql.unsafe<PostgresJobRow>(
|
|
1125
|
+
`UPDATE ${this.tableName}
|
|
1126
|
+
SET ${this.col("status")} = '${this.toDbStatus("running")}',
|
|
1127
|
+
${this.col("attemptCount")} = ${this.col("attemptCount")} + 1,
|
|
1128
|
+
${this.setSql("startedAt", "now(),")}
|
|
1129
|
+
${this.col("updatedAt")} = now(),
|
|
1130
|
+
${this.setSql("lastError", "NULL")}
|
|
1131
|
+
WHERE ${this.col("jobId")} = $1
|
|
1132
|
+
${queueClause}
|
|
1133
|
+
AND ${this.col("status")} = '${this.toDbStatus("queued")}'
|
|
1134
|
+
AND ${this.col("availableAt")} <= now()
|
|
1135
|
+
AND (
|
|
1136
|
+
${this.nullableColSql("concurrencyKey", "NULL")} IS NULL
|
|
1137
|
+
OR (
|
|
1138
|
+
SELECT COUNT(*)
|
|
1139
|
+
FROM ${this.tableName} active
|
|
1140
|
+
WHERE active.${this.colRaw("concurrencyKey")} = ${this.tableName}.${this.colRaw("concurrencyKey")}
|
|
1141
|
+
AND active.${this.colRaw("status")} IN ('${this.toDbStatus("running")}', '${this.toDbStatus("stopping")}')
|
|
1142
|
+
AND active.${this.colRaw("jobId")} <> ${this.tableName}.${this.colRaw("jobId")}
|
|
1143
|
+
) < COALESCE(${this.nullableColSql("concurrencyLimit", "1")}, 1)
|
|
1144
|
+
)
|
|
1145
|
+
RETURNING ${this.returningSql()}`,
|
|
1146
|
+
values,
|
|
1147
|
+
);
|
|
1148
|
+
return rows[0] ? this.rowToJob<TPayload>(rows[0]) : null;
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
async succeed<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
|
|
1152
|
+
await this.complete(jobId, "succeeded", result, null);
|
|
1153
|
+
}
|
|
1154
|
+
|
|
1155
|
+
async requeue<TPayload = unknown, TResult = unknown>(
|
|
187
1156
|
jobId: string,
|
|
188
|
-
|
|
189
|
-
): Promise<
|
|
190
|
-
const
|
|
191
|
-
|
|
1157
|
+
input: { payload?: TPayload; result?: TResult | null; availableAt?: string | null; lastError?: string | null } = {},
|
|
1158
|
+
): Promise<void> {
|
|
1159
|
+
const payloadSet = input.payload === undefined ? "" : `${this.col("payloadJson")} = $5::jsonb,`;
|
|
1160
|
+
const values = [
|
|
1161
|
+
jsonParam(input.result ?? null),
|
|
1162
|
+
input.availableAt ?? null,
|
|
1163
|
+
input.lastError ?? null,
|
|
1164
|
+
jobId,
|
|
1165
|
+
...(input.payload === undefined ? [] : [jsonParam(input.payload)]),
|
|
1166
|
+
];
|
|
1167
|
+
await this.sql.unsafe(
|
|
1168
|
+
`UPDATE ${this.tableName}
|
|
1169
|
+
SET ${this.col("status")} = '${this.toDbStatus("queued")}',
|
|
1170
|
+
${payloadSet}
|
|
1171
|
+
${this.col("resultJson")} = $1::jsonb,
|
|
1172
|
+
${this.col("availableAt")} = COALESCE($2::timestamptz, now()),
|
|
1173
|
+
${this.setSql("lastError", "$3,")}
|
|
1174
|
+
${this.col("updatedAt")} = now()
|
|
1175
|
+
WHERE ${this.col("jobId")} = $4`,
|
|
1176
|
+
values,
|
|
1177
|
+
);
|
|
1178
|
+
}
|
|
192
1179
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
1180
|
+
async fail(jobId: string, error: unknown): Promise<void> {
|
|
1181
|
+
await this.complete(jobId, "failed", null, errorMessage(error));
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
async stop<TResult = unknown>(jobId: string, result: TResult | null = null): Promise<void> {
|
|
1185
|
+
await this.complete(jobId, "stopped", result, null);
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
async requestStop(jobId: string): Promise<void> {
|
|
1189
|
+
await this.sql.unsafe(
|
|
1190
|
+
`UPDATE ${this.tableName}
|
|
1191
|
+
SET ${this.col("status")} = CASE
|
|
1192
|
+
WHEN ${this.col("status")} = '${this.toDbStatus("queued")}' THEN '${this.toDbStatus("stopped")}'
|
|
1193
|
+
ELSE '${this.toDbStatus("stopping")}'
|
|
1194
|
+
END,
|
|
1195
|
+
${this.setSql("completedAt", `CASE
|
|
1196
|
+
WHEN ${this.col("status")} = '${this.toDbStatus("queued")}' THEN now()
|
|
1197
|
+
ELSE ${this.col("completedAt")}
|
|
1198
|
+
END,`)}
|
|
1199
|
+
${this.col("updatedAt")} = now()
|
|
1200
|
+
WHERE ${this.col("jobId")} = $1
|
|
1201
|
+
AND ${this.col("status")} IN ('${this.toDbStatus("queued")}', '${this.toDbStatus("running")}')`,
|
|
1202
|
+
[jobId],
|
|
1203
|
+
);
|
|
1204
|
+
}
|
|
1205
|
+
|
|
1206
|
+
async listChildren(parentJobId: string): Promise<JobRecord[]> {
|
|
1207
|
+
const rows = await this.sql.unsafe<PostgresJobRow>(
|
|
1208
|
+
`${this.selectSql()} WHERE ${this.col("parentJobId")} = $1 ORDER BY ${this.col("createdAt")} ASC`,
|
|
1209
|
+
[parentJobId],
|
|
1210
|
+
);
|
|
1211
|
+
return rows.map((row) => this.rowToJob(row));
|
|
1212
|
+
}
|
|
1213
|
+
|
|
1214
|
+
async summarizeChildren(parentJobId: string): Promise<JobChildrenSummary> {
|
|
1215
|
+
return summarizeJobs(parentJobId, await this.listChildren(parentJobId));
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
async markQueuedChildrenTerminal(parentJobId: string, status: "failed" | "stopped", error?: unknown): Promise<number> {
|
|
1219
|
+
const rows = await this.sql.unsafe<{ changed: number }>(
|
|
1220
|
+
`WITH updated AS (
|
|
1221
|
+
UPDATE ${this.tableName}
|
|
1222
|
+
SET ${this.col("status")} = $1,
|
|
1223
|
+
${this.setSql("lastError", "$2,")}
|
|
1224
|
+
${this.setSql("completedAt", "now(),")}
|
|
1225
|
+
${this.col("updatedAt")} = now()
|
|
1226
|
+
WHERE ${this.col("parentJobId")} = $3
|
|
1227
|
+
AND ${this.col("status")} = '${this.toDbStatus("queued")}'
|
|
1228
|
+
RETURNING ${this.col("jobId")}
|
|
1229
|
+
)
|
|
1230
|
+
SELECT COUNT(*)::int AS changed FROM updated`,
|
|
1231
|
+
[this.toDbStatus(status), status === "failed" && error ? errorMessage(error) : null, parentJobId],
|
|
1232
|
+
);
|
|
1233
|
+
return Number(rows[0]?.changed ?? 0);
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
async requestStopActiveChildren(parentJobId: string): Promise<number> {
|
|
1237
|
+
const rows = await this.sql.unsafe<{ changed: number }>(
|
|
1238
|
+
`WITH updated AS (
|
|
1239
|
+
UPDATE ${this.tableName}
|
|
1240
|
+
SET ${this.col("status")} = '${this.toDbStatus("stopping")}',
|
|
1241
|
+
${this.col("updatedAt")} = now()
|
|
1242
|
+
WHERE ${this.col("parentJobId")} = $1
|
|
1243
|
+
AND ${this.col("status")} = '${this.toDbStatus("running")}'
|
|
1244
|
+
RETURNING ${this.col("jobId")}
|
|
1245
|
+
)
|
|
1246
|
+
SELECT COUNT(*)::int AS changed FROM updated`,
|
|
1247
|
+
[parentJobId],
|
|
1248
|
+
);
|
|
1249
|
+
return Number(rows[0]?.changed ?? 0);
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
private async getActiveForLockKey<TPayload = unknown>(lockKey: string): Promise<JobRecord<TPayload> | null> {
|
|
1253
|
+
const rows = await this.sql.unsafe<PostgresJobRow>(
|
|
1254
|
+
`${this.selectSql()}
|
|
1255
|
+
WHERE ${this.col("lockKey")} = $1
|
|
1256
|
+
AND ${this.col("status")} IN ('${this.toDbStatus("queued")}', '${this.toDbStatus("running")}', '${this.toDbStatus("stopping")}')
|
|
1257
|
+
ORDER BY ${this.col("createdAt")} DESC
|
|
1258
|
+
LIMIT 1`,
|
|
1259
|
+
[lockKey],
|
|
1260
|
+
);
|
|
1261
|
+
return rows[0] ? this.rowToJob<TPayload>(rows[0]) : null;
|
|
1262
|
+
}
|
|
1263
|
+
|
|
1264
|
+
private async complete<TResult>(jobId: string, status: TerminalJobStatus, result: TResult | null, lastError: string | null): Promise<void> {
|
|
1265
|
+
await this.sql.unsafe(
|
|
1266
|
+
`UPDATE ${this.tableName}
|
|
1267
|
+
SET ${this.col("status")} = $1,
|
|
1268
|
+
${this.col("resultJson")} = $2::jsonb,
|
|
1269
|
+
${this.setSql("lastError", "$3,")}
|
|
1270
|
+
${this.setSql("completedAt", "now(),")}
|
|
1271
|
+
${this.col("updatedAt")} = now()
|
|
1272
|
+
WHERE ${this.col("jobId")} = $4`,
|
|
1273
|
+
[this.toDbStatus(status), jsonParam(result), lastError, jobId],
|
|
1274
|
+
);
|
|
1275
|
+
}
|
|
1276
|
+
|
|
1277
|
+
private selectSql(): string {
|
|
1278
|
+
return `SELECT ${this.returningSql()} FROM ${this.tableName}`;
|
|
1279
|
+
}
|
|
1280
|
+
|
|
1281
|
+
private returningSql(): string {
|
|
1282
|
+
return ([
|
|
1283
|
+
["jobId", "jobId"],
|
|
1284
|
+
["jobType", "jobType"],
|
|
1285
|
+
["status", "status"],
|
|
1286
|
+
["orgId", "orgId"],
|
|
1287
|
+
["siteId", "siteId"],
|
|
1288
|
+
["payloadJson", "payloadJson"],
|
|
1289
|
+
["resultJson", "resultJson"],
|
|
1290
|
+
["metadataJson", "metadataJson"],
|
|
1291
|
+
["parentJobId", "parentJobId"],
|
|
1292
|
+
["lockKey", "lockKey"],
|
|
1293
|
+
["concurrencyKey", "concurrencyKey"],
|
|
1294
|
+
["concurrencyLimit", "concurrencyLimit"],
|
|
1295
|
+
["enqueuePolicy", "enqueuePolicy"],
|
|
1296
|
+
["attemptCount", "attemptCount"],
|
|
1297
|
+
["maxAttempts", "maxAttempts"],
|
|
1298
|
+
["availableAt", "availableAt"],
|
|
1299
|
+
["startedAt", "startedAt"],
|
|
1300
|
+
["completedAt", "completedAt"],
|
|
1301
|
+
["lastError", "lastError"],
|
|
1302
|
+
["createdAt", "createdAt"],
|
|
1303
|
+
["updatedAt", "updatedAt"],
|
|
1304
|
+
] as Array<[keyof PostgresJobColumns, string]>)
|
|
1305
|
+
.map(([key, alias]) => `${this.selectExpr(key)} AS "${alias}"`)
|
|
1306
|
+
.join(", ");
|
|
1307
|
+
}
|
|
1308
|
+
|
|
1309
|
+
private rowToJob<TPayload = unknown, TResult = unknown>(row: PostgresJobRow): JobRecord<TPayload, TResult> {
|
|
1310
|
+
return rowToJob({
|
|
1311
|
+
...row,
|
|
1312
|
+
status: this.fromDbStatus(String(row.status)),
|
|
1313
|
+
payloadJson: jsonText(row.payloadJson) ?? "{}",
|
|
1314
|
+
resultJson: jsonText(row.resultJson),
|
|
1315
|
+
metadataJson: jsonText(row.metadataJson),
|
|
1316
|
+
availableAt: dateText(row.availableAt),
|
|
1317
|
+
startedAt: dateText(row.startedAt),
|
|
1318
|
+
completedAt: dateText(row.completedAt),
|
|
1319
|
+
createdAt: dateText(row.createdAt) ?? "",
|
|
1320
|
+
updatedAt: dateText(row.updatedAt) ?? "",
|
|
1321
|
+
});
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
private toDbStatus(status: JobStatus): string {
|
|
1325
|
+
return this.options.statusMap?.[status] ?? status;
|
|
1326
|
+
}
|
|
1327
|
+
|
|
1328
|
+
private fromDbStatus(status: string): JobStatus {
|
|
1329
|
+
return this.options.reverseStatusMap?.[status] ?? (status as JobStatus);
|
|
1330
|
+
}
|
|
1331
|
+
|
|
1332
|
+
private col(key: keyof PostgresJobColumns): string {
|
|
1333
|
+
const column = this.columns[key];
|
|
1334
|
+
if (!column) throw new Error(`Postgres job column is not configured: ${String(key)}`);
|
|
1335
|
+
return quotePgIdent(column);
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
private colRaw(key: keyof PostgresJobColumns): string {
|
|
1339
|
+
const column = this.columns[key];
|
|
1340
|
+
if (!column) throw new Error(`Postgres job column is not configured: ${String(key)}`);
|
|
1341
|
+
return column;
|
|
1342
|
+
}
|
|
1343
|
+
|
|
1344
|
+
private insertPlaceholder(key: keyof PostgresJobColumns, index: number): string {
|
|
1345
|
+
if (key === "payloadJson" || key === "resultJson" || key === "metadataJson") return `$${index}::jsonb`;
|
|
1346
|
+
if (key === "availableAt" || key === "startedAt" || key === "completedAt") return `COALESCE($${index}::timestamptz, now())`;
|
|
1347
|
+
return `$${index}`;
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
private nullableColSql(key: keyof PostgresJobColumns, fallback: string): string {
|
|
1351
|
+
return this.columns[key] ? this.col(key) : fallback;
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
private selectExpr(key: keyof PostgresJobColumns): string {
|
|
1355
|
+
if (this.columns[key]) return this.col(key);
|
|
1356
|
+
if (key === "metadataJson") return `'{}'::jsonb`;
|
|
1357
|
+
if (key === "resultJson") return `'null'::jsonb`;
|
|
1358
|
+
if (key === "enqueuePolicy") return `'enqueue'`;
|
|
1359
|
+
if (key === "concurrencyLimit" || key === "attemptCount" || key === "maxAttempts") return "0";
|
|
1360
|
+
return "NULL";
|
|
1361
|
+
}
|
|
1362
|
+
|
|
1363
|
+
private setSql(key: keyof PostgresJobColumns, assignment: string): string {
|
|
1364
|
+
return this.columns[key] ? `${this.col(key)} = ${assignment}` : "";
|
|
1365
|
+
}
|
|
1366
|
+
}
|
|
1367
|
+
|
|
1368
|
+
export type PipelineJob<StepName extends string, State> = {
|
|
1369
|
+
pipeline: string;
|
|
1370
|
+
runId: string;
|
|
1371
|
+
step: StepName;
|
|
1372
|
+
continuationId?: string;
|
|
1373
|
+
state: State;
|
|
1374
|
+
};
|
|
1375
|
+
|
|
1376
|
+
export type PipelineStepHandlerInput<StepName extends string, State> = {
|
|
1377
|
+
job: PipelineJob<StepName, State>;
|
|
1378
|
+
stepIndex: number;
|
|
1379
|
+
isFinalStep: boolean;
|
|
1380
|
+
};
|
|
1381
|
+
|
|
1382
|
+
export type PipelineCompletion = {
|
|
1383
|
+
status: string;
|
|
1384
|
+
targetsTotal: number;
|
|
1385
|
+
targetsSucceeded: number;
|
|
1386
|
+
targetsFailed: number;
|
|
1387
|
+
message?: string | null;
|
|
1388
|
+
};
|
|
1389
|
+
|
|
1390
|
+
export type PipelineStepHandlerResult<StepName extends string, State> = {
|
|
1391
|
+
state?: State;
|
|
1392
|
+
nextStep?: { step: StepName; state?: State } | null;
|
|
1393
|
+
nextStepDelaySeconds?: number;
|
|
1394
|
+
completion?: PipelineCompletion | null;
|
|
1395
|
+
};
|
|
1396
|
+
|
|
1397
|
+
export type PipelineStepHandler<StepName extends string, State> = (
|
|
1398
|
+
input: PipelineStepHandlerInput<StepName, State>,
|
|
1399
|
+
) => Promise<PipelineStepHandlerResult<StepName, State> | undefined> | Promise<void>;
|
|
1400
|
+
|
|
1401
|
+
export type PipelineManagerOptions<StepName extends string, State> = {
|
|
1402
|
+
pipeline: string;
|
|
1403
|
+
steps: readonly StepName[];
|
|
1404
|
+
handlers: Record<StepName, PipelineStepHandler<StepName, State>>;
|
|
1405
|
+
enqueue: (job: PipelineJob<StepName, State>, options?: { delaySeconds?: number }) => Promise<void>;
|
|
1406
|
+
makeContinuationId?: () => string;
|
|
1407
|
+
};
|
|
1408
|
+
|
|
1409
|
+
export class PipelineManager<StepName extends string, State> {
|
|
1410
|
+
constructor(private readonly options: PipelineManagerOptions<StepName, State>) {}
|
|
1411
|
+
|
|
1412
|
+
getNextStep(step: StepName): StepName | null {
|
|
1413
|
+
const index = this.options.steps.indexOf(step);
|
|
1414
|
+
if (index === -1) throw new Error(`Unknown pipeline step: ${step}`);
|
|
1415
|
+
return this.options.steps[index + 1] ?? null;
|
|
1416
|
+
}
|
|
1417
|
+
|
|
1418
|
+
async run(job: PipelineJob<StepName, State>): Promise<{
|
|
1419
|
+
nextStep: StepName | null;
|
|
1420
|
+
state: State;
|
|
1421
|
+
completion: PipelineCompletion | null;
|
|
1422
|
+
}> {
|
|
1423
|
+
if (job.pipeline !== this.options.pipeline) throw new Error(`Unexpected pipeline: ${job.pipeline}`);
|
|
1424
|
+
const stepIndex = this.options.steps.indexOf(job.step);
|
|
1425
|
+
if (stepIndex === -1) throw new Error(`Unknown pipeline step: ${job.step}`);
|
|
1426
|
+
|
|
1427
|
+
const defaultNextStep = this.options.steps[stepIndex + 1] ?? null;
|
|
1428
|
+
const result = await this.options.handlers[job.step]({
|
|
1429
|
+
job,
|
|
1430
|
+
stepIndex,
|
|
1431
|
+
isFinalStep: defaultNextStep === null,
|
|
1432
|
+
});
|
|
1433
|
+
const nextState = result?.state ?? job.state;
|
|
1434
|
+
const explicitNextStep = result?.nextStep;
|
|
1435
|
+
const nextStep =
|
|
1436
|
+
explicitNextStep === undefined
|
|
1437
|
+
? defaultNextStep
|
|
1438
|
+
: explicitNextStep === null
|
|
1439
|
+
? null
|
|
1440
|
+
: (explicitNextStep.step as StepName);
|
|
1441
|
+
const enqueuedState = explicitNextStep && explicitNextStep !== null ? (explicitNextStep.state ?? nextState) : nextState;
|
|
1442
|
+
|
|
1443
|
+
if (nextStep) {
|
|
1444
|
+
const continuationId = this.options.makeContinuationId?.();
|
|
1445
|
+
const nextJob = {
|
|
1446
|
+
pipeline: job.pipeline,
|
|
1447
|
+
runId: job.runId,
|
|
1448
|
+
step: nextStep,
|
|
1449
|
+
...(continuationId ? { continuationId } : {}),
|
|
1450
|
+
state: enqueuedState,
|
|
1451
|
+
};
|
|
1452
|
+
if (result?.nextStepDelaySeconds === undefined) await this.options.enqueue(nextJob);
|
|
1453
|
+
else await this.options.enqueue(nextJob, { delaySeconds: result.nextStepDelaySeconds });
|
|
210
1454
|
}
|
|
1455
|
+
|
|
1456
|
+
const completion =
|
|
1457
|
+
result?.completion ??
|
|
1458
|
+
(nextStep === null && defaultNextStep === null
|
|
1459
|
+
? {
|
|
1460
|
+
status: "completed",
|
|
1461
|
+
targetsTotal: 0,
|
|
1462
|
+
targetsSucceeded: 0,
|
|
1463
|
+
targetsFailed: 0,
|
|
1464
|
+
message: `${job.pipeline} pipeline completed`,
|
|
1465
|
+
}
|
|
1466
|
+
: null);
|
|
1467
|
+
|
|
1468
|
+
return { nextStep, state: enqueuedState, completion };
|
|
211
1469
|
}
|
|
212
1470
|
}
|
|
213
1471
|
|
|
1472
|
+
export type QueueSendOptions = { delaySeconds?: number };
|
|
1473
|
+
export type QueueBindingLike<TMessage> = {
|
|
1474
|
+
send(message: TMessage, options?: QueueSendOptions): Promise<void>;
|
|
1475
|
+
};
|
|
1476
|
+
|
|
1477
|
+
export type JobPublisher<TMessage> = {
|
|
1478
|
+
put(message: TMessage, options?: QueueSendOptions): Promise<void>;
|
|
1479
|
+
};
|
|
1480
|
+
|
|
1481
|
+
export type JobPublisherOptions<TMessage> = {
|
|
1482
|
+
queue?: QueueBindingLike<TMessage>;
|
|
1483
|
+
inline?: boolean;
|
|
1484
|
+
handleInline?: (message: TMessage) => Promise<void>;
|
|
1485
|
+
onQueued?: (message: TMessage, options: QueueSendOptions & { mode: "inline" | "queue" }) => Promise<void> | void;
|
|
1486
|
+
};
|
|
1487
|
+
|
|
1488
|
+
export function createJobPublisher<TMessage>(options: JobPublisherOptions<TMessage>): JobPublisher<TMessage> {
|
|
1489
|
+
return {
|
|
1490
|
+
async put(message, sendOptions = {}) {
|
|
1491
|
+
if (options.inline) {
|
|
1492
|
+
if (!options.handleInline) throw new Error("Inline job publisher requires handleInline.");
|
|
1493
|
+
await options.onQueued?.(message, { ...sendOptions, mode: "inline" });
|
|
1494
|
+
await options.handleInline(message);
|
|
1495
|
+
return;
|
|
1496
|
+
}
|
|
1497
|
+
if (!options.queue) throw new Error("Job queue is not configured.");
|
|
1498
|
+
await options.onQueued?.(message, { ...sendOptions, mode: "queue" });
|
|
1499
|
+
await options.queue.send(message, sendOptions);
|
|
1500
|
+
},
|
|
1501
|
+
};
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
export async function enqueueJobOperation<TPayload extends Record<string, unknown> = Record<string, unknown>>(
|
|
1505
|
+
jobs: Pick<DurableJobRepository, "enqueue">,
|
|
1506
|
+
input: EnqueueJobOperationInput<TPayload>,
|
|
1507
|
+
): Promise<JobRecord> {
|
|
1508
|
+
const keys = jobOperationKeys(input);
|
|
1509
|
+
return jobs.enqueue({
|
|
1510
|
+
jobType: input.jobType,
|
|
1511
|
+
orgId: input.orgId ?? null,
|
|
1512
|
+
siteId: input.siteId ?? null,
|
|
1513
|
+
parentJobId: input.parentJobId ?? null,
|
|
1514
|
+
payload: jobOperationPayload(input),
|
|
1515
|
+
lockKey: keys.lockKey,
|
|
1516
|
+
concurrencyKey: keys.concurrencyKey,
|
|
1517
|
+
concurrencyLimit: keys.concurrencyLimit,
|
|
1518
|
+
enqueuePolicy: input.dedupeActive === false ? "enqueue" : "dedupe_active",
|
|
1519
|
+
maxAttempts: input.maxAttempts ?? undefined,
|
|
1520
|
+
availableAt: input.availableAt ?? null,
|
|
1521
|
+
});
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
export function jobOperationKeys(input: {
|
|
1525
|
+
operationKey: string;
|
|
1526
|
+
siteId?: string | null;
|
|
1527
|
+
entity?: JobOperationEntity | null;
|
|
1528
|
+
dedupeActive?: boolean;
|
|
1529
|
+
concurrencyScope?: string | null;
|
|
1530
|
+
concurrencyLimit?: number | null;
|
|
1531
|
+
}): JobOperationKeys {
|
|
1532
|
+
const scope = keyPart(input.siteId ? `site:${input.siteId}` : "global");
|
|
1533
|
+
const operation = keyPart(input.operationKey);
|
|
1534
|
+
const entityKey = input.entity ? jobOperationEntityKey(input.entity) : null;
|
|
1535
|
+
const concurrencyScope = keyPart(input.concurrencyScope ?? input.operationKey);
|
|
1536
|
+
return {
|
|
1537
|
+
lockKey:
|
|
1538
|
+
input.dedupeActive === false
|
|
1539
|
+
? null
|
|
1540
|
+
: ["jobop", scope, operation, entityKey ? keyPart(entityKey) : "root"].join(":"),
|
|
1541
|
+
concurrencyKey: ["jobop", scope, concurrencyScope].join(":"),
|
|
1542
|
+
concurrencyLimit: normalizeOperationConcurrencyLimit(input.concurrencyLimit),
|
|
1543
|
+
};
|
|
1544
|
+
}
|
|
1545
|
+
|
|
214
1546
|
export const D1_JOBS_SCHEMA = `
|
|
215
1547
|
CREATE TABLE IF NOT EXISTS jobs (
|
|
216
1548
|
job_id TEXT PRIMARY KEY,
|
|
@@ -218,7 +1550,14 @@ CREATE TABLE IF NOT EXISTS jobs (
|
|
|
218
1550
|
status TEXT NOT NULL,
|
|
219
1551
|
payload_json TEXT NOT NULL DEFAULT '{}',
|
|
220
1552
|
result_json TEXT,
|
|
1553
|
+
org_id TEXT,
|
|
1554
|
+
site_id TEXT,
|
|
1555
|
+
metadata_json TEXT NOT NULL DEFAULT '{}',
|
|
1556
|
+
parent_job_id TEXT,
|
|
221
1557
|
lock_key TEXT,
|
|
1558
|
+
concurrency_key TEXT,
|
|
1559
|
+
concurrency_limit INTEGER,
|
|
1560
|
+
enqueue_policy TEXT NOT NULL DEFAULT 'enqueue',
|
|
222
1561
|
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
223
1562
|
max_attempts INTEGER NOT NULL DEFAULT 3,
|
|
224
1563
|
available_at TEXT,
|
|
@@ -230,17 +1569,29 @@ CREATE TABLE IF NOT EXISTS jobs (
|
|
|
230
1569
|
);
|
|
231
1570
|
CREATE INDEX IF NOT EXISTS jobs_status_available_idx ON jobs(status, available_at, created_at);
|
|
232
1571
|
CREATE INDEX IF NOT EXISTS jobs_type_status_idx ON jobs(job_type, status, created_at);
|
|
1572
|
+
CREATE INDEX IF NOT EXISTS jobs_org_status_idx ON jobs(org_id, status, created_at);
|
|
1573
|
+
CREATE INDEX IF NOT EXISTS jobs_site_status_idx ON jobs(site_id, status, created_at);
|
|
1574
|
+
CREATE INDEX IF NOT EXISTS jobs_parent_status_idx ON jobs(parent_job_id, status, created_at);
|
|
1575
|
+
CREATE INDEX IF NOT EXISTS jobs_concurrency_active_idx ON jobs(concurrency_key, status)
|
|
1576
|
+
WHERE concurrency_key IS NOT NULL AND status IN ('running', 'stopping');
|
|
233
1577
|
CREATE UNIQUE INDEX IF NOT EXISTS jobs_active_lock_key_idx ON jobs(lock_key)
|
|
234
|
-
WHERE lock_key IS NOT NULL AND status IN ('queued', 'running');
|
|
1578
|
+
WHERE lock_key IS NOT NULL AND status IN ('queued', 'running', 'stopping');
|
|
235
1579
|
`;
|
|
236
1580
|
|
|
237
1581
|
type JobRow = {
|
|
238
1582
|
jobId: string;
|
|
239
1583
|
jobType: string;
|
|
240
1584
|
status: JobStatus;
|
|
1585
|
+
orgId?: string | null;
|
|
1586
|
+
siteId?: string | null;
|
|
241
1587
|
payloadJson: string;
|
|
242
1588
|
resultJson: string | null;
|
|
1589
|
+
metadataJson?: string | null;
|
|
1590
|
+
parentJobId?: string | null;
|
|
243
1591
|
lockKey: string | null;
|
|
1592
|
+
concurrencyKey?: string | null;
|
|
1593
|
+
concurrencyLimit?: number | null;
|
|
1594
|
+
enqueuePolicy?: JobEnqueuePolicy | null;
|
|
244
1595
|
attemptCount: number;
|
|
245
1596
|
maxAttempts: number;
|
|
246
1597
|
availableAt: string | null;
|
|
@@ -251,6 +1602,20 @@ type JobRow = {
|
|
|
251
1602
|
updatedAt: string;
|
|
252
1603
|
};
|
|
253
1604
|
|
|
1605
|
+
type JobTreeRow = JobRow & {
|
|
1606
|
+
depth: number | string;
|
|
1607
|
+
};
|
|
1608
|
+
|
|
1609
|
+
type JobActivityEventRow = {
|
|
1610
|
+
opsEventId: string;
|
|
1611
|
+
jobId: string | null;
|
|
1612
|
+
level: string;
|
|
1613
|
+
eventType: string;
|
|
1614
|
+
message: string;
|
|
1615
|
+
metadataJson: string | null;
|
|
1616
|
+
createdAt: string;
|
|
1617
|
+
};
|
|
1618
|
+
|
|
254
1619
|
function rowToJob<TPayload = unknown, TResult = unknown>(row: JobRow): JobRecord<TPayload, TResult> {
|
|
255
1620
|
return {
|
|
256
1621
|
jobId: row.jobId,
|
|
@@ -258,7 +1623,14 @@ function rowToJob<TPayload = unknown, TResult = unknown>(row: JobRow): JobRecord
|
|
|
258
1623
|
status: row.status,
|
|
259
1624
|
payload: parseJsonColumn<TPayload>(row.payloadJson, {} as TPayload),
|
|
260
1625
|
result: parseJsonColumn<TResult | null>(row.resultJson, null),
|
|
1626
|
+
orgId: row.orgId ?? null,
|
|
1627
|
+
siteId: row.siteId ?? null,
|
|
1628
|
+
parentJobId: row.parentJobId ?? null,
|
|
261
1629
|
lockKey: row.lockKey,
|
|
1630
|
+
concurrencyKey: row.concurrencyKey ?? null,
|
|
1631
|
+
concurrencyLimit: row.concurrencyLimit === undefined || row.concurrencyLimit === null ? null : Number(row.concurrencyLimit),
|
|
1632
|
+
enqueuePolicy: row.enqueuePolicy ?? "enqueue",
|
|
1633
|
+
metadata: parseJsonColumn<Record<string, unknown>>(row.metadataJson, {}),
|
|
262
1634
|
attemptCount: Number(row.attemptCount),
|
|
263
1635
|
maxAttempts: Number(row.maxAttempts),
|
|
264
1636
|
availableAt: row.availableAt,
|
|
@@ -274,3 +1646,225 @@ function normalizeJobResult<TResult>(output: JobHandlerResult<TResult> | TResult
|
|
|
274
1646
|
if (output && typeof output === "object" && "kind" in output) return output as JobHandlerResult<TResult>;
|
|
275
1647
|
return { kind: "done", result: (output ?? null) as TResult | null };
|
|
276
1648
|
}
|
|
1649
|
+
|
|
1650
|
+
function quotePgIdent(value: string): string {
|
|
1651
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(value)) throw new Error(`Invalid Postgres identifier: ${value}`);
|
|
1652
|
+
return `"${value}"`;
|
|
1653
|
+
}
|
|
1654
|
+
|
|
1655
|
+
function jsonText(value: unknown): string | null {
|
|
1656
|
+
if (value === null || value === undefined) return null;
|
|
1657
|
+
return typeof value === "string" ? value : JSON.stringify(value);
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
function jsonParam(value: unknown): string {
|
|
1661
|
+
return JSON.stringify(value ?? null);
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
function queueNameFromPayload(value: unknown): string | null {
|
|
1665
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
1666
|
+
return stringValue((value as Record<string, unknown>).queueName);
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
function dateText(value: unknown): string | null {
|
|
1670
|
+
if (value === null || value === undefined) return null;
|
|
1671
|
+
if (value instanceof Date) return value.toISOString();
|
|
1672
|
+
if (typeof value === "string") return value;
|
|
1673
|
+
return String(value);
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
function rowToActivityEvent(row: JobActivityEventRow): JobActivityEvent {
|
|
1677
|
+
const level = row.level === "warn" || row.level === "error" ? row.level : "info";
|
|
1678
|
+
return {
|
|
1679
|
+
opsEventId: row.opsEventId,
|
|
1680
|
+
jobId: row.jobId,
|
|
1681
|
+
level,
|
|
1682
|
+
eventType: row.eventType,
|
|
1683
|
+
message: row.message,
|
|
1684
|
+
metadata: parseJsonColumn<Record<string, unknown> | null>(row.metadataJson, null),
|
|
1685
|
+
createdAt: row.createdAt,
|
|
1686
|
+
};
|
|
1687
|
+
}
|
|
1688
|
+
|
|
1689
|
+
function summarizeJobs(parentJobId: string, jobs: readonly JobRecord[]): JobChildrenSummary {
|
|
1690
|
+
const summary: JobChildrenSummary = {
|
|
1691
|
+
parentJobId,
|
|
1692
|
+
total: jobs.length,
|
|
1693
|
+
queued: 0,
|
|
1694
|
+
running: 0,
|
|
1695
|
+
stopping: 0,
|
|
1696
|
+
succeeded: 0,
|
|
1697
|
+
failed: 0,
|
|
1698
|
+
stopped: 0,
|
|
1699
|
+
active: 0,
|
|
1700
|
+
terminal: 0,
|
|
1701
|
+
allTerminal: jobs.length > 0,
|
|
1702
|
+
};
|
|
1703
|
+
for (const job of jobs) {
|
|
1704
|
+
summary[job.status] += 1;
|
|
1705
|
+
if (isTerminalJobStatus(job.status)) summary.terminal += 1;
|
|
1706
|
+
else summary.active += 1;
|
|
1707
|
+
}
|
|
1708
|
+
summary.allTerminal = summary.total > 0 && summary.terminal === summary.total;
|
|
1709
|
+
return summary;
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
function rollupJobTreeStatus(jobs: readonly JobRecord[]): JobTreeRollupStatus {
|
|
1713
|
+
if (jobs.some((job) => job.status === "failed")) return "failed";
|
|
1714
|
+
if (jobs.some((job) => job.status === "stopped")) return "stopped";
|
|
1715
|
+
if (jobs.some((job) => job.status === "running" || job.status === "stopping")) return "running";
|
|
1716
|
+
if (jobs.some((job) => job.status === "queued")) return "queued";
|
|
1717
|
+
return "succeeded";
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
function latestJobError(jobs: readonly JobRecord[]): string | null {
|
|
1721
|
+
return (
|
|
1722
|
+
[...jobs]
|
|
1723
|
+
.sort((a, b) => b.updatedAt.localeCompare(a.updatedAt))
|
|
1724
|
+
.find((job) => job.lastError && job.lastError.trim().length > 0)?.lastError ?? null
|
|
1725
|
+
);
|
|
1726
|
+
}
|
|
1727
|
+
|
|
1728
|
+
function jobEntityState(job: JobRecord, depth: number): JobEntityState {
|
|
1729
|
+
const payload = isRecord(job.payload) ? job.payload : {};
|
|
1730
|
+
const entityType = stringValue(payload.entityType);
|
|
1731
|
+
const entityId = stringValue(payload.entityId);
|
|
1732
|
+
const entityKey = stringValue(payload.entityKey) ?? (entityType && entityId ? `${entityType}:${entityId}` : job.jobId);
|
|
1733
|
+
return {
|
|
1734
|
+
jobId: job.jobId,
|
|
1735
|
+
jobType: job.jobType,
|
|
1736
|
+
status: job.status,
|
|
1737
|
+
entityKey,
|
|
1738
|
+
entityType,
|
|
1739
|
+
entityId,
|
|
1740
|
+
label: stringValue(payload.entityLabel),
|
|
1741
|
+
lastError: job.lastError,
|
|
1742
|
+
result: job.result,
|
|
1743
|
+
depth,
|
|
1744
|
+
};
|
|
1745
|
+
}
|
|
1746
|
+
|
|
1747
|
+
function emptyChildrenSummary(parentJobId: string): JobChildrenSummary {
|
|
1748
|
+
return summarizeJobs(parentJobId, []);
|
|
1749
|
+
}
|
|
1750
|
+
|
|
1751
|
+
function requeueParent<TPayload>(
|
|
1752
|
+
payload: TPayload,
|
|
1753
|
+
children: readonly JobRecord[],
|
|
1754
|
+
summary: JobChildrenSummary,
|
|
1755
|
+
input: ParentJobOrchestrationInput<TPayload>,
|
|
1756
|
+
): JobHandlerResult {
|
|
1757
|
+
const pollSeconds = Math.max(1, Math.floor(input.pollSeconds ?? 10));
|
|
1758
|
+
const now = input.now?.() ?? new Date();
|
|
1759
|
+
return {
|
|
1760
|
+
kind: "requeue",
|
|
1761
|
+
payload,
|
|
1762
|
+
result: {
|
|
1763
|
+
stage: "waiting_children",
|
|
1764
|
+
failureMode: input.failureMode ?? "fail_fast",
|
|
1765
|
+
queuedSiblingPolicy: input.queuedSiblingPolicy ?? "fail",
|
|
1766
|
+
childJobIds: children.map((child) => child.jobId),
|
|
1767
|
+
summary,
|
|
1768
|
+
} satisfies ParentJobOrchestrationResult,
|
|
1769
|
+
availableAt: new Date(now.getTime() + pollSeconds * 1000).toISOString(),
|
|
1770
|
+
};
|
|
1771
|
+
}
|
|
1772
|
+
|
|
1773
|
+
function isTerminalJobStatus(status: JobStatus): status is TerminalJobStatus {
|
|
1774
|
+
return status === "succeeded" || status === "failed" || status === "stopped";
|
|
1775
|
+
}
|
|
1776
|
+
|
|
1777
|
+
function normalizeConcurrencyLimit(value: number | null | undefined): number | null {
|
|
1778
|
+
if (value === null || value === undefined) return null;
|
|
1779
|
+
return Math.max(1, Math.floor(value));
|
|
1780
|
+
}
|
|
1781
|
+
|
|
1782
|
+
function jobOperationPayload<TPayload extends Record<string, unknown>>(
|
|
1783
|
+
input: EnqueueJobOperationInput<TPayload>,
|
|
1784
|
+
): Record<string, unknown> {
|
|
1785
|
+
return {
|
|
1786
|
+
...(input.payload ?? {}),
|
|
1787
|
+
operationKey: input.operationKey,
|
|
1788
|
+
...(input.entity
|
|
1789
|
+
? {
|
|
1790
|
+
entityType: input.entity.entityType,
|
|
1791
|
+
entityId: input.entity.entityId,
|
|
1792
|
+
entityKey: jobOperationEntityKey(input.entity),
|
|
1793
|
+
...(input.entity.entityLabel ? { entityLabel: input.entity.entityLabel } : {}),
|
|
1794
|
+
}
|
|
1795
|
+
: {}),
|
|
1796
|
+
};
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
function jobOperationEntityKey(entity: JobOperationEntity): string {
|
|
1800
|
+
return entity.entityKey?.trim() || `${entity.entityType}:${entity.entityId}`;
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
function keyPart(value: string): string {
|
|
1804
|
+
return value
|
|
1805
|
+
.trim()
|
|
1806
|
+
.toLowerCase()
|
|
1807
|
+
.replace(/[^a-z0-9:_-]+/g, "-")
|
|
1808
|
+
.replace(/-+/g, "-")
|
|
1809
|
+
.replace(/^-|-$/g, "");
|
|
1810
|
+
}
|
|
1811
|
+
|
|
1812
|
+
function normalizeOperationConcurrencyLimit(value: number | null | undefined): number {
|
|
1813
|
+
if (!Number.isFinite(value) || !value) return 1;
|
|
1814
|
+
return Math.max(1, Math.floor(value));
|
|
1815
|
+
}
|
|
1816
|
+
|
|
1817
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
1818
|
+
return !!value && typeof value === "object" && !Array.isArray(value);
|
|
1819
|
+
}
|
|
1820
|
+
|
|
1821
|
+
function stringValue(value: unknown): string | null {
|
|
1822
|
+
return typeof value === "string" && value.trim().length > 0 ? value : null;
|
|
1823
|
+
}
|
|
1824
|
+
|
|
1825
|
+
function isRequeueInput(value: unknown): value is {
|
|
1826
|
+
payload?: unknown;
|
|
1827
|
+
result?: unknown;
|
|
1828
|
+
availableAt?: string | null;
|
|
1829
|
+
lastError?: string | null;
|
|
1830
|
+
} {
|
|
1831
|
+
return (
|
|
1832
|
+
!!value &&
|
|
1833
|
+
typeof value === "object" &&
|
|
1834
|
+
("payload" in value || "result" in value || "availableAt" in value || "lastError" in value)
|
|
1835
|
+
);
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
function isStaleRunningJob(job: JobRecord, staleRunningAfterSeconds: number | undefined): boolean {
|
|
1839
|
+
if (!staleRunningAfterSeconds || staleRunningAfterSeconds <= 0 || !job.startedAt) return false;
|
|
1840
|
+
const startedAt = Date.parse(job.startedAt);
|
|
1841
|
+
if (!Number.isFinite(startedAt)) return false;
|
|
1842
|
+
return Date.now() - startedAt >= staleRunningAfterSeconds * 1000;
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
function errorMessage(error: unknown): string {
|
|
1846
|
+
return error instanceof Error ? error.message : String(error);
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
class DurableJobEvents<TPayload = unknown, TResult = unknown> {
|
|
1850
|
+
constructor(
|
|
1851
|
+
private readonly job: JobRecord<TPayload, TResult>,
|
|
1852
|
+
private readonly sink: DurableJobEventSink<TPayload, TResult> | undefined,
|
|
1853
|
+
) {}
|
|
1854
|
+
|
|
1855
|
+
async info(input: Omit<DurableJobEventInput, "severity">): Promise<void> {
|
|
1856
|
+
await this.emit({ ...input, severity: "info" });
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
async warn(input: Omit<DurableJobEventInput, "severity">): Promise<void> {
|
|
1860
|
+
await this.emit({ ...input, severity: "warn" });
|
|
1861
|
+
}
|
|
1862
|
+
|
|
1863
|
+
async error(input: Omit<DurableJobEventInput, "severity">): Promise<void> {
|
|
1864
|
+
await this.emit({ ...input, severity: "error" });
|
|
1865
|
+
}
|
|
1866
|
+
|
|
1867
|
+
private async emit(input: DurableJobEventInput): Promise<void> {
|
|
1868
|
+
await this.sink?.(this.job, input);
|
|
1869
|
+
}
|
|
1870
|
+
}
|