@coji/durably 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -1,9 +1,9 @@
1
1
  import {
2
2
  withLogPersistence
3
- } from "./chunk-UCUP6NMJ.js";
3
+ } from "./chunk-L42OCQEV.js";
4
4
 
5
5
  // src/durably.ts
6
- import { Kysely } from "kysely";
6
+ import { Kysely, sql as sql5 } from "kysely";
7
7
  import { monotonicFactory as monotonicFactory2 } from "ulidx";
8
8
 
9
9
  // src/errors.ts
@@ -19,197 +19,54 @@ var LeaseLostError = class extends Error {
19
19
  this.name = "LeaseLostError";
20
20
  }
21
21
  };
22
+ var DurablyError = class extends Error {
23
+ statusCode;
24
+ constructor(message, statusCode) {
25
+ super(message);
26
+ this.name = "DurablyError";
27
+ this.statusCode = statusCode;
28
+ }
29
+ };
30
+ var NotFoundError = class extends DurablyError {
31
+ constructor(message) {
32
+ super(message, 404);
33
+ this.name = "NotFoundError";
34
+ }
35
+ };
36
+ var ValidationError = class extends DurablyError {
37
+ constructor(message) {
38
+ super(message, 400);
39
+ this.name = "ValidationError";
40
+ }
41
+ };
42
+ var ConflictError = class extends DurablyError {
43
+ constructor(message) {
44
+ super(message, 409);
45
+ this.name = "ConflictError";
46
+ }
47
+ };
22
48
  function getErrorMessage(error) {
23
49
  return error instanceof Error ? error.message : String(error);
24
50
  }
25
-
26
- // src/context.ts
27
- var LEASE_LOST = "lease-lost";
28
- function createStepContext(run, jobName, leaseGeneration, storage, eventEmitter) {
29
- let stepIndex = run.currentStepIndex;
30
- let currentStepName = null;
31
- const controller = new AbortController();
32
- function abortForLeaseLoss() {
33
- if (!controller.signal.aborted) {
34
- controller.abort(LEASE_LOST);
35
- }
36
- }
37
- function throwIfAborted() {
38
- if (!controller.signal.aborted) {
39
- return;
40
- }
41
- if (controller.signal.reason === LEASE_LOST) {
42
- throw new LeaseLostError(run.id);
43
- }
44
- throw new CancelledError(run.id);
45
- }
46
- const unsubscribe = eventEmitter.on("run:cancel", (event) => {
47
- if (event.runId === run.id) {
48
- controller.abort();
49
- }
50
- });
51
- const step = {
52
- get runId() {
53
- return run.id;
54
- },
55
- get signal() {
56
- return controller.signal;
57
- },
58
- isAborted() {
59
- return controller.signal.aborted;
60
- },
61
- throwIfAborted() {
62
- throwIfAborted();
63
- },
64
- async run(name, fn) {
65
- throwIfAborted();
66
- const currentRun = await storage.getRun(run.id);
67
- if (currentRun?.status === "cancelled") {
68
- controller.abort();
69
- throwIfAborted();
70
- }
71
- if (currentRun && (currentRun.status === "leased" && currentRun.leaseGeneration !== leaseGeneration || currentRun.status === "completed" || currentRun.status === "failed")) {
72
- abortForLeaseLoss();
73
- throwIfAborted();
74
- }
75
- throwIfAborted();
76
- const existingStep = await storage.getCompletedStep(run.id, name);
77
- if (existingStep) {
78
- stepIndex++;
79
- return existingStep.output;
80
- }
81
- currentStepName = name;
82
- const startedAt = (/* @__PURE__ */ new Date()).toISOString();
83
- const startTime = Date.now();
84
- eventEmitter.emit({
85
- type: "step:start",
86
- runId: run.id,
87
- jobName,
88
- stepName: name,
89
- stepIndex,
90
- labels: run.labels
91
- });
92
- try {
93
- const result = await fn(controller.signal);
94
- throwIfAborted();
95
- const savedStep = await storage.persistStep(run.id, leaseGeneration, {
96
- name,
97
- index: stepIndex,
98
- status: "completed",
99
- output: result,
100
- startedAt
101
- });
102
- if (!savedStep) {
103
- abortForLeaseLoss();
104
- throwIfAborted();
105
- }
106
- stepIndex++;
107
- eventEmitter.emit({
108
- type: "step:complete",
109
- runId: run.id,
110
- jobName,
111
- stepName: name,
112
- stepIndex: stepIndex - 1,
113
- output: result,
114
- duration: Date.now() - startTime,
115
- labels: run.labels
116
- });
117
- return result;
118
- } catch (error) {
119
- if (error instanceof LeaseLostError) {
120
- throw error;
121
- }
122
- const isLeaseLost = controller.signal.aborted && controller.signal.reason === LEASE_LOST;
123
- if (isLeaseLost) {
124
- throw new LeaseLostError(run.id);
125
- }
126
- const isCancelled = controller.signal.aborted;
127
- const errorMessage = getErrorMessage(error);
128
- const savedStep = await storage.persistStep(run.id, leaseGeneration, {
129
- name,
130
- index: stepIndex,
131
- status: isCancelled ? "cancelled" : "failed",
132
- error: errorMessage,
133
- startedAt
134
- });
135
- if (!savedStep) {
136
- abortForLeaseLoss();
137
- throw new LeaseLostError(run.id);
138
- }
139
- eventEmitter.emit({
140
- ...isCancelled ? { type: "step:cancel" } : { type: "step:fail", error: errorMessage },
141
- runId: run.id,
142
- jobName,
143
- stepName: name,
144
- stepIndex,
145
- labels: run.labels
146
- });
147
- if (isCancelled) {
148
- throwIfAborted();
149
- }
150
- throw error;
151
- } finally {
152
- currentStepName = null;
153
- }
154
- },
155
- progress(current, total, message) {
156
- const progressData = { current, total, message };
157
- storage.updateProgress(run.id, leaseGeneration, progressData);
158
- eventEmitter.emit({
159
- type: "run:progress",
160
- runId: run.id,
161
- jobName,
162
- progress: progressData,
163
- labels: run.labels
164
- });
165
- },
166
- log: {
167
- info(message, data) {
168
- eventEmitter.emit({
169
- type: "log:write",
170
- runId: run.id,
171
- jobName,
172
- labels: run.labels,
173
- stepName: currentStepName,
174
- level: "info",
175
- message,
176
- data
177
- });
178
- },
179
- warn(message, data) {
180
- eventEmitter.emit({
181
- type: "log:write",
182
- runId: run.id,
183
- jobName,
184
- labels: run.labels,
185
- stepName: currentStepName,
186
- level: "warn",
187
- message,
188
- data
189
- });
190
- },
191
- error(message, data) {
192
- eventEmitter.emit({
193
- type: "log:write",
194
- runId: run.id,
195
- jobName,
196
- labels: run.labels,
197
- stepName: currentStepName,
198
- level: "error",
199
- message,
200
- data
201
- });
202
- }
203
- }
204
- };
205
- return {
206
- step,
207
- abortLeaseOwnership: abortForLeaseLoss,
208
- dispose: unsubscribe
209
- };
51
+ function toError(error) {
52
+ return error instanceof Error ? error : new Error(String(error));
210
53
  }
211
54
 
212
55
  // src/events.ts
56
+ var DOMAIN_EVENT_TYPE_VALUES = [
57
+ "run:trigger",
58
+ "run:coalesced",
59
+ "run:complete",
60
+ "run:fail",
61
+ "run:cancel",
62
+ "run:delete"
63
+ ];
64
+ var DOMAIN_EVENT_TYPES = new Set(
65
+ DOMAIN_EVENT_TYPE_VALUES
66
+ );
67
+ function isDomainEvent(event) {
68
+ return DOMAIN_EVENT_TYPES.has(event.type);
69
+ }
213
70
  function createEventEmitter() {
214
71
  const listeners = /* @__PURE__ */ new Map();
215
72
  let sequence = 0;
@@ -239,16 +96,16 @@ function createEventEmitter() {
239
96
  if (!typeListeners) {
240
97
  return;
241
98
  }
99
+ const reportError = (error) => errorHandler?.(toError(error), fullEvent);
242
100
  for (const listener of typeListeners) {
243
101
  try {
244
- listener(fullEvent);
245
- } catch (error) {
246
- if (errorHandler) {
247
- errorHandler(
248
- error instanceof Error ? error : new Error(String(error)),
249
- fullEvent
250
- );
102
+ const result = listener(fullEvent);
103
+ if (result != null && typeof result.then === "function") {
104
+ ;
105
+ result.catch(reportError);
251
106
  }
107
+ } catch (error) {
108
+ reportError(error);
252
109
  }
253
110
  }
254
111
  }
@@ -257,13 +114,16 @@ function createEventEmitter() {
257
114
 
258
115
  // src/job.ts
259
116
  import { prettifyError } from "zod";
117
+ var DEFAULT_WAIT_POLLING_INTERVAL_MS = 1e3;
260
118
  var noop = () => {
261
119
  };
262
120
  function validateJobInputOrThrow(schema, input, context) {
263
121
  const result = schema.safeParse(input);
264
122
  if (!result.success) {
265
123
  const prefix = context ? `${context}: ` : "";
266
- throw new Error(`${prefix}Invalid input: ${prettifyError(result.error)}`);
124
+ throw new ValidationError(
125
+ `${prefix}Invalid input: ${prettifyError(result.error)}`
126
+ );
267
127
  }
268
128
  return result.data;
269
129
  }
@@ -281,7 +141,157 @@ function createJobRegistry() {
281
141
  }
282
142
  };
283
143
  }
284
- function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema) {
144
+ function waitForRunCompletion(runId, storage, eventEmitter, options, timeoutMessagePrefix = "waitForRun") {
145
+ return new Promise((resolve, reject) => {
146
+ let timeoutId;
147
+ let pollIntervalId;
148
+ let resolved = false;
149
+ let pollInFlight = false;
150
+ const unsubscribes = [];
151
+ const pollingMs = options?.pollingIntervalMs ?? DEFAULT_WAIT_POLLING_INTERVAL_MS;
152
+ if (!Number.isFinite(pollingMs) || pollingMs <= 0) {
153
+ throw new ValidationError(
154
+ "pollingIntervalMs must be a positive finite number"
155
+ );
156
+ }
157
+ const cleanup = () => {
158
+ if (resolved) return;
159
+ resolved = true;
160
+ for (const unsub of unsubscribes) unsub();
161
+ if (timeoutId !== void 0) {
162
+ clearTimeout(timeoutId);
163
+ timeoutId = void 0;
164
+ }
165
+ if (pollIntervalId !== void 0) {
166
+ clearInterval(pollIntervalId);
167
+ pollIntervalId = void 0;
168
+ }
169
+ };
170
+ const settleFromStorage = (run) => {
171
+ if (resolved) return;
172
+ if (!run) {
173
+ cleanup();
174
+ reject(new NotFoundError(`Run not found: ${runId}`));
175
+ return;
176
+ }
177
+ if (run.status === "completed") {
178
+ cleanup();
179
+ resolve(run);
180
+ return;
181
+ }
182
+ if (run.status === "failed") {
183
+ cleanup();
184
+ reject(new Error(run.error || "Run failed"));
185
+ return;
186
+ }
187
+ if (run.status === "cancelled") {
188
+ cleanup();
189
+ reject(new CancelledError(runId));
190
+ return;
191
+ }
192
+ };
193
+ const poll = () => {
194
+ if (resolved || pollInFlight) return;
195
+ pollInFlight = true;
196
+ void storage.getRun(runId).then((run) => {
197
+ if (resolved) return;
198
+ settleFromStorage(run);
199
+ }).catch((err) => {
200
+ if (resolved) return;
201
+ cleanup();
202
+ reject(toError(err));
203
+ }).finally(() => {
204
+ pollInFlight = false;
205
+ });
206
+ };
207
+ unsubscribes.push(
208
+ eventEmitter.on("run:complete", (event) => {
209
+ if (event.runId !== runId || resolved) return;
210
+ cleanup();
211
+ storage.getRun(runId).then((run) => {
212
+ if (run) resolve(run);
213
+ else reject(new NotFoundError(`Run not found: ${runId}`));
214
+ }).catch((err) => reject(toError(err)));
215
+ })
216
+ );
217
+ unsubscribes.push(
218
+ eventEmitter.on("run:fail", (event) => {
219
+ if (event.runId !== runId || resolved) return;
220
+ cleanup();
221
+ reject(new Error(event.error));
222
+ })
223
+ );
224
+ unsubscribes.push(
225
+ eventEmitter.on("run:cancel", (event) => {
226
+ if (event.runId !== runId || resolved) return;
227
+ cleanup();
228
+ reject(new CancelledError(runId));
229
+ })
230
+ );
231
+ if (options?.onProgress) {
232
+ const onProgress = options.onProgress;
233
+ unsubscribes.push(
234
+ eventEmitter.on("run:progress", (event) => {
235
+ if (event.runId !== runId || resolved) return;
236
+ void Promise.resolve(onProgress(event.progress)).catch(noop);
237
+ })
238
+ );
239
+ }
240
+ if (options?.onLog) {
241
+ const onLog = options.onLog;
242
+ unsubscribes.push(
243
+ eventEmitter.on("log:write", (event) => {
244
+ if (event.runId !== runId || resolved) return;
245
+ const { level, message, data, stepName } = event;
246
+ void Promise.resolve(onLog({ level, message, data, stepName })).catch(
247
+ noop
248
+ );
249
+ })
250
+ );
251
+ }
252
+ storage.getRun(runId).then((currentRun) => {
253
+ if (resolved) return;
254
+ if (!currentRun) {
255
+ cleanup();
256
+ reject(new NotFoundError(`Run not found: ${runId}`));
257
+ return;
258
+ }
259
+ if (currentRun.status === "completed") {
260
+ cleanup();
261
+ resolve(currentRun);
262
+ return;
263
+ }
264
+ if (currentRun.status === "failed") {
265
+ cleanup();
266
+ reject(new Error(currentRun.error || "Run failed"));
267
+ return;
268
+ }
269
+ if (currentRun.status === "cancelled") {
270
+ cleanup();
271
+ reject(new CancelledError(runId));
272
+ return;
273
+ }
274
+ pollIntervalId = setInterval(poll, pollingMs);
275
+ }).catch((error) => {
276
+ if (resolved) return;
277
+ cleanup();
278
+ reject(toError(error));
279
+ });
280
+ if (options?.timeout !== void 0) {
281
+ timeoutId = setTimeout(() => {
282
+ if (!resolved) {
283
+ cleanup();
284
+ reject(
285
+ new Error(
286
+ `${timeoutMessagePrefix} timeout after ${options.timeout}ms`
287
+ )
288
+ );
289
+ }
290
+ }, options.timeout);
291
+ }
292
+ });
293
+ }
294
+ function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema, pollingIntervalMs) {
285
295
  const existingJob = registry.get(jobDef.name);
286
296
  if (existingJob) {
287
297
  if (existingJob.jobDef === jobDef) {
@@ -293,113 +303,77 @@ function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema)
293
303
  }
294
304
  const inputSchema = jobDef.input;
295
305
  const outputSchema = jobDef.output;
296
- const handle = {
297
- name: jobDef.name,
298
- async trigger(input, options) {
299
- const validatedInput = validateJobInputOrThrow(inputSchema, input);
300
- if (labelsSchema && options?.labels) {
301
- validateJobInputOrThrow(labelsSchema, options.labels, "labels");
302
- }
303
- const run = await storage.enqueue({
304
- jobName: jobDef.name,
305
- input: validatedInput,
306
- idempotencyKey: options?.idempotencyKey,
307
- concurrencyKey: options?.concurrencyKey,
308
- labels: options?.labels
309
- });
306
+ function validateCoalesceOption(coalesce, concurrencyKey, context) {
307
+ if (coalesce === void 0) return;
308
+ const suffix = context ? ` ${context}` : "";
309
+ if (coalesce !== "skip") {
310
+ throw new ValidationError(
311
+ `Invalid coalesce value${suffix}: '${coalesce}'. Valid values: 'skip'`
312
+ );
313
+ }
314
+ if (!concurrencyKey) {
315
+ throw new ValidationError(`coalesce requires concurrencyKey${suffix}`);
316
+ }
317
+ }
318
+ function emitDispositionEvent(disposition, run, input, labels) {
319
+ if (disposition === "created") {
310
320
  eventEmitter.emit({
311
321
  type: "run:trigger",
312
322
  runId: run.id,
313
323
  jobName: jobDef.name,
314
- input: validatedInput,
324
+ input,
315
325
  labels: run.labels
316
326
  });
317
- return run;
327
+ } else if (disposition === "coalesced") {
328
+ eventEmitter.emit({
329
+ type: "run:coalesced",
330
+ runId: run.id,
331
+ jobName: jobDef.name,
332
+ labels: run.labels,
333
+ skippedInput: input,
334
+ skippedLabels: labels ?? {}
335
+ });
336
+ }
337
+ }
338
+ const handle = {
339
+ name: jobDef.name,
340
+ async trigger(input, options) {
341
+ validateCoalesceOption(options?.coalesce, options?.concurrencyKey);
342
+ const validatedInput = validateJobInputOrThrow(inputSchema, input);
343
+ const validatedLabels = labelsSchema && options?.labels ? validateJobInputOrThrow(labelsSchema, options.labels, "labels") : options?.labels;
344
+ const { run, disposition } = await storage.enqueue({
345
+ jobName: jobDef.name,
346
+ input: validatedInput,
347
+ idempotencyKey: options?.idempotencyKey,
348
+ concurrencyKey: options?.concurrencyKey,
349
+ labels: validatedLabels,
350
+ coalesce: options?.coalesce
351
+ });
352
+ emitDispositionEvent(
353
+ disposition,
354
+ run,
355
+ validatedInput,
356
+ validatedLabels
357
+ );
358
+ return { ...run, disposition };
318
359
  },
319
360
  async triggerAndWait(input, options) {
320
361
  const run = await this.trigger(input, options);
321
- return new Promise((resolve, reject) => {
322
- let timeoutId;
323
- let resolved = false;
324
- const unsubscribes = [];
325
- const cleanup = () => {
326
- if (resolved) return;
327
- resolved = true;
328
- for (const unsub of unsubscribes) unsub();
329
- if (timeoutId) {
330
- clearTimeout(timeoutId);
331
- }
332
- };
333
- unsubscribes.push(
334
- eventEmitter.on("run:complete", (event) => {
335
- if (event.runId === run.id && !resolved) {
336
- cleanup();
337
- resolve({
338
- id: run.id,
339
- output: event.output
340
- });
341
- }
342
- })
343
- );
344
- unsubscribes.push(
345
- eventEmitter.on("run:fail", (event) => {
346
- if (event.runId === run.id && !resolved) {
347
- cleanup();
348
- reject(new Error(event.error));
349
- }
350
- })
351
- );
352
- if (options?.onProgress) {
353
- const onProgress = options.onProgress;
354
- unsubscribes.push(
355
- eventEmitter.on("run:progress", (event) => {
356
- if (event.runId === run.id && !resolved) {
357
- void Promise.resolve(onProgress(event.progress)).catch(noop);
358
- }
359
- })
360
- );
361
- }
362
- if (options?.onLog) {
363
- const onLog = options.onLog;
364
- unsubscribes.push(
365
- eventEmitter.on("log:write", (event) => {
366
- if (event.runId === run.id && !resolved) {
367
- const { level, message, data, stepName } = event;
368
- void Promise.resolve(
369
- onLog({ level, message, data, stepName })
370
- ).catch(noop);
371
- }
372
- })
373
- );
374
- }
375
- storage.getRun(run.id).then((currentRun) => {
376
- if (resolved || !currentRun) return;
377
- if (currentRun.status === "completed") {
378
- cleanup();
379
- resolve({
380
- id: run.id,
381
- output: currentRun.output
382
- });
383
- } else if (currentRun.status === "failed") {
384
- cleanup();
385
- reject(new Error(currentRun.error || "Run failed"));
386
- }
387
- }).catch((error) => {
388
- if (resolved) return;
389
- cleanup();
390
- reject(error instanceof Error ? error : new Error(String(error)));
391
- });
392
- if (options?.timeout !== void 0) {
393
- timeoutId = setTimeout(() => {
394
- if (!resolved) {
395
- cleanup();
396
- reject(
397
- new Error(`triggerAndWait timeout after ${options.timeout}ms`)
398
- );
399
- }
400
- }, options.timeout);
401
- }
402
- });
362
+ const completedRun = await waitForRunCompletion(
363
+ run.id,
364
+ storage,
365
+ eventEmitter,
366
+ {
367
+ ...options,
368
+ pollingIntervalMs: options?.pollingIntervalMs ?? pollingIntervalMs
369
+ },
370
+ "triggerAndWait"
371
+ );
372
+ return {
373
+ id: run.id,
374
+ output: completedRun.output,
375
+ disposition: run.disposition
376
+ };
403
377
  },
404
378
  async batchTrigger(inputs) {
405
379
  if (inputs.length === 0) {
@@ -413,42 +387,51 @@ function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema)
413
387
  });
414
388
  const validated = [];
415
389
  for (let i = 0; i < normalized.length; i++) {
390
+ const opts = normalized[i].options;
391
+ validateCoalesceOption(
392
+ opts?.coalesce,
393
+ opts?.concurrencyKey,
394
+ `at index ${i}`
395
+ );
416
396
  const validatedInput = validateJobInputOrThrow(
417
397
  inputSchema,
418
398
  normalized[i].input,
419
399
  `at index ${i}`
420
400
  );
421
- if (labelsSchema && normalized[i].options?.labels) {
422
- validateJobInputOrThrow(
423
- labelsSchema,
424
- normalized[i].options?.labels,
425
- `labels at index ${i}`
426
- );
427
- }
401
+ const validatedLabels = labelsSchema && opts?.labels ? validateJobInputOrThrow(
402
+ labelsSchema,
403
+ opts.labels,
404
+ `labels at index ${i}`
405
+ ) : opts?.labels;
428
406
  validated.push({
429
407
  input: validatedInput,
430
- options: normalized[i].options
408
+ options: opts ? { ...opts, labels: validatedLabels } : opts
431
409
  });
432
410
  }
433
- const runs = await storage.enqueueMany(
411
+ const results = await storage.enqueueMany(
434
412
  validated.map((v) => ({
435
413
  jobName: jobDef.name,
436
414
  input: v.input,
437
415
  idempotencyKey: v.options?.idempotencyKey,
438
416
  concurrencyKey: v.options?.concurrencyKey,
439
- labels: v.options?.labels
417
+ labels: v.options?.labels,
418
+ coalesce: v.options?.coalesce
440
419
  }))
441
420
  );
442
- for (let i = 0; i < runs.length; i++) {
443
- eventEmitter.emit({
444
- type: "run:trigger",
445
- runId: runs[i].id,
446
- jobName: jobDef.name,
447
- input: validated[i].input,
448
- labels: runs[i].labels
449
- });
421
+ for (let i = 0; i < results.length; i++) {
422
+ emitDispositionEvent(
423
+ results[i].disposition,
424
+ results[i].run,
425
+ validated[i].input,
426
+ validated[i].options?.labels
427
+ );
450
428
  }
451
- return runs;
429
+ return results.map(
430
+ (r) => ({
431
+ ...r.run,
432
+ disposition: r.disposition
433
+ })
434
+ );
452
435
  },
453
436
  async getRun(id) {
454
437
  const run = await storage.getRun(id);
@@ -487,6 +470,10 @@ var migrations = [
487
470
  "current_step_index",
488
471
  "integer",
489
472
  (col) => col.notNull().defaultTo(0)
473
+ ).addColumn(
474
+ "completed_step_count",
475
+ "integer",
476
+ (col) => col.notNull().defaultTo(0)
490
477
  ).addColumn("progress", "text").addColumn("output", "text").addColumn("error", "text").addColumn("lease_owner", "text").addColumn("lease_expires_at", "text").addColumn(
491
478
  "lease_generation",
492
479
  "integer",
@@ -510,6 +497,11 @@ var migrations = [
510
497
  await db.schema.createTable("durably_logs").ifNotExists().addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("step_name", "text").addColumn("level", "text", (col) => col.notNull()).addColumn("message", "text", (col) => col.notNull()).addColumn("data", "text").addColumn("created_at", "text", (col) => col.notNull()).execute();
511
498
  await db.schema.createIndex("idx_durably_logs_run_created").ifNotExists().on("durably_logs").columns(["run_id", "created_at"]).execute();
512
499
  await db.schema.createTable("durably_schema_versions").ifNotExists().addColumn("version", "integer", (col) => col.primaryKey()).addColumn("applied_at", "text", (col) => col.notNull()).execute();
500
+ await sql`
501
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_durably_runs_pending_concurrency
502
+ ON durably_runs (job_name, concurrency_key)
503
+ WHERE status = 'pending' AND concurrency_key IS NOT NULL
504
+ `.execute(db);
513
505
  }
514
506
  }
515
507
  ];
@@ -536,34 +528,365 @@ async function runMigrations(db) {
536
528
  }
537
529
  }
538
530
 
539
- // src/storage.ts
540
- import { sql as sql2 } from "kysely";
541
- import { monotonicFactory } from "ulidx";
542
- var ulid = monotonicFactory();
543
- var TERMINAL_STATUSES = ["completed", "failed", "cancelled"];
544
- function toClientRun(run) {
545
- const {
546
- idempotencyKey,
547
- concurrencyKey,
548
- leaseOwner,
549
- leaseExpiresAt,
550
- leaseGeneration,
551
- updatedAt,
552
- ...clientRun
553
- } = run;
554
- return clientRun;
531
+ // src/context.ts
532
+ var LEASE_LOST = "lease-lost";
533
+ function createStepContext(run, jobName, leaseGeneration, storage, eventEmitter) {
534
+ let stepIndex = run.currentStepIndex;
535
+ let currentStepName = null;
536
+ const controller = new AbortController();
537
+ function abortForLeaseLoss() {
538
+ if (!controller.signal.aborted) {
539
+ controller.abort(LEASE_LOST);
540
+ }
541
+ }
542
+ function throwIfAborted() {
543
+ if (!controller.signal.aborted) {
544
+ return;
545
+ }
546
+ if (controller.signal.reason === LEASE_LOST) {
547
+ throw new LeaseLostError(run.id);
548
+ }
549
+ throw new CancelledError(run.id);
550
+ }
551
+ async function throwForRefusedStep(stepName, stepIndex2) {
552
+ const latestRun = await storage.getRun(run.id);
553
+ if (latestRun?.status === "cancelled") {
554
+ eventEmitter.emit({
555
+ type: "step:cancel",
556
+ runId: run.id,
557
+ jobName,
558
+ stepName,
559
+ stepIndex: stepIndex2,
560
+ labels: run.labels
561
+ });
562
+ throw new CancelledError(run.id);
563
+ }
564
+ abortForLeaseLoss();
565
+ throw new LeaseLostError(run.id);
566
+ }
567
+ const unsubscribe = eventEmitter.on("run:cancel", (event) => {
568
+ if (event.runId === run.id) {
569
+ controller.abort();
570
+ }
571
+ });
572
+ const step = {
573
+ get runId() {
574
+ return run.id;
575
+ },
576
+ get signal() {
577
+ return controller.signal;
578
+ },
579
+ isAborted() {
580
+ return controller.signal.aborted;
581
+ },
582
+ throwIfAborted() {
583
+ throwIfAborted();
584
+ },
585
+ async run(name, fn) {
586
+ throwIfAborted();
587
+ const currentRun = await storage.getRun(run.id);
588
+ if (currentRun?.status === "cancelled") {
589
+ controller.abort();
590
+ throwIfAborted();
591
+ }
592
+ if (currentRun && (currentRun.status === "leased" && currentRun.leaseGeneration !== leaseGeneration || currentRun.status === "completed" || currentRun.status === "failed")) {
593
+ abortForLeaseLoss();
594
+ throwIfAborted();
595
+ }
596
+ throwIfAborted();
597
+ const existingStep = await storage.getCompletedStep(run.id, name);
598
+ if (existingStep) {
599
+ stepIndex++;
600
+ return existingStep.output;
601
+ }
602
+ currentStepName = name;
603
+ const startedAt = (/* @__PURE__ */ new Date()).toISOString();
604
+ const startTime = Date.now();
605
+ eventEmitter.emit({
606
+ type: "step:start",
607
+ runId: run.id,
608
+ jobName,
609
+ stepName: name,
610
+ stepIndex,
611
+ labels: run.labels
612
+ });
613
+ try {
614
+ const result = await fn(controller.signal);
615
+ throwIfAborted();
616
+ const savedStep = await storage.persistStep(run.id, leaseGeneration, {
617
+ name,
618
+ index: stepIndex,
619
+ status: "completed",
620
+ output: result,
621
+ startedAt
622
+ });
623
+ if (!savedStep) {
624
+ await throwForRefusedStep(name, stepIndex);
625
+ }
626
+ stepIndex++;
627
+ eventEmitter.emit({
628
+ type: "step:complete",
629
+ runId: run.id,
630
+ jobName,
631
+ stepName: name,
632
+ stepIndex: stepIndex - 1,
633
+ output: result,
634
+ duration: Date.now() - startTime,
635
+ labels: run.labels
636
+ });
637
+ return result;
638
+ } catch (error) {
639
+ if (error instanceof LeaseLostError) {
640
+ throw error;
641
+ }
642
+ const isLeaseLost = controller.signal.aborted && controller.signal.reason === LEASE_LOST;
643
+ if (isLeaseLost) {
644
+ throw new LeaseLostError(run.id);
645
+ }
646
+ const isCancelled = controller.signal.aborted;
647
+ const errorMessage = getErrorMessage(error);
648
+ const savedStep = await storage.persistStep(run.id, leaseGeneration, {
649
+ name,
650
+ index: stepIndex,
651
+ status: isCancelled ? "cancelled" : "failed",
652
+ error: errorMessage,
653
+ startedAt
654
+ });
655
+ if (!savedStep) {
656
+ await throwForRefusedStep(name, stepIndex);
657
+ }
658
+ eventEmitter.emit({
659
+ type: "step:fail",
660
+ error: errorMessage,
661
+ runId: run.id,
662
+ jobName,
663
+ stepName: name,
664
+ stepIndex,
665
+ labels: run.labels
666
+ });
667
+ throw error;
668
+ } finally {
669
+ currentStepName = null;
670
+ }
671
+ },
672
+ progress(current, total, message) {
673
+ const progressData = { current, total, message };
674
+ storage.updateProgress(run.id, leaseGeneration, progressData);
675
+ eventEmitter.emit({
676
+ type: "run:progress",
677
+ runId: run.id,
678
+ jobName,
679
+ progress: progressData,
680
+ labels: run.labels
681
+ });
682
+ },
683
+ log: {
684
+ info(message, data) {
685
+ eventEmitter.emit({
686
+ type: "log:write",
687
+ runId: run.id,
688
+ jobName,
689
+ labels: run.labels,
690
+ stepName: currentStepName,
691
+ level: "info",
692
+ message,
693
+ data
694
+ });
695
+ },
696
+ warn(message, data) {
697
+ eventEmitter.emit({
698
+ type: "log:write",
699
+ runId: run.id,
700
+ jobName,
701
+ labels: run.labels,
702
+ stepName: currentStepName,
703
+ level: "warn",
704
+ message,
705
+ data
706
+ });
707
+ },
708
+ error(message, data) {
709
+ eventEmitter.emit({
710
+ type: "log:write",
711
+ runId: run.id,
712
+ jobName,
713
+ labels: run.labels,
714
+ stepName: currentStepName,
715
+ level: "error",
716
+ message,
717
+ data
718
+ });
719
+ }
720
+ }
721
+ };
722
+ return {
723
+ step,
724
+ abortLeaseOwnership: abortForLeaseLoss,
725
+ dispose: unsubscribe
726
+ };
555
727
  }
556
- var LABEL_KEY_PATTERN = /^[a-zA-Z0-9\-_./]+$/;
557
- function validateLabels(labels) {
558
- if (!labels) return;
559
- for (const key of Object.keys(labels)) {
560
- if (!LABEL_KEY_PATTERN.test(key)) {
561
- throw new Error(
562
- `Invalid label key "${key}": must contain only alphanumeric characters, dashes, underscores, dots, and slashes`
563
- );
728
+
729
+ // src/runtime.ts
730
+ function isoNow(clock) {
731
+ return new Date(clock.now()).toISOString();
732
+ }
733
+ async function executeRun(run, job, config, environment) {
734
+ const { storage, eventEmitter, clock } = environment;
735
+ const { step, abortLeaseOwnership, dispose } = createStepContext(
736
+ run,
737
+ run.jobName,
738
+ run.leaseGeneration,
739
+ storage,
740
+ eventEmitter
741
+ );
742
+ let leaseDeadlineTimer = null;
743
+ const scheduleLeaseDeadline = (leaseExpiresAt) => {
744
+ if (leaseDeadlineTimer) {
745
+ clock.clearTimeout(leaseDeadlineTimer);
746
+ leaseDeadlineTimer = null;
747
+ }
748
+ if (!leaseExpiresAt) {
749
+ return;
750
+ }
751
+ const delay = Math.max(0, Date.parse(leaseExpiresAt) - clock.now());
752
+ leaseDeadlineTimer = clock.setTimeout(() => {
753
+ abortLeaseOwnership();
754
+ }, delay);
755
+ };
756
+ scheduleLeaseDeadline(run.leaseExpiresAt);
757
+ const leaseTimer = clock.setInterval(() => {
758
+ const now = isoNow(clock);
759
+ storage.renewLease(run.id, run.leaseGeneration, now, config.leaseMs).then((renewed) => {
760
+ if (!renewed) {
761
+ abortLeaseOwnership();
762
+ eventEmitter.emit({
763
+ type: "worker:error",
764
+ error: `Lease renewal lost ownership for run ${run.id}`,
765
+ context: "lease-renewal",
766
+ runId: run.id
767
+ });
768
+ return;
769
+ }
770
+ const renewedLeaseExpiresAt = new Date(
771
+ Date.parse(now) + config.leaseMs
772
+ ).toISOString();
773
+ scheduleLeaseDeadline(renewedLeaseExpiresAt);
774
+ eventEmitter.emit({
775
+ type: "run:lease-renewed",
776
+ runId: run.id,
777
+ jobName: run.jobName,
778
+ leaseOwner: run.leaseOwner ?? "",
779
+ leaseExpiresAt: renewedLeaseExpiresAt,
780
+ labels: run.labels
781
+ });
782
+ }).catch((error) => {
783
+ eventEmitter.emit({
784
+ type: "worker:error",
785
+ error: getErrorMessage(error),
786
+ context: "lease-renewal",
787
+ runId: run.id
788
+ });
789
+ });
790
+ }, config.leaseRenewIntervalMs);
791
+ const started = clock.now();
792
+ let reachedTerminalState = false;
793
+ try {
794
+ eventEmitter.emit({
795
+ type: "run:leased",
796
+ runId: run.id,
797
+ jobName: run.jobName,
798
+ input: run.input,
799
+ leaseOwner: run.leaseOwner ?? "",
800
+ leaseExpiresAt: run.leaseExpiresAt ?? isoNow(clock),
801
+ labels: run.labels
802
+ });
803
+ const output = await job.fn(step, run.input);
804
+ if (job.outputSchema) {
805
+ const parseResult = job.outputSchema.safeParse(output);
806
+ if (!parseResult.success) {
807
+ throw new Error(`Invalid output: ${parseResult.error.message}`);
808
+ }
809
+ }
810
+ const completedAt = isoNow(clock);
811
+ const completed = await storage.completeRun(
812
+ run.id,
813
+ run.leaseGeneration,
814
+ output,
815
+ completedAt
816
+ );
817
+ if (completed) {
818
+ reachedTerminalState = true;
819
+ eventEmitter.emit({
820
+ type: "run:complete",
821
+ runId: run.id,
822
+ jobName: run.jobName,
823
+ output,
824
+ duration: clock.now() - started,
825
+ labels: run.labels
826
+ });
827
+ return { kind: "completed" };
828
+ }
829
+ eventEmitter.emit({
830
+ type: "worker:error",
831
+ error: `Lease lost before completing run ${run.id}`,
832
+ context: "run-completion"
833
+ });
834
+ return { kind: "lease-lost" };
835
+ } catch (error) {
836
+ if (error instanceof LeaseLostError) {
837
+ return { kind: "lease-lost" };
838
+ }
839
+ if (error instanceof CancelledError) {
840
+ return { kind: "cancelled" };
841
+ }
842
+ const errorMessage = getErrorMessage(error);
843
+ const completedAt = isoNow(clock);
844
+ const failed = await storage.failRun(
845
+ run.id,
846
+ run.leaseGeneration,
847
+ errorMessage,
848
+ completedAt
849
+ );
850
+ if (failed) {
851
+ reachedTerminalState = true;
852
+ const steps = await storage.getSteps(run.id);
853
+ const failedStep = steps.find((entry) => entry.status === "failed");
854
+ eventEmitter.emit({
855
+ type: "run:fail",
856
+ runId: run.id,
857
+ jobName: run.jobName,
858
+ error: errorMessage,
859
+ failedStepName: failedStep?.name ?? "unknown",
860
+ labels: run.labels
861
+ });
862
+ return { kind: "failed" };
863
+ }
864
+ eventEmitter.emit({
865
+ type: "worker:error",
866
+ error: `Lease lost before recording failure for run ${run.id}`,
867
+ context: "run-failure"
868
+ });
869
+ return { kind: "lease-lost" };
870
+ } finally {
871
+ clock.clearInterval(leaseTimer);
872
+ if (leaseDeadlineTimer) {
873
+ clock.clearTimeout(leaseDeadlineTimer);
874
+ }
875
+ dispose();
876
+ if (!config.preserveSteps && reachedTerminalState) {
877
+ await storage.deleteSteps(run.id);
564
878
  }
565
879
  }
566
880
  }
881
+
882
+ // src/storage.ts
883
+ import { sql as sql4 } from "kysely";
884
+ import { monotonicFactory } from "ulidx";
885
+
886
+ // src/claim-postgres.ts
887
+ import { sql as sql2 } from "kysely";
888
+
889
+ // src/transformers.ts
567
890
  function rowToRun(row) {
568
891
  return {
569
892
  id: row.id,
@@ -573,7 +896,7 @@ function rowToRun(row) {
573
896
  idempotencyKey: row.idempotency_key,
574
897
  concurrencyKey: row.concurrency_key,
575
898
  currentStepIndex: row.current_step_index,
576
- stepCount: Number(row.step_count ?? 0),
899
+ completedStepCount: row.completed_step_count,
577
900
  progress: row.progress ? JSON.parse(row.progress) : null,
578
901
  output: row.output ? JSON.parse(row.output) : null,
579
902
  error: row.error,
@@ -600,15 +923,135 @@ function rowToStep(row) {
600
923
  completedAt: row.completed_at
601
924
  };
602
925
  }
603
- function rowToLog(row) {
926
+ function rowToLog(row) {
927
+ return {
928
+ id: row.id,
929
+ runId: row.run_id,
930
+ stepName: row.step_name,
931
+ level: row.level,
932
+ message: row.message,
933
+ data: row.data ? JSON.parse(row.data) : null,
934
+ createdAt: row.created_at
935
+ };
936
+ }
937
+ var LABEL_KEY_PATTERN = /^[a-zA-Z0-9\-_./]+$/;
938
+ function validateLabels(labels) {
939
+ if (!labels) return;
940
+ for (const key of Object.keys(labels)) {
941
+ if (!LABEL_KEY_PATTERN.test(key)) {
942
+ throw new Error(
943
+ `Invalid label key "${key}": must contain only alphanumeric characters, dashes, underscores, dots, and slashes`
944
+ );
945
+ }
946
+ }
947
+ }
948
+
949
+ // src/claim-postgres.ts
950
+ async function claimNextPostgres(db, workerId, now, leaseExpiresAt, activeLeaseGuard) {
951
+ return await db.transaction().execute(async (trx) => {
952
+ const skipKeys = [];
953
+ for (; ; ) {
954
+ const concurrencyCondition = skipKeys.length > 0 ? sql2`
955
+ AND (
956
+ concurrency_key IS NULL
957
+ OR concurrency_key NOT IN (${sql2.join(skipKeys)})
958
+ )
959
+ ` : sql2``;
960
+ const candidateResult = await sql2`
961
+ SELECT id, concurrency_key
962
+ FROM durably_runs
963
+ WHERE
964
+ (
965
+ status = 'pending'
966
+ OR (status = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ${now})
967
+ )
968
+ AND ${activeLeaseGuard}
969
+ ${concurrencyCondition}
970
+ ORDER BY created_at ASC, id ASC
971
+ FOR UPDATE SKIP LOCKED
972
+ LIMIT 1
973
+ `.execute(trx);
974
+ const candidate = candidateResult.rows[0];
975
+ if (!candidate) return null;
976
+ if (candidate.concurrency_key) {
977
+ await sql2`SELECT pg_advisory_xact_lock(hashtext(${candidate.concurrency_key}))`.execute(
978
+ trx
979
+ );
980
+ const conflict = await sql2`
981
+ SELECT 1 FROM durably_runs
982
+ WHERE concurrency_key = ${candidate.concurrency_key}
983
+ AND id <> ${candidate.id}
984
+ AND status = 'leased'
985
+ AND lease_expires_at IS NOT NULL
986
+ AND lease_expires_at > ${now}
987
+ LIMIT 1
988
+ `.execute(trx);
989
+ if (conflict.rows.length > 0) {
990
+ skipKeys.push(candidate.concurrency_key);
991
+ continue;
992
+ }
993
+ }
994
+ const result = await sql2`
995
+ UPDATE durably_runs
996
+ SET
997
+ status = 'leased',
998
+ lease_owner = ${workerId},
999
+ lease_expires_at = ${leaseExpiresAt},
1000
+ lease_generation = lease_generation + 1,
1001
+ started_at = COALESCE(started_at, ${now}),
1002
+ updated_at = ${now}
1003
+ WHERE id = ${candidate.id}
1004
+ RETURNING *
1005
+ `.execute(trx);
1006
+ const row = result.rows[0];
1007
+ if (!row) return null;
1008
+ return rowToRun(row);
1009
+ }
1010
+ });
1011
+ }
1012
+
1013
+ // src/claim-sqlite.ts
1014
+ import { sql as sql3 } from "kysely";
1015
+ async function claimNextSqlite(db, workerId, now, leaseExpiresAt, activeLeaseGuard) {
1016
+ const subquery = db.selectFrom("durably_runs").select("durably_runs.id").where(
1017
+ (eb) => eb.or([
1018
+ eb("status", "=", "pending"),
1019
+ eb.and([
1020
+ eb("status", "=", "leased"),
1021
+ eb("lease_expires_at", "is not", null),
1022
+ eb("lease_expires_at", "<=", now)
1023
+ ])
1024
+ ])
1025
+ ).where(activeLeaseGuard).orderBy("created_at", "asc").orderBy("id", "asc").limit(1);
1026
+ const row = await db.updateTable("durably_runs").set({
1027
+ status: "leased",
1028
+ lease_owner: workerId,
1029
+ lease_expires_at: leaseExpiresAt,
1030
+ lease_generation: sql3`lease_generation + 1`,
1031
+ started_at: sql3`COALESCE(started_at, ${now})`,
1032
+ updated_at: now
1033
+ }).where("id", "=", (eb) => eb.selectFrom(subquery.as("sub")).select("id")).returningAll().executeTakeFirst();
1034
+ if (!row) return null;
1035
+ return rowToRun(row);
1036
+ }
1037
+
1038
+ // src/storage.ts
1039
+ var ulid = monotonicFactory();
1040
+ var TERMINAL_STATUSES = ["completed", "failed", "cancelled"];
1041
+ function toClientRun(run) {
1042
+ const {
1043
+ idempotencyKey,
1044
+ concurrencyKey,
1045
+ leaseOwner,
1046
+ leaseExpiresAt,
1047
+ leaseGeneration,
1048
+ updatedAt,
1049
+ ...clientRun
1050
+ } = run;
604
1051
  return {
605
- id: row.id,
606
- runId: row.run_id,
607
- stepName: row.step_name,
608
- level: row.level,
609
- message: row.message,
610
- data: row.data ? JSON.parse(row.data) : null,
611
- createdAt: row.created_at
1052
+ ...clientRun,
1053
+ isTerminal: TERMINAL_STATUSES.includes(run.status),
1054
+ isActive: run.status === "pending" || run.status === "leased"
612
1055
  };
613
1056
  }
614
1057
  function createWriteMutex() {
@@ -628,6 +1071,29 @@ function createWriteMutex() {
628
1071
  }
629
1072
  };
630
1073
  }
1074
+ function isUniqueViolation(err) {
1075
+ if (!(err instanceof Error)) return false;
1076
+ const pgCode = err.code;
1077
+ if (pgCode === "23505") return true;
1078
+ if (err.constraint) return true;
1079
+ if (/unique constraint/i.test(err.message)) return true;
1080
+ return false;
1081
+ }
1082
+ function parseUniqueViolation(err) {
1083
+ if (!(err instanceof Error)) return null;
1084
+ const msg = err.message;
1085
+ const pgConstraint = err.constraint;
1086
+ if (pgConstraint) {
1087
+ if (pgConstraint.includes("idempotency")) return "idempotency";
1088
+ if (pgConstraint.includes("pending_concurrency"))
1089
+ return "pending_concurrency";
1090
+ }
1091
+ if (/unique constraint/i.test(msg)) {
1092
+ if (msg.includes("idempotency_key")) return "idempotency";
1093
+ if (msg.includes("concurrency_key")) return "pending_concurrency";
1094
+ }
1095
+ return null;
1096
+ }
631
1097
  function createKyselyStore(db, backend = "generic") {
632
1098
  const withWriteLock = createWriteMutex();
633
1099
  async function cascadeDeleteRuns(trx, ids) {
@@ -653,119 +1119,138 @@ function createKyselyStore(db, backend = "generic") {
653
1119
  }).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).executeTakeFirst();
654
1120
  return Number(result.numUpdatedRows) > 0;
655
1121
  }
656
- const store = {
657
- async enqueue(input) {
658
- const now = (/* @__PURE__ */ new Date()).toISOString();
1122
+ function findPendingByConcurrencyKey(queryDb, jobName, concurrencyKey) {
1123
+ return queryDb.selectFrom("durably_runs").selectAll().where("job_name", "=", jobName).where("concurrency_key", "=", concurrencyKey).where("status", "=", "pending").orderBy("created_at", "asc").orderBy("id", "asc").limit(1).executeTakeFirst();
1124
+ }
1125
+ async function enqueueInTx(trx, input, retried = false) {
1126
+ const queryDb = trx ?? db;
1127
+ const now = (/* @__PURE__ */ new Date()).toISOString();
1128
+ if (input.idempotencyKey) {
1129
+ const existing = await queryDb.selectFrom("durably_runs").selectAll().where("job_name", "=", input.jobName).where("idempotency_key", "=", input.idempotencyKey).executeTakeFirst();
1130
+ if (existing) {
1131
+ return { run: rowToRun(existing), disposition: "idempotent" };
1132
+ }
1133
+ }
1134
+ validateLabels(input.labels);
1135
+ const id = ulid();
1136
+ const row = {
1137
+ id,
1138
+ job_name: input.jobName,
1139
+ input: JSON.stringify(input.input),
1140
+ status: "pending",
1141
+ idempotency_key: input.idempotencyKey ?? null,
1142
+ concurrency_key: input.concurrencyKey ?? null,
1143
+ current_step_index: 0,
1144
+ completed_step_count: 0,
1145
+ progress: null,
1146
+ output: null,
1147
+ error: null,
1148
+ labels: JSON.stringify(input.labels ?? {}),
1149
+ lease_owner: null,
1150
+ lease_expires_at: null,
1151
+ lease_generation: 0,
1152
+ started_at: null,
1153
+ completed_at: null,
1154
+ created_at: now,
1155
+ updated_at: now
1156
+ };
1157
+ const doInsert = async (insertDb) => {
1158
+ await sql4`SAVEPOINT sp_enqueue`.execute(insertDb);
1159
+ try {
1160
+ await insertDb.insertInto("durably_runs").values(row).execute();
1161
+ await insertLabelRows(insertDb, id, input.labels);
1162
+ await sql4`RELEASE SAVEPOINT sp_enqueue`.execute(insertDb);
1163
+ } catch (err) {
1164
+ await sql4`ROLLBACK TO SAVEPOINT sp_enqueue`.execute(insertDb);
1165
+ throw err;
1166
+ }
1167
+ };
1168
+ try {
1169
+ if (trx) {
1170
+ await doInsert(trx);
1171
+ } else {
1172
+ await db.transaction().execute(doInsert);
1173
+ }
1174
+ return { run: rowToRun(row), disposition: "created" };
1175
+ } catch (err) {
1176
+ if (!isUniqueViolation(err)) throw err;
1177
+ const violation = parseUniqueViolation(err);
659
1178
  if (input.idempotencyKey) {
660
- const existing = await db.selectFrom("durably_runs").selectAll().where("job_name", "=", input.jobName).where("idempotency_key", "=", input.idempotencyKey).executeTakeFirst();
661
- if (existing) {
662
- return rowToRun(existing);
1179
+ const idempotent = await queryDb.selectFrom("durably_runs").selectAll().where("job_name", "=", input.jobName).where("idempotency_key", "=", input.idempotencyKey).executeTakeFirst();
1180
+ if (idempotent) {
1181
+ return { run: rowToRun(idempotent), disposition: "idempotent" };
663
1182
  }
664
1183
  }
665
- validateLabels(input.labels);
666
- const id = ulid();
667
- const run = {
668
- id,
669
- job_name: input.jobName,
670
- input: JSON.stringify(input.input),
671
- status: "pending",
672
- idempotency_key: input.idempotencyKey ?? null,
673
- concurrency_key: input.concurrencyKey ?? null,
674
- current_step_index: 0,
675
- progress: null,
676
- output: null,
677
- error: null,
678
- labels: JSON.stringify(input.labels ?? {}),
679
- lease_owner: null,
680
- lease_expires_at: null,
681
- lease_generation: 0,
682
- started_at: null,
683
- completed_at: null,
684
- created_at: now,
685
- updated_at: now
686
- };
687
- await db.transaction().execute(async (trx) => {
688
- await trx.insertInto("durably_runs").values(run).execute();
689
- await insertLabelRows(trx, id, input.labels);
690
- });
691
- return rowToRun(run);
1184
+ if ((violation === "pending_concurrency" || violation === null) && input.concurrencyKey) {
1185
+ if (input.coalesce === "skip") {
1186
+ const pending = await findPendingByConcurrencyKey(
1187
+ queryDb,
1188
+ input.jobName,
1189
+ input.concurrencyKey
1190
+ );
1191
+ if (pending) {
1192
+ return { run: rowToRun(pending), disposition: "coalesced" };
1193
+ }
1194
+ if (!retried) {
1195
+ return enqueueInTx(trx, input, true);
1196
+ }
1197
+ const lastChance = await findPendingByConcurrencyKey(
1198
+ queryDb,
1199
+ input.jobName,
1200
+ input.concurrencyKey
1201
+ );
1202
+ if (lastChance) {
1203
+ return { run: rowToRun(lastChance), disposition: "coalesced" };
1204
+ }
1205
+ throw new ConflictError(
1206
+ `Conflict after retry for concurrency key "${input.concurrencyKey}" in job "${input.jobName}". Concurrent modification detected.`
1207
+ );
1208
+ }
1209
+ throw new ConflictError(
1210
+ `A pending run already exists for concurrency key "${input.concurrencyKey}" in job "${input.jobName}". Use coalesce: 'skip' to return the existing run instead.`
1211
+ );
1212
+ }
1213
+ throw err;
1214
+ }
1215
+ }
1216
+ const store = {
1217
+ async enqueue(input) {
1218
+ return enqueueInTx(null, input);
692
1219
  },
693
1220
  async enqueueMany(inputs) {
694
1221
  if (inputs.length === 0) {
695
1222
  return [];
696
1223
  }
697
- return await db.transaction().execute(async (trx) => {
698
- const now = (/* @__PURE__ */ new Date()).toISOString();
699
- const runs = [];
1224
+ return db.transaction().execute(async (trx) => {
1225
+ const results = [];
700
1226
  for (const input of inputs) {
701
- validateLabels(input.labels);
702
- }
703
- const allLabelRows = [];
704
- for (const input of inputs) {
705
- if (input.idempotencyKey) {
706
- const existing = await trx.selectFrom("durably_runs").selectAll().where("job_name", "=", input.jobName).where("idempotency_key", "=", input.idempotencyKey).executeTakeFirst();
707
- if (existing) {
708
- runs.push(existing);
709
- continue;
710
- }
711
- }
712
- const id = ulid();
713
- if (input.labels) {
714
- for (const [key, value] of Object.entries(input.labels)) {
715
- allLabelRows.push({ run_id: id, key, value });
716
- }
717
- }
718
- runs.push({
719
- id,
720
- job_name: input.jobName,
721
- input: JSON.stringify(input.input),
722
- status: "pending",
723
- idempotency_key: input.idempotencyKey ?? null,
724
- concurrency_key: input.concurrencyKey ?? null,
725
- current_step_index: 0,
726
- progress: null,
727
- output: null,
728
- error: null,
729
- labels: JSON.stringify(input.labels ?? {}),
730
- lease_owner: null,
731
- lease_expires_at: null,
732
- lease_generation: 0,
733
- started_at: null,
734
- completed_at: null,
735
- created_at: now,
736
- updated_at: now
737
- });
738
- }
739
- const newRuns = runs.filter((r) => r.created_at === now);
740
- if (newRuns.length > 0) {
741
- await trx.insertInto("durably_runs").values(newRuns).execute();
742
- if (allLabelRows.length > 0) {
743
- await trx.insertInto("durably_run_labels").values(allLabelRows).execute();
744
- }
1227
+ results.push(await enqueueInTx(trx, input));
745
1228
  }
746
- return runs.map(rowToRun);
1229
+ return results;
747
1230
  });
748
1231
  },
749
1232
  async getRun(runId) {
750
- const row = await db.selectFrom("durably_runs").leftJoin("durably_steps", "durably_runs.id", "durably_steps.run_id").selectAll("durably_runs").select(
751
- (eb) => eb.fn.count("durably_steps.id").as("step_count")
752
- ).where("durably_runs.id", "=", runId).groupBy("durably_runs.id").executeTakeFirst();
1233
+ const row = await db.selectFrom("durably_runs").selectAll().where("id", "=", runId).executeTakeFirst();
753
1234
  return row ? rowToRun(row) : null;
754
1235
  },
755
1236
  async getRuns(filter) {
756
- let query = db.selectFrom("durably_runs").leftJoin("durably_steps", "durably_runs.id", "durably_steps.run_id").selectAll("durably_runs").select(
757
- (eb) => eb.fn.count("durably_steps.id").as("step_count")
758
- ).groupBy("durably_runs.id");
1237
+ let query = db.selectFrom("durably_runs").selectAll();
759
1238
  if (filter?.status) {
760
- query = query.where("durably_runs.status", "=", filter.status);
1239
+ if (Array.isArray(filter.status)) {
1240
+ if (filter.status.length > 0) {
1241
+ query = query.where("status", "in", filter.status);
1242
+ }
1243
+ } else {
1244
+ query = query.where("status", "=", filter.status);
1245
+ }
761
1246
  }
762
1247
  if (filter?.jobName) {
763
1248
  if (Array.isArray(filter.jobName)) {
764
1249
  if (filter.jobName.length > 0) {
765
- query = query.where("durably_runs.job_name", "in", filter.jobName);
1250
+ query = query.where("job_name", "in", filter.jobName);
766
1251
  }
767
1252
  } else {
768
- query = query.where("durably_runs.job_name", "=", filter.jobName);
1253
+ query = query.where("job_name", "=", filter.jobName);
769
1254
  }
770
1255
  }
771
1256
  if (filter?.labels) {
@@ -773,18 +1258,14 @@ function createKyselyStore(db, backend = "generic") {
773
1258
  validateLabels(labels);
774
1259
  for (const [key, value] of Object.entries(labels)) {
775
1260
  if (value === void 0) continue;
776
- const jsonFallback = backend === "postgres" ? sql2`durably_runs.labels ->> ${key} = ${value}` : sql2`json_extract(durably_runs.labels, ${`$.${key}`}) = ${value}`;
777
1261
  query = query.where(
778
- (eb) => eb.or([
779
- eb.exists(
780
- eb.selectFrom("durably_run_labels").select(sql2.lit(1).as("one")).whereRef("durably_run_labels.run_id", "=", "durably_runs.id").where("durably_run_labels.key", "=", key).where("durably_run_labels.value", "=", value)
781
- ),
782
- jsonFallback
783
- ])
1262
+ (eb) => eb.exists(
1263
+ eb.selectFrom("durably_run_labels").select(sql4.lit(1).as("one")).whereRef("durably_run_labels.run_id", "=", "durably_runs.id").where("durably_run_labels.key", "=", key).where("durably_run_labels.value", "=", value)
1264
+ )
784
1265
  );
785
1266
  }
786
1267
  }
787
- query = query.orderBy("durably_runs.created_at", "desc");
1268
+ query = query.orderBy("created_at", "desc");
788
1269
  if (filter?.limit !== void 0) {
789
1270
  query = query.limit(filter.limit);
790
1271
  }
@@ -830,7 +1311,7 @@ function createKyselyStore(db, backend = "generic") {
830
1311
  },
831
1312
  async claimNext(workerId, now, leaseMs) {
832
1313
  const leaseExpiresAt = new Date(Date.parse(now) + leaseMs).toISOString();
833
- const activeLeaseGuard = sql2`
1314
+ const activeLeaseGuard = sql4`
834
1315
  (
835
1316
  concurrency_key IS NULL
836
1317
  OR NOT EXISTS (
@@ -844,92 +1325,7 @@ function createKyselyStore(db, backend = "generic") {
844
1325
  )
845
1326
  )
846
1327
  `;
847
- if (backend === "postgres") {
848
- return await db.transaction().execute(async (trx) => {
849
- const skipKeys = [];
850
- for (; ; ) {
851
- const concurrencyCondition = skipKeys.length > 0 ? sql2`
852
- AND (
853
- concurrency_key IS NULL
854
- OR concurrency_key NOT IN (${sql2.join(skipKeys)})
855
- )
856
- ` : sql2``;
857
- const candidateResult = await sql2`
858
- SELECT id, concurrency_key
859
- FROM durably_runs
860
- WHERE
861
- (
862
- status = 'pending'
863
- OR (status = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ${now})
864
- )
865
- AND ${activeLeaseGuard}
866
- ${concurrencyCondition}
867
- ORDER BY created_at ASC, id ASC
868
- FOR UPDATE SKIP LOCKED
869
- LIMIT 1
870
- `.execute(trx);
871
- const candidate = candidateResult.rows[0];
872
- if (!candidate) return null;
873
- if (candidate.concurrency_key) {
874
- await sql2`SELECT pg_advisory_xact_lock(hashtext(${candidate.concurrency_key}))`.execute(
875
- trx
876
- );
877
- const conflict = await sql2`
878
- SELECT 1 FROM durably_runs
879
- WHERE concurrency_key = ${candidate.concurrency_key}
880
- AND id <> ${candidate.id}
881
- AND status = 'leased'
882
- AND lease_expires_at IS NOT NULL
883
- AND lease_expires_at > ${now}
884
- LIMIT 1
885
- `.execute(trx);
886
- if (conflict.rows.length > 0) {
887
- skipKeys.push(candidate.concurrency_key);
888
- continue;
889
- }
890
- }
891
- const result = await sql2`
892
- UPDATE durably_runs
893
- SET
894
- status = 'leased',
895
- lease_owner = ${workerId},
896
- lease_expires_at = ${leaseExpiresAt},
897
- lease_generation = lease_generation + 1,
898
- started_at = COALESCE(started_at, ${now}),
899
- updated_at = ${now}
900
- WHERE id = ${candidate.id}
901
- RETURNING *
902
- `.execute(trx);
903
- const row2 = result.rows[0];
904
- if (!row2) return null;
905
- return rowToRun({ ...row2, step_count: 0 });
906
- }
907
- });
908
- }
909
- let subquery = db.selectFrom("durably_runs").select("durably_runs.id").where(
910
- (eb) => eb.or([
911
- eb("status", "=", "pending"),
912
- eb.and([
913
- eb("status", "=", "leased"),
914
- eb("lease_expires_at", "is not", null),
915
- eb("lease_expires_at", "<=", now)
916
- ])
917
- ])
918
- ).where(activeLeaseGuard).orderBy("created_at", "asc").orderBy("id", "asc").limit(1);
919
- const row = await db.updateTable("durably_runs").set({
920
- status: "leased",
921
- lease_owner: workerId,
922
- lease_expires_at: leaseExpiresAt,
923
- lease_generation: sql2`lease_generation + 1`,
924
- started_at: sql2`COALESCE(started_at, ${now})`,
925
- updated_at: now
926
- }).where(
927
- "id",
928
- "=",
929
- (eb) => eb.selectFrom(subquery.as("sub")).select("id")
930
- ).returningAll().executeTakeFirst();
931
- if (!row) return null;
932
- return rowToRun({ ...row, step_count: 0 });
1328
+ return backend === "postgres" ? claimNextPostgres(db, workerId, now, leaseExpiresAt, activeLeaseGuard) : claimNextSqlite(db, workerId, now, leaseExpiresAt, activeLeaseGuard);
933
1329
  },
934
1330
  async renewLease(runId, leaseGeneration, now, leaseMs) {
935
1331
  const leaseExpiresAt = new Date(Date.parse(now) + leaseMs).toISOString();
@@ -940,13 +1336,54 @@ function createKyselyStore(db, backend = "generic") {
940
1336
  return Number(result.numUpdatedRows) > 0;
941
1337
  },
942
1338
  async releaseExpiredLeases(now) {
943
- const result = await db.updateTable("durably_runs").set({
944
- status: "pending",
1339
+ const conflicting = await db.updateTable("durably_runs").set({
1340
+ status: "failed",
1341
+ error: "Lease expired; pending run already exists",
945
1342
  lease_owner: null,
946
1343
  lease_expires_at: null,
1344
+ completed_at: now,
947
1345
  updated_at: now
948
- }).where("status", "=", "leased").where("lease_expires_at", "is not", null).where("lease_expires_at", "<=", now).executeTakeFirst();
949
- return Number(result.numUpdatedRows);
1346
+ }).where("status", "=", "leased").where("lease_expires_at", "is not", null).where("lease_expires_at", "<=", now).where(
1347
+ ({ exists, selectFrom }) => exists(
1348
+ selectFrom("durably_runs as other").select(sql4.lit(1).as("one")).whereRef("other.job_name", "=", "durably_runs.job_name").whereRef(
1349
+ "other.concurrency_key",
1350
+ "=",
1351
+ "durably_runs.concurrency_key"
1352
+ ).where("other.status", "=", "pending").whereRef("other.id", "<>", "durably_runs.id")
1353
+ )
1354
+ ).executeTakeFirst();
1355
+ let count = Number(conflicting.numUpdatedRows);
1356
+ const remaining = await db.selectFrom("durably_runs").select("id").where("status", "=", "leased").where("lease_expires_at", "is not", null).where("lease_expires_at", "<=", now).execute();
1357
+ if (remaining.length > 0) {
1358
+ await db.transaction().execute(async (trx) => {
1359
+ for (const row of remaining) {
1360
+ try {
1361
+ await sql4`SAVEPOINT sp_release`.execute(trx);
1362
+ const reset = await trx.updateTable("durably_runs").set({
1363
+ status: "pending",
1364
+ lease_owner: null,
1365
+ lease_expires_at: null,
1366
+ updated_at: now
1367
+ }).where("id", "=", row.id).where("status", "=", "leased").where("lease_expires_at", "<=", now).executeTakeFirst();
1368
+ await sql4`RELEASE SAVEPOINT sp_release`.execute(trx);
1369
+ count += Number(reset.numUpdatedRows);
1370
+ } catch (err) {
1371
+ await sql4`ROLLBACK TO SAVEPOINT sp_release`.execute(trx);
1372
+ if (!isUniqueViolation(err)) throw err;
1373
+ const failed = await trx.updateTable("durably_runs").set({
1374
+ status: "failed",
1375
+ error: "Lease expired; pending run already exists",
1376
+ lease_owner: null,
1377
+ lease_expires_at: null,
1378
+ completed_at: now,
1379
+ updated_at: now
1380
+ }).where("id", "=", row.id).where("status", "=", "leased").executeTakeFirst();
1381
+ count += Number(failed.numUpdatedRows);
1382
+ }
1383
+ }
1384
+ });
1385
+ }
1386
+ return count;
950
1387
  },
951
1388
  async completeRun(runId, leaseGeneration, output, completedAt) {
952
1389
  return terminateRun(runId, leaseGeneration, completedAt, {
@@ -977,19 +1414,20 @@ function createKyselyStore(db, backend = "generic") {
977
1414
  const outputJson = input.output !== void 0 ? JSON.stringify(input.output) : null;
978
1415
  const errorValue = input.error ?? null;
979
1416
  return await db.transaction().execute(async (trx) => {
980
- const insertResult = await sql2`
1417
+ const insertResult = await sql4`
981
1418
  INSERT INTO durably_steps (id, run_id, name, "index", status, output, error, started_at, completed_at)
982
1419
  SELECT ${id}, ${runId}, ${input.name}, ${input.index}, ${input.status},
983
1420
  ${outputJson}, ${errorValue}, ${input.startedAt}, ${completedAt}
984
1421
  FROM durably_runs
985
- WHERE id = ${runId} AND lease_generation = ${leaseGeneration}
1422
+ WHERE id = ${runId} AND status = 'leased' AND lease_generation = ${leaseGeneration}
986
1423
  `.execute(trx);
987
1424
  if (Number(insertResult.numAffectedRows) === 0) return null;
988
1425
  if (input.status === "completed") {
989
1426
  await trx.updateTable("durably_runs").set({
990
1427
  current_step_index: input.index + 1,
1428
+ completed_step_count: sql4`completed_step_count + 1`,
991
1429
  updated_at: completedAt
992
- }).where("id", "=", runId).where("lease_generation", "=", leaseGeneration).execute();
1430
+ }).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).execute();
993
1431
  }
994
1432
  return {
995
1433
  id,
@@ -1020,7 +1458,7 @@ function createKyselyStore(db, backend = "generic") {
1020
1458
  await db.updateTable("durably_runs").set({
1021
1459
  progress: progress ? JSON.stringify(progress) : null,
1022
1460
  updated_at: (/* @__PURE__ */ new Date()).toISOString()
1023
- }).where("id", "=", runId).where("lease_generation", "=", leaseGeneration).execute();
1461
+ }).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).execute();
1024
1462
  },
1025
1463
  async createLog(input) {
1026
1464
  const now = (/* @__PURE__ */ new Date()).toISOString();
@@ -1042,56 +1480,107 @@ function createKyselyStore(db, backend = "generic") {
1042
1480
  return rows.map(rowToLog);
1043
1481
  }
1044
1482
  };
1045
- const mutatingKeys = [
1046
- "enqueue",
1047
- "enqueueMany",
1048
- "updateRun",
1049
- "deleteRun",
1050
- "purgeRuns",
1051
- "claimNext",
1052
- "renewLease",
1053
- "releaseExpiredLeases",
1054
- "completeRun",
1055
- "failRun",
1056
- "cancelRun",
1057
- "persistStep",
1058
- "deleteSteps",
1059
- "updateProgress",
1060
- "createLog"
1061
- ];
1062
- for (const key of mutatingKeys) {
1063
- const original = store[key];
1064
- store[key] = (...args) => withWriteLock(() => original.apply(store, args));
1483
+ if (backend !== "postgres") {
1484
+ const mutatingKeys = [
1485
+ "enqueue",
1486
+ "enqueueMany",
1487
+ "updateRun",
1488
+ "deleteRun",
1489
+ "purgeRuns",
1490
+ "claimNext",
1491
+ "renewLease",
1492
+ "releaseExpiredLeases",
1493
+ "completeRun",
1494
+ "failRun",
1495
+ "cancelRun",
1496
+ "persistStep",
1497
+ "deleteSteps",
1498
+ "updateProgress",
1499
+ "createLog"
1500
+ ];
1501
+ for (const key of mutatingKeys) {
1502
+ const original = store[key];
1503
+ store[key] = (...args) => withWriteLock(() => original.apply(store, args));
1504
+ }
1065
1505
  }
1066
1506
  return store;
1067
1507
  }
1068
1508
 
1069
1509
  // src/worker.ts
1070
- function createWorker(config, processOne) {
1510
+ function createWorker(config, processOne, onIdle) {
1511
+ const maxConcurrentRuns = config.maxConcurrentRuns;
1071
1512
  let running = false;
1072
1513
  let pollingTimeout = null;
1073
- let inFlight = null;
1074
- let stopResolver = null;
1514
+ let activeCount = 0;
1075
1515
  let activeWorkerId;
1076
- async function poll() {
1516
+ const activePromises = /* @__PURE__ */ new Set();
1517
+ let idleMaintenanceInFlight = null;
1518
+ function scheduleDelayedPoll() {
1077
1519
  if (!running) {
1078
1520
  return;
1079
1521
  }
1522
+ if (pollingTimeout) {
1523
+ clearTimeout(pollingTimeout);
1524
+ pollingTimeout = null;
1525
+ }
1526
+ pollingTimeout = setTimeout(() => {
1527
+ pollingTimeout = null;
1528
+ if (running) {
1529
+ fillSlots();
1530
+ }
1531
+ }, config.pollingIntervalMs);
1532
+ }
1533
+ async function runIdleMaintenanceSafe() {
1534
+ if (!onIdle) {
1535
+ return;
1536
+ }
1537
+ const cycle = (async () => {
1538
+ try {
1539
+ await onIdle();
1540
+ } catch {
1541
+ }
1542
+ })();
1543
+ idleMaintenanceInFlight = cycle;
1080
1544
  try {
1081
- inFlight = processOne({ workerId: activeWorkerId }).then(() => void 0);
1082
- await inFlight;
1545
+ await cycle;
1083
1546
  } finally {
1084
- inFlight = null;
1547
+ if (idleMaintenanceInFlight === cycle) {
1548
+ idleMaintenanceInFlight = null;
1549
+ }
1550
+ }
1551
+ }
1552
+ async function processSlotCycle() {
1553
+ try {
1554
+ const didProcess = await processOne({ workerId: activeWorkerId });
1555
+ activeCount--;
1556
+ if (didProcess && running) {
1557
+ fillSlots();
1558
+ } else if (!didProcess && running) {
1559
+ if (activeCount === 0) {
1560
+ await runIdleMaintenanceSafe();
1561
+ }
1562
+ scheduleDelayedPoll();
1563
+ }
1564
+ } catch (err) {
1565
+ activeCount--;
1566
+ if (running) {
1567
+ fillSlots();
1568
+ }
1569
+ throw err;
1085
1570
  }
1086
- if (running) {
1087
- pollingTimeout = setTimeout(() => {
1088
- void poll();
1089
- }, config.pollingIntervalMs);
1571
+ }
1572
+ function fillSlots() {
1573
+ if (!running) {
1090
1574
  return;
1091
1575
  }
1092
- if (stopResolver) {
1093
- stopResolver();
1094
- stopResolver = null;
1576
+ while (running && activeCount < maxConcurrentRuns) {
1577
+ activeCount++;
1578
+ const p = processSlotCycle();
1579
+ activePromises.add(p);
1580
+ void p.finally(() => {
1581
+ activePromises.delete(p);
1582
+ }).catch(() => {
1583
+ });
1095
1584
  }
1096
1585
  }
1097
1586
  return {
@@ -1104,7 +1593,7 @@ function createWorker(config, processOne) {
1104
1593
  }
1105
1594
  activeWorkerId = options?.workerId;
1106
1595
  running = true;
1107
- void poll();
1596
+ fillSlots();
1108
1597
  },
1109
1598
  async stop() {
1110
1599
  if (!running) {
@@ -1115,11 +1604,14 @@ function createWorker(config, processOne) {
1115
1604
  clearTimeout(pollingTimeout);
1116
1605
  pollingTimeout = null;
1117
1606
  }
1118
- if (inFlight) {
1119
- return new Promise((resolve) => {
1120
- stopResolver = resolve;
1121
- });
1607
+ const pending = [...activePromises];
1608
+ if (idleMaintenanceInFlight) {
1609
+ pending.push(idleMaintenanceInFlight);
1610
+ }
1611
+ if (pending.length === 0) {
1612
+ return;
1122
1613
  }
1614
+ await Promise.allSettled(pending);
1123
1615
  }
1124
1616
  };
1125
1617
  }
@@ -1127,10 +1619,20 @@ function createWorker(config, processOne) {
1127
1619
  // src/durably.ts
1128
1620
  var DEFAULTS = {
1129
1621
  pollingIntervalMs: 1e3,
1622
+ maxConcurrentRuns: 1,
1130
1623
  leaseRenewIntervalMs: 5e3,
1131
1624
  leaseMs: 3e4,
1132
1625
  preserveSteps: false
1133
1626
  };
1627
+ var MAX_CONCURRENT_RUNS = 1e3;
1628
+ function validateMaxConcurrentRuns(value) {
1629
+ if (!Number.isSafeInteger(value) || value < 1 || value > MAX_CONCURRENT_RUNS) {
1630
+ throw new ValidationError(
1631
+ `maxConcurrentRuns must be between 1 and ${MAX_CONCURRENT_RUNS}`
1632
+ );
1633
+ }
1634
+ return value;
1635
+ }
1134
1636
  function parseDuration(value) {
1135
1637
  const match = value.match(/^(\d+)(d|h|m)$/);
1136
1638
  if (!match) {
@@ -1148,6 +1650,14 @@ function parseDuration(value) {
1148
1650
  return num * multipliers[unit];
1149
1651
  }
1150
1652
  var PURGE_INTERVAL_MS = 6e4;
1653
+ var CHECKPOINT_INTERVAL_MS = 6e4;
1654
+ var realClock = {
1655
+ now: () => Date.now(),
1656
+ setTimeout: (fn, ms) => globalThis.setTimeout(fn, ms),
1657
+ clearTimeout: (id) => globalThis.clearTimeout(id),
1658
+ setInterval: (fn, ms) => globalThis.setInterval(fn, ms),
1659
+ clearInterval: (id) => globalThis.clearInterval(id)
1660
+ };
1151
1661
  var ulid2 = monotonicFactory2();
1152
1662
  var BROWSER_SINGLETON_REGISTRY_KEY = "__durablyBrowserSingletonRegistry";
1153
1663
  var BROWSER_LOCAL_DIALECT_KEY = "__durablyBrowserLocalKey";
@@ -1213,11 +1723,11 @@ function createDurablyInstance(state, jobs) {
1213
1723
  async function getRunOrThrow(runId) {
1214
1724
  const run = await storage.getRun(runId);
1215
1725
  if (!run) {
1216
- throw new Error(`Run not found: ${runId}`);
1726
+ throw new NotFoundError(`Run not found: ${runId}`);
1217
1727
  }
1218
1728
  return run;
1219
1729
  }
1220
- async function executeRun(run, workerId) {
1730
+ async function executeRun2(run, _workerId) {
1221
1731
  const job = jobRegistry.get(run.jobName);
1222
1732
  if (!job) {
1223
1733
  await storage.failRun(
@@ -1228,146 +1738,16 @@ function createDurablyInstance(state, jobs) {
1228
1738
  );
1229
1739
  return;
1230
1740
  }
1231
- const { step, abortLeaseOwnership, dispose } = createStepContext(
1741
+ await executeRun(
1232
1742
  run,
1233
- run.jobName,
1234
- run.leaseGeneration,
1235
- storage,
1236
- eventEmitter
1743
+ job,
1744
+ {
1745
+ leaseMs: state.leaseMs,
1746
+ leaseRenewIntervalMs: state.leaseRenewIntervalMs,
1747
+ preserveSteps: state.preserveSteps
1748
+ },
1749
+ { storage, eventEmitter, clock: realClock }
1237
1750
  );
1238
- let leaseDeadlineTimer = null;
1239
- const scheduleLeaseDeadline = (leaseExpiresAt) => {
1240
- if (leaseDeadlineTimer) {
1241
- clearTimeout(leaseDeadlineTimer);
1242
- leaseDeadlineTimer = null;
1243
- }
1244
- if (!leaseExpiresAt) {
1245
- return;
1246
- }
1247
- const delay = Math.max(0, Date.parse(leaseExpiresAt) - Date.now());
1248
- leaseDeadlineTimer = setTimeout(() => {
1249
- abortLeaseOwnership();
1250
- }, delay);
1251
- };
1252
- scheduleLeaseDeadline(run.leaseExpiresAt);
1253
- const leaseTimer = setInterval(() => {
1254
- const now = (/* @__PURE__ */ new Date()).toISOString();
1255
- storage.renewLease(run.id, run.leaseGeneration, now, state.leaseMs).then((renewed) => {
1256
- if (!renewed) {
1257
- abortLeaseOwnership();
1258
- eventEmitter.emit({
1259
- type: "worker:error",
1260
- error: `Lease renewal lost ownership for run ${run.id}`,
1261
- context: "lease-renewal",
1262
- runId: run.id
1263
- });
1264
- return;
1265
- }
1266
- const renewedLeaseExpiresAt = new Date(
1267
- Date.parse(now) + state.leaseMs
1268
- ).toISOString();
1269
- scheduleLeaseDeadline(renewedLeaseExpiresAt);
1270
- eventEmitter.emit({
1271
- type: "run:lease-renewed",
1272
- runId: run.id,
1273
- jobName: run.jobName,
1274
- leaseOwner: workerId,
1275
- leaseExpiresAt: renewedLeaseExpiresAt,
1276
- labels: run.labels
1277
- });
1278
- }).catch((error) => {
1279
- eventEmitter.emit({
1280
- type: "worker:error",
1281
- error: getErrorMessage(error),
1282
- context: "lease-renewal",
1283
- runId: run.id
1284
- });
1285
- });
1286
- }, state.leaseRenewIntervalMs);
1287
- const started = Date.now();
1288
- let reachedTerminalState = false;
1289
- try {
1290
- eventEmitter.emit({
1291
- type: "run:leased",
1292
- runId: run.id,
1293
- jobName: run.jobName,
1294
- input: run.input,
1295
- leaseOwner: workerId,
1296
- leaseExpiresAt: run.leaseExpiresAt ?? (/* @__PURE__ */ new Date()).toISOString(),
1297
- labels: run.labels
1298
- });
1299
- const output = await job.fn(step, run.input);
1300
- if (job.outputSchema) {
1301
- const parseResult = job.outputSchema.safeParse(output);
1302
- if (!parseResult.success) {
1303
- throw new Error(`Invalid output: ${parseResult.error.message}`);
1304
- }
1305
- }
1306
- const completedAt = (/* @__PURE__ */ new Date()).toISOString();
1307
- const completed = await storage.completeRun(
1308
- run.id,
1309
- run.leaseGeneration,
1310
- output,
1311
- completedAt
1312
- );
1313
- if (completed) {
1314
- reachedTerminalState = true;
1315
- eventEmitter.emit({
1316
- type: "run:complete",
1317
- runId: run.id,
1318
- jobName: run.jobName,
1319
- output,
1320
- duration: Date.now() - started,
1321
- labels: run.labels
1322
- });
1323
- } else {
1324
- eventEmitter.emit({
1325
- type: "worker:error",
1326
- error: `Lease lost before completing run ${run.id}`,
1327
- context: "run-completion"
1328
- });
1329
- }
1330
- } catch (error) {
1331
- if (error instanceof LeaseLostError || error instanceof CancelledError) {
1332
- return;
1333
- }
1334
- const errorMessage = getErrorMessage(error);
1335
- const completedAt = (/* @__PURE__ */ new Date()).toISOString();
1336
- const failed = await storage.failRun(
1337
- run.id,
1338
- run.leaseGeneration,
1339
- errorMessage,
1340
- completedAt
1341
- );
1342
- if (failed) {
1343
- reachedTerminalState = true;
1344
- const steps = await storage.getSteps(run.id);
1345
- const failedStep = steps.find((entry) => entry.status === "failed");
1346
- eventEmitter.emit({
1347
- type: "run:fail",
1348
- runId: run.id,
1349
- jobName: run.jobName,
1350
- error: errorMessage,
1351
- failedStepName: failedStep?.name ?? "unknown",
1352
- labels: run.labels
1353
- });
1354
- } else {
1355
- eventEmitter.emit({
1356
- type: "worker:error",
1357
- error: `Lease lost before recording failure for run ${run.id}`,
1358
- context: "run-failure"
1359
- });
1360
- }
1361
- } finally {
1362
- clearInterval(leaseTimer);
1363
- if (leaseDeadlineTimer) {
1364
- clearTimeout(leaseDeadlineTimer);
1365
- }
1366
- dispose();
1367
- if (!state.preserveSteps && reachedTerminalState) {
1368
- await storage.deleteSteps(run.id);
1369
- }
1370
- }
1371
1751
  }
1372
1752
  const durably = {
1373
1753
  db,
@@ -1391,7 +1771,8 @@ function createDurablyInstance(state, jobs) {
1391
1771
  storage,
1392
1772
  eventEmitter,
1393
1773
  jobRegistry,
1394
- state.labelsSchema
1774
+ state.labelsSchema,
1775
+ state.pollingIntervalMs
1395
1776
  );
1396
1777
  newHandles[key] = handle;
1397
1778
  }
@@ -1403,6 +1784,19 @@ function createDurablyInstance(state, jobs) {
1403
1784
  },
1404
1785
  getRun: storage.getRun.bind(storage),
1405
1786
  getRuns: storage.getRuns.bind(storage),
1787
+ async waitForRun(runId, options) {
1788
+ const run = await waitForRunCompletion(
1789
+ runId,
1790
+ storage,
1791
+ eventEmitter,
1792
+ {
1793
+ ...options,
1794
+ pollingIntervalMs: options?.pollingIntervalMs ?? state.pollingIntervalMs
1795
+ },
1796
+ "waitForRun"
1797
+ );
1798
+ return run;
1799
+ },
1406
1800
  use(plugin) {
1407
1801
  plugin.install(durably);
1408
1802
  },
@@ -1520,21 +1914,21 @@ function createDurablyInstance(state, jobs) {
1520
1914
  async retrigger(runId) {
1521
1915
  const run = await getRunOrThrow(runId);
1522
1916
  if (run.status === "pending") {
1523
- throw new Error(`Cannot retrigger pending run: ${runId}`);
1917
+ throw new ConflictError(`Cannot retrigger pending run: ${runId}`);
1524
1918
  }
1525
1919
  if (run.status === "leased") {
1526
- throw new Error(`Cannot retrigger leased run: ${runId}`);
1920
+ throw new ConflictError(`Cannot retrigger leased run: ${runId}`);
1527
1921
  }
1528
1922
  const job = jobRegistry.get(run.jobName);
1529
1923
  if (!job) {
1530
- throw new Error(`Unknown job: ${run.jobName}`);
1924
+ throw new NotFoundError(`Unknown job: ${run.jobName}`);
1531
1925
  }
1532
1926
  const validatedInput = validateJobInputOrThrow(
1533
1927
  job.inputSchema,
1534
1928
  run.input,
1535
1929
  `Cannot retrigger run ${runId}`
1536
1930
  );
1537
- const nextRun = await storage.enqueue({
1931
+ const { run: nextRun } = await storage.enqueue({
1538
1932
  jobName: run.jobName,
1539
1933
  input: validatedInput,
1540
1934
  concurrencyKey: run.concurrencyKey ?? void 0,
@@ -1552,19 +1946,19 @@ function createDurablyInstance(state, jobs) {
1552
1946
  async cancel(runId) {
1553
1947
  const run = await getRunOrThrow(runId);
1554
1948
  if (run.status === "completed") {
1555
- throw new Error(`Cannot cancel completed run: ${runId}`);
1949
+ throw new ConflictError(`Cannot cancel completed run: ${runId}`);
1556
1950
  }
1557
1951
  if (run.status === "failed") {
1558
- throw new Error(`Cannot cancel failed run: ${runId}`);
1952
+ throw new ConflictError(`Cannot cancel failed run: ${runId}`);
1559
1953
  }
1560
1954
  if (run.status === "cancelled") {
1561
- throw new Error(`Cannot cancel already cancelled run: ${runId}`);
1955
+ throw new ConflictError(`Cannot cancel already cancelled run: ${runId}`);
1562
1956
  }
1563
1957
  const wasPending = run.status === "pending";
1564
1958
  const cancelled = await storage.cancelRun(runId, (/* @__PURE__ */ new Date()).toISOString());
1565
1959
  if (!cancelled) {
1566
1960
  const current = await getRunOrThrow(runId);
1567
- throw new Error(
1961
+ throw new ConflictError(
1568
1962
  `Cannot cancel run ${runId}: status changed to ${current.status}`
1569
1963
  );
1570
1964
  }
@@ -1581,10 +1975,10 @@ function createDurablyInstance(state, jobs) {
1581
1975
  async deleteRun(runId) {
1582
1976
  const run = await getRunOrThrow(runId);
1583
1977
  if (run.status === "pending") {
1584
- throw new Error(`Cannot delete pending run: ${runId}`);
1978
+ throw new ConflictError(`Cannot delete pending run: ${runId}`);
1585
1979
  }
1586
1980
  if (run.status === "leased") {
1587
- throw new Error(`Cannot delete leased run: ${runId}`);
1981
+ throw new ConflictError(`Cannot delete leased run: ${runId}`);
1588
1982
  }
1589
1983
  await storage.deleteRun(runId);
1590
1984
  eventEmitter.emit({
@@ -1603,37 +1997,29 @@ function createDurablyInstance(state, jobs) {
1603
1997
  async processOne(options) {
1604
1998
  const workerId = options?.workerId ?? defaultWorkerId();
1605
1999
  const now = (/* @__PURE__ */ new Date()).toISOString();
1606
- await storage.releaseExpiredLeases(now);
1607
2000
  const run = await storage.claimNext(workerId, now, state.leaseMs);
1608
2001
  if (!run) {
1609
- if (state.retainRunsMs !== null && Date.now() - state.lastPurgeAt >= PURGE_INTERVAL_MS) {
1610
- const purgeNow = Date.now();
1611
- state.lastPurgeAt = purgeNow;
1612
- const cutoff = new Date(purgeNow - state.retainRunsMs).toISOString();
1613
- storage.purgeRuns({ olderThan: cutoff, limit: 100 }).catch((error) => {
1614
- eventEmitter.emit({
1615
- type: "worker:error",
1616
- error: getErrorMessage(error),
1617
- context: "auto-purge"
1618
- });
1619
- });
1620
- }
1621
2002
  return false;
1622
2003
  }
1623
- await executeRun(run, workerId);
2004
+ await executeRun2(run, workerId);
1624
2005
  return true;
1625
2006
  },
1626
2007
  async processUntilIdle(options) {
1627
2008
  const workerId = options?.workerId ?? defaultWorkerId();
1628
2009
  const maxRuns = options?.maxRuns ?? Number.POSITIVE_INFINITY;
1629
2010
  let processed = 0;
2011
+ let reachedIdle = false;
1630
2012
  while (processed < maxRuns) {
1631
2013
  const didProcess = await this.processOne({ workerId });
1632
2014
  if (!didProcess) {
2015
+ reachedIdle = true;
1633
2016
  break;
1634
2017
  }
1635
2018
  processed++;
1636
2019
  }
2020
+ if (reachedIdle) {
2021
+ await state.runIdleMaintenance();
2022
+ }
1637
2023
  return processed;
1638
2024
  },
1639
2025
  async migrate() {
@@ -1643,8 +2029,10 @@ function createDurablyInstance(state, jobs) {
1643
2029
  if (state.migrating) {
1644
2030
  return state.migrating;
1645
2031
  }
1646
- state.migrating = runMigrations(db).then(() => {
2032
+ state.migrating = runMigrations(db).then(async () => {
1647
2033
  state.migrated = true;
2034
+ await state.probeWalCheckpoint?.();
2035
+ state.probeWalCheckpoint = null;
1648
2036
  }).finally(() => {
1649
2037
  state.migrating = null;
1650
2038
  });
@@ -1658,8 +2046,10 @@ function createDurablyInstance(state, jobs) {
1658
2046
  return durably;
1659
2047
  }
1660
2048
  function createDurably(options) {
2049
+ const maxConcurrentRuns = options.maxConcurrentRuns !== void 0 ? validateMaxConcurrentRuns(options.maxConcurrentRuns) : DEFAULTS.maxConcurrentRuns;
1661
2050
  const config = {
1662
2051
  pollingIntervalMs: options.pollingIntervalMs ?? DEFAULTS.pollingIntervalMs,
2052
+ maxConcurrentRuns,
1663
2053
  leaseRenewIntervalMs: options.leaseRenewIntervalMs ?? DEFAULTS.leaseRenewIntervalMs,
1664
2054
  leaseMs: options.leaseMs ?? DEFAULTS.leaseMs,
1665
2055
  preserveSteps: options.preserveSteps ?? DEFAULTS.preserveSteps,
@@ -1681,15 +2071,60 @@ function createDurably(options) {
1681
2071
  });
1682
2072
  const eventEmitter = createEventEmitter();
1683
2073
  const jobRegistry = createJobRegistry();
2074
+ let lastPurgeAt = 0;
2075
+ let lastCheckpointAt = 0;
2076
+ const runIdleMaintenance = async () => {
2077
+ try {
2078
+ const nowMs = Date.now();
2079
+ await storage.releaseExpiredLeases(new Date(nowMs).toISOString());
2080
+ if (config.retainRunsMs !== null) {
2081
+ if (nowMs - lastPurgeAt >= PURGE_INTERVAL_MS) {
2082
+ lastPurgeAt = nowMs;
2083
+ const cutoff = new Date(nowMs - config.retainRunsMs).toISOString();
2084
+ await storage.purgeRuns({ olderThan: cutoff, limit: 100 });
2085
+ }
2086
+ }
2087
+ if (state.walCheckpointSupported) {
2088
+ if (nowMs - lastCheckpointAt >= CHECKPOINT_INTERVAL_MS) {
2089
+ lastCheckpointAt = nowMs;
2090
+ try {
2091
+ const result = await sql5`PRAGMA wal_checkpoint(TRUNCATE)`.execute(
2092
+ db
2093
+ );
2094
+ const row = result.rows[0];
2095
+ if (row?.busy !== 0) {
2096
+ lastCheckpointAt = nowMs - CHECKPOINT_INTERVAL_MS / 2;
2097
+ }
2098
+ } catch (checkpointError) {
2099
+ eventEmitter.emit({
2100
+ type: "worker:error",
2101
+ error: getErrorMessage(checkpointError),
2102
+ context: "wal-checkpoint"
2103
+ });
2104
+ }
2105
+ }
2106
+ }
2107
+ } catch (error) {
2108
+ eventEmitter.emit({
2109
+ type: "worker:error",
2110
+ error: getErrorMessage(error),
2111
+ context: "idle-maintenance"
2112
+ });
2113
+ }
2114
+ };
1684
2115
  let processOneImpl = null;
1685
2116
  const worker = createWorker(
1686
- { pollingIntervalMs: config.pollingIntervalMs },
2117
+ {
2118
+ pollingIntervalMs: config.pollingIntervalMs,
2119
+ maxConcurrentRuns: config.maxConcurrentRuns
2120
+ },
1687
2121
  (runtimeOptions) => {
1688
2122
  if (!processOneImpl) {
1689
2123
  throw new Error("Durably runtime is not initialized");
1690
2124
  }
1691
2125
  return processOneImpl(runtimeOptions);
1692
- }
2126
+ },
2127
+ runIdleMaintenance
1693
2128
  );
1694
2129
  const state = {
1695
2130
  db,
@@ -1701,12 +2136,27 @@ function createDurably(options) {
1701
2136
  preserveSteps: config.preserveSteps,
1702
2137
  migrating: null,
1703
2138
  migrated: false,
2139
+ walCheckpointSupported: false,
2140
+ probeWalCheckpoint: null,
1704
2141
  leaseMs: config.leaseMs,
1705
2142
  leaseRenewIntervalMs: config.leaseRenewIntervalMs,
2143
+ pollingIntervalMs: config.pollingIntervalMs,
1706
2144
  retainRunsMs: config.retainRunsMs,
1707
- lastPurgeAt: 0,
1708
- releaseBrowserSingleton
2145
+ releaseBrowserSingleton,
2146
+ runIdleMaintenance
1709
2147
  };
2148
+ if (backend === "generic" && !isBrowserLikeEnvironment()) {
2149
+ state.probeWalCheckpoint = async () => {
2150
+ try {
2151
+ const result = await sql5`PRAGMA wal_checkpoint(PASSIVE)`.execute(db);
2152
+ const row = result.rows[0];
2153
+ if (row && row.log !== -1) {
2154
+ state.walCheckpointSupported = true;
2155
+ }
2156
+ } catch {
2157
+ }
2158
+ };
2159
+ }
1710
2160
  const instance = createDurablyInstance(
1711
2161
  state,
1712
2162
  {}
@@ -1973,15 +2423,27 @@ function parseLabelsFromParams(searchParams) {
1973
2423
  }
1974
2424
  function parseRunFilter(url) {
1975
2425
  const jobNames = url.searchParams.getAll("jobName");
1976
- const statusParam = url.searchParams.get("status");
2426
+ const statusParams = url.searchParams.getAll("status");
1977
2427
  const limitParam = url.searchParams.get("limit");
1978
2428
  const offsetParam = url.searchParams.get("offset");
1979
2429
  const labels = parseLabelsFromParams(url.searchParams);
1980
- if (statusParam && !VALID_STATUSES_SET.has(statusParam)) {
1981
- return errorResponse(
1982
- `Invalid status: ${statusParam}. Must be one of: ${VALID_STATUSES.join(", ")}`,
1983
- 400
1984
- );
2430
+ let status;
2431
+ if (statusParams.length > 0) {
2432
+ for (const s of statusParams) {
2433
+ if (s === "") {
2434
+ return errorResponse(
2435
+ `Invalid status: empty value. Must be one of: ${VALID_STATUSES.join(", ")}`,
2436
+ 400
2437
+ );
2438
+ }
2439
+ if (!VALID_STATUSES_SET.has(s)) {
2440
+ return errorResponse(
2441
+ `Invalid status: ${s}. Must be one of: ${VALID_STATUSES.join(", ")}`,
2442
+ 400
2443
+ );
2444
+ }
2445
+ }
2446
+ status = statusParams.length === 1 ? statusParams[0] : statusParams;
1985
2447
  }
1986
2448
  let limit;
1987
2449
  if (limitParam) {
@@ -2002,7 +2464,7 @@ function parseRunFilter(url) {
2002
2464
  }
2003
2465
  return {
2004
2466
  jobName: jobNames.length > 0 ? jobNames : void 0,
2005
- status: statusParam,
2467
+ status,
2006
2468
  labels,
2007
2469
  limit,
2008
2470
  offset
@@ -2035,6 +2497,12 @@ function createDurablyHandler(durably, options) {
2035
2497
  return await fn();
2036
2498
  } catch (error) {
2037
2499
  if (error instanceof Response) throw error;
2500
+ if (error instanceof DurablyError) {
2501
+ return errorResponse(
2502
+ error.message,
2503
+ error.statusCode
2504
+ );
2505
+ }
2038
2506
  return errorResponse(getErrorMessage(error), 500);
2039
2507
  }
2040
2508
  }
@@ -2063,15 +2531,16 @@ function createDurablyHandler(durably, options) {
2063
2531
  if (auth?.onTrigger && ctx !== void 0) {
2064
2532
  await auth.onTrigger(ctx, body);
2065
2533
  }
2066
- const run = await job.trigger(
2067
- body.input ?? {},
2068
- {
2069
- idempotencyKey: body.idempotencyKey,
2070
- concurrencyKey: body.concurrencyKey,
2071
- labels: body.labels
2072
- }
2073
- );
2074
- const response = { runId: run.id };
2534
+ const run = await job.trigger(body.input, {
2535
+ idempotencyKey: body.idempotencyKey,
2536
+ concurrencyKey: body.concurrencyKey,
2537
+ labels: body.labels,
2538
+ coalesce: body.coalesce
2539
+ });
2540
+ const response = {
2541
+ runId: run.id,
2542
+ disposition: run.disposition
2543
+ };
2075
2544
  return jsonResponse(response);
2076
2545
  });
2077
2546
  }
@@ -2186,6 +2655,18 @@ function createDurablyHandler(durably, options) {
2186
2655
  });
2187
2656
  }
2188
2657
  }),
2658
+ durably.on("run:coalesced", (event) => {
2659
+ if (matchesFilter(event.jobName, event.labels)) {
2660
+ ctrl.enqueue({
2661
+ type: "run:coalesced",
2662
+ runId: event.runId,
2663
+ jobName: event.jobName,
2664
+ labels: event.labels,
2665
+ skippedInput: event.skippedInput,
2666
+ skippedLabels: event.skippedLabels
2667
+ });
2668
+ }
2669
+ }),
2189
2670
  durably.on("run:leased", (event) => {
2190
2671
  if (matchesFilter(event.jobName, event.labels)) {
2191
2672
  ctrl.enqueue({
@@ -2355,11 +2836,16 @@ function createDurablyHandler(durably, options) {
2355
2836
  }
2356
2837
  export {
2357
2838
  CancelledError,
2839
+ ConflictError,
2840
+ DurablyError,
2358
2841
  LeaseLostError,
2842
+ NotFoundError,
2843
+ ValidationError,
2359
2844
  createDurably,
2360
2845
  createDurablyHandler,
2361
2846
  createKyselyStore,
2362
2847
  defineJob,
2848
+ isDomainEvent,
2363
2849
  toClientRun,
2364
2850
  withLogPersistence
2365
2851
  };