@coji/durably 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/{chunk-UCUP6NMJ.js → chunk-L42OCQEV.js} +3 -3
- package/dist/chunk-L42OCQEV.js.map +1 -0
- package/dist/{index-DWsJlgyh.d.ts → index-CDCdrLgw.d.ts} +4 -2
- package/dist/index.d.ts +23 -3
- package/dist/index.js +295 -210
- package/dist/index.js.map +1 -1
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/docs/llms.md +50 -5
- package/package.json +1 -1
- package/dist/chunk-UCUP6NMJ.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
withLogPersistence
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-L42OCQEV.js";
|
|
4
4
|
|
|
5
5
|
// src/durably.ts
|
|
6
6
|
import { Kysely } from "kysely";
|
|
@@ -19,9 +19,38 @@ var LeaseLostError = class extends Error {
|
|
|
19
19
|
this.name = "LeaseLostError";
|
|
20
20
|
}
|
|
21
21
|
};
|
|
22
|
+
var DurablyError = class extends Error {
|
|
23
|
+
statusCode;
|
|
24
|
+
constructor(message, statusCode) {
|
|
25
|
+
super(message);
|
|
26
|
+
this.name = "DurablyError";
|
|
27
|
+
this.statusCode = statusCode;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
var NotFoundError = class extends DurablyError {
|
|
31
|
+
constructor(message) {
|
|
32
|
+
super(message, 404);
|
|
33
|
+
this.name = "NotFoundError";
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
var ValidationError = class extends DurablyError {
|
|
37
|
+
constructor(message) {
|
|
38
|
+
super(message, 400);
|
|
39
|
+
this.name = "ValidationError";
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
var ConflictError = class extends DurablyError {
|
|
43
|
+
constructor(message) {
|
|
44
|
+
super(message, 409);
|
|
45
|
+
this.name = "ConflictError";
|
|
46
|
+
}
|
|
47
|
+
};
|
|
22
48
|
function getErrorMessage(error) {
|
|
23
49
|
return error instanceof Error ? error.message : String(error);
|
|
24
50
|
}
|
|
51
|
+
function toError(error) {
|
|
52
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
53
|
+
}
|
|
25
54
|
|
|
26
55
|
// src/context.ts
|
|
27
56
|
var LEASE_LOST = "lease-lost";
|
|
@@ -43,6 +72,22 @@ function createStepContext(run, jobName, leaseGeneration, storage, eventEmitter)
|
|
|
43
72
|
}
|
|
44
73
|
throw new CancelledError(run.id);
|
|
45
74
|
}
|
|
75
|
+
async function throwForRefusedStep(stepName, stepIndex2) {
|
|
76
|
+
const latestRun = await storage.getRun(run.id);
|
|
77
|
+
if (latestRun?.status === "cancelled") {
|
|
78
|
+
eventEmitter.emit({
|
|
79
|
+
type: "step:cancel",
|
|
80
|
+
runId: run.id,
|
|
81
|
+
jobName,
|
|
82
|
+
stepName,
|
|
83
|
+
stepIndex: stepIndex2,
|
|
84
|
+
labels: run.labels
|
|
85
|
+
});
|
|
86
|
+
throw new CancelledError(run.id);
|
|
87
|
+
}
|
|
88
|
+
abortForLeaseLoss();
|
|
89
|
+
throw new LeaseLostError(run.id);
|
|
90
|
+
}
|
|
46
91
|
const unsubscribe = eventEmitter.on("run:cancel", (event) => {
|
|
47
92
|
if (event.runId === run.id) {
|
|
48
93
|
controller.abort();
|
|
@@ -100,8 +145,7 @@ function createStepContext(run, jobName, leaseGeneration, storage, eventEmitter)
|
|
|
100
145
|
startedAt
|
|
101
146
|
});
|
|
102
147
|
if (!savedStep) {
|
|
103
|
-
|
|
104
|
-
throwIfAborted();
|
|
148
|
+
await throwForRefusedStep(name, stepIndex);
|
|
105
149
|
}
|
|
106
150
|
stepIndex++;
|
|
107
151
|
eventEmitter.emit({
|
|
@@ -133,20 +177,17 @@ function createStepContext(run, jobName, leaseGeneration, storage, eventEmitter)
|
|
|
133
177
|
startedAt
|
|
134
178
|
});
|
|
135
179
|
if (!savedStep) {
|
|
136
|
-
|
|
137
|
-
throw new LeaseLostError(run.id);
|
|
180
|
+
await throwForRefusedStep(name, stepIndex);
|
|
138
181
|
}
|
|
139
182
|
eventEmitter.emit({
|
|
140
|
-
|
|
183
|
+
type: "step:fail",
|
|
184
|
+
error: errorMessage,
|
|
141
185
|
runId: run.id,
|
|
142
186
|
jobName,
|
|
143
187
|
stepName: name,
|
|
144
188
|
stepIndex,
|
|
145
189
|
labels: run.labels
|
|
146
190
|
});
|
|
147
|
-
if (isCancelled) {
|
|
148
|
-
throwIfAborted();
|
|
149
|
-
}
|
|
150
191
|
throw error;
|
|
151
192
|
} finally {
|
|
152
193
|
currentStepName = null;
|
|
@@ -239,16 +280,16 @@ function createEventEmitter() {
|
|
|
239
280
|
if (!typeListeners) {
|
|
240
281
|
return;
|
|
241
282
|
}
|
|
283
|
+
const reportError = (error) => errorHandler?.(toError(error), fullEvent);
|
|
242
284
|
for (const listener of typeListeners) {
|
|
243
285
|
try {
|
|
244
|
-
listener(fullEvent);
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
249
|
-
fullEvent
|
|
250
|
-
);
|
|
286
|
+
const result = listener(fullEvent);
|
|
287
|
+
if (result != null && typeof result.then === "function") {
|
|
288
|
+
;
|
|
289
|
+
result.catch(reportError);
|
|
251
290
|
}
|
|
291
|
+
} catch (error) {
|
|
292
|
+
reportError(error);
|
|
252
293
|
}
|
|
253
294
|
}
|
|
254
295
|
}
|
|
@@ -263,7 +304,9 @@ function validateJobInputOrThrow(schema, input, context) {
|
|
|
263
304
|
const result = schema.safeParse(input);
|
|
264
305
|
if (!result.success) {
|
|
265
306
|
const prefix = context ? `${context}: ` : "";
|
|
266
|
-
throw new
|
|
307
|
+
throw new ValidationError(
|
|
308
|
+
`${prefix}Invalid input: ${prettifyError(result.error)}`
|
|
309
|
+
);
|
|
267
310
|
}
|
|
268
311
|
return result.data;
|
|
269
312
|
}
|
|
@@ -387,7 +430,7 @@ function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema)
|
|
|
387
430
|
}).catch((error) => {
|
|
388
431
|
if (resolved) return;
|
|
389
432
|
cleanup();
|
|
390
|
-
reject(
|
|
433
|
+
reject(toError(error));
|
|
391
434
|
});
|
|
392
435
|
if (options?.timeout !== void 0) {
|
|
393
436
|
timeoutId = setTimeout(() => {
|
|
@@ -487,6 +530,10 @@ var migrations = [
|
|
|
487
530
|
"current_step_index",
|
|
488
531
|
"integer",
|
|
489
532
|
(col) => col.notNull().defaultTo(0)
|
|
533
|
+
).addColumn(
|
|
534
|
+
"completed_step_count",
|
|
535
|
+
"integer",
|
|
536
|
+
(col) => col.notNull().defaultTo(0)
|
|
490
537
|
).addColumn("progress", "text").addColumn("output", "text").addColumn("error", "text").addColumn("lease_owner", "text").addColumn("lease_expires_at", "text").addColumn(
|
|
491
538
|
"lease_generation",
|
|
492
539
|
"integer",
|
|
@@ -537,33 +584,13 @@ async function runMigrations(db) {
|
|
|
537
584
|
}
|
|
538
585
|
|
|
539
586
|
// src/storage.ts
|
|
540
|
-
import { sql as
|
|
587
|
+
import { sql as sql4 } from "kysely";
|
|
541
588
|
import { monotonicFactory } from "ulidx";
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
concurrencyKey,
|
|
548
|
-
leaseOwner,
|
|
549
|
-
leaseExpiresAt,
|
|
550
|
-
leaseGeneration,
|
|
551
|
-
updatedAt,
|
|
552
|
-
...clientRun
|
|
553
|
-
} = run;
|
|
554
|
-
return clientRun;
|
|
555
|
-
}
|
|
556
|
-
var LABEL_KEY_PATTERN = /^[a-zA-Z0-9\-_./]+$/;
|
|
557
|
-
function validateLabels(labels) {
|
|
558
|
-
if (!labels) return;
|
|
559
|
-
for (const key of Object.keys(labels)) {
|
|
560
|
-
if (!LABEL_KEY_PATTERN.test(key)) {
|
|
561
|
-
throw new Error(
|
|
562
|
-
`Invalid label key "${key}": must contain only alphanumeric characters, dashes, underscores, dots, and slashes`
|
|
563
|
-
);
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
}
|
|
589
|
+
|
|
590
|
+
// src/claim-postgres.ts
|
|
591
|
+
import { sql as sql2 } from "kysely";
|
|
592
|
+
|
|
593
|
+
// src/transformers.ts
|
|
567
594
|
function rowToRun(row) {
|
|
568
595
|
return {
|
|
569
596
|
id: row.id,
|
|
@@ -573,7 +600,7 @@ function rowToRun(row) {
|
|
|
573
600
|
idempotencyKey: row.idempotency_key,
|
|
574
601
|
concurrencyKey: row.concurrency_key,
|
|
575
602
|
currentStepIndex: row.current_step_index,
|
|
576
|
-
|
|
603
|
+
completedStepCount: row.completed_step_count,
|
|
577
604
|
progress: row.progress ? JSON.parse(row.progress) : null,
|
|
578
605
|
output: row.output ? JSON.parse(row.output) : null,
|
|
579
606
|
error: row.error,
|
|
@@ -611,6 +638,122 @@ function rowToLog(row) {
|
|
|
611
638
|
createdAt: row.created_at
|
|
612
639
|
};
|
|
613
640
|
}
|
|
641
|
+
var LABEL_KEY_PATTERN = /^[a-zA-Z0-9\-_./]+$/;
|
|
642
|
+
function validateLabels(labels) {
|
|
643
|
+
if (!labels) return;
|
|
644
|
+
for (const key of Object.keys(labels)) {
|
|
645
|
+
if (!LABEL_KEY_PATTERN.test(key)) {
|
|
646
|
+
throw new Error(
|
|
647
|
+
`Invalid label key "${key}": must contain only alphanumeric characters, dashes, underscores, dots, and slashes`
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
// src/claim-postgres.ts
|
|
654
|
+
async function claimNextPostgres(db, workerId, now, leaseExpiresAt, activeLeaseGuard) {
|
|
655
|
+
return await db.transaction().execute(async (trx) => {
|
|
656
|
+
const skipKeys = [];
|
|
657
|
+
for (; ; ) {
|
|
658
|
+
const concurrencyCondition = skipKeys.length > 0 ? sql2`
|
|
659
|
+
AND (
|
|
660
|
+
concurrency_key IS NULL
|
|
661
|
+
OR concurrency_key NOT IN (${sql2.join(skipKeys)})
|
|
662
|
+
)
|
|
663
|
+
` : sql2``;
|
|
664
|
+
const candidateResult = await sql2`
|
|
665
|
+
SELECT id, concurrency_key
|
|
666
|
+
FROM durably_runs
|
|
667
|
+
WHERE
|
|
668
|
+
(
|
|
669
|
+
status = 'pending'
|
|
670
|
+
OR (status = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ${now})
|
|
671
|
+
)
|
|
672
|
+
AND ${activeLeaseGuard}
|
|
673
|
+
${concurrencyCondition}
|
|
674
|
+
ORDER BY created_at ASC, id ASC
|
|
675
|
+
FOR UPDATE SKIP LOCKED
|
|
676
|
+
LIMIT 1
|
|
677
|
+
`.execute(trx);
|
|
678
|
+
const candidate = candidateResult.rows[0];
|
|
679
|
+
if (!candidate) return null;
|
|
680
|
+
if (candidate.concurrency_key) {
|
|
681
|
+
await sql2`SELECT pg_advisory_xact_lock(hashtext(${candidate.concurrency_key}))`.execute(
|
|
682
|
+
trx
|
|
683
|
+
);
|
|
684
|
+
const conflict = await sql2`
|
|
685
|
+
SELECT 1 FROM durably_runs
|
|
686
|
+
WHERE concurrency_key = ${candidate.concurrency_key}
|
|
687
|
+
AND id <> ${candidate.id}
|
|
688
|
+
AND status = 'leased'
|
|
689
|
+
AND lease_expires_at IS NOT NULL
|
|
690
|
+
AND lease_expires_at > ${now}
|
|
691
|
+
LIMIT 1
|
|
692
|
+
`.execute(trx);
|
|
693
|
+
if (conflict.rows.length > 0) {
|
|
694
|
+
skipKeys.push(candidate.concurrency_key);
|
|
695
|
+
continue;
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
const result = await sql2`
|
|
699
|
+
UPDATE durably_runs
|
|
700
|
+
SET
|
|
701
|
+
status = 'leased',
|
|
702
|
+
lease_owner = ${workerId},
|
|
703
|
+
lease_expires_at = ${leaseExpiresAt},
|
|
704
|
+
lease_generation = lease_generation + 1,
|
|
705
|
+
started_at = COALESCE(started_at, ${now}),
|
|
706
|
+
updated_at = ${now}
|
|
707
|
+
WHERE id = ${candidate.id}
|
|
708
|
+
RETURNING *
|
|
709
|
+
`.execute(trx);
|
|
710
|
+
const row = result.rows[0];
|
|
711
|
+
if (!row) return null;
|
|
712
|
+
return rowToRun(row);
|
|
713
|
+
}
|
|
714
|
+
});
|
|
715
|
+
}
|
|
716
|
+
|
|
717
|
+
// src/claim-sqlite.ts
|
|
718
|
+
import { sql as sql3 } from "kysely";
|
|
719
|
+
async function claimNextSqlite(db, workerId, now, leaseExpiresAt, activeLeaseGuard) {
|
|
720
|
+
const subquery = db.selectFrom("durably_runs").select("durably_runs.id").where(
|
|
721
|
+
(eb) => eb.or([
|
|
722
|
+
eb("status", "=", "pending"),
|
|
723
|
+
eb.and([
|
|
724
|
+
eb("status", "=", "leased"),
|
|
725
|
+
eb("lease_expires_at", "is not", null),
|
|
726
|
+
eb("lease_expires_at", "<=", now)
|
|
727
|
+
])
|
|
728
|
+
])
|
|
729
|
+
).where(activeLeaseGuard).orderBy("created_at", "asc").orderBy("id", "asc").limit(1);
|
|
730
|
+
const row = await db.updateTable("durably_runs").set({
|
|
731
|
+
status: "leased",
|
|
732
|
+
lease_owner: workerId,
|
|
733
|
+
lease_expires_at: leaseExpiresAt,
|
|
734
|
+
lease_generation: sql3`lease_generation + 1`,
|
|
735
|
+
started_at: sql3`COALESCE(started_at, ${now})`,
|
|
736
|
+
updated_at: now
|
|
737
|
+
}).where("id", "=", (eb) => eb.selectFrom(subquery.as("sub")).select("id")).returningAll().executeTakeFirst();
|
|
738
|
+
if (!row) return null;
|
|
739
|
+
return rowToRun(row);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
// src/storage.ts
|
|
743
|
+
var ulid = monotonicFactory();
|
|
744
|
+
var TERMINAL_STATUSES = ["completed", "failed", "cancelled"];
|
|
745
|
+
function toClientRun(run) {
|
|
746
|
+
const {
|
|
747
|
+
idempotencyKey,
|
|
748
|
+
concurrencyKey,
|
|
749
|
+
leaseOwner,
|
|
750
|
+
leaseExpiresAt,
|
|
751
|
+
leaseGeneration,
|
|
752
|
+
updatedAt,
|
|
753
|
+
...clientRun
|
|
754
|
+
} = run;
|
|
755
|
+
return clientRun;
|
|
756
|
+
}
|
|
614
757
|
function createWriteMutex() {
|
|
615
758
|
let queue = Promise.resolve();
|
|
616
759
|
return async function withWriteLock(fn) {
|
|
@@ -672,6 +815,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
672
815
|
idempotency_key: input.idempotencyKey ?? null,
|
|
673
816
|
concurrency_key: input.concurrencyKey ?? null,
|
|
674
817
|
current_step_index: 0,
|
|
818
|
+
completed_step_count: 0,
|
|
675
819
|
progress: null,
|
|
676
820
|
output: null,
|
|
677
821
|
error: null,
|
|
@@ -723,6 +867,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
723
867
|
idempotency_key: input.idempotencyKey ?? null,
|
|
724
868
|
concurrency_key: input.concurrencyKey ?? null,
|
|
725
869
|
current_step_index: 0,
|
|
870
|
+
completed_step_count: 0,
|
|
726
871
|
progress: null,
|
|
727
872
|
output: null,
|
|
728
873
|
error: null,
|
|
@@ -747,25 +892,21 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
747
892
|
});
|
|
748
893
|
},
|
|
749
894
|
async getRun(runId) {
|
|
750
|
-
const row = await db.selectFrom("durably_runs").
|
|
751
|
-
(eb) => eb.fn.count("durably_steps.id").as("step_count")
|
|
752
|
-
).where("durably_runs.id", "=", runId).groupBy("durably_runs.id").executeTakeFirst();
|
|
895
|
+
const row = await db.selectFrom("durably_runs").selectAll().where("id", "=", runId).executeTakeFirst();
|
|
753
896
|
return row ? rowToRun(row) : null;
|
|
754
897
|
},
|
|
755
898
|
async getRuns(filter) {
|
|
756
|
-
let query = db.selectFrom("durably_runs").
|
|
757
|
-
(eb) => eb.fn.count("durably_steps.id").as("step_count")
|
|
758
|
-
).groupBy("durably_runs.id");
|
|
899
|
+
let query = db.selectFrom("durably_runs").selectAll();
|
|
759
900
|
if (filter?.status) {
|
|
760
|
-
query = query.where("
|
|
901
|
+
query = query.where("status", "=", filter.status);
|
|
761
902
|
}
|
|
762
903
|
if (filter?.jobName) {
|
|
763
904
|
if (Array.isArray(filter.jobName)) {
|
|
764
905
|
if (filter.jobName.length > 0) {
|
|
765
|
-
query = query.where("
|
|
906
|
+
query = query.where("job_name", "in", filter.jobName);
|
|
766
907
|
}
|
|
767
908
|
} else {
|
|
768
|
-
query = query.where("
|
|
909
|
+
query = query.where("job_name", "=", filter.jobName);
|
|
769
910
|
}
|
|
770
911
|
}
|
|
771
912
|
if (filter?.labels) {
|
|
@@ -773,18 +914,14 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
773
914
|
validateLabels(labels);
|
|
774
915
|
for (const [key, value] of Object.entries(labels)) {
|
|
775
916
|
if (value === void 0) continue;
|
|
776
|
-
const jsonFallback = backend === "postgres" ? sql2`durably_runs.labels ->> ${key} = ${value}` : sql2`json_extract(durably_runs.labels, ${`$.${key}`}) = ${value}`;
|
|
777
917
|
query = query.where(
|
|
778
|
-
(eb) => eb.
|
|
779
|
-
eb.
|
|
780
|
-
|
|
781
|
-
),
|
|
782
|
-
jsonFallback
|
|
783
|
-
])
|
|
918
|
+
(eb) => eb.exists(
|
|
919
|
+
eb.selectFrom("durably_run_labels").select(sql4.lit(1).as("one")).whereRef("durably_run_labels.run_id", "=", "durably_runs.id").where("durably_run_labels.key", "=", key).where("durably_run_labels.value", "=", value)
|
|
920
|
+
)
|
|
784
921
|
);
|
|
785
922
|
}
|
|
786
923
|
}
|
|
787
|
-
query = query.orderBy("
|
|
924
|
+
query = query.orderBy("created_at", "desc");
|
|
788
925
|
if (filter?.limit !== void 0) {
|
|
789
926
|
query = query.limit(filter.limit);
|
|
790
927
|
}
|
|
@@ -830,7 +967,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
830
967
|
},
|
|
831
968
|
async claimNext(workerId, now, leaseMs) {
|
|
832
969
|
const leaseExpiresAt = new Date(Date.parse(now) + leaseMs).toISOString();
|
|
833
|
-
const activeLeaseGuard =
|
|
970
|
+
const activeLeaseGuard = sql4`
|
|
834
971
|
(
|
|
835
972
|
concurrency_key IS NULL
|
|
836
973
|
OR NOT EXISTS (
|
|
@@ -844,92 +981,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
844
981
|
)
|
|
845
982
|
)
|
|
846
983
|
`;
|
|
847
|
-
|
|
848
|
-
return await db.transaction().execute(async (trx) => {
|
|
849
|
-
const skipKeys = [];
|
|
850
|
-
for (; ; ) {
|
|
851
|
-
const concurrencyCondition = skipKeys.length > 0 ? sql2`
|
|
852
|
-
AND (
|
|
853
|
-
concurrency_key IS NULL
|
|
854
|
-
OR concurrency_key NOT IN (${sql2.join(skipKeys)})
|
|
855
|
-
)
|
|
856
|
-
` : sql2``;
|
|
857
|
-
const candidateResult = await sql2`
|
|
858
|
-
SELECT id, concurrency_key
|
|
859
|
-
FROM durably_runs
|
|
860
|
-
WHERE
|
|
861
|
-
(
|
|
862
|
-
status = 'pending'
|
|
863
|
-
OR (status = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ${now})
|
|
864
|
-
)
|
|
865
|
-
AND ${activeLeaseGuard}
|
|
866
|
-
${concurrencyCondition}
|
|
867
|
-
ORDER BY created_at ASC, id ASC
|
|
868
|
-
FOR UPDATE SKIP LOCKED
|
|
869
|
-
LIMIT 1
|
|
870
|
-
`.execute(trx);
|
|
871
|
-
const candidate = candidateResult.rows[0];
|
|
872
|
-
if (!candidate) return null;
|
|
873
|
-
if (candidate.concurrency_key) {
|
|
874
|
-
await sql2`SELECT pg_advisory_xact_lock(hashtext(${candidate.concurrency_key}))`.execute(
|
|
875
|
-
trx
|
|
876
|
-
);
|
|
877
|
-
const conflict = await sql2`
|
|
878
|
-
SELECT 1 FROM durably_runs
|
|
879
|
-
WHERE concurrency_key = ${candidate.concurrency_key}
|
|
880
|
-
AND id <> ${candidate.id}
|
|
881
|
-
AND status = 'leased'
|
|
882
|
-
AND lease_expires_at IS NOT NULL
|
|
883
|
-
AND lease_expires_at > ${now}
|
|
884
|
-
LIMIT 1
|
|
885
|
-
`.execute(trx);
|
|
886
|
-
if (conflict.rows.length > 0) {
|
|
887
|
-
skipKeys.push(candidate.concurrency_key);
|
|
888
|
-
continue;
|
|
889
|
-
}
|
|
890
|
-
}
|
|
891
|
-
const result = await sql2`
|
|
892
|
-
UPDATE durably_runs
|
|
893
|
-
SET
|
|
894
|
-
status = 'leased',
|
|
895
|
-
lease_owner = ${workerId},
|
|
896
|
-
lease_expires_at = ${leaseExpiresAt},
|
|
897
|
-
lease_generation = lease_generation + 1,
|
|
898
|
-
started_at = COALESCE(started_at, ${now}),
|
|
899
|
-
updated_at = ${now}
|
|
900
|
-
WHERE id = ${candidate.id}
|
|
901
|
-
RETURNING *
|
|
902
|
-
`.execute(trx);
|
|
903
|
-
const row2 = result.rows[0];
|
|
904
|
-
if (!row2) return null;
|
|
905
|
-
return rowToRun({ ...row2, step_count: 0 });
|
|
906
|
-
}
|
|
907
|
-
});
|
|
908
|
-
}
|
|
909
|
-
let subquery = db.selectFrom("durably_runs").select("durably_runs.id").where(
|
|
910
|
-
(eb) => eb.or([
|
|
911
|
-
eb("status", "=", "pending"),
|
|
912
|
-
eb.and([
|
|
913
|
-
eb("status", "=", "leased"),
|
|
914
|
-
eb("lease_expires_at", "is not", null),
|
|
915
|
-
eb("lease_expires_at", "<=", now)
|
|
916
|
-
])
|
|
917
|
-
])
|
|
918
|
-
).where(activeLeaseGuard).orderBy("created_at", "asc").orderBy("id", "asc").limit(1);
|
|
919
|
-
const row = await db.updateTable("durably_runs").set({
|
|
920
|
-
status: "leased",
|
|
921
|
-
lease_owner: workerId,
|
|
922
|
-
lease_expires_at: leaseExpiresAt,
|
|
923
|
-
lease_generation: sql2`lease_generation + 1`,
|
|
924
|
-
started_at: sql2`COALESCE(started_at, ${now})`,
|
|
925
|
-
updated_at: now
|
|
926
|
-
}).where(
|
|
927
|
-
"id",
|
|
928
|
-
"=",
|
|
929
|
-
(eb) => eb.selectFrom(subquery.as("sub")).select("id")
|
|
930
|
-
).returningAll().executeTakeFirst();
|
|
931
|
-
if (!row) return null;
|
|
932
|
-
return rowToRun({ ...row, step_count: 0 });
|
|
984
|
+
return backend === "postgres" ? claimNextPostgres(db, workerId, now, leaseExpiresAt, activeLeaseGuard) : claimNextSqlite(db, workerId, now, leaseExpiresAt, activeLeaseGuard);
|
|
933
985
|
},
|
|
934
986
|
async renewLease(runId, leaseGeneration, now, leaseMs) {
|
|
935
987
|
const leaseExpiresAt = new Date(Date.parse(now) + leaseMs).toISOString();
|
|
@@ -977,19 +1029,20 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
977
1029
|
const outputJson = input.output !== void 0 ? JSON.stringify(input.output) : null;
|
|
978
1030
|
const errorValue = input.error ?? null;
|
|
979
1031
|
return await db.transaction().execute(async (trx) => {
|
|
980
|
-
const insertResult = await
|
|
1032
|
+
const insertResult = await sql4`
|
|
981
1033
|
INSERT INTO durably_steps (id, run_id, name, "index", status, output, error, started_at, completed_at)
|
|
982
1034
|
SELECT ${id}, ${runId}, ${input.name}, ${input.index}, ${input.status},
|
|
983
1035
|
${outputJson}, ${errorValue}, ${input.startedAt}, ${completedAt}
|
|
984
1036
|
FROM durably_runs
|
|
985
|
-
WHERE id = ${runId} AND lease_generation = ${leaseGeneration}
|
|
1037
|
+
WHERE id = ${runId} AND status = 'leased' AND lease_generation = ${leaseGeneration}
|
|
986
1038
|
`.execute(trx);
|
|
987
1039
|
if (Number(insertResult.numAffectedRows) === 0) return null;
|
|
988
1040
|
if (input.status === "completed") {
|
|
989
1041
|
await trx.updateTable("durably_runs").set({
|
|
990
1042
|
current_step_index: input.index + 1,
|
|
1043
|
+
completed_step_count: sql4`completed_step_count + 1`,
|
|
991
1044
|
updated_at: completedAt
|
|
992
|
-
}).where("id", "=", runId).where("lease_generation", "=", leaseGeneration).execute();
|
|
1045
|
+
}).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).execute();
|
|
993
1046
|
}
|
|
994
1047
|
return {
|
|
995
1048
|
id,
|
|
@@ -1020,7 +1073,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
1020
1073
|
await db.updateTable("durably_runs").set({
|
|
1021
1074
|
progress: progress ? JSON.stringify(progress) : null,
|
|
1022
1075
|
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1023
|
-
}).where("id", "=", runId).where("lease_generation", "=", leaseGeneration).execute();
|
|
1076
|
+
}).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).execute();
|
|
1024
1077
|
},
|
|
1025
1078
|
async createLog(input) {
|
|
1026
1079
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -1042,32 +1095,34 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
1042
1095
|
return rows.map(rowToLog);
|
|
1043
1096
|
}
|
|
1044
1097
|
};
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
const
|
|
1064
|
-
|
|
1098
|
+
if (backend !== "postgres") {
|
|
1099
|
+
const mutatingKeys = [
|
|
1100
|
+
"enqueue",
|
|
1101
|
+
"enqueueMany",
|
|
1102
|
+
"updateRun",
|
|
1103
|
+
"deleteRun",
|
|
1104
|
+
"purgeRuns",
|
|
1105
|
+
"claimNext",
|
|
1106
|
+
"renewLease",
|
|
1107
|
+
"releaseExpiredLeases",
|
|
1108
|
+
"completeRun",
|
|
1109
|
+
"failRun",
|
|
1110
|
+
"cancelRun",
|
|
1111
|
+
"persistStep",
|
|
1112
|
+
"deleteSteps",
|
|
1113
|
+
"updateProgress",
|
|
1114
|
+
"createLog"
|
|
1115
|
+
];
|
|
1116
|
+
for (const key of mutatingKeys) {
|
|
1117
|
+
const original = store[key];
|
|
1118
|
+
store[key] = (...args) => withWriteLock(() => original.apply(store, args));
|
|
1119
|
+
}
|
|
1065
1120
|
}
|
|
1066
1121
|
return store;
|
|
1067
1122
|
}
|
|
1068
1123
|
|
|
1069
1124
|
// src/worker.ts
|
|
1070
|
-
function createWorker(config, processOne) {
|
|
1125
|
+
function createWorker(config, processOne, onIdle) {
|
|
1071
1126
|
let running = false;
|
|
1072
1127
|
let pollingTimeout = null;
|
|
1073
1128
|
let inFlight = null;
|
|
@@ -1077,9 +1132,18 @@ function createWorker(config, processOne) {
|
|
|
1077
1132
|
if (!running) {
|
|
1078
1133
|
return;
|
|
1079
1134
|
}
|
|
1135
|
+
const cycle = (async () => {
|
|
1136
|
+
const didProcess = await processOne({ workerId: activeWorkerId });
|
|
1137
|
+
if (!didProcess && onIdle && running) {
|
|
1138
|
+
try {
|
|
1139
|
+
await onIdle();
|
|
1140
|
+
} catch {
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
})();
|
|
1144
|
+
inFlight = cycle;
|
|
1080
1145
|
try {
|
|
1081
|
-
|
|
1082
|
-
await inFlight;
|
|
1146
|
+
await cycle;
|
|
1083
1147
|
} finally {
|
|
1084
1148
|
inFlight = null;
|
|
1085
1149
|
}
|
|
@@ -1213,7 +1277,7 @@ function createDurablyInstance(state, jobs) {
|
|
|
1213
1277
|
async function getRunOrThrow(runId) {
|
|
1214
1278
|
const run = await storage.getRun(runId);
|
|
1215
1279
|
if (!run) {
|
|
1216
|
-
throw new
|
|
1280
|
+
throw new NotFoundError(`Run not found: ${runId}`);
|
|
1217
1281
|
}
|
|
1218
1282
|
return run;
|
|
1219
1283
|
}
|
|
@@ -1520,14 +1584,14 @@ function createDurablyInstance(state, jobs) {
|
|
|
1520
1584
|
async retrigger(runId) {
|
|
1521
1585
|
const run = await getRunOrThrow(runId);
|
|
1522
1586
|
if (run.status === "pending") {
|
|
1523
|
-
throw new
|
|
1587
|
+
throw new ConflictError(`Cannot retrigger pending run: ${runId}`);
|
|
1524
1588
|
}
|
|
1525
1589
|
if (run.status === "leased") {
|
|
1526
|
-
throw new
|
|
1590
|
+
throw new ConflictError(`Cannot retrigger leased run: ${runId}`);
|
|
1527
1591
|
}
|
|
1528
1592
|
const job = jobRegistry.get(run.jobName);
|
|
1529
1593
|
if (!job) {
|
|
1530
|
-
throw new
|
|
1594
|
+
throw new NotFoundError(`Unknown job: ${run.jobName}`);
|
|
1531
1595
|
}
|
|
1532
1596
|
const validatedInput = validateJobInputOrThrow(
|
|
1533
1597
|
job.inputSchema,
|
|
@@ -1552,19 +1616,19 @@ function createDurablyInstance(state, jobs) {
|
|
|
1552
1616
|
async cancel(runId) {
|
|
1553
1617
|
const run = await getRunOrThrow(runId);
|
|
1554
1618
|
if (run.status === "completed") {
|
|
1555
|
-
throw new
|
|
1619
|
+
throw new ConflictError(`Cannot cancel completed run: ${runId}`);
|
|
1556
1620
|
}
|
|
1557
1621
|
if (run.status === "failed") {
|
|
1558
|
-
throw new
|
|
1622
|
+
throw new ConflictError(`Cannot cancel failed run: ${runId}`);
|
|
1559
1623
|
}
|
|
1560
1624
|
if (run.status === "cancelled") {
|
|
1561
|
-
throw new
|
|
1625
|
+
throw new ConflictError(`Cannot cancel already cancelled run: ${runId}`);
|
|
1562
1626
|
}
|
|
1563
1627
|
const wasPending = run.status === "pending";
|
|
1564
1628
|
const cancelled = await storage.cancelRun(runId, (/* @__PURE__ */ new Date()).toISOString());
|
|
1565
1629
|
if (!cancelled) {
|
|
1566
1630
|
const current = await getRunOrThrow(runId);
|
|
1567
|
-
throw new
|
|
1631
|
+
throw new ConflictError(
|
|
1568
1632
|
`Cannot cancel run ${runId}: status changed to ${current.status}`
|
|
1569
1633
|
);
|
|
1570
1634
|
}
|
|
@@ -1581,10 +1645,10 @@ function createDurablyInstance(state, jobs) {
|
|
|
1581
1645
|
async deleteRun(runId) {
|
|
1582
1646
|
const run = await getRunOrThrow(runId);
|
|
1583
1647
|
if (run.status === "pending") {
|
|
1584
|
-
throw new
|
|
1648
|
+
throw new ConflictError(`Cannot delete pending run: ${runId}`);
|
|
1585
1649
|
}
|
|
1586
1650
|
if (run.status === "leased") {
|
|
1587
|
-
throw new
|
|
1651
|
+
throw new ConflictError(`Cannot delete leased run: ${runId}`);
|
|
1588
1652
|
}
|
|
1589
1653
|
await storage.deleteRun(runId);
|
|
1590
1654
|
eventEmitter.emit({
|
|
@@ -1603,21 +1667,8 @@ function createDurablyInstance(state, jobs) {
|
|
|
1603
1667
|
async processOne(options) {
|
|
1604
1668
|
const workerId = options?.workerId ?? defaultWorkerId();
|
|
1605
1669
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1606
|
-
await storage.releaseExpiredLeases(now);
|
|
1607
1670
|
const run = await storage.claimNext(workerId, now, state.leaseMs);
|
|
1608
1671
|
if (!run) {
|
|
1609
|
-
if (state.retainRunsMs !== null && Date.now() - state.lastPurgeAt >= PURGE_INTERVAL_MS) {
|
|
1610
|
-
const purgeNow = Date.now();
|
|
1611
|
-
state.lastPurgeAt = purgeNow;
|
|
1612
|
-
const cutoff = new Date(purgeNow - state.retainRunsMs).toISOString();
|
|
1613
|
-
storage.purgeRuns({ olderThan: cutoff, limit: 100 }).catch((error) => {
|
|
1614
|
-
eventEmitter.emit({
|
|
1615
|
-
type: "worker:error",
|
|
1616
|
-
error: getErrorMessage(error),
|
|
1617
|
-
context: "auto-purge"
|
|
1618
|
-
});
|
|
1619
|
-
});
|
|
1620
|
-
}
|
|
1621
1672
|
return false;
|
|
1622
1673
|
}
|
|
1623
1674
|
await executeRun(run, workerId);
|
|
@@ -1627,13 +1678,18 @@ function createDurablyInstance(state, jobs) {
|
|
|
1627
1678
|
const workerId = options?.workerId ?? defaultWorkerId();
|
|
1628
1679
|
const maxRuns = options?.maxRuns ?? Number.POSITIVE_INFINITY;
|
|
1629
1680
|
let processed = 0;
|
|
1681
|
+
let reachedIdle = false;
|
|
1630
1682
|
while (processed < maxRuns) {
|
|
1631
1683
|
const didProcess = await this.processOne({ workerId });
|
|
1632
1684
|
if (!didProcess) {
|
|
1685
|
+
reachedIdle = true;
|
|
1633
1686
|
break;
|
|
1634
1687
|
}
|
|
1635
1688
|
processed++;
|
|
1636
1689
|
}
|
|
1690
|
+
if (reachedIdle) {
|
|
1691
|
+
await state.runIdleMaintenance();
|
|
1692
|
+
}
|
|
1637
1693
|
return processed;
|
|
1638
1694
|
},
|
|
1639
1695
|
async migrate() {
|
|
@@ -1681,6 +1737,27 @@ function createDurably(options) {
|
|
|
1681
1737
|
});
|
|
1682
1738
|
const eventEmitter = createEventEmitter();
|
|
1683
1739
|
const jobRegistry = createJobRegistry();
|
|
1740
|
+
let lastPurgeAt = 0;
|
|
1741
|
+
const runIdleMaintenance = async () => {
|
|
1742
|
+
try {
|
|
1743
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1744
|
+
await storage.releaseExpiredLeases(now);
|
|
1745
|
+
if (config.retainRunsMs !== null) {
|
|
1746
|
+
const purgeNow = Date.now();
|
|
1747
|
+
if (purgeNow - lastPurgeAt >= PURGE_INTERVAL_MS) {
|
|
1748
|
+
lastPurgeAt = purgeNow;
|
|
1749
|
+
const cutoff = new Date(purgeNow - config.retainRunsMs).toISOString();
|
|
1750
|
+
await storage.purgeRuns({ olderThan: cutoff, limit: 100 });
|
|
1751
|
+
}
|
|
1752
|
+
}
|
|
1753
|
+
} catch (error) {
|
|
1754
|
+
eventEmitter.emit({
|
|
1755
|
+
type: "worker:error",
|
|
1756
|
+
error: getErrorMessage(error),
|
|
1757
|
+
context: "idle-maintenance"
|
|
1758
|
+
});
|
|
1759
|
+
}
|
|
1760
|
+
};
|
|
1684
1761
|
let processOneImpl = null;
|
|
1685
1762
|
const worker = createWorker(
|
|
1686
1763
|
{ pollingIntervalMs: config.pollingIntervalMs },
|
|
@@ -1689,7 +1766,8 @@ function createDurably(options) {
|
|
|
1689
1766
|
throw new Error("Durably runtime is not initialized");
|
|
1690
1767
|
}
|
|
1691
1768
|
return processOneImpl(runtimeOptions);
|
|
1692
|
-
}
|
|
1769
|
+
},
|
|
1770
|
+
runIdleMaintenance
|
|
1693
1771
|
);
|
|
1694
1772
|
const state = {
|
|
1695
1773
|
db,
|
|
@@ -1704,8 +1782,8 @@ function createDurably(options) {
|
|
|
1704
1782
|
leaseMs: config.leaseMs,
|
|
1705
1783
|
leaseRenewIntervalMs: config.leaseRenewIntervalMs,
|
|
1706
1784
|
retainRunsMs: config.retainRunsMs,
|
|
1707
|
-
|
|
1708
|
-
|
|
1785
|
+
releaseBrowserSingleton,
|
|
1786
|
+
runIdleMaintenance
|
|
1709
1787
|
};
|
|
1710
1788
|
const instance = createDurablyInstance(
|
|
1711
1789
|
state,
|
|
@@ -2035,6 +2113,12 @@ function createDurablyHandler(durably, options) {
|
|
|
2035
2113
|
return await fn();
|
|
2036
2114
|
} catch (error) {
|
|
2037
2115
|
if (error instanceof Response) throw error;
|
|
2116
|
+
if (error instanceof DurablyError) {
|
|
2117
|
+
return errorResponse(
|
|
2118
|
+
error.message,
|
|
2119
|
+
error.statusCode
|
|
2120
|
+
);
|
|
2121
|
+
}
|
|
2038
2122
|
return errorResponse(getErrorMessage(error), 500);
|
|
2039
2123
|
}
|
|
2040
2124
|
}
|
|
@@ -2063,14 +2147,11 @@ function createDurablyHandler(durably, options) {
|
|
|
2063
2147
|
if (auth?.onTrigger && ctx !== void 0) {
|
|
2064
2148
|
await auth.onTrigger(ctx, body);
|
|
2065
2149
|
}
|
|
2066
|
-
const run = await job.trigger(
|
|
2067
|
-
body.
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
labels: body.labels
|
|
2072
|
-
}
|
|
2073
|
-
);
|
|
2150
|
+
const run = await job.trigger(body.input, {
|
|
2151
|
+
idempotencyKey: body.idempotencyKey,
|
|
2152
|
+
concurrencyKey: body.concurrencyKey,
|
|
2153
|
+
labels: body.labels
|
|
2154
|
+
});
|
|
2074
2155
|
const response = { runId: run.id };
|
|
2075
2156
|
return jsonResponse(response);
|
|
2076
2157
|
});
|
|
@@ -2355,7 +2436,11 @@ function createDurablyHandler(durably, options) {
|
|
|
2355
2436
|
}
|
|
2356
2437
|
export {
|
|
2357
2438
|
CancelledError,
|
|
2439
|
+
ConflictError,
|
|
2440
|
+
DurablyError,
|
|
2358
2441
|
LeaseLostError,
|
|
2442
|
+
NotFoundError,
|
|
2443
|
+
ValidationError,
|
|
2359
2444
|
createDurably,
|
|
2360
2445
|
createDurablyHandler,
|
|
2361
2446
|
createKyselyStore,
|