@iterativeflow/conformance 2.0.0-alpha.9 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/index.d.mts +25 -1
- package/dist/index.mjs +547 -73
- package/package.json +6 -6
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ahmed Rowaihi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/dist/index.d.mts
CHANGED
|
@@ -82,4 +82,28 @@ declare const engineConformance: (label: string, makeBackend: () => Backend | Pr
|
|
|
82
82
|
*/
|
|
83
83
|
declare const claimFilterConformance: (label: string, makeBackend: () => Backend$1 | Promise<Backend$1>) => void;
|
|
84
84
|
//#endregion
|
|
85
|
-
|
|
85
|
+
//#region src/sharded-claim.d.ts
|
|
86
|
+
/**
|
|
87
|
+
* A horizontally-sharded worker pool where each pod claims only the flow names it registered.
|
|
88
|
+
* Proves that two pods claiming ONE backend concurrently are SAFE — no run is ever leased by both,
|
|
89
|
+
* and no pod ever leases a run outside its shard — and LIVE: together they drain both shards. Server
|
|
90
|
+
* backends only; single-writer backends (sqlite / memory / durable-objects) can't issue concurrent
|
|
91
|
+
* transactions, so their name-shard claim is covered sequentially by {@link claimFilterConformance}.
|
|
92
|
+
*
|
|
93
|
+
* Per-round throughput is backend-specific: Postgres locks only the name-matched rows, so both shards
|
|
94
|
+
* drain in one round; MySQL's `FOR UPDATE OF j SKIP LOCKED` locks the eligible `job` head before the
|
|
95
|
+
* `run`-side name filter applies, so the first claimer transiently blocks the other's shard, which
|
|
96
|
+
* fills on the next round. Both are safe — the loop asserts safety every round and liveness overall.
|
|
97
|
+
*/
|
|
98
|
+
declare const shardedClaimConformance: (label: string, makeBackend: () => Backend$1 | Promise<Backend$1>) => void;
|
|
99
|
+
//#endregion
|
|
100
|
+
//#region src/pending-work.d.ts
|
|
101
|
+
/**
|
|
102
|
+
* Pins the autoscaling-backlog reads — `Queue.depth(now, names)`, `Timer.dueCount(now, names)`, and
|
|
103
|
+
* `Store.dueCronCount(now, names)` — that `engine.pendingWork` composes. Every backend must count the
|
|
104
|
+
* same due work (claimable jobs, due timers, due crons), exclude leased/future work, and filter by
|
|
105
|
+
* flow name identically.
|
|
106
|
+
*/
|
|
107
|
+
declare const pendingWorkConformance: (label: string, makeBackend: () => Backend$1 | Promise<Backend$1>) => void;
|
|
108
|
+
//#endregion
|
|
109
|
+
export { claimFilterConformance, cronConformance, engineConformance, outboxConformance, pendingWorkConformance, queueConformance, reconcileConformance, shardedClaimConformance, signalConformance, storeConformance, timerConformance, wakeupConformance };
|
package/dist/index.mjs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
|
-
import { builder, cancelRun, defineFlow, reconcile, registry, signalRun, signalType, submit, tickOnce } from "@iterativeflow/core";
|
|
2
|
+
import { builder, cancelRun, defineFlow, reconcile, registry, runTick, serverlessTick, signalRun, signalType, submit, tickOnce } from "@iterativeflow/core";
|
|
3
3
|
import { isTerminal } from "@iterativeflow/core/backend";
|
|
4
4
|
//#region src/store.ts
|
|
5
5
|
/**
|
|
@@ -119,15 +119,17 @@ const storeConformance = (label, makeStore) => {
|
|
|
119
119
|
input: {},
|
|
120
120
|
idempotencyKey: "k"
|
|
121
121
|
});
|
|
122
|
-
|
|
122
|
+
const b = await s.startRun({
|
|
123
123
|
name: "f",
|
|
124
124
|
version: 2,
|
|
125
125
|
input: {},
|
|
126
126
|
idempotencyKey: "k"
|
|
127
|
-
})
|
|
127
|
+
});
|
|
128
|
+
expect(b.runId).not.toBe(a.runId);
|
|
128
129
|
});
|
|
129
130
|
it("loadRun returns undefined for an unknown run", async () => {
|
|
130
|
-
|
|
131
|
+
const s = await makeStore();
|
|
132
|
+
expect(await s.loadRun("00000000-0000-0000-0000-000000000000")).toBeUndefined();
|
|
131
133
|
});
|
|
132
134
|
it("loadRunRow returns just the run row (no step/signal maps), undefined if gone", async () => {
|
|
133
135
|
const s = await makeStore();
|
|
@@ -210,7 +212,8 @@ const storeConformance = (label, makeStore) => {
|
|
|
210
212
|
result: 42,
|
|
211
213
|
attempts: 1
|
|
212
214
|
});
|
|
213
|
-
|
|
215
|
+
const snap = await s.loadRun(runId);
|
|
216
|
+
expect(snap?.steps.get("a")).toMatchObject({
|
|
214
217
|
status: "ok",
|
|
215
218
|
result: 42,
|
|
216
219
|
attempts: 1
|
|
@@ -390,7 +393,8 @@ const storeConformance = (label, makeStore) => {
|
|
|
390
393
|
expect(await s.deleteRunsOlderThan(after, 1)).toBe(0);
|
|
391
394
|
});
|
|
392
395
|
it("checkpointStep on an unknown run is rejected (no orphan step)", async () => {
|
|
393
|
-
|
|
396
|
+
const s = await makeStore();
|
|
397
|
+
await expect(s.checkpointStep({
|
|
394
398
|
runId: "nope",
|
|
395
399
|
cursorKey: "a",
|
|
396
400
|
status: "ok",
|
|
@@ -489,9 +493,12 @@ const storeConformance = (label, makeStore) => {
|
|
|
489
493
|
const all = await s.listRuns({}, { limit: 10 });
|
|
490
494
|
expect(all.runs).toHaveLength(3);
|
|
491
495
|
expect(all.runs[0].id).not.toBe(a.runId);
|
|
492
|
-
|
|
493
|
-
expect(
|
|
494
|
-
|
|
496
|
+
const byName = await s.listRuns({ name: "alpha" }, { limit: 10 });
|
|
497
|
+
expect(byName.runs.map((r) => r.name)).toEqual(["alpha", "alpha"]);
|
|
498
|
+
const byStatus = await s.listRuns({ status: "done" }, { limit: 10 });
|
|
499
|
+
expect(byStatus.runs.map((r) => r.id)).toEqual([b.runId]);
|
|
500
|
+
const byTag = await s.listRuns({ tag: "t2" }, { limit: 10 });
|
|
501
|
+
expect(byTag.runs.map((r) => r.id)).toEqual([b.runId]);
|
|
495
502
|
});
|
|
496
503
|
it("listRuns paginates with a cursor", async () => {
|
|
497
504
|
const s = await makeStore();
|
|
@@ -508,7 +515,8 @@ const storeConformance = (label, makeStore) => {
|
|
|
508
515
|
cursor: first.cursor
|
|
509
516
|
});
|
|
510
517
|
expect(second.runs).toHaveLength(2);
|
|
511
|
-
|
|
518
|
+
const ids = new Set([...first.runs, ...second.runs].map((r) => r.id));
|
|
519
|
+
expect(ids.size).toBe(4);
|
|
512
520
|
});
|
|
513
521
|
it("childrenOf returns the runs spawned under a parent", async () => {
|
|
514
522
|
const s = await makeStore();
|
|
@@ -558,12 +566,14 @@ const storeConformance = (label, makeStore) => {
|
|
|
558
566
|
message: "boom"
|
|
559
567
|
}
|
|
560
568
|
});
|
|
561
|
-
|
|
569
|
+
const res = await s.retryRun(runId);
|
|
570
|
+
expect(res.retried).toBe(true);
|
|
562
571
|
const snap = await s.loadRun(runId);
|
|
563
572
|
expect(snap?.run.status).toBe("pending");
|
|
564
573
|
expect(snap?.run.error).toBeUndefined();
|
|
565
574
|
expect(snap?.steps.get("a")?.result).toBe(7);
|
|
566
|
-
|
|
575
|
+
const again = await s.retryRun(runId);
|
|
576
|
+
expect(again.retried).toBe(false);
|
|
567
577
|
});
|
|
568
578
|
it("runStats counts runs per status", async () => {
|
|
569
579
|
const s = await makeStore();
|
|
@@ -625,16 +635,18 @@ const queueConformance = (label, makeQueue) => {
|
|
|
625
635
|
});
|
|
626
636
|
await q.enqueue("r1", { runAt: at(0) });
|
|
627
637
|
await q.enqueue("r2", { runAt: at(5e3) });
|
|
628
|
-
|
|
638
|
+
const backlog = await q.depth(at(1e3));
|
|
639
|
+
expect(backlog).toEqual({
|
|
629
640
|
claimable: 1,
|
|
630
641
|
leased: 0,
|
|
631
642
|
oldestClaimableAgeMs: 1e3
|
|
632
643
|
});
|
|
633
|
-
|
|
644
|
+
const leases = await q.claim({
|
|
634
645
|
limit: 10,
|
|
635
646
|
leaseMs: 1e3,
|
|
636
647
|
now: at(1e3)
|
|
637
|
-
})
|
|
648
|
+
});
|
|
649
|
+
expect(leases.map((l) => l.runId)).toEqual(["r1"]);
|
|
638
650
|
expect(await q.depth(at(1e3))).toEqual({
|
|
639
651
|
claimable: 0,
|
|
640
652
|
leased: 1,
|
|
@@ -649,25 +661,28 @@ const queueConformance = (label, makeQueue) => {
|
|
|
649
661
|
leaseMs: 1e3,
|
|
650
662
|
now: at(0)
|
|
651
663
|
})).toEqual([]);
|
|
652
|
-
|
|
664
|
+
const due = await q.claim({
|
|
653
665
|
limit: 10,
|
|
654
666
|
leaseMs: 1e3,
|
|
655
667
|
now: at(5e3)
|
|
656
|
-
})
|
|
668
|
+
});
|
|
669
|
+
expect(due.map((l) => l.runId)).toEqual(["r1"]);
|
|
657
670
|
});
|
|
658
671
|
it("a claimed run is leased exclusively — a second claim before expiry gets nothing", async () => {
|
|
659
672
|
const q = await makeQueue();
|
|
660
673
|
await q.enqueue("r1");
|
|
661
|
-
|
|
674
|
+
const first = await q.claim({
|
|
662
675
|
limit: 10,
|
|
663
676
|
leaseMs: 1e3,
|
|
664
677
|
now: at(0)
|
|
665
|
-
})
|
|
666
|
-
expect(
|
|
678
|
+
});
|
|
679
|
+
expect(first).toHaveLength(1);
|
|
680
|
+
const second = await q.claim({
|
|
667
681
|
limit: 10,
|
|
668
682
|
leaseMs: 1e3,
|
|
669
683
|
now: at(500)
|
|
670
|
-
})
|
|
684
|
+
});
|
|
685
|
+
expect(second).toEqual([]);
|
|
671
686
|
});
|
|
672
687
|
it("an expired lease is re-claimable (crash recovery), with a fresh token", async () => {
|
|
673
688
|
const q = await makeQueue();
|
|
@@ -702,11 +717,12 @@ const queueConformance = (label, makeQueue) => {
|
|
|
702
717
|
leaseMs: 1e3,
|
|
703
718
|
now: at(1001)
|
|
704
719
|
})).toEqual([]);
|
|
705
|
-
|
|
720
|
+
const after = await q.claim({
|
|
706
721
|
limit: 10,
|
|
707
722
|
leaseMs: 1e3,
|
|
708
723
|
now: at(1501)
|
|
709
|
-
})
|
|
724
|
+
});
|
|
725
|
+
expect(after.map((l) => l.runId)).toEqual(["r1"]);
|
|
710
726
|
});
|
|
711
727
|
it("heartbeat on a lost (re-claimed) lease throws", async () => {
|
|
712
728
|
const q = await makeQueue();
|
|
@@ -755,11 +771,12 @@ const queueConformance = (label, makeQueue) => {
|
|
|
755
771
|
now: at(1001)
|
|
756
772
|
});
|
|
757
773
|
await q.ack(stale, { now: at(1002) });
|
|
758
|
-
|
|
774
|
+
const still = await q.claim({
|
|
759
775
|
limit: 10,
|
|
760
776
|
leaseMs: 1e3,
|
|
761
777
|
now: at(2002)
|
|
762
|
-
})
|
|
778
|
+
});
|
|
779
|
+
expect(still.map((l) => l.runId)).toEqual(["r1"]);
|
|
763
780
|
});
|
|
764
781
|
it("a wake that re-enqueues during a lease survives the ack — the job is released, not deleted", async () => {
|
|
765
782
|
const q = await makeQueue();
|
|
@@ -771,11 +788,12 @@ const queueConformance = (label, makeQueue) => {
|
|
|
771
788
|
});
|
|
772
789
|
await q.enqueue("r1");
|
|
773
790
|
await q.ack(lease, { now: at(1) });
|
|
774
|
-
|
|
791
|
+
const again = await q.claim({
|
|
775
792
|
limit: 10,
|
|
776
793
|
leaseMs: 1e3,
|
|
777
794
|
now: at(2)
|
|
778
|
-
})
|
|
795
|
+
});
|
|
796
|
+
expect(again.map((l) => l.runId)).toEqual(["r1"]);
|
|
779
797
|
});
|
|
780
798
|
it("an expired ack is a no-op — an expired owner can't delete a re-claimable run", async () => {
|
|
781
799
|
const q = await makeQueue();
|
|
@@ -786,11 +804,12 @@ const queueConformance = (label, makeQueue) => {
|
|
|
786
804
|
now: at(0)
|
|
787
805
|
});
|
|
788
806
|
await q.ack(lease, { now: at(1001) });
|
|
789
|
-
|
|
807
|
+
const still = await q.claim({
|
|
790
808
|
limit: 10,
|
|
791
809
|
leaseMs: 1e3,
|
|
792
810
|
now: at(1002)
|
|
793
|
-
})
|
|
811
|
+
});
|
|
812
|
+
expect(still.map((l) => l.runId)).toEqual(["r1"]);
|
|
794
813
|
});
|
|
795
814
|
it("heartbeat after expiry throws (lease lost even without a re-claim)", async () => {
|
|
796
815
|
const q = await makeQueue();
|
|
@@ -823,11 +842,12 @@ const queueConformance = (label, makeQueue) => {
|
|
|
823
842
|
it("claim honours the batch limit", async () => {
|
|
824
843
|
const q = await makeQueue();
|
|
825
844
|
for (let i = 0; i < 5; i++) await q.enqueue(`r${i}`);
|
|
826
|
-
|
|
845
|
+
const leases = await q.claim({
|
|
827
846
|
limit: 2,
|
|
828
847
|
leaseMs: 1e3,
|
|
829
848
|
now: at(0)
|
|
830
|
-
})
|
|
849
|
+
});
|
|
850
|
+
expect(leases).toHaveLength(2);
|
|
831
851
|
});
|
|
832
852
|
it("lower priority is claimed first", async () => {
|
|
833
853
|
const q = await makeQueue();
|
|
@@ -844,11 +864,12 @@ const queueConformance = (label, makeQueue) => {
|
|
|
844
864
|
const q = await makeQueue();
|
|
845
865
|
await q.enqueue("r1");
|
|
846
866
|
await q.enqueue("r1", { priority: 5 });
|
|
847
|
-
|
|
867
|
+
const leases = await q.claim({
|
|
848
868
|
limit: 10,
|
|
849
869
|
leaseMs: 1e3,
|
|
850
870
|
now: at(0)
|
|
851
|
-
})
|
|
871
|
+
});
|
|
872
|
+
expect(leases.map((l) => l.runId)).toEqual(["r1"]);
|
|
852
873
|
});
|
|
853
874
|
});
|
|
854
875
|
};
|
|
@@ -920,13 +941,15 @@ const timerConformance = (label, makeTimer) => {
|
|
|
920
941
|
await tm.schedule("late", at(300));
|
|
921
942
|
await tm.schedule("early", at(100));
|
|
922
943
|
await tm.schedule("mid", at(200));
|
|
923
|
-
|
|
944
|
+
const due = await tm.dueBatch({
|
|
924
945
|
now: at(1e3),
|
|
925
946
|
limit: 2
|
|
926
|
-
})
|
|
947
|
+
});
|
|
948
|
+
expect(due).toEqual(["early", "mid"]);
|
|
927
949
|
});
|
|
928
950
|
it("nextDueAt is null when nothing is pending", async () => {
|
|
929
|
-
|
|
951
|
+
const tm = await makeTimer();
|
|
952
|
+
expect(await tm.nextDueAt(at(0))).toBeNull();
|
|
930
953
|
});
|
|
931
954
|
it("nextDueAt returns the earliest timer due strictly after now", async () => {
|
|
932
955
|
const tm = await makeTimer();
|
|
@@ -986,7 +1009,8 @@ const wakeupConformance = (label, makeWakeup) => {
|
|
|
986
1009
|
expect(resolved).toBe(true);
|
|
987
1010
|
});
|
|
988
1011
|
it("signal is a no-op when there are no waiters", async () => {
|
|
989
|
-
|
|
1012
|
+
const w = await makeWakeup();
|
|
1013
|
+
await expect(w.signal("nobody")).resolves.toBeUndefined();
|
|
990
1014
|
});
|
|
991
1015
|
it("is edge-triggered — a signal before any waiter does not latch", async () => {
|
|
992
1016
|
const w = await makeWakeup();
|
|
@@ -1032,11 +1056,12 @@ const outboxConformance = (label, makeBackend) => {
|
|
|
1032
1056
|
result: 1,
|
|
1033
1057
|
attempts: 1
|
|
1034
1058
|
}, { enqueue: [{ runId }] });
|
|
1035
|
-
|
|
1059
|
+
const leases = await queue.claim({
|
|
1036
1060
|
limit: 10,
|
|
1037
1061
|
leaseMs: 1e3,
|
|
1038
1062
|
now: at(0)
|
|
1039
|
-
})
|
|
1063
|
+
});
|
|
1064
|
+
expect(leases.map((l) => l.runId)).toEqual([runId]);
|
|
1040
1065
|
});
|
|
1041
1066
|
it("checkpointStep spawns a child run + enqueues it atomically", async () => {
|
|
1042
1067
|
const { store, queue } = await makeBackend();
|
|
@@ -1061,11 +1086,12 @@ const outboxConformance = (label, makeBackend) => {
|
|
|
1061
1086
|
}
|
|
1062
1087
|
}] });
|
|
1063
1088
|
expect((await store.loadRun(childId))?.run.input).toEqual({ n: 1 });
|
|
1064
|
-
|
|
1089
|
+
const leases = await queue.claim({
|
|
1065
1090
|
limit: 10,
|
|
1066
1091
|
leaseMs: 1e3,
|
|
1067
1092
|
now: at(0)
|
|
1068
|
-
})
|
|
1093
|
+
});
|
|
1094
|
+
expect(leases.map((l) => l.runId)).toContain(childId);
|
|
1069
1095
|
});
|
|
1070
1096
|
it("a replayed checkpoint does NOT re-fire the outbox — the child spawns exactly once", async () => {
|
|
1071
1097
|
const { store, queue } = await makeBackend();
|
|
@@ -1097,11 +1123,12 @@ const outboxConformance = (label, makeBackend) => {
|
|
|
1097
1123
|
result: childId,
|
|
1098
1124
|
attempts: 2
|
|
1099
1125
|
}, spawn);
|
|
1100
|
-
|
|
1126
|
+
const leases = await queue.claim({
|
|
1101
1127
|
limit: 10,
|
|
1102
1128
|
leaseMs: 1e3,
|
|
1103
1129
|
now: at(0)
|
|
1104
|
-
})
|
|
1130
|
+
});
|
|
1131
|
+
expect(leases.filter((l) => l.runId === childId)).toHaveLength(1);
|
|
1105
1132
|
});
|
|
1106
1133
|
it("suspendRun(sleeping) sets status + wake timer atomically", async () => {
|
|
1107
1134
|
const { store, timer } = await makeBackend();
|
|
@@ -1163,11 +1190,12 @@ const outboxConformance = (label, makeBackend) => {
|
|
|
1163
1190
|
status: "done",
|
|
1164
1191
|
output: 7
|
|
1165
1192
|
}, { enqueue: [{ runId: parent.runId }] });
|
|
1166
|
-
|
|
1193
|
+
const leases = await queue.claim({
|
|
1167
1194
|
limit: 10,
|
|
1168
1195
|
leaseMs: 1e3,
|
|
1169
1196
|
now: at(0)
|
|
1170
|
-
})
|
|
1197
|
+
});
|
|
1198
|
+
expect(leases.map((l) => l.runId)).toContain(parent.runId);
|
|
1171
1199
|
});
|
|
1172
1200
|
it("markTerminal cancels a pending timer atomically (no stale wake after completion)", async () => {
|
|
1173
1201
|
const { store, timer } = await makeBackend();
|
|
@@ -1227,19 +1255,22 @@ const signalConformance = (label, makeBackend) => {
|
|
|
1227
1255
|
version: 1,
|
|
1228
1256
|
input: {}
|
|
1229
1257
|
});
|
|
1230
|
-
|
|
1231
|
-
expect(
|
|
1258
|
+
const res = await store.postSignal(runId, "approve", { ok: true });
|
|
1259
|
+
expect(res.delivered).toBe(true);
|
|
1260
|
+
const snap = await store.loadRun(runId);
|
|
1261
|
+
expect(snap?.signals.map((s) => ({
|
|
1232
1262
|
name: s.name,
|
|
1233
1263
|
payload: s.payload
|
|
1234
1264
|
}))).toEqual([{
|
|
1235
1265
|
name: "approve",
|
|
1236
1266
|
payload: { ok: true }
|
|
1237
1267
|
}]);
|
|
1238
|
-
|
|
1268
|
+
const leases = await queue.claim({
|
|
1239
1269
|
limit: 10,
|
|
1240
1270
|
leaseMs: 1e3,
|
|
1241
1271
|
now: at(0)
|
|
1242
|
-
})
|
|
1272
|
+
});
|
|
1273
|
+
expect(leases.map((l) => l.runId)).toEqual([runId]);
|
|
1243
1274
|
});
|
|
1244
1275
|
it("postSignal is idempotent on its key — a retried delivery lands once", async () => {
|
|
1245
1276
|
const { store } = await makeBackend();
|
|
@@ -1288,22 +1319,24 @@ const signalConformance = (label, makeBackend) => {
|
|
|
1288
1319
|
leaseMs: 1e3,
|
|
1289
1320
|
now: at(0)
|
|
1290
1321
|
});
|
|
1291
|
-
|
|
1322
|
+
const committed = await store.checkpointStep({
|
|
1292
1323
|
runId,
|
|
1293
1324
|
cursorKey: "res",
|
|
1294
1325
|
status: "ok",
|
|
1295
1326
|
result: { received: false },
|
|
1296
1327
|
attempts: 1
|
|
1297
|
-
}, { requireVersion: lease.version })
|
|
1328
|
+
}, { requireVersion: lease.version });
|
|
1329
|
+
expect(committed.committed).not.toBe(false);
|
|
1298
1330
|
expect((await store.loadRun(runId))?.steps.get("res")?.result).toEqual({ received: false });
|
|
1299
1331
|
await store.postSignal(runId, "go", "payload");
|
|
1300
|
-
|
|
1332
|
+
const blocked = await store.checkpointStep({
|
|
1301
1333
|
runId,
|
|
1302
1334
|
cursorKey: "res2",
|
|
1303
1335
|
status: "ok",
|
|
1304
1336
|
result: { received: false },
|
|
1305
1337
|
attempts: 1
|
|
1306
|
-
}, { requireVersion: lease.version })
|
|
1338
|
+
}, { requireVersion: lease.version });
|
|
1339
|
+
expect(blocked.committed).toBe(false);
|
|
1307
1340
|
expect((await store.loadRun(runId))?.steps.get("res2")).toBeUndefined();
|
|
1308
1341
|
});
|
|
1309
1342
|
it("distinct signals to a run preserve delivery order in the inbox", async () => {
|
|
@@ -1538,8 +1571,8 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1538
1571
|
batchMax: 16,
|
|
1539
1572
|
leaseMs: 6e5
|
|
1540
1573
|
};
|
|
1541
|
-
const drive = async (backend, flows, runId, retry) => {
|
|
1542
|
-
let clock = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
1574
|
+
const drive = async (backend, flows, runId, retry, start) => {
|
|
1575
|
+
let clock = start ?? /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
1543
1576
|
const now = () => clock;
|
|
1544
1577
|
for (let i = 0; i < 200; i++) {
|
|
1545
1578
|
await tickOnce(backend, flows, {
|
|
@@ -1564,11 +1597,12 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1564
1597
|
}
|
|
1565
1598
|
});
|
|
1566
1599
|
const runId = await submit(backend, flow, {});
|
|
1567
|
-
|
|
1600
|
+
const run = await drive(backend, registry([flow]), runId, {
|
|
1568
1601
|
maxAttempts: 3,
|
|
1569
1602
|
baseDelayMs: 1,
|
|
1570
1603
|
maxDelayMs: 1
|
|
1571
|
-
})
|
|
1604
|
+
});
|
|
1605
|
+
expect(run).toMatchObject({
|
|
1572
1606
|
status: "done",
|
|
1573
1607
|
output: "settled"
|
|
1574
1608
|
});
|
|
@@ -1583,7 +1617,8 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1583
1617
|
nap: acc.nap
|
|
1584
1618
|
}));
|
|
1585
1619
|
const runId = await submit(backend, flow, { x: 21 });
|
|
1586
|
-
|
|
1620
|
+
const run = await drive(backend, registry([flow]), runId);
|
|
1621
|
+
expect(run).toMatchObject({
|
|
1587
1622
|
status: "done",
|
|
1588
1623
|
output: {
|
|
1589
1624
|
doubled: 42,
|
|
@@ -1601,11 +1636,12 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1601
1636
|
})
|
|
1602
1637
|
});
|
|
1603
1638
|
const runId = await submit(backend, flow, {});
|
|
1604
|
-
|
|
1639
|
+
const run = await drive(backend, registry([flow]), runId, {
|
|
1605
1640
|
maxAttempts: 2,
|
|
1606
1641
|
baseDelayMs: 1,
|
|
1607
1642
|
maxDelayMs: 1
|
|
1608
|
-
})
|
|
1643
|
+
});
|
|
1644
|
+
expect(run.status).toBe("failed");
|
|
1609
1645
|
});
|
|
1610
1646
|
it("dead-letters a poison-pill (uncatchable-crash) run once attempts exceed the cap", async () => {
|
|
1611
1647
|
const backend = await makeBackend();
|
|
@@ -1647,7 +1683,8 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1647
1683
|
});
|
|
1648
1684
|
expect((await backend.store.loadRunRow(runId))?.status).toBe("awaiting_signal");
|
|
1649
1685
|
await signalRun(backend, runId, "go", { ok: true });
|
|
1650
|
-
|
|
1686
|
+
const run = await drive(backend, flows, runId);
|
|
1687
|
+
expect(run).toMatchObject({
|
|
1651
1688
|
status: "done",
|
|
1652
1689
|
output: "shipped"
|
|
1653
1690
|
});
|
|
@@ -1800,7 +1837,8 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1800
1837
|
])
|
|
1801
1838
|
});
|
|
1802
1839
|
const runId = await submit(backend, parent, {});
|
|
1803
|
-
|
|
1840
|
+
const run = await drive(backend, registry([parent, dbl]), runId);
|
|
1841
|
+
expect(run).toMatchObject({
|
|
1804
1842
|
status: "done",
|
|
1805
1843
|
output: [
|
|
1806
1844
|
2,
|
|
@@ -1845,11 +1883,12 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1845
1883
|
run: async (ctx, n) => ctx.invoke(recurse, n + 1)
|
|
1846
1884
|
});
|
|
1847
1885
|
const runId = await submit(backend, recurse, 0);
|
|
1848
|
-
|
|
1886
|
+
const top = await drive(backend, registry([recurse]), runId, {
|
|
1849
1887
|
maxAttempts: 1,
|
|
1850
1888
|
baseDelayMs: 1,
|
|
1851
1889
|
maxDelayMs: 1
|
|
1852
|
-
})
|
|
1890
|
+
});
|
|
1891
|
+
expect(top.status).toBe("failed");
|
|
1853
1892
|
let cur = runId;
|
|
1854
1893
|
let deepest = await backend.store.loadRunRow(cur);
|
|
1855
1894
|
for (;;) {
|
|
@@ -1910,7 +1949,7 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1910
1949
|
}])
|
|
1911
1950
|
});
|
|
1912
1951
|
const runId = await submit(backend, parent, {});
|
|
1913
|
-
|
|
1952
|
+
const run = await drive(backend, registry([
|
|
1914
1953
|
parent,
|
|
1915
1954
|
boom,
|
|
1916
1955
|
slow
|
|
@@ -1918,12 +1957,316 @@ const engineConformance = (label, makeBackend) => {
|
|
|
1918
1957
|
maxAttempts: 1,
|
|
1919
1958
|
baseDelayMs: 1,
|
|
1920
1959
|
maxDelayMs: 1
|
|
1921
|
-
})
|
|
1960
|
+
});
|
|
1961
|
+
expect(run.status).toBe("failed");
|
|
1922
1962
|
const kids = await backend.store.childrenOf(runId);
|
|
1923
1963
|
expect(kids).toHaveLength(2);
|
|
1924
1964
|
expect(kids.every((k) => isTerminal(k.status))).toBe(true);
|
|
1925
1965
|
expect(kids.some((k) => k.status === "canceled")).toBe(true);
|
|
1926
1966
|
});
|
|
1967
|
+
it("a claimed run for an unregistered flow version parks and resumes once that version deploys", async () => {
|
|
1968
|
+
const backend = await makeBackend();
|
|
1969
|
+
const v1 = defineFlow({
|
|
1970
|
+
name: "wf",
|
|
1971
|
+
version: 1,
|
|
1972
|
+
run: async () => "v1"
|
|
1973
|
+
});
|
|
1974
|
+
const v2 = defineFlow({
|
|
1975
|
+
name: "wf",
|
|
1976
|
+
version: 2,
|
|
1977
|
+
run: async () => "v2"
|
|
1978
|
+
});
|
|
1979
|
+
const fast = {
|
|
1980
|
+
maxAttempts: 3,
|
|
1981
|
+
baseDelayMs: 1,
|
|
1982
|
+
maxDelayMs: 1
|
|
1983
|
+
};
|
|
1984
|
+
const runId = await submit(backend, v2, {});
|
|
1985
|
+
let clock = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
1986
|
+
const now = () => clock;
|
|
1987
|
+
await tickOnce(backend, registry([v1]), {
|
|
1988
|
+
...base,
|
|
1989
|
+
now,
|
|
1990
|
+
retry: fast
|
|
1991
|
+
});
|
|
1992
|
+
expect((await backend.store.loadRunRow(runId))?.status).toBe("retrying");
|
|
1993
|
+
for (let i = 0; i < 4; i++) {
|
|
1994
|
+
clock = new Date(clock.getTime() + 6e4);
|
|
1995
|
+
await tickOnce(backend, registry([v1]), {
|
|
1996
|
+
...base,
|
|
1997
|
+
now,
|
|
1998
|
+
retry: fast
|
|
1999
|
+
});
|
|
2000
|
+
expect((await backend.store.loadRunRow(runId))?.status).toBe("retrying");
|
|
2001
|
+
}
|
|
2002
|
+
const run = await drive(backend, registry([v2]), runId, fast, clock);
|
|
2003
|
+
expect(run).toMatchObject({
|
|
2004
|
+
status: "done",
|
|
2005
|
+
output: "v2"
|
|
2006
|
+
});
|
|
2007
|
+
});
|
|
2008
|
+
it("a signal delivered at the timeout deadline wins the race — never lost to the timeout", async () => {
|
|
2009
|
+
const backend = await makeBackend();
|
|
2010
|
+
const flow = defineFlow({
|
|
2011
|
+
name: "raced",
|
|
2012
|
+
version: 1,
|
|
2013
|
+
signals: { go: signalType() },
|
|
2014
|
+
run: async (ctx) => {
|
|
2015
|
+
const r = await ctx.signal("go", { timeoutMs: 5e3 });
|
|
2016
|
+
return r.received ? `signal:${r.payload}` : "timeout";
|
|
2017
|
+
}
|
|
2018
|
+
});
|
|
2019
|
+
const flows = registry([flow]);
|
|
2020
|
+
const t0 = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2021
|
+
const runId = await submit(backend, flow, {});
|
|
2022
|
+
await tickOnce(backend, flows, {
|
|
2023
|
+
...base,
|
|
2024
|
+
now: () => t0
|
|
2025
|
+
});
|
|
2026
|
+
expect((await backend.store.loadRunRow(runId))?.status).toBe("awaiting_signal");
|
|
2027
|
+
const deadline = new Date(t0.getTime() + 5e3);
|
|
2028
|
+
await signalRun(backend, runId, "go", "ok");
|
|
2029
|
+
const run = await drive(backend, flows, runId, void 0, deadline);
|
|
2030
|
+
expect(run).toMatchObject({
|
|
2031
|
+
status: "done",
|
|
2032
|
+
output: "signal:ok"
|
|
2033
|
+
});
|
|
2034
|
+
});
|
|
2035
|
+
it("a bounded signal wait commits { received: false } when nothing arrives before the deadline", async () => {
|
|
2036
|
+
const backend = await makeBackend();
|
|
2037
|
+
const flow = defineFlow({
|
|
2038
|
+
name: "waits-out",
|
|
2039
|
+
version: 1,
|
|
2040
|
+
signals: { go: signalType() },
|
|
2041
|
+
run: async (ctx) => {
|
|
2042
|
+
return (await ctx.signal("go", { timeoutMs: 5e3 })).received ? "signal" : "timeout";
|
|
2043
|
+
}
|
|
2044
|
+
});
|
|
2045
|
+
const runId = await submit(backend, flow, {});
|
|
2046
|
+
await tickOnce(backend, registry([flow]), {
|
|
2047
|
+
...base,
|
|
2048
|
+
now: () => /* @__PURE__ */ new Date("2030-01-01T00:00:00Z")
|
|
2049
|
+
});
|
|
2050
|
+
const run = await drive(backend, registry([flow]), runId);
|
|
2051
|
+
expect(run).toMatchObject({
|
|
2052
|
+
status: "done",
|
|
2053
|
+
output: "timeout"
|
|
2054
|
+
});
|
|
2055
|
+
});
|
|
2056
|
+
it("memoizes a completed step across a crash-retry — the step fn runs once, the run still completes", async () => {
|
|
2057
|
+
const backend = await makeBackend();
|
|
2058
|
+
let stepCalls = 0;
|
|
2059
|
+
let invocations = 0;
|
|
2060
|
+
const flow = defineFlow({
|
|
2061
|
+
name: "flaky",
|
|
2062
|
+
version: 1,
|
|
2063
|
+
run: async (ctx) => {
|
|
2064
|
+
const v = await ctx.step("charge", () => {
|
|
2065
|
+
stepCalls += 1;
|
|
2066
|
+
return 42;
|
|
2067
|
+
});
|
|
2068
|
+
invocations += 1;
|
|
2069
|
+
if (invocations === 1) throw new Error("crash after the charge committed");
|
|
2070
|
+
return v;
|
|
2071
|
+
}
|
|
2072
|
+
});
|
|
2073
|
+
const runId = await submit(backend, flow, {});
|
|
2074
|
+
const run = await drive(backend, registry([flow]), runId, {
|
|
2075
|
+
maxAttempts: 5,
|
|
2076
|
+
baseDelayMs: 1,
|
|
2077
|
+
maxDelayMs: 1
|
|
2078
|
+
});
|
|
2079
|
+
expect(run).toMatchObject({
|
|
2080
|
+
status: "done",
|
|
2081
|
+
output: 42
|
|
2082
|
+
});
|
|
2083
|
+
expect(stepCalls).toBe(1);
|
|
2084
|
+
});
|
|
2085
|
+
it("reconcile re-drives a run stranded off the queue (crash between start and enqueue)", async () => {
|
|
2086
|
+
const backend = await makeBackend();
|
|
2087
|
+
const flow = defineFlow({
|
|
2088
|
+
name: "stranded",
|
|
2089
|
+
version: 1,
|
|
2090
|
+
run: async () => "recovered"
|
|
2091
|
+
});
|
|
2092
|
+
const flows = registry([flow]);
|
|
2093
|
+
const { runId } = await backend.store.startRun({
|
|
2094
|
+
name: "stranded",
|
|
2095
|
+
version: 1,
|
|
2096
|
+
input: {}
|
|
2097
|
+
});
|
|
2098
|
+
let clock = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2099
|
+
const now = () => clock;
|
|
2100
|
+
for (let i = 0; i < 10; i++) {
|
|
2101
|
+
await reconcile(backend, { limit: 16 });
|
|
2102
|
+
await tickOnce(backend, flows, {
|
|
2103
|
+
...base,
|
|
2104
|
+
now
|
|
2105
|
+
});
|
|
2106
|
+
const r = await backend.store.loadRunRow(runId);
|
|
2107
|
+
if (r && isTerminal(r.status)) break;
|
|
2108
|
+
clock = new Date(clock.getTime() + 6e4);
|
|
2109
|
+
}
|
|
2110
|
+
expect(await backend.store.loadRunRow(runId)).toMatchObject({
|
|
2111
|
+
status: "done",
|
|
2112
|
+
output: "recovered"
|
|
2113
|
+
});
|
|
2114
|
+
});
|
|
2115
|
+
it("a reclaimed run fences the crashed worker — its stale ack and duplicate commit don't corrupt the new owner", async () => {
|
|
2116
|
+
const { store, queue } = await makeBackend();
|
|
2117
|
+
const { runId } = await store.startRun({
|
|
2118
|
+
name: "f",
|
|
2119
|
+
version: 1,
|
|
2120
|
+
input: {}
|
|
2121
|
+
});
|
|
2122
|
+
await queue.enqueue(runId);
|
|
2123
|
+
const t0 = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2124
|
+
const [a] = await queue.claim({
|
|
2125
|
+
limit: 1,
|
|
2126
|
+
leaseMs: 1e3,
|
|
2127
|
+
now: t0
|
|
2128
|
+
});
|
|
2129
|
+
const t1 = new Date(t0.getTime() + 2e3);
|
|
2130
|
+
const [b] = await queue.claim({
|
|
2131
|
+
limit: 1,
|
|
2132
|
+
leaseMs: 6e5,
|
|
2133
|
+
now: t1
|
|
2134
|
+
});
|
|
2135
|
+
expect(b?.runId).toBe(runId);
|
|
2136
|
+
await store.checkpointStep({
|
|
2137
|
+
runId,
|
|
2138
|
+
cursorKey: "s0",
|
|
2139
|
+
status: "ok",
|
|
2140
|
+
result: "B",
|
|
2141
|
+
attempts: 1
|
|
2142
|
+
});
|
|
2143
|
+
const late = await store.checkpointStep({
|
|
2144
|
+
runId,
|
|
2145
|
+
cursorKey: "s0",
|
|
2146
|
+
status: "ok",
|
|
2147
|
+
result: "A",
|
|
2148
|
+
attempts: 1
|
|
2149
|
+
});
|
|
2150
|
+
expect(late.result).toBe("B");
|
|
2151
|
+
await queue.ack(a, { now: new Date(t1.getTime() + 1e3) });
|
|
2152
|
+
expect((await store.loadRun(runId))?.steps.get("s0")?.result).toBe("B");
|
|
2153
|
+
await expect(queue.heartbeat(b, {
|
|
2154
|
+
leaseMs: 6e5,
|
|
2155
|
+
now: new Date(t1.getTime() + 1e3)
|
|
2156
|
+
})).resolves.toBeDefined();
|
|
2157
|
+
});
|
|
2158
|
+
it("a cancel landing while a worker holds a live lease still cancels — the in-flight tick doesn't resurrect it", async () => {
|
|
2159
|
+
const backend = await makeBackend();
|
|
2160
|
+
let bodyRan = 0;
|
|
2161
|
+
const flow = defineFlow({
|
|
2162
|
+
name: "cancelable",
|
|
2163
|
+
version: 1,
|
|
2164
|
+
run: async (ctx) => {
|
|
2165
|
+
bodyRan += 1;
|
|
2166
|
+
await ctx.sleep(1e4);
|
|
2167
|
+
return "done";
|
|
2168
|
+
}
|
|
2169
|
+
});
|
|
2170
|
+
const flows = registry([flow]);
|
|
2171
|
+
const { store, queue } = backend;
|
|
2172
|
+
const { runId } = await store.startRun({
|
|
2173
|
+
name: "cancelable",
|
|
2174
|
+
version: 1,
|
|
2175
|
+
input: {}
|
|
2176
|
+
});
|
|
2177
|
+
await queue.enqueue(runId);
|
|
2178
|
+
const t0 = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2179
|
+
const [lease] = await queue.claim({
|
|
2180
|
+
limit: 1,
|
|
2181
|
+
leaseMs: 6e5,
|
|
2182
|
+
now: t0
|
|
2183
|
+
});
|
|
2184
|
+
await cancelRun(backend, runId);
|
|
2185
|
+
const result = await runTick(backend, flows, lease, {
|
|
2186
|
+
...base,
|
|
2187
|
+
now: () => t0
|
|
2188
|
+
});
|
|
2189
|
+
expect(result.status).toBe("already_terminal");
|
|
2190
|
+
expect((await store.loadRunRow(runId))?.status).toBe("canceled");
|
|
2191
|
+
expect(bodyRan).toBe(0);
|
|
2192
|
+
const after = await queue.claim({
|
|
2193
|
+
limit: 1,
|
|
2194
|
+
leaseMs: 1e3,
|
|
2195
|
+
now: new Date(t0.getTime() + 1e3)
|
|
2196
|
+
});
|
|
2197
|
+
expect(after).toHaveLength(0);
|
|
2198
|
+
});
|
|
2199
|
+
it("two disjoint worker registries drive their own flows to completion against one backend", async () => {
|
|
2200
|
+
const backend = await makeBackend();
|
|
2201
|
+
const a = defineFlow({
|
|
2202
|
+
name: "sa",
|
|
2203
|
+
version: 1,
|
|
2204
|
+
run: async () => "a-done"
|
|
2205
|
+
});
|
|
2206
|
+
const b = defineFlow({
|
|
2207
|
+
name: "sb",
|
|
2208
|
+
version: 1,
|
|
2209
|
+
run: async () => "b-done"
|
|
2210
|
+
});
|
|
2211
|
+
const podA = registry([a]);
|
|
2212
|
+
const podB = registry([b]);
|
|
2213
|
+
const idA = await submit(backend, a, {});
|
|
2214
|
+
const idB = await submit(backend, b, {});
|
|
2215
|
+
let clock = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2216
|
+
const now = () => clock;
|
|
2217
|
+
for (let i = 0; i < 10; i++) {
|
|
2218
|
+
await tickOnce(backend, podA, {
|
|
2219
|
+
...base,
|
|
2220
|
+
now
|
|
2221
|
+
});
|
|
2222
|
+
await tickOnce(backend, podB, {
|
|
2223
|
+
...base,
|
|
2224
|
+
now
|
|
2225
|
+
});
|
|
2226
|
+
const ra = await backend.store.loadRunRow(idA);
|
|
2227
|
+
const rb = await backend.store.loadRunRow(idB);
|
|
2228
|
+
if (ra && rb && isTerminal(ra.status) && isTerminal(rb.status)) break;
|
|
2229
|
+
clock = new Date(clock.getTime() + 6e4);
|
|
2230
|
+
}
|
|
2231
|
+
expect(await backend.store.loadRunRow(idA)).toMatchObject({
|
|
2232
|
+
status: "done",
|
|
2233
|
+
output: "a-done"
|
|
2234
|
+
});
|
|
2235
|
+
expect(await backend.store.loadRunRow(idB)).toMatchObject({
|
|
2236
|
+
status: "done",
|
|
2237
|
+
output: "b-done"
|
|
2238
|
+
});
|
|
2239
|
+
});
|
|
2240
|
+
it("serverlessTick advances a durable sleep and reports the next wake horizon (no resident loop)", async () => {
|
|
2241
|
+
const backend = await makeBackend();
|
|
2242
|
+
const flow = defineFlow({
|
|
2243
|
+
name: "napper",
|
|
2244
|
+
version: 1,
|
|
2245
|
+
run: async (ctx) => {
|
|
2246
|
+
await ctx.sleep(6e4);
|
|
2247
|
+
return "woke";
|
|
2248
|
+
}
|
|
2249
|
+
});
|
|
2250
|
+
const flows = registry([flow]);
|
|
2251
|
+
let clock = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2252
|
+
const now = () => clock;
|
|
2253
|
+
const runId = await submit(backend, flow, {});
|
|
2254
|
+
const s1 = await serverlessTick(backend, flows, {
|
|
2255
|
+
...base,
|
|
2256
|
+
now
|
|
2257
|
+
});
|
|
2258
|
+
expect(s1.results.map((r) => r.status)).toContain("sleeping");
|
|
2259
|
+
expect(s1.nextWakeAt?.getTime()).toBe(clock.getTime() + 6e4);
|
|
2260
|
+
clock = new Date(clock.getTime() + 6e4);
|
|
2261
|
+
await serverlessTick(backend, flows, {
|
|
2262
|
+
...base,
|
|
2263
|
+
now
|
|
2264
|
+
});
|
|
2265
|
+
expect(await backend.store.loadRunRow(runId)).toMatchObject({
|
|
2266
|
+
status: "done",
|
|
2267
|
+
output: "woke"
|
|
2268
|
+
});
|
|
2269
|
+
});
|
|
1927
2270
|
});
|
|
1928
2271
|
};
|
|
1929
2272
|
//#endregion
|
|
@@ -1948,18 +2291,20 @@ const claimFilterConformance = (label, makeBackend) => {
|
|
|
1948
2291
|
});
|
|
1949
2292
|
await queue.enqueue(a.runId);
|
|
1950
2293
|
await queue.enqueue(b.runId);
|
|
1951
|
-
|
|
2294
|
+
const onlyA = await queue.claim({
|
|
1952
2295
|
limit: 10,
|
|
1953
2296
|
leaseMs: 1e3,
|
|
1954
2297
|
names: ["flow-a"],
|
|
1955
2298
|
now: at(0)
|
|
1956
|
-
})
|
|
1957
|
-
expect((
|
|
2299
|
+
});
|
|
2300
|
+
expect(onlyA.map((l) => l.runId)).toEqual([a.runId]);
|
|
2301
|
+
const onlyB = await queue.claim({
|
|
1958
2302
|
limit: 10,
|
|
1959
2303
|
leaseMs: 1e3,
|
|
1960
2304
|
names: ["flow-b"],
|
|
1961
2305
|
now: at(0)
|
|
1962
|
-
})
|
|
2306
|
+
});
|
|
2307
|
+
expect(onlyB.map((l) => l.runId)).toEqual([b.runId]);
|
|
1963
2308
|
});
|
|
1964
2309
|
it("an empty set leases nothing; an omitted set leases everything", async () => {
|
|
1965
2310
|
const { store, queue } = await makeBackend();
|
|
@@ -1975,13 +2320,142 @@ const claimFilterConformance = (label, makeBackend) => {
|
|
|
1975
2320
|
names: [],
|
|
1976
2321
|
now: at(0)
|
|
1977
2322
|
})).toEqual([]);
|
|
1978
|
-
|
|
2323
|
+
const all = await queue.claim({
|
|
1979
2324
|
limit: 10,
|
|
1980
2325
|
leaseMs: 1e3,
|
|
1981
2326
|
now: at(0)
|
|
1982
|
-
})
|
|
2327
|
+
});
|
|
2328
|
+
expect(all.map((l) => l.runId)).toEqual([a.runId]);
|
|
2329
|
+
});
|
|
2330
|
+
});
|
|
2331
|
+
};
|
|
2332
|
+
//#endregion
|
|
2333
|
+
//#region src/sharded-claim.ts
|
|
2334
|
+
/**
|
|
2335
|
+
* A horizontally-sharded worker pool where each pod claims only the flow names it registered.
|
|
2336
|
+
* Proves that two pods claiming ONE backend concurrently are SAFE — no run is ever leased by both,
|
|
2337
|
+
* and no pod ever leases a run outside its shard — and LIVE: together they drain both shards. Server
|
|
2338
|
+
* backends only; single-writer backends (sqlite / memory / durable-objects) can't issue concurrent
|
|
2339
|
+
* transactions, so their name-shard claim is covered sequentially by {@link claimFilterConformance}.
|
|
2340
|
+
*
|
|
2341
|
+
* Per-round throughput is backend-specific: Postgres locks only the name-matched rows, so both shards
|
|
2342
|
+
* drain in one round; MySQL's `FOR UPDATE OF j SKIP LOCKED` locks the eligible `job` head before the
|
|
2343
|
+
* `run`-side name filter applies, so the first claimer transiently blocks the other's shard, which
|
|
2344
|
+
* fills on the next round. Both are safe — the loop asserts safety every round and liveness overall.
|
|
2345
|
+
*/
|
|
2346
|
+
const shardedClaimConformance = (label, makeBackend) => {
|
|
2347
|
+
describe(`sharded-claim conformance (${label})`, () => {
|
|
2348
|
+
it("two pods claim disjoint flow-name shards concurrently — no overlap, each drains its own shard", async () => {
|
|
2349
|
+
const backend = await makeBackend();
|
|
2350
|
+
const mk = async (name) => {
|
|
2351
|
+
const { runId } = await backend.store.startRun({
|
|
2352
|
+
name,
|
|
2353
|
+
version: 1,
|
|
2354
|
+
input: {}
|
|
2355
|
+
});
|
|
2356
|
+
await backend.queue.enqueue(runId);
|
|
2357
|
+
return runId;
|
|
2358
|
+
};
|
|
2359
|
+
const aIds = new Set(await Promise.all([
|
|
2360
|
+
mk("a"),
|
|
2361
|
+
mk("a"),
|
|
2362
|
+
mk("a")
|
|
2363
|
+
]));
|
|
2364
|
+
const bIds = new Set(await Promise.all([
|
|
2365
|
+
mk("b"),
|
|
2366
|
+
mk("b"),
|
|
2367
|
+
mk("b")
|
|
2368
|
+
]));
|
|
2369
|
+
const now = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2370
|
+
const gotA = /* @__PURE__ */ new Set();
|
|
2371
|
+
const gotB = /* @__PURE__ */ new Set();
|
|
2372
|
+
const claim = async (name, into) => {
|
|
2373
|
+
for (const l of await backend.queue.claim({
|
|
2374
|
+
limit: 10,
|
|
2375
|
+
leaseMs: 6e5,
|
|
2376
|
+
now,
|
|
2377
|
+
names: [name]
|
|
2378
|
+
})) into.add(l.runId);
|
|
2379
|
+
};
|
|
2380
|
+
for (let round = 0; round < 6 && (gotA.size < 3 || gotB.size < 3); round++) {
|
|
2381
|
+
await Promise.all([gotA.size < 3 ? claim("a", gotA) : Promise.resolve(), gotB.size < 3 ? claim("b", gotB) : Promise.resolve()]);
|
|
2382
|
+
expect([...gotA].filter((id) => gotB.has(id))).toEqual([]);
|
|
2383
|
+
}
|
|
2384
|
+
expect([...gotA].every((id) => aIds.has(id))).toBe(true);
|
|
2385
|
+
expect([...gotB].every((id) => bIds.has(id))).toBe(true);
|
|
2386
|
+
expect(gotA.size).toBe(3);
|
|
2387
|
+
expect(gotB.size).toBe(3);
|
|
2388
|
+
});
|
|
2389
|
+
});
|
|
2390
|
+
};
|
|
2391
|
+
//#endregion
|
|
2392
|
+
//#region src/pending-work.ts
|
|
2393
|
+
/**
|
|
2394
|
+
* Pins the autoscaling-backlog reads — `Queue.depth(now, names)`, `Timer.dueCount(now, names)`, and
|
|
2395
|
+
* `Store.dueCronCount(now, names)` — that `engine.pendingWork` composes. Every backend must count the
|
|
2396
|
+
* same due work (claimable jobs, due timers, due crons), exclude leased/future work, and filter by
|
|
2397
|
+
* flow name identically.
|
|
2398
|
+
*/
|
|
2399
|
+
const pendingWorkConformance = (label, makeBackend) => {
|
|
2400
|
+
describe(`pending-work conformance (${label})`, () => {
|
|
2401
|
+
const now = /* @__PURE__ */ new Date("2030-01-01T00:00:00Z");
|
|
2402
|
+
const past = /* @__PURE__ */ new Date(now.getTime() - 6e4);
|
|
2403
|
+
const future = new Date(now.getTime() + 36e5);
|
|
2404
|
+
const seed = async () => {
|
|
2405
|
+
const be = await makeBackend();
|
|
2406
|
+
const start = async (name) => (await be.store.startRun({
|
|
2407
|
+
name,
|
|
2408
|
+
version: 1,
|
|
2409
|
+
input: {}
|
|
2410
|
+
})).runId;
|
|
2411
|
+
const leased = await start("a");
|
|
2412
|
+
await be.queue.enqueue(leased);
|
|
2413
|
+
await be.queue.claim({
|
|
2414
|
+
limit: 1,
|
|
2415
|
+
leaseMs: 6e5,
|
|
2416
|
+
now
|
|
2417
|
+
});
|
|
2418
|
+
const a1 = await start("a");
|
|
2419
|
+
await be.queue.enqueue(a1);
|
|
2420
|
+
const b1 = await start("b");
|
|
2421
|
+
await be.queue.enqueue(b1);
|
|
2422
|
+
const sleeping = await start("a");
|
|
2423
|
+
await be.timer.schedule(sleeping, past);
|
|
2424
|
+
const napping = await start("a");
|
|
2425
|
+
await be.timer.schedule(napping, future);
|
|
2426
|
+
await be.timer.schedule("orphan-run", past);
|
|
2427
|
+
const futureJob = await start("a");
|
|
2428
|
+
await be.queue.enqueue(futureJob, { runAt: future });
|
|
2429
|
+
await be.store.upsertCron({
|
|
2430
|
+
name: "cron-a",
|
|
2431
|
+
schedule: "* * * * *",
|
|
2432
|
+
flowName: "a",
|
|
2433
|
+
flowVersion: 1,
|
|
2434
|
+
input: {},
|
|
2435
|
+
nextRunAt: past
|
|
2436
|
+
});
|
|
2437
|
+
return be;
|
|
2438
|
+
};
|
|
2439
|
+
it("depth counts claimable jobs (not leased/future), filtered by flow name", async () => {
|
|
2440
|
+
const be = await seed();
|
|
2441
|
+
expect((await be.queue.depth(now)).claimable).toBe(2);
|
|
2442
|
+
expect((await be.queue.depth(now, ["a"])).claimable).toBe(1);
|
|
2443
|
+
expect((await be.queue.depth(now, ["b"])).claimable).toBe(1);
|
|
2444
|
+
expect((await be.queue.depth(now, [])).claimable).toBe(0);
|
|
2445
|
+
});
|
|
2446
|
+
it("dueCount counts due timers incl. run-less (not future), filtered by flow name", async () => {
|
|
2447
|
+
const be = await seed();
|
|
2448
|
+
expect(await be.timer.dueCount(now)).toBe(2);
|
|
2449
|
+
expect(await be.timer.dueCount(now, ["a"])).toBe(1);
|
|
2450
|
+
expect(await be.timer.dueCount(now, ["b"])).toBe(0);
|
|
2451
|
+
});
|
|
2452
|
+
it("dueCronCount counts due crons, filtered by flow name", async () => {
|
|
2453
|
+
const be = await seed();
|
|
2454
|
+
expect(await be.store.dueCronCount(now)).toBe(1);
|
|
2455
|
+
expect(await be.store.dueCronCount(now, ["a"])).toBe(1);
|
|
2456
|
+
expect(await be.store.dueCronCount(now, ["b"])).toBe(0);
|
|
1983
2457
|
});
|
|
1984
2458
|
});
|
|
1985
2459
|
};
|
|
1986
2460
|
//#endregion
|
|
1987
|
-
export { claimFilterConformance, cronConformance, engineConformance, outboxConformance, queueConformance, reconcileConformance, signalConformance, storeConformance, timerConformance, wakeupConformance };
|
|
2461
|
+
export { claimFilterConformance, cronConformance, engineConformance, outboxConformance, pendingWorkConformance, queueConformance, reconcileConformance, shardedClaimConformance, signalConformance, storeConformance, timerConformance, wakeupConformance };
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@iterativeflow/conformance",
|
|
3
|
-
"version": "2.0.0
|
|
3
|
+
"version": "2.0.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
@@ -24,13 +24,13 @@
|
|
|
24
24
|
"publishConfig": {
|
|
25
25
|
"access": "public"
|
|
26
26
|
},
|
|
27
|
-
"scripts": {
|
|
28
|
-
"build": "tsdown"
|
|
29
|
-
},
|
|
30
27
|
"dependencies": {
|
|
31
|
-
"@iterativeflow/core": "2.0.0
|
|
28
|
+
"@iterativeflow/core": "2.0.0"
|
|
32
29
|
},
|
|
33
30
|
"peerDependencies": {
|
|
34
31
|
"vitest": ">=3"
|
|
32
|
+
},
|
|
33
|
+
"scripts": {
|
|
34
|
+
"build": "tsdown"
|
|
35
35
|
}
|
|
36
|
-
}
|
|
36
|
+
}
|