effect-mq 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +85 -17
  2. package/dist/Flow.d.ts +381 -0
  3. package/dist/Flow.d.ts.map +1 -0
  4. package/dist/Flow.js +340 -0
  5. package/dist/Flow.js.map +1 -0
  6. package/dist/Job.d.ts +31 -6
  7. package/dist/Job.d.ts.map +1 -1
  8. package/dist/Job.js +16 -2
  9. package/dist/Job.js.map +1 -1
  10. package/dist/JobStore.d.ts +353 -13
  11. package/dist/JobStore.d.ts.map +1 -1
  12. package/dist/JobStore.js +10 -0
  13. package/dist/JobStore.js.map +1 -1
  14. package/dist/MemoryJobStore.d.ts.map +1 -1
  15. package/dist/MemoryJobStore.js +361 -21
  16. package/dist/MemoryJobStore.js.map +1 -1
  17. package/dist/Metrics.d.ts +31 -0
  18. package/dist/Metrics.d.ts.map +1 -1
  19. package/dist/Metrics.js +39 -0
  20. package/dist/Metrics.js.map +1 -1
  21. package/dist/Worker.d.ts +120 -11
  22. package/dist/Worker.d.ts.map +1 -1
  23. package/dist/Worker.js +452 -26
  24. package/dist/Worker.js.map +1 -1
  25. package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
  26. package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
  27. package/dist/drizzle-postgres/DrizzleJobStore.js +678 -80
  28. package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
  29. package/dist/drizzle-postgres/schema.d.ts +293 -3
  30. package/dist/drizzle-postgres/schema.d.ts.map +1 -1
  31. package/dist/drizzle-postgres/schema.js +66 -1
  32. package/dist/drizzle-postgres/schema.js.map +1 -1
  33. package/dist/index.d.ts +7 -0
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +7 -0
  36. package/dist/index.js.map +1 -1
  37. package/dist/redis/RedisJobStore.d.ts +53 -0
  38. package/dist/redis/RedisJobStore.d.ts.map +1 -1
  39. package/dist/redis/RedisJobStore.js +402 -50
  40. package/dist/redis/RedisJobStore.js.map +1 -1
  41. package/dist/redis/scripts.d.ts +213 -35
  42. package/dist/redis/scripts.d.ts.map +1 -1
  43. package/dist/redis/scripts.js +689 -73
  44. package/dist/redis/scripts.js.map +1 -1
  45. package/dist/testing/conformance.d.ts +6 -0
  46. package/dist/testing/conformance.d.ts.map +1 -1
  47. package/dist/testing/conformance.js +855 -12
  48. package/dist/testing/conformance.js.map +1 -1
  49. package/package.json +1 -1
  50. package/src/Flow.ts +778 -0
  51. package/src/Job.ts +35 -11
  52. package/src/JobStore.ts +377 -12
  53. package/src/MemoryJobStore.ts +396 -25
  54. package/src/Metrics.ts +43 -0
  55. package/src/Worker.ts +726 -37
  56. package/src/drizzle-postgres/DrizzleJobStore.ts +844 -81
  57. package/src/drizzle-postgres/schema.ts +92 -0
  58. package/src/index.ts +8 -0
  59. package/src/redis/RedisJobStore.ts +540 -39
  60. package/src/redis/scripts.ts +751 -78
  61. package/src/testing/conformance.ts +1088 -12
@@ -14,6 +14,12 @@
14
14
  * Effect `Clock` (e.g. pass `now` into queries as a bind parameter) — never
15
15
  * from the database server's clock — so this works against real storage too.
16
16
  *
17
+ * Beyond the core queue contract, dedicated sections pin the flow contract:
18
+ * parent-side ownership (the FanOut ack, dependency rows, batched
19
+ * `recordChildResults`, every settle decision) and the child-side outbox
20
+ * (terminal transitions of envelope-carrying jobs staging reports for the
21
+ * relay to another store).
22
+ *
17
23
  * @since 0.1.0
18
24
  */
19
25
  import * as JobStore from "../JobStore.ts"
@@ -38,6 +44,7 @@ const baseRequest = (
38
44
  timeoutMs: undefined,
39
45
  dedupe: undefined,
40
46
  trace: undefined,
47
+ parent: undefined,
41
48
  delayMs: 0,
42
49
  ...overrides
43
50
  })
@@ -521,21 +528,153 @@ export const jobStoreConformance = (
521
528
  withStore((store) =>
522
529
  Effect.gen(function*() {
523
530
  // No clock adjustment: every record shares one enqueuedAt, so
524
- // ordering and the cursor fall back entirely to the id tie-break.
531
+ // ordering and the cursor fall back entirely to the id tie-break
532
+ // in both directions.
525
533
  for (let i = 0; i < 7; i++) {
526
534
  yield* store.enqueue(baseRequest({ payload: { n: i } }))
527
535
  }
528
- const seen = new Set<string>()
529
- let cursor: string | undefined
530
- do {
531
- const page: JobStore.ListResult = yield* store.list({ limit: 3, cursor })
532
- for (const item of page.items) {
533
- expect(seen.has(item.id)).toBe(false)
534
- seen.add(item.id)
536
+ for (const order of ["desc", "asc"] as const) {
537
+ const seen: Array<string> = []
538
+ let cursor: string | undefined
539
+ do {
540
+ const page: JobStore.ListResult = yield* store.list({ limit: 3, order, cursor })
541
+ for (const item of page.items) {
542
+ expect(seen.includes(item.id)).toBe(false)
543
+ seen.push(item.id)
544
+ }
545
+ cursor = page.cursor
546
+ } while (cursor !== undefined)
547
+ expect(seen.length).toBe(7)
548
+ // Opposite directions walk exact mirror orders.
549
+ if (order === "asc") {
550
+ const descAll = yield* store.list({ limit: 7, order: "desc" })
551
+ expect(seen).toEqual(descAll.items.map((item) => item.id).toReversed())
535
552
  }
536
- cursor = page.cursor
537
- } while (cursor !== undefined)
538
- expect(seen.size).toBe(7)
553
+ }
554
+ })
555
+ ))
556
+
557
+ it.effect("list orders by enqueuedAt ascending on request, cursor included", () =>
558
+ withStore((store) =>
559
+ Effect.gen(function*() {
560
+ const ids: Array<JobStore.JobId> = []
561
+ for (let i = 0; i < 3; i++) {
562
+ const { id } = yield* store.enqueue(baseRequest({ payload: { n: i } }))
563
+ ids.push(id)
564
+ yield* TestClock.adjust(1_000)
565
+ }
566
+ const first = yield* store.list({ orderBy: "enqueuedAt", order: "asc", limit: 2 })
567
+ expect(first.items.map((job) => job.id)).toEqual([ids[0], ids[1]])
568
+ assert(first.cursor !== undefined)
569
+ const second = yield* store.list({
570
+ orderBy: "enqueuedAt",
571
+ order: "asc",
572
+ limit: 2,
573
+ cursor: first.cursor
574
+ })
575
+ expect(second.items.map((job) => job.id)).toEqual([ids[2]])
576
+ expect(second.cursor).toBeUndefined()
577
+ })
578
+ ))
579
+
580
+ it.effect("list orders delayed jobs by runAt within a queue", () =>
581
+ withStore((store) =>
582
+ Effect.gen(function*() {
583
+ const late = yield* store.enqueue(baseRequest({ payload: { n: 1 }, delayMs: 30_000 }))
584
+ const soon = yield* store.enqueue(baseRequest({ payload: { n: 2 }, delayMs: 5_000 }))
585
+ const middle = yield* store.enqueue(baseRequest({ payload: { n: 3 }, delayMs: 10_000 }))
586
+ // An immediate job in the same queue must not appear.
587
+ yield* store.enqueue(baseRequest({ payload: { n: 4 } }))
588
+
589
+ const upcoming = yield* store.list({
590
+ queue: QueueName("default"),
591
+ states: ["delayed"],
592
+ orderBy: "runAt",
593
+ order: "asc",
594
+ limit: 2
595
+ })
596
+ expect(upcoming.items.map((job) => job.id)).toEqual([soon.id, middle.id])
597
+ assert(upcoming.cursor !== undefined)
598
+ const rest = yield* store.list({
599
+ queue: QueueName("default"),
600
+ states: ["delayed"],
601
+ orderBy: "runAt",
602
+ order: "asc",
603
+ limit: 2,
604
+ cursor: upcoming.cursor
605
+ })
606
+ expect(rest.items.map((job) => job.id)).toEqual([late.id])
607
+ expect(rest.cursor).toBeUndefined()
608
+ })
609
+ ))
610
+
611
+ it.effect("list orders terminal jobs by finishedAt, with and without a name", () =>
612
+ withStore((store) =>
613
+ Effect.gen(function*() {
614
+ // Enqueue in one order, finish in the REVERSE order, so finishedAt
615
+ // ordering and enqueuedAt ordering disagree — a driver that quietly
616
+ // ignores orderBy fails here instead of passing by coincidence.
617
+ const enqueueAs = (name: string, queue?: string) => {
618
+ const request = queue === undefined
619
+ ? baseRequest({ name })
620
+ : baseRequest({ name, queue: QueueName(queue) })
621
+ return Effect.map(store.enqueue(request), (result) => result.id)
622
+ }
623
+ const first = yield* enqueueAs("TestJob")
624
+ const second = yield* enqueueAs("OtherJob")
625
+ const third = yield* enqueueAs("TestJob")
626
+ const elsewhere = yield* enqueueAs("TestJob", "other")
627
+ const claims = new Map<JobStore.JobId, string>()
628
+ for (const token of ["t-1", "t-2", "t-3"]) {
629
+ const claim = yield* store.claim(claimOptions({
630
+ names: ["TestJob", "OtherJob"],
631
+ token
632
+ }))
633
+ assert(claim._tag === "Claimed")
634
+ claims.set(claim.job.id, token)
635
+ }
636
+ const otherClaim = yield* store.claim(claimOptions({
637
+ queue: QueueName("other"),
638
+ token: "t-4"
639
+ }))
640
+ assert(otherClaim._tag === "Claimed")
641
+ claims.set(elsewhere, "t-4")
642
+ for (const id of [elsewhere, third, second, first]) {
643
+ const token = claims.get(id)
644
+ assert(token !== undefined)
645
+ yield* store.ack(id, token, {
646
+ _tag: id === second ? "Fail" : "Complete",
647
+ exit: undefined
648
+ })
649
+ yield* TestClock.adjust(1_000)
650
+ }
651
+
652
+ // Across terminal states, newest FINISHED first: the reverse of
653
+ // enqueue order.
654
+ const recent = yield* store.list({
655
+ states: ["completed", "failed"],
656
+ orderBy: "finishedAt",
657
+ order: "desc"
658
+ })
659
+ expect(recent.items.map((job) => job.id)).toEqual([first, second, third, elsewhere])
660
+
661
+ // Scoped to one name and state.
662
+ const byName = yield* store.list({
663
+ name: "TestJob",
664
+ states: ["completed"],
665
+ orderBy: "finishedAt",
666
+ order: "desc"
667
+ })
668
+ expect(byName.items.map((job) => job.id)).toEqual([first, third, elsewhere])
669
+
670
+ // The queue filter applies on top of the finishedAt route.
671
+ const byQueue = yield* store.list({
672
+ queue: QueueName("other"),
673
+ states: ["completed"],
674
+ orderBy: "finishedAt",
675
+ order: "desc"
676
+ })
677
+ expect(byQueue.items.map((job) => job.id)).toEqual([elsewhere])
539
678
  })
540
679
  ))
541
680
 
@@ -623,6 +762,7 @@ export const jobStoreConformance = (
623
762
  waiting: 1,
624
763
  delayed: 1,
625
764
  active: 1,
765
+ "waiting-children": 0,
626
766
  completed: 0,
627
767
  failed: 0,
628
768
  cancelled: 0
@@ -1108,7 +1248,14 @@ export const jobStoreConformance = (
1108
1248
  backoff: { _tag: "fixed", delayMs: 2_000 },
1109
1249
  keep: { completed: { count: 2, ageMs: undefined } },
1110
1250
  timeoutMs: 9_000,
1111
- trace: { traceId: "trace-1", spanId: "span-1", sampled: true, delayed: false }
1251
+ trace: { traceId: "trace-1", spanId: "span-1", sampled: true, delayed: false },
1252
+ parent: {
1253
+ flowName: "rich-flow",
1254
+ flowId: JobId("rich-parent"),
1255
+ childKey: "rich-child",
1256
+ parentStoreKey: "effect-mq/JobStore",
1257
+ depth: 1
1258
+ }
1112
1259
  })
1113
1260
  yield* store.enqueue(richRequest("rich-single"))
1114
1261
  expect(yield* store.enqueueMany([richRequest("rich-batch")]))
@@ -1136,6 +1283,8 @@ export const jobStoreConformance = (
1136
1283
  timeoutMs: job.timeoutMs,
1137
1284
  cancelRequested: job.cancelRequested,
1138
1285
  trace: job.trace,
1286
+ parent: job.parent,
1287
+ flow: job.flow,
1139
1288
  runAt: job.runAt,
1140
1289
  enqueuedAt: job.enqueuedAt
1141
1290
  })
@@ -1154,11 +1303,28 @@ export const jobStoreConformance = (
1154
1303
  delayed: false
1155
1304
  })
1156
1305
  expect(expected.payload).toEqual({ big: 1234567890123456, nested: { arr: [1, 2, 3] } })
1306
+ expect(expected.parent).toEqual({
1307
+ flowName: "rich-flow",
1308
+ flowId: "rich-parent",
1309
+ childKey: "rich-child",
1310
+ parentStoreKey: "effect-mq/JobStore",
1311
+ depth: 1
1312
+ })
1313
+ expect(expected.flow).toBeUndefined()
1157
1314
  for (const id of [JobId("rich-batch"), JobId("rich-tick")]) {
1158
1315
  const job = yield* store.getJob(id)
1159
1316
  assert(Option.isSome(job))
1160
1317
  expect(project(job.value)).toEqual(expected)
1161
1318
  }
1319
+
1320
+ // `list` must return the same complete records as `getJob` —
1321
+ // drivers with hand-written SELECT lists can silently drop fields
1322
+ // there while every getJob-path test stays green.
1323
+ const listed = yield* store.list({ name: "TestJob" })
1324
+ expect(listed.items).toHaveLength(3)
1325
+ for (const job of listed.items) {
1326
+ expect(project(job)).toEqual(expected)
1327
+ }
1162
1328
  })
1163
1329
  ))
1164
1330
 
@@ -1370,6 +1536,7 @@ export const jobStoreConformance = (
1370
1536
  waiting: 1,
1371
1537
  delayed: 0,
1372
1538
  active: 0,
1539
+ "waiting-children": 0,
1373
1540
  completed: 0,
1374
1541
  failed: 0,
1375
1542
  cancelled: 1
@@ -1701,5 +1868,914 @@ export const jobStoreConformance = (
1701
1868
  expect(job.value.state).toBe("waiting")
1702
1869
  })
1703
1870
  ))
1871
+
1872
+ // ----------------------------------------------------------------------
1873
+ // Flows (parent-child). The parent store owns the flow: the FanOut ack,
1874
+ // dependency rows, pending counter, and every settle decision are pinned
1875
+ // here. `storeKey` strings are opaque to the store.
1876
+ // ----------------------------------------------------------------------
1877
+
1878
+ const parentEnvelope = (
1879
+ flowId: JobStore.JobId,
1880
+ key: string
1881
+ ): JobStore.ParentEnvelope => ({
1882
+ flowName: "test-flow",
1883
+ flowId,
1884
+ childKey: key,
1885
+ parentStoreKey: "main",
1886
+ depth: 1
1887
+ })
1888
+
1889
+ const childSpec = (
1890
+ flowId: JobStore.JobId,
1891
+ key: string,
1892
+ overrides?: Partial<JobStore.EnqueueRequest>
1893
+ ): JobStore.FlowChildSpec => ({
1894
+ childKey: key,
1895
+ storeKey: "effect-mq/JobStore/children",
1896
+ request: baseRequest({
1897
+ id: JobId(`flow/main/${flowId}/${key}`),
1898
+ name: "ChildJob",
1899
+ parent: parentEnvelope(flowId, key),
1900
+ ...overrides
1901
+ })
1902
+ })
1903
+
1904
+ const fanOutParent = (
1905
+ store: JobStore.Service,
1906
+ options?: {
1907
+ readonly children?: ReadonlyArray<string> | undefined
1908
+ readonly failFast?: boolean | undefined
1909
+ }
1910
+ ) =>
1911
+ Effect.gen(function*() {
1912
+ const { id } = yield* store.enqueue(baseRequest({ payload: { parent: true } }))
1913
+ const claim = yield* store.claim(claimOptions({ token: "t-parent" }))
1914
+ assert(claim._tag === "Claimed")
1915
+ expect(claim.job.id).toBe(id)
1916
+ const keys = options?.children ?? ["a", "b"]
1917
+ yield* store.ack(id, "t-parent", {
1918
+ _tag: "FanOut",
1919
+ failFast: options?.failFast ?? false,
1920
+ children: keys.map((key) => childSpec(id, key))
1921
+ })
1922
+ return id
1923
+ })
1924
+
1925
+ const report = (
1926
+ flowId: JobStore.JobId,
1927
+ key: string,
1928
+ outcome: JobStore.FlowChildReport["outcome"],
1929
+ overrides?: Partial<JobStore.FlowChildReport>
1930
+ ): JobStore.FlowChildReport => ({
1931
+ flowId,
1932
+ childKey: key,
1933
+ outcome,
1934
+ exit: { ok: outcome === "completed" },
1935
+ failedReason: undefined,
1936
+ ...overrides
1937
+ })
1938
+
1939
+ // Batch-of-one sugar for the single-report pins below; the batch
1940
+ // semantics get their own section.
1941
+ const recordOne = (store: JobStore.Service, value: JobStore.FlowChildReport) =>
1942
+ Effect.map(
1943
+ store.recordChildResults([value]),
1944
+ (results) => results[0] ?? { applied: false, parentSettled: false }
1945
+ )
1946
+
1947
+ const flowCounts = (
1948
+ overrides?: Partial<Omit<JobStore.FlowState, "failFast">> & { readonly failFast?: boolean }
1949
+ ): JobStore.FlowState => ({
1950
+ failFast: false,
1951
+ pending: 0,
1952
+ completed: 0,
1953
+ failed: 0,
1954
+ cancelled: 0,
1955
+ ...overrides
1956
+ })
1957
+
1958
+ it.effect("FanOut parks the parent with its manifest, rows, and ledger entry", () =>
1959
+ withStore((store) =>
1960
+ Effect.gen(function*() {
1961
+ const flowId = yield* fanOutParent(store)
1962
+
1963
+ const parent = yield* store.getJob(flowId)
1964
+ assert(Option.isSome(parent))
1965
+ expect(parent.value.state).toBe("waiting-children")
1966
+ expect(parent.value.flow).toEqual(flowCounts({ pending: 2 }))
1967
+ // A fan-out is a phase transition, not a completed run.
1968
+ expect(parent.value.attemptsMade).toBe(0)
1969
+ const attempts = yield* store.getAttempts(flowId)
1970
+ expect(attempts.map((attempt) => attempt.outcome)).toEqual(["fanned-out"])
1971
+
1972
+ const rows = yield* store.listChildResults(flowId)
1973
+ expect(rows.cursor).toBeUndefined()
1974
+ expect(rows.items.map((row) => ({
1975
+ childKey: row.childKey,
1976
+ name: row.name,
1977
+ storeKey: row.storeKey,
1978
+ childJobId: row.childJobId,
1979
+ status: row.status,
1980
+ cascaded: row.cascaded
1981
+ }))).toEqual([
1982
+ {
1983
+ childKey: "a",
1984
+ name: "ChildJob",
1985
+ storeKey: "effect-mq/JobStore/children",
1986
+ childJobId: `flow/main/${flowId}/a`,
1987
+ status: "pending",
1988
+ cascaded: false
1989
+ },
1990
+ {
1991
+ childKey: "b",
1992
+ name: "ChildJob",
1993
+ storeKey: "effect-mq/JobStore/children",
1994
+ childJobId: `flow/main/${flowId}/b`,
1995
+ status: "pending",
1996
+ cascaded: false
1997
+ }
1998
+ ])
1999
+
2000
+ // Parked parents are never claimable and show in counts.
2001
+ const claim = yield* store.claim(claimOptions({ token: "t-again" }))
2002
+ expect(claim._tag).toBe("Empty")
2003
+ expect((yield* store.counts())["waiting-children"]).toBe(1)
2004
+ })
2005
+ ))
2006
+
2007
+ it.effect("FanOut is lock-token-guarded and validates child ids", () =>
2008
+ withStore((store) =>
2009
+ Effect.gen(function*() {
2010
+ const { id } = yield* store.enqueue(baseRequest())
2011
+ const claim = yield* store.claim(claimOptions({ token: "t-owner" }))
2012
+ assert(claim._tag === "Claimed")
2013
+
2014
+ const stale = yield* Effect.exit(store.ack(id, "t-wrong", {
2015
+ _tag: "FanOut",
2016
+ failFast: false,
2017
+ children: [childSpec(id, "a")]
2018
+ }))
2019
+ assert(Exit.isFailure(stale))
2020
+
2021
+ // A spec without an explicit id fails loudly and leaves the job
2022
+ // active (the ack can be retried with a fixed spec).
2023
+ const bad = yield* Effect.exit(store.ack(id, "t-owner", {
2024
+ _tag: "FanOut",
2025
+ failFast: false,
2026
+ children: [{ ...childSpec(id, "a"), request: baseRequest({ id: undefined }) }]
2027
+ }))
2028
+ assert(Exit.isFailure(bad))
2029
+ const job = yield* store.getJob(id)
2030
+ assert(Option.isSome(job))
2031
+ expect(job.value.state).toBe("active")
2032
+
2033
+ yield* store.ack(id, "t-owner", {
2034
+ _tag: "FanOut",
2035
+ failFast: false,
2036
+ children: [childSpec(id, "a")]
2037
+ })
2038
+ })
2039
+ ))
2040
+
2041
+ it.effect("an empty FanOut settles straight to runnable collect", () =>
2042
+ withStore((store) =>
2043
+ Effect.gen(function*() {
2044
+ const flowId = yield* fanOutParent(store, { children: [] })
2045
+ const parent = yield* store.getJob(flowId)
2046
+ assert(Option.isSome(parent))
2047
+ expect(parent.value.state).toBe("waiting")
2048
+ expect(parent.value.flow).toEqual(flowCounts())
2049
+
2050
+ const claim = yield* store.claim(claimOptions({ token: "t-resume" }))
2051
+ assert(claim._tag === "Claimed")
2052
+ expect(claim.job.id).toBe(flowId)
2053
+ expect(claim.job.flow).toEqual(flowCounts())
2054
+ })
2055
+ ))
2056
+
2057
+ it.effect("recordChildResults applies once, decrements, and settles on the last report", () =>
2058
+ withStore((store) =>
2059
+ Effect.gen(function*() {
2060
+ const flowId = yield* fanOutParent(store)
2061
+
2062
+ const first = yield* recordOne(store, report(flowId, "a", "completed"))
2063
+ expect(first).toEqual({ applied: true, parentSettled: false })
2064
+ const midway = yield* store.getJob(flowId)
2065
+ assert(Option.isSome(midway))
2066
+ expect(midway.value.state).toBe("waiting-children")
2067
+ expect(midway.value.flow?.pending).toBe(1)
2068
+
2069
+ // Duplicates and unknowns drop on the dependency row.
2070
+ expect(yield* recordOne(store, report(flowId, "a", "failed")))
2071
+ .toEqual({ applied: false, parentSettled: false })
2072
+ expect(yield* recordOne(store, report(flowId, "ghost", "completed")))
2073
+ .toEqual({ applied: false, parentSettled: false })
2074
+ expect(yield* recordOne(store, report(JobId("no-such-flow"), "a", "completed")))
2075
+ .toEqual({ applied: false, parentSettled: false })
2076
+
2077
+ const last = yield* recordOne(store, report(flowId, "b", "failed", {
2078
+ exit: { boom: true }
2079
+ }))
2080
+ expect(last).toEqual({ applied: true, parentSettled: true })
2081
+
2082
+ // Settled: runnable now, phase collect, results recorded exactly.
2083
+ const parent = yield* store.getJob(flowId)
2084
+ assert(Option.isSome(parent))
2085
+ expect(parent.value.state).toBe("waiting")
2086
+ // The counters mirror the recorded outcomes exactly — via getJob,
2087
+ // via the claimed record (what `collect` reads its counts from),
2088
+ // and via list (what dashboards read). A driver whose claim/list
2089
+ // projections drop the counter columns fails here, not in prod.
2090
+ expect(parent.value.flow).toEqual(flowCounts({ completed: 1, failed: 1 }))
2091
+ const claim = yield* store.claim(claimOptions({ token: "t-resume" }))
2092
+ assert(claim._tag === "Claimed")
2093
+ expect(claim.job.id).toBe(flowId)
2094
+ expect(claim.job.flow).toEqual(flowCounts({ completed: 1, failed: 1 }))
2095
+ const listed = yield* store.list({ name: "TestJob" })
2096
+ expect(listed.items.find((job) => job.id === flowId)?.flow)
2097
+ .toEqual(flowCounts({ completed: 1, failed: 1 }))
2098
+
2099
+ const rows = yield* store.listChildResults(flowId)
2100
+ const byKey = new Map(rows.items.map((row) => [row.childKey, row]))
2101
+ expect(byKey.get("a")?.status).toBe("completed")
2102
+ expect(byKey.get("a")?.exit).toEqual({ ok: true })
2103
+ // A recorded outcome came FROM the child's store: nothing to cascade.
2104
+ expect(byKey.get("a")?.cascaded).toBe(true)
2105
+ expect(byKey.get("b")?.status).toBe("failed")
2106
+ expect(byKey.get("b")?.exit).toEqual({ boom: true })
2107
+ })
2108
+ ))
2109
+
2110
+ it.effect("recordChildResults wakes a taker parked on the parent's queue", () =>
2111
+ withStore((store) =>
2112
+ Effect.gen(function*() {
2113
+ const flowId = yield* fanOutParent(store, { children: ["only"] })
2114
+ const empty = yield* store.claim(claimOptions({ token: "t-idle" }))
2115
+ assert(empty._tag === "Empty")
2116
+ const waiter = yield* Effect.forkChild(
2117
+ store.awaitWake([QueueName("default")], empty.wakeToken)
2118
+ )
2119
+ yield* TestClock.adjust(1)
2120
+ yield* recordOne(store, report(flowId, "only", "completed"))
2121
+ yield* TestClock.adjust(1)
2122
+ expect(yield* Fiber.join(waiter)).toBeUndefined()
2123
+ })
2124
+ ))
2125
+
2126
+ it.effect("concurrent last reports settle the parent exactly once", () =>
2127
+ withStore((store) =>
2128
+ Effect.gen(function*() {
2129
+ const flowId = yield* fanOutParent(store)
2130
+ const results = yield* Effect.all([
2131
+ recordOne(store, report(flowId, "a", "completed")),
2132
+ recordOne(store, report(flowId, "b", "completed")),
2133
+ recordOne(store, report(flowId, "a", "completed")),
2134
+ recordOne(store, report(flowId, "b", "completed"))
2135
+ ], { concurrency: 4 })
2136
+ expect(results.filter((result) => result.applied).length).toBe(2)
2137
+ expect(results.filter((result) => result.parentSettled).length).toBe(1)
2138
+ })
2139
+ ))
2140
+
2141
+ it.effect("fail-fast settles the parent terminally and marks remaining rows", () =>
2142
+ withStore((store) =>
2143
+ Effect.gen(function*() {
2144
+ const flowId = yield* fanOutParent(store, { children: ["a", "b", "c"], failFast: true })
2145
+ yield* recordOne(store, report(flowId, "a", "completed"))
2146
+ const settle = yield* recordOne(store, report(flowId, "b", "failed"))
2147
+ expect(settle).toEqual({ applied: true, parentSettled: true })
2148
+
2149
+ const parent = yield* store.getJob(flowId)
2150
+ assert(Option.isSome(parent))
2151
+ expect(parent.value.state).toBe("failed")
2152
+ expect(parent.value.failedReason).toContain("b")
2153
+ expect(parent.value.exit).toBeUndefined()
2154
+
2155
+ const rows = yield* store.listChildResults(flowId)
2156
+ const remaining = rows.items.find((row) => row.childKey === "c")
2157
+ expect(remaining?.status).toBe("cancelled")
2158
+ // Marked by the settle — the sweeper still owes a real cancel.
2159
+ expect(remaining?.cascaded).toBe(false)
2160
+ // Settle-time marking lands in the counters too.
2161
+ expect(parent.value.flow).toEqual(
2162
+ flowCounts({ failFast: true, completed: 1, failed: 1, cancelled: 1 })
2163
+ )
2164
+
2165
+ // A late completion finds its row terminal and drops.
2166
+ expect(yield* recordOne(store, report(flowId, "c", "completed")))
2167
+ .toEqual({ applied: false, parentSettled: false })
2168
+ })
2169
+ ))
2170
+
2171
+ it.effect("cancelling a waiting-children parent settles and marks its pending rows", () =>
2172
+ withStore((store) =>
2173
+ Effect.gen(function*() {
2174
+ const flowId = yield* fanOutParent(store)
2175
+ yield* recordOne(store, report(flowId, "a", "completed"))
2176
+ yield* store.cancel(flowId)
2177
+
2178
+ const parent = yield* store.getJob(flowId)
2179
+ assert(Option.isSome(parent))
2180
+ expect(parent.value.state).toBe("cancelled")
2181
+ // The settle marking moves the counters too.
2182
+ expect(parent.value.flow).toEqual(flowCounts({ completed: 1, cancelled: 1 }))
2183
+
2184
+ const rows = yield* store.listChildResults(flowId)
2185
+ const byKey = new Map(rows.items.map((row) => [row.childKey, row]))
2186
+ expect(byKey.get("a")?.status).toBe("completed")
2187
+ expect(byKey.get("b")?.status).toBe("cancelled")
2188
+ expect(byKey.get("b")?.cascaded).toBe(false)
2189
+
2190
+ expect(yield* recordOne(store, report(flowId, "b", "completed")))
2191
+ .toEqual({ applied: false, parentSettled: false })
2192
+ })
2193
+ ))
2194
+
2195
+ it.effect("a cancel that races the fan-out wins and marks the rows", () =>
2196
+ withStore((store) =>
2197
+ Effect.gen(function*() {
2198
+ const { id } = yield* store.enqueue(baseRequest())
2199
+ const claim = yield* store.claim(claimOptions({ token: "t-parent" }))
2200
+ assert(claim._tag === "Claimed")
2201
+ // Cancel the ACTIVE parent (sets cancelRequested), then the worker
2202
+ // acks its fan-out: cancellation wins over parking.
2203
+ yield* store.cancel(id)
2204
+ yield* store.ack(id, "t-parent", {
2205
+ _tag: "FanOut",
2206
+ failFast: false,
2207
+ children: [childSpec(id, "a"), childSpec(id, "b")]
2208
+ })
2209
+
2210
+ const parent = yield* store.getJob(id)
2211
+ assert(Option.isSome(parent))
2212
+ expect(parent.value.state).toBe("cancelled")
2213
+ expect(parent.value.flow).toEqual(flowCounts({ cancelled: 2 }))
2214
+
2215
+ // The manifest landed and every row was marked for cascade, so
2216
+ // the sweeper delivers (mostly no-op) cancels to the child store.
2217
+ const rows = yield* store.listChildResults(id)
2218
+ expect(rows.items.map((row) => row.status)).toEqual(["cancelled", "cancelled"])
2219
+ expect(rows.items.every((row) => !row.cascaded)).toBe(true)
2220
+ const work = yield* store.flowSweepWork({ pendingAgeMs: 0 })
2221
+ expect(work.cascade[0]?.children.map((child) => child.childKey)).toEqual(["a", "b"])
2222
+ })
2223
+ ))
2224
+
2225
+ it.effect("promote and retry reject a waiting-children parent", () =>
2226
+ withStore((store) =>
2227
+ Effect.gen(function*() {
2228
+ const flowId = yield* fanOutParent(store)
2229
+ const promoted = yield* Effect.exit(store.promote(flowId))
2230
+ assert(Exit.isFailure(promoted))
2231
+ const retried = yield* Effect.exit(store.retry(flowId))
2232
+ assert(Exit.isFailure(retried))
2233
+ })
2234
+ ))
2235
+
2236
+ it.effect("retrying a fail-fast-failed parent re-enters collect with its manifest", () =>
2237
+ withStore((store) =>
2238
+ Effect.gen(function*() {
2239
+ const flowId = yield* fanOutParent(store, { children: ["a", "b"], failFast: true })
2240
+ yield* recordOne(store, report(flowId, "a", "failed"))
2241
+ yield* store.retry(flowId)
2242
+
2243
+ const parent = yield* store.getJob(flowId)
2244
+ assert(Option.isSome(parent))
2245
+ expect(parent.value.state).toBe("waiting")
2246
+ // The manifest survives: a re-claimed parent dispatches collect,
2247
+ // never a second fan-out.
2248
+ expect(parent.value.flow).toBeDefined()
2249
+ expect((yield* store.listChildResults(flowId)).items.length).toBe(2)
2250
+ })
2251
+ ))
2252
+
2253
+ it.effect("a second FanOut converges on the persisted manifest", () =>
2254
+ withStore((store) =>
2255
+ Effect.gen(function*() {
2256
+ const flowId = yield* fanOutParent(store, { children: ["a"] })
2257
+ yield* recordOne(store, report(flowId, "a", "completed"))
2258
+ // Parent settled to waiting; claim and (bug-path) fan out again
2259
+ // with DIFFERENT children.
2260
+ const claim = yield* store.claim(claimOptions({ token: "t-double" }))
2261
+ assert(claim._tag === "Claimed")
2262
+ yield* store.ack(flowId, "t-double", {
2263
+ _tag: "FanOut",
2264
+ failFast: false,
2265
+ children: [childSpec(flowId, "x"), childSpec(flowId, "y")]
2266
+ })
2267
+
2268
+ // The original manifest is untouched; state follows its pending
2269
+ // count (0 → runnable collect again).
2270
+ const rows = yield* store.listChildResults(flowId)
2271
+ expect(rows.items.map((row) => row.childKey)).toEqual(["a"])
2272
+ const parent = yield* store.getJob(flowId)
2273
+ assert(Option.isSome(parent))
2274
+ expect(parent.value.state).toBe("waiting")
2275
+ })
2276
+ ))
2277
+
2278
+ it.effect("flowSweepWork scopes reconcile by parent state and pending age", () =>
2279
+ withStore((store) =>
2280
+ Effect.gen(function*() {
2281
+ const flowId = yield* fanOutParent(store)
2282
+
2283
+ // Fresh rows are the push path's business.
2284
+ const fresh = yield* store.flowSweepWork({ pendingAgeMs: 30_000 })
2285
+ expect(fresh.reconcile).toEqual([])
2286
+ expect(fresh.cascade).toEqual([])
2287
+
2288
+ yield* TestClock.adjust(30_000)
2289
+ const due = yield* store.flowSweepWork({ pendingAgeMs: 30_000 })
2290
+ expect(due.reconcile.length).toBe(1)
2291
+ expect(due.reconcile[0]?.flowId).toBe(flowId)
2292
+ expect(due.reconcile[0]?.children.map((child) => child.childKey)).toEqual(["a", "b"])
2293
+ // The stored spec is the complete original request.
2294
+ expect(due.reconcile[0]?.children[0]?.request).toEqual(
2295
+ childSpec(flowId, "a").request
2296
+ )
2297
+
2298
+ // Returned rows are re-armed: they leave the page for another full
2299
+ // age, so a sweep page rotates instead of pinning its head.
2300
+ const rearmed = yield* store.flowSweepWork({ pendingAgeMs: 30_000 })
2301
+ expect(rearmed.reconcile).toEqual([])
2302
+
2303
+ // A recorded row leaves the reconcile set for good; a settled
2304
+ // parent leaves it entirely.
2305
+ yield* recordOne(store, report(flowId, "a", "completed"))
2306
+ yield* TestClock.adjust(30_000)
2307
+ const partial = yield* store.flowSweepWork({ pendingAgeMs: 30_000 })
2308
+ expect(partial.reconcile[0]?.children.map((child) => child.childKey)).toEqual(["b"])
2309
+ yield* recordOne(store, report(flowId, "b", "completed"))
2310
+ yield* TestClock.adjust(30_000)
2311
+ const settled = yield* store.flowSweepWork({ pendingAgeMs: 30_000 })
2312
+ expect(settled.reconcile).toEqual([])
2313
+ })
2314
+ ))
2315
+
2316
+ it.effect("a fail-fast report that is also the last pending row settles as failed", () =>
2317
+ withStore((store) =>
2318
+ Effect.gen(function*() {
2319
+ const flowId = yield* fanOutParent(store, { children: ["a", "b"], failFast: true })
2320
+ yield* recordOne(store, report(flowId, "a", "completed"))
2321
+ // This report triggers BOTH settle rules: pending hits zero AND it
2322
+ // is the first failure under fail-fast. Fail-fast wins: terminal
2323
+ // `failed`, never a resume into collect.
2324
+ const last = yield* recordOne(store, report(flowId, "b", "failed"))
2325
+ expect(last).toEqual({ applied: true, parentSettled: true })
2326
+ const parent = yield* store.getJob(flowId)
2327
+ assert(Option.isSome(parent))
2328
+ expect(parent.value.state).toBe("failed")
2329
+ expect(parent.value.failedReason).toContain("b")
2330
+ })
2331
+ ))
2332
+
2333
+ it.effect("an empty FanOut wakes takers parked on the parent's queue", () =>
2334
+ withStore((store) =>
2335
+ Effect.gen(function*() {
2336
+ const { id } = yield* store.enqueue(baseRequest())
2337
+ const claim = yield* store.claim(claimOptions({ token: "t-parent" }))
2338
+ assert(claim._tag === "Claimed")
2339
+ const empty = yield* store.claim(claimOptions({ token: "t-idle" }))
2340
+ assert(empty._tag === "Empty")
2341
+ const waiter = yield* Effect.forkChild(
2342
+ store.awaitWake([QueueName("default")], empty.wakeToken)
2343
+ )
2344
+ yield* TestClock.adjust(1)
2345
+ yield* store.ack(id, "t-parent", { _tag: "FanOut", failFast: false, children: [] })
2346
+ yield* TestClock.adjust(1)
2347
+ expect(yield* Fiber.join(waiter)).toBeUndefined()
2348
+ })
2349
+ ))
2350
+
2351
+ it.effect("automatic retention spares a settled parent that still owes cascades", () =>
2352
+ withStore((store) =>
2353
+ Effect.gen(function*() {
2354
+ // A fail-fast settle marks rows for cascade in the same op that
2355
+ // makes the parent prunable — retention must not race the sweeper
2356
+ // out of its only record that cancels are still owed.
2357
+ const keep = { failed: { count: 1, ageMs: undefined } }
2358
+ const { id: flowId } = yield* store.enqueue(baseRequest({ keep }))
2359
+ const claim = yield* store.claim(claimOptions({ token: "t-parent" }))
2360
+ assert(claim._tag === "Claimed")
2361
+ yield* store.ack(flowId, "t-parent", {
2362
+ _tag: "FanOut",
2363
+ failFast: true,
2364
+ children: [childSpec(flowId, "a"), childSpec(flowId, "b")]
2365
+ })
2366
+ yield* recordOne(store, report(flowId, "a", "failed"))
2367
+
2368
+ // A newer failed peer would evict the flow parent under count: 1 —
2369
+ // but its "b" row is cancelled and not yet cascaded.
2370
+ const { id: peer1 } = yield* store.enqueue(baseRequest({ keep }))
2371
+ const claim1 = yield* store.claim(claimOptions({ token: "t-p1" }))
2372
+ assert(claim1._tag === "Claimed")
2373
+ yield* store.ack(peer1, "t-p1", { _tag: "Fail", exit: undefined })
2374
+ const spared = yield* store.getJob(flowId)
2375
+ assert(Option.isSome(spared))
2376
+ expect(spared.value.state).toBe("failed")
2377
+ expect((yield* store.listChildResults(flowId)).items.length).toBe(2)
2378
+
2379
+ // Once the cascade is delivered, retention applies normally.
2380
+ yield* store.markChildrenCascaded(flowId, ["b"])
2381
+ const { id: peer2 } = yield* store.enqueue(baseRequest({ keep }))
2382
+ const claim2 = yield* store.claim(claimOptions({ token: "t-p2" }))
2383
+ assert(claim2._tag === "Claimed")
2384
+ yield* store.ack(peer2, "t-p2", { _tag: "Fail", exit: undefined })
2385
+ expect(Option.isNone(yield* store.getJob(flowId))).toBe(true)
2386
+ expect((yield* store.listChildResults(flowId)).items).toEqual([])
2387
+ })
2388
+ ))
2389
+
2390
+ it.effect("flowSweepWork yields cascade work until rows are marked cascaded", () =>
2391
+ withStore((store) =>
2392
+ Effect.gen(function*() {
2393
+ const flowId = yield* fanOutParent(store, { children: ["a", "b"], failFast: true })
2394
+ yield* recordOne(store, report(flowId, "a", "failed"))
2395
+
2396
+ const work = yield* store.flowSweepWork({ pendingAgeMs: 30_000 })
2397
+ // Settled flow: nothing to reconcile, but "b" owes a cascade.
2398
+ expect(work.reconcile).toEqual([])
2399
+ expect(work.cascade.length).toBe(1)
2400
+ expect(work.cascade[0]?.children).toEqual([{
2401
+ childKey: "b",
2402
+ storeKey: "effect-mq/JobStore/children",
2403
+ childJobId: `flow/main/${flowId}/b`
2404
+ }])
2405
+
2406
+ yield* store.markChildrenCascaded(flowId, ["b"])
2407
+ const after = yield* store.flowSweepWork({ pendingAgeMs: 30_000 })
2408
+ expect(after.cascade).toEqual([])
2409
+ // Idempotent (unknown keys included).
2410
+ yield* store.markChildrenCascaded(flowId, ["b", "ghost"])
2411
+ })
2412
+ ))
2413
+
2414
+ it.effect("listChildResults paginates in child-key order", () =>
2415
+ withStore((store) =>
2416
+ Effect.gen(function*() {
2417
+ const flowId = yield* fanOutParent(store, { children: ["c", "a", "b"] })
2418
+ const first = yield* store.listChildResults(flowId, { limit: 2 })
2419
+ expect(first.items.map((row) => row.childKey)).toEqual(["a", "b"])
2420
+ expect(first.cursor).toBeDefined()
2421
+ const second = yield* store.listChildResults(flowId, {
2422
+ cursor: first.cursor,
2423
+ limit: 2
2424
+ })
2425
+ expect(second.items.map((row) => row.childKey)).toEqual(["c"])
2426
+ expect(second.cursor).toBeUndefined()
2427
+ })
2428
+ ))
2429
+
2430
+ it.effect("remove refuses a waiting-children parent and deletes rows with a settled one", () =>
2431
+ withStore((store) =>
2432
+ Effect.gen(function*() {
2433
+ const flowId = yield* fanOutParent(store, { children: ["a"] })
2434
+ expect(yield* store.remove(flowId)).toBe(false)
2435
+
2436
+ yield* recordOne(store, report(flowId, "a", "failed", { exit: undefined }))
2437
+ // continue-policy: the parent settled to waiting; cancel it so it
2438
+ // is removable, then remove it — the dependency rows go with it.
2439
+ yield* store.cancel(flowId)
2440
+ expect(yield* store.remove(flowId)).toBe(true)
2441
+ expect((yield* store.listChildResults(flowId)).items).toEqual([])
2442
+ })
2443
+ ))
2444
+
2445
+ it.effect("store-side child failures carry failedReason on the row", () =>
2446
+ withStore((store) =>
2447
+ Effect.gen(function*() {
2448
+ const flowId = yield* fanOutParent(store, { children: ["a"] })
2449
+ yield* recordOne(store, report(flowId, "a", "failed", {
2450
+ exit: undefined,
2451
+ failedReason: "job stalled more than allowable limit"
2452
+ }))
2453
+ const rows = yield* store.listChildResults(flowId)
2454
+ expect(rows.items[0]?.exit).toBeUndefined()
2455
+ expect(rows.items[0]?.failedReason).toBe("job stalled more than allowable limit")
2456
+ })
2457
+ ))
2458
+
2459
+ // ----------------------------------------------------------------------
2460
+ // Batched reports + the child-side outbox. The outbox is how a CHILD
2461
+ // store reports terminal transitions to a parent living in another
2462
+ // store: append on the transition, peek/delete from the relay.
2463
+ // ----------------------------------------------------------------------
2464
+
2465
+ it.effect("recordChildResults applies a batch positionally and keeps counters exact", () =>
2466
+ withStore((store) =>
2467
+ Effect.gen(function*() {
2468
+ const flowId = yield* fanOutParent(store, { children: ["a", "b", "c"] })
2469
+ const results = yield* store.recordChildResults([
2470
+ report(flowId, "a", "completed"),
2471
+ report(flowId, "a", "completed"), // duplicate inside the batch
2472
+ report(flowId, "ghost", "completed"),
2473
+ report(flowId, "b", "failed")
2474
+ ])
2475
+ expect(results).toEqual([
2476
+ { applied: true, parentSettled: false },
2477
+ { applied: false, parentSettled: false },
2478
+ { applied: false, parentSettled: false },
2479
+ { applied: true, parentSettled: false }
2480
+ ])
2481
+ const parent = yield* store.getJob(flowId)
2482
+ assert(Option.isSome(parent))
2483
+ expect(parent.value.state).toBe("waiting-children")
2484
+ expect(parent.value.flow).toEqual(flowCounts({ pending: 1, completed: 1, failed: 1 }))
2485
+ })
2486
+ ))
2487
+
2488
+ it.effect("a batch that empties pending settles once, on its last applied report", () =>
2489
+ withStore((store) =>
2490
+ Effect.gen(function*() {
2491
+ const flowId = yield* fanOutParent(store)
2492
+ const results = yield* store.recordChildResults([
2493
+ report(flowId, "a", "completed"),
2494
+ report(flowId, "b", "completed")
2495
+ ])
2496
+ expect(results).toEqual([
2497
+ { applied: true, parentSettled: false },
2498
+ { applied: true, parentSettled: true }
2499
+ ])
2500
+ const parent = yield* store.getJob(flowId)
2501
+ assert(Option.isSome(parent))
2502
+ expect(parent.value.state).toBe("waiting")
2503
+ })
2504
+ ))
2505
+
2506
+ it.effect("fail-fast wins inside a batch, after every batch-mate applied", () =>
2507
+ withStore((store) =>
2508
+ Effect.gen(function*() {
2509
+ const flowId = yield* fanOutParent(store, { children: ["a", "b", "c"], failFast: true })
2510
+ const results = yield* store.recordChildResults([
2511
+ report(flowId, "b", "failed"),
2512
+ report(flowId, "c", "completed")
2513
+ ])
2514
+ // Row updates apply BEFORE the settle decision: "c" keeps its real
2515
+ // completed outcome even though "b" settles the flow.
2516
+ expect(results).toEqual([
2517
+ { applied: true, parentSettled: true },
2518
+ { applied: true, parentSettled: false }
2519
+ ])
2520
+ const parent = yield* store.getJob(flowId)
2521
+ assert(Option.isSome(parent))
2522
+ expect(parent.value.state).toBe("failed")
2523
+ expect(parent.value.failedReason).toContain("b")
2524
+ expect(parent.value.flow).toEqual(
2525
+ flowCounts({ failFast: true, completed: 1, failed: 1, cancelled: 1 })
2526
+ )
2527
+ const rows = yield* store.listChildResults(flowId)
2528
+ const byKey = new Map(rows.items.map((row) => [row.childKey, row.status]))
2529
+ expect(byKey.get("c")).toBe("completed")
2530
+ expect(byKey.get("a")).toBe("cancelled")
2531
+ })
2532
+ ))
2533
+
2534
+ it.effect("a batch may span flows and settles each independently", () =>
2535
+ withStore((store) =>
2536
+ Effect.gen(function*() {
2537
+ const first = yield* fanOutParent(store, { children: ["a"] })
2538
+ const second = yield* fanOutParent(store, { children: ["b"] })
2539
+ const results = yield* store.recordChildResults([
2540
+ report(first, "a", "completed"),
2541
+ report(second, "b", "completed")
2542
+ ])
2543
+ expect(results).toEqual([
2544
+ { applied: true, parentSettled: true },
2545
+ { applied: true, parentSettled: true }
2546
+ ])
2547
+ })
2548
+ ))
2549
+
2550
+ it.effect("a cancelled child report moves the cancelled counter", () =>
2551
+ withStore((store) =>
2552
+ Effect.gen(function*() {
2553
+ const flowId = yield* fanOutParent(store)
2554
+ yield* recordOne(store, report(flowId, "a", "cancelled", { exit: undefined }))
2555
+ const parent = yield* store.getJob(flowId)
2556
+ assert(Option.isSome(parent))
2557
+ expect(parent.value.flow).toEqual(flowCounts({ pending: 1, cancelled: 1 }))
2558
+ })
2559
+ ))
2560
+
2561
+ it.effect("peekOutbox pages past prior entries with `after`, even deleted ones", () =>
2562
+ withStore((store) =>
2563
+ Effect.gen(function*() {
2564
+ const enqueueChild = (key: string) =>
2565
+ Effect.gen(function*() {
2566
+ const { id } = yield* store.enqueue(baseRequest({
2567
+ parent: parentEnvelope(JobId("remote-flow-3"), key)
2568
+ }))
2569
+ const claim = yield* store.claim(claimOptions({ token: `t-${key}` }))
2570
+ assert(claim._tag === "Claimed")
2571
+ yield* store.ack(id, `t-${key}`, { _tag: "Complete", exit: { key } })
2572
+ })
2573
+ yield* enqueueChild("one")
2574
+ yield* enqueueChild("two")
2575
+ yield* enqueueChild("three")
2576
+
2577
+ const first = yield* store.peekOutbox({ limit: 2 })
2578
+ expect(first.map((entry) => entry.report.childKey)).toEqual(["one", "two"])
2579
+ const cursor = first[first.length - 1]?.id
2580
+ assert(cursor !== undefined)
2581
+ const rest = yield* store.peekOutbox({ limit: 2, after: cursor })
2582
+ expect(rest.map((entry) => entry.report.childKey)).toEqual(["three"])
2583
+
2584
+ // The cursor keeps working when the entry it names is gone.
2585
+ yield* store.deleteOutbox([cursor])
2586
+ const restAgain = yield* store.peekOutbox({ limit: 2, after: cursor })
2587
+ expect(restAgain.map((entry) => entry.report.childKey)).toEqual(["three"])
2588
+ })
2589
+ ))
2590
+
2591
+ it.effect("cancels honoured by retry acks, the stall sweep, and on a parked parent land in the outbox", () =>
2592
+ withStore((store) =>
2593
+ Effect.gen(function*() {
2594
+ const envelope = (key: string) => parentEnvelope(JobId("remote-flow-4"), key)
2595
+ // A cancel honoured when a RETRY ack finds the flag set.
2596
+ const retried = yield* store.enqueue(baseRequest({ parent: envelope("retry-cancel") }))
2597
+ const claimA = yield* store.claim(claimOptions({ token: "t-a" }))
2598
+ assert(claimA._tag === "Claimed")
2599
+ yield* store.cancel(retried.id)
2600
+ yield* store.ack(retried.id, "t-a", { _tag: "Retry", delayMs: 0, exit: undefined })
2601
+
2602
+ // A cancel honoured when the stall sweep recovers a dead worker.
2603
+ const stalled = yield* store.enqueue(baseRequest({ parent: envelope("stall-cancel") }))
2604
+ const claimB = yield* store.claim(claimOptions({ token: "t-b", lockDurationMs: 1_000 }))
2605
+ assert(claimB._tag === "Claimed")
2606
+ yield* store.cancel(stalled.id)
2607
+ yield* TestClock.adjust(2_000)
2608
+ yield* store.recoverStalled({ maxStalledCount: 5 })
2609
+
2610
+ // A direct cancel of a PARKED nested parent (waiting-children).
2611
+ const parked = yield* store.enqueue(baseRequest({ parent: envelope("parked-cancel") }))
2612
+ const claimC = yield* store.claim(claimOptions({ token: "t-c" }))
2613
+ assert(claimC._tag === "Claimed")
2614
+ yield* store.ack(parked.id, "t-c", {
2615
+ _tag: "FanOut",
2616
+ failFast: false,
2617
+ children: [childSpec(parked.id, "a")]
2618
+ })
2619
+ yield* store.cancel(parked.id)
2620
+
2621
+ const entries = yield* store.peekOutbox({ limit: 10 })
2622
+ expect(entries.map((entry) => [entry.report.childKey, entry.report.outcome])).toEqual([
2623
+ ["retry-cancel", "cancelled"],
2624
+ ["stall-cancel", "cancelled"],
2625
+ ["parked-cancel", "cancelled"]
2626
+ ])
2627
+ })
2628
+ ))
2629
+
2630
+ it.effect("terminal transitions of envelope-carrying jobs land in the outbox", () =>
2631
+ withStore((store) =>
2632
+ Effect.gen(function*() {
2633
+ const envelope = (key: string) => parentEnvelope(JobId("remote-flow-1"), key)
2634
+ // A plain job's terminal ack appends nothing.
2635
+ const plain = yield* store.enqueue(baseRequest())
2636
+ const plainClaim = yield* store.claim(claimOptions({ token: "t-plain" }))
2637
+ assert(plainClaim._tag === "Claimed")
2638
+ yield* store.ack(plain.id, "t-plain", { _tag: "Complete", exit: { ok: true } })
2639
+ expect(yield* store.peekOutbox({ limit: 10 })).toEqual([])
2640
+
2641
+ // Ack Complete → outbox entry with the exit.
2642
+ const acked = yield* store.enqueue(baseRequest({ parent: envelope("acked") }))
2643
+ const claim = yield* store.claim(claimOptions({ token: "t-child" }))
2644
+ assert(claim._tag === "Claimed")
2645
+ expect(claim.job.id).toBe(acked.id)
2646
+ yield* store.ack(acked.id, "t-child", { _tag: "Complete", exit: { sent: 1 } })
2647
+
2648
+ // Direct cancel of a delayed child → outbox entry.
2649
+ const cancelled = yield* store.enqueue(
2650
+ baseRequest({ parent: envelope("cancelled"), delayMs: 60_000 })
2651
+ )
2652
+ yield* store.cancel(cancelled.id)
2653
+
2654
+ // Stall exhaustion → outbox entry carrying the failedReason.
2655
+ yield* store.enqueue(baseRequest({ parent: envelope("stalled") }))
2656
+ const stalledClaim = yield* store.claim(
2657
+ claimOptions({ token: "t-stall", lockDurationMs: 1_000 })
2658
+ )
2659
+ assert(stalledClaim._tag === "Claimed")
2660
+ yield* TestClock.adjust(2_000)
2661
+ const recovered = yield* store.recoverStalled({ maxStalledCount: 0 })
2662
+ expect(recovered).toEqual([{ id: stalledClaim.job.id, failed: true }])
2663
+
2664
+ // Oldest first, `limit` respected, full entry shape.
2665
+ const firstPage = yield* store.peekOutbox({ limit: 2 })
2666
+ expect(firstPage.map((entry) => entry.report.childKey)).toEqual(["acked", "cancelled"])
2667
+ const head = firstPage[0]
2668
+ assert(head !== undefined)
2669
+ expect(head.flowName).toBe("test-flow")
2670
+ expect(head.parentStoreKey).toBe("main")
2671
+ expect(head.report.flowId).toBe("remote-flow-1")
2672
+ expect(head.report.outcome).toBe("completed")
2673
+ expect(head.report.exit).toEqual({ sent: 1 })
2674
+ const all = yield* store.peekOutbox({ limit: 10 })
2675
+ expect(all.map((entry) => entry.report.outcome)).toEqual([
2676
+ "completed",
2677
+ "cancelled",
2678
+ "failed"
2679
+ ])
2680
+ expect(all[2]?.report.exit).toBeUndefined()
2681
+ expect(all[2]?.report.failedReason).toBe("job stalled more than allowable limit")
2682
+
2683
+ // Peek does not consume; delete does, idempotently.
2684
+ yield* store.deleteOutbox([head.id, "ghost-id"])
2685
+ const rest = yield* store.peekOutbox({ limit: 10 })
2686
+ expect(rest.map((entry) => entry.report.childKey)).toEqual(["cancelled", "stalled"])
2687
+ yield* store.deleteOutbox([head.id])
2688
+ expect((yield* store.peekOutbox({ limit: 10 })).length).toBe(2)
2689
+ })
2690
+ ))
2691
+
2692
+ it.effect("cancels honoured off the ack path still land in the outbox", () =>
2693
+ withStore((store) =>
2694
+ Effect.gen(function*() {
2695
+ // A cancel that arrives while the child runs, honoured when the
2696
+ // worker RELEASES the job (shutdown) instead of acking it.
2697
+ const { id } = yield* store.enqueue(baseRequest({
2698
+ parent: parentEnvelope(JobId("remote-flow-2"), "released")
2699
+ }))
2700
+ const claim = yield* store.claim(claimOptions({ token: "t-run" }))
2701
+ assert(claim._tag === "Claimed")
2702
+ yield* store.cancel(id)
2703
+ yield* store.release(id, "t-run")
2704
+
2705
+ const job = yield* store.getJob(id)
2706
+ assert(Option.isSome(job))
2707
+ expect(job.value.state).toBe("cancelled")
2708
+ const entries = yield* store.peekOutbox({ limit: 10 })
2709
+ expect(entries.map((entry) => entry.report.outcome)).toEqual(["cancelled"])
2710
+ expect(entries[0]?.report.childKey).toBe("released")
2711
+ })
2712
+ ))
2713
+
2714
+ it.effect("a cancel that races a nested parent's fan-out reports upward through the outbox", () =>
2715
+ withStore((store) =>
2716
+ Effect.gen(function*() {
2717
+ // The parent being fanned out is itself a flow child; the raced
2718
+ // cancel settles it terminally inside the FanOut ack.
2719
+ const inner = yield* store.enqueue(baseRequest({
2720
+ parent: {
2721
+ flowName: "outer-flow",
2722
+ flowId: JobId("outer-2"),
2723
+ childKey: "inner-raced",
2724
+ parentStoreKey: "outer-store",
2725
+ depth: 1
2726
+ }
2727
+ }))
2728
+ const claim = yield* store.claim(claimOptions({ token: "t-race" }))
2729
+ assert(claim._tag === "Claimed")
2730
+ yield* store.cancel(inner.id)
2731
+ yield* store.ack(inner.id, "t-race", {
2732
+ _tag: "FanOut",
2733
+ failFast: false,
2734
+ children: [childSpec(inner.id, "a")]
2735
+ })
2736
+
2737
+ const parent = yield* store.getJob(inner.id)
2738
+ assert(Option.isSome(parent))
2739
+ expect(parent.value.state).toBe("cancelled")
2740
+ const entries = yield* store.peekOutbox({ limit: 10 })
2741
+ expect(entries.map((entry) => entry.report.outcome)).toEqual(["cancelled"])
2742
+ expect(entries[0]?.report.childKey).toBe("inner-raced")
2743
+ expect(entries[0]?.flowName).toBe("outer-flow")
2744
+ })
2745
+ ))
2746
+
2747
+ it.effect("a fail-fast settle of a nested parent reports upward through the outbox", () =>
2748
+ withStore((store) =>
2749
+ Effect.gen(function*() {
2750
+ // The inner parent is itself a flow child; its terminal transition
2751
+ // happens store-side (the settle), with no worker ack to hook.
2752
+ const inner = yield* store.enqueue(baseRequest({
2753
+ parent: {
2754
+ flowName: "outer-flow",
2755
+ flowId: JobId("outer-1"),
2756
+ childKey: "inner",
2757
+ parentStoreKey: "outer-store",
2758
+ depth: 1
2759
+ }
2760
+ }))
2761
+ const claim = yield* store.claim(claimOptions({ token: "t-inner" }))
2762
+ assert(claim._tag === "Claimed")
2763
+ yield* store.ack(inner.id, "t-inner", {
2764
+ _tag: "FanOut",
2765
+ failFast: true,
2766
+ children: [childSpec(inner.id, "a")]
2767
+ })
2768
+ // Parking is not terminal: nothing in the outbox yet.
2769
+ expect(yield* store.peekOutbox({ limit: 10 })).toEqual([])
2770
+
2771
+ yield* recordOne(store, report(inner.id, "a", "failed"))
2772
+ const entries = yield* store.peekOutbox({ limit: 10 })
2773
+ expect(entries.length).toBe(1)
2774
+ expect(entries[0]?.flowName).toBe("outer-flow")
2775
+ expect(entries[0]?.report.childKey).toBe("inner")
2776
+ expect(entries[0]?.report.outcome).toBe("failed")
2777
+ expect(entries[0]?.report.failedReason).toContain('"a" failed')
2778
+ })
2779
+ ))
1704
2780
  })
1705
2781
  }