effect-mq 0.5.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +85 -17
  2. package/dist/Flow.d.ts +381 -0
  3. package/dist/Flow.d.ts.map +1 -0
  4. package/dist/Flow.js +340 -0
  5. package/dist/Flow.js.map +1 -0
  6. package/dist/Job.d.ts +31 -6
  7. package/dist/Job.d.ts.map +1 -1
  8. package/dist/Job.js +16 -2
  9. package/dist/Job.js.map +1 -1
  10. package/dist/JobStore.d.ts +312 -10
  11. package/dist/JobStore.d.ts.map +1 -1
  12. package/dist/JobStore.js.map +1 -1
  13. package/dist/MemoryJobStore.d.ts.map +1 -1
  14. package/dist/MemoryJobStore.js +334 -7
  15. package/dist/MemoryJobStore.js.map +1 -1
  16. package/dist/Metrics.d.ts +31 -0
  17. package/dist/Metrics.d.ts.map +1 -1
  18. package/dist/Metrics.js +39 -0
  19. package/dist/Metrics.js.map +1 -1
  20. package/dist/Worker.d.ts +120 -11
  21. package/dist/Worker.d.ts.map +1 -1
  22. package/dist/Worker.js +452 -26
  23. package/dist/Worker.js.map +1 -1
  24. package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
  25. package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
  26. package/dist/drizzle-postgres/DrizzleJobStore.js +653 -77
  27. package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
  28. package/dist/drizzle-postgres/schema.d.ts +293 -3
  29. package/dist/drizzle-postgres/schema.d.ts.map +1 -1
  30. package/dist/drizzle-postgres/schema.js +66 -1
  31. package/dist/drizzle-postgres/schema.js.map +1 -1
  32. package/dist/index.d.ts +7 -0
  33. package/dist/index.d.ts.map +1 -1
  34. package/dist/index.js +7 -0
  35. package/dist/index.js.map +1 -1
  36. package/dist/redis/RedisJobStore.d.ts.map +1 -1
  37. package/dist/redis/RedisJobStore.js +219 -18
  38. package/dist/redis/RedisJobStore.js.map +1 -1
  39. package/dist/redis/scripts.d.ts +117 -10
  40. package/dist/redis/scripts.d.ts.map +1 -1
  41. package/dist/redis/scripts.js +492 -25
  42. package/dist/redis/scripts.js.map +1 -1
  43. package/dist/testing/conformance.d.ts +6 -0
  44. package/dist/testing/conformance.d.ts.map +1 -1
  45. package/dist/testing/conformance.js +728 -1
  46. package/dist/testing/conformance.js.map +1 -1
  47. package/package.json +1 -1
  48. package/src/Flow.ts +778 -0
  49. package/src/Job.ts +35 -11
  50. package/src/JobStore.ts +339 -9
  51. package/src/MemoryJobStore.ts +370 -7
  52. package/src/Metrics.ts +43 -0
  53. package/src/Worker.ts +726 -37
  54. package/src/drizzle-postgres/DrizzleJobStore.ts +817 -78
  55. package/src/drizzle-postgres/schema.ts +92 -0
  56. package/src/index.ts +8 -0
  57. package/src/redis/RedisJobStore.ts +289 -8
  58. package/src/redis/scripts.ts +524 -24
  59. package/src/testing/conformance.ts +945 -1
@@ -18,11 +18,19 @@
18
18
  */
19
19
  import * as JobStore from "../JobStore.ts"
20
20
  import type { PgClient } from "@effect/sql-pg"
21
- import { asc, eq, getTableColumns, sql } from "drizzle-orm"
21
+ import { asc, eq, getTableColumns, type SQL, sql } from "drizzle-orm"
22
22
  import * as PgDrizzle from "drizzle-orm/effect-postgres"
23
23
  import { getTableConfig } from "drizzle-orm/pg-core"
24
- import { Clock, type Context, Deferred, Duration, Effect, Layer, Option, type Scope, Stream } from "effect"
25
- import type { MqDedupeTable, MqJobAttemptsTable, MqJobsTable, MqQueueControlTable, MqSchedulesTable } from "./schema.ts"
24
+ import { Cause, Clock, type Context, Deferred, Duration, Effect, Layer, Option, Predicate, type Scope, Stream } from "effect"
25
+ import type {
26
+ MqDedupeTable,
27
+ MqFlowChildrenTable,
28
+ MqFlowOutboxTable,
29
+ MqJobAttemptsTable,
30
+ MqJobsTable,
31
+ MqQueueControlTable,
32
+ MqSchedulesTable
33
+ } from "./schema.ts"
26
34
 
27
35
  const { JobId } = JobStore
28
36
 
@@ -40,6 +48,10 @@ export interface DrizzleJobStoreOptions<StoreId = JobStore.JobStore> {
40
48
  readonly queues: MqQueueControlTable
41
49
  /** The dedup-key registry table (from `mqDedupe`). */
42
50
  readonly dedupe: MqDedupeTable
51
+ /** The flow dependency-rows table (from `mqFlowChildren`). */
52
+ readonly flowChildren: MqFlowChildrenTable
53
+ /** The child-report outbox table (from `mqFlowOutbox`). */
54
+ readonly flowOutbox: MqFlowOutboxTable
43
55
  /**
44
56
  * Values for columns added via `mqJobs({ extend })`, evaluated at enqueue
45
57
  * (and on dedupe `replace`). Keys are the extended columns' TS names.
@@ -84,6 +96,50 @@ export type ExtraColumnValue = string | number | boolean | Date | null
84
96
  const storeError = (message: string) => (cause: unknown) =>
85
97
  new JobStore.JobStoreError({ message, cause })
86
98
 
99
+ // The exact spellings a bigserial ever produces: no leading zeros, no signs,
100
+ // no whitespace — what outbox ids look like and nothing Postgres would
101
+ // silently normalize into one.
102
+ const CANONICAL_BIGSERIAL = /^[1-9]\d*$/
103
+
104
+ /**
105
+ * Whether an error surfaced by drizzle's Effect driver is a Postgres
106
+ * deadlock (40P01), i.e. safe to retry.
107
+ *
108
+ * The deadlock travels wrapped: drizzle's session fails with an
109
+ * `EffectDrizzleQueryError` whose `cause` field is `Cause.fail(SqlError)`
110
+ * (drizzle-orm `pg-core/effect/session.ts`), and `@effect/sql-pg` classifies
111
+ * pg code 40P01 into the `SqlError`'s `reason: DeadlockError` (PgClient.ts
112
+ * `classifyError`). Neither layer's `toString` renders the pg code, so this
113
+ * unwraps structurally instead of string-matching the rendered error.
114
+ *
115
+ * @internal
116
+ */
117
+ // oxlint-disable-next-line anti-slop/no-unknown-parameters -- a retry-predicate classifier over whatever the driver threw IS the boundary parser
118
+ export const isDeadlockError = (error: unknown): boolean => {
119
+ // Unwrap drizzle's envelope: its `cause` is an Effect Cause holding the
120
+ // original failure. A bare SqlError (non-drizzle path) passes through.
121
+ const unwrapped = Predicate.hasProperty(error, "cause") && Cause.isCause(error.cause)
122
+ ? Option.getOrUndefined(Cause.findErrorOption(error.cause))
123
+ : error
124
+ if (
125
+ !Predicate.hasProperty(unwrapped, "_tag") || unwrapped._tag !== "SqlError" ||
126
+ !Predicate.hasProperty(unwrapped, "reason")
127
+ ) {
128
+ return false
129
+ }
130
+ const reason = unwrapped.reason
131
+ if (Predicate.hasProperty(reason, "_tag") && reason._tag === "DeadlockError") {
132
+ return true
133
+ }
134
+ // Fallback for classifiers that missed the code: the reason keeps the raw
135
+ // pg error, whose message for 40P01 is "deadlock detected".
136
+ if (Predicate.hasProperty(reason, "cause")) {
137
+ const raw = String(reason.cause)
138
+ return raw.includes("40P01") || raw.includes("deadlock detected")
139
+ }
140
+ return false
141
+ }
142
+
87
143
  /**
88
144
  * drizzle's Effect driver types raw `execute` results as row arrays, but at
89
145
  * runtime the raw path yields the node-postgres `QueryResult` envelope
@@ -114,6 +170,12 @@ type JobRow = {
114
170
  readonly cancelRequested: boolean
115
171
  readonly dedupeKey: string | null
116
172
  readonly trace: JobStore.TraceContext | null
173
+ readonly parent: JobStore.ParentEnvelope | null
174
+ readonly flowFailFast: boolean | null
175
+ readonly flowPending: number | null
176
+ readonly flowCompleted: number | null
177
+ readonly flowFailed: number | null
178
+ readonly flowCancelled: number | null
117
179
  readonly runAt: Date
118
180
  readonly enqueuedAt: Date
119
181
  readonly processedAt: Date | null
@@ -175,6 +237,17 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
175
237
  cancelRequested: row.cancelRequested,
176
238
  dedupeKey: row.dedupeKey ?? undefined,
177
239
  trace: row.trace ?? undefined,
240
+ parent: row.parent ?? undefined,
241
+ // The flow columns are NULL together; `flowPending` is the presence marker.
242
+ flow: row.flowPending === null || row.flowPending === undefined
243
+ ? undefined
244
+ : {
245
+ failFast: row.flowFailFast === true,
246
+ pending: Number(row.flowPending),
247
+ completed: Number(row.flowCompleted ?? 0),
248
+ failed: Number(row.flowFailed ?? 0),
249
+ cancelled: Number(row.flowCancelled ?? 0)
250
+ },
178
251
  runAt: row.runAt.getTime(),
179
252
  enqueuedAt: row.enqueuedAt.getTime(),
180
253
  processedAt: row.processedAt?.getTime(),
@@ -183,6 +256,31 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
183
256
  failedReason: row.failedReason ?? undefined
184
257
  })
185
258
 
259
+ type FlowChildRow = {
260
+ readonly flowId: string
261
+ readonly childKey: string
262
+ readonly name: string
263
+ readonly storeKey: string
264
+ /** Projected from `spec->>'id'` (never NULL: the FanOut ack validated it). */
265
+ readonly childJobId: string
266
+ readonly status: JobStore.FlowChildRecord["status"]
267
+ readonly exit: unknown
268
+ readonly failedReason: string | null
269
+ readonly cascaded: boolean
270
+ }
271
+
272
+ const toFlowChildRecord = (row: FlowChildRow): JobStore.FlowChildRecord => ({
273
+ flowId: JobId(row.flowId),
274
+ childKey: row.childKey,
275
+ name: row.name,
276
+ storeKey: row.storeKey,
277
+ childJobId: JobId(row.childJobId),
278
+ status: row.status,
279
+ exit: row.exit ?? undefined,
280
+ failedReason: row.failedReason ?? undefined,
281
+ cascaded: row.cascaded
282
+ })
283
+
186
284
  /**
187
285
  * Build the store implementation. Requires `PgClient` and a `Scope` (for the
188
286
  * LISTEN subscription).
@@ -200,6 +298,8 @@ export const make = (
200
298
  const schedules = options.schedules
201
299
  const queues = options.queues
202
300
  const dedupe = options.dedupe
301
+ const flowChildren = options.flowChildren
302
+ const flowOutbox = options.flowOutbox
203
303
  const jobsName = getTableConfig(jobs).name
204
304
  const attemptsName = getTableConfig(attempts).name
205
305
  const wakeChannel = `effect_mq_wake_${jobsName}`
@@ -225,6 +325,12 @@ export const make = (
225
325
  "cancelRequested",
226
326
  "dedupeKey",
227
327
  "trace",
328
+ "parent",
329
+ "flowFailFast",
330
+ "flowPending",
331
+ "flowCompleted",
332
+ "flowFailed",
333
+ "flowCancelled",
228
334
  "runAt",
229
335
  "enqueuedAt",
230
336
  "processedAt",
@@ -263,7 +369,9 @@ export const make = (
263
369
  db.select({ jobId: attempts.jobId }).from(attempts).limit(0),
264
370
  db.select({ key: schedules.key }).from(schedules).limit(0),
265
371
  db.select({ queue: queues.queue }).from(queues).limit(0),
266
- db.select({ key: dedupe.key }).from(dedupe).limit(0)
372
+ db.select({ key: dedupe.key }).from(dedupe).limit(0),
373
+ db.select({ flowId: flowChildren.flowId }).from(flowChildren).limit(0),
374
+ db.select({ id: flowOutbox.id }).from(flowOutbox).limit(0)
267
375
  ]).pipe(
268
376
  Effect.mapError(storeError(
269
377
  `effect-mq: tables "${jobsName}"/"${attemptsName}" are missing or mismatched — ` +
@@ -324,6 +432,28 @@ export const make = (
324
432
  Effect.forever,
325
433
  Effect.forkScoped
326
434
  )
435
+ // Automatic retention (the store-level sweep and per-job `keep`) never
436
+ // prunes a flow parent that still owes cascade cancels: its dependency
437
+ // rows marked `cancelled` and not `cascaded` are the only record that
438
+ // real cancels are still due in the child stores. The explicit `remove`
439
+ // verb is not exempted.
440
+ const owesCascades = sql`EXISTS (
441
+ SELECT 1 FROM ${flowChildren}
442
+ WHERE ${flowChildren.flowId} = ${jobs.id}
443
+ AND ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
444
+ )`
445
+ // A pruned flow parent's dependency rows go with it in the same statement.
446
+ const purgeJobsWhere = (predicate: SQL) =>
447
+ sql`
448
+ WITH deleted AS (
449
+ DELETE FROM ${jobs}
450
+ WHERE ${predicate}
451
+ RETURNING ${jobs.id} AS id
452
+ )
453
+ DELETE FROM ${flowChildren}
454
+ WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
455
+ `
456
+
327
457
  if (options.historyTtl !== undefined) {
328
458
  const ttlByState = JobStore.normalizeHistoryTtl(options.historyTtl)
329
459
  const sweepMs = Duration.toMillis(options.historySweepInterval ?? "1 minute")
@@ -334,9 +464,8 @@ export const make = (
334
464
  // job name is pruned on the timer, not only when its group is acked.
335
465
  for (const state of ["completed", "failed", "cancelled"] as const) {
336
466
  const ttl = ttlByState[state]
337
- yield* db.execute(sql`
338
- DELETE FROM ${jobs}
339
- WHERE ${jobs.state} = ${state} AND (
467
+ yield* db.execute(purgeJobsWhere(sql`
468
+ ${jobs.state} = ${state} AND NOT ${owesCascades} AND (
340
469
  ${ttl !== undefined ? sql`${jobs.finishedAt} <= ${new Date(now.getTime() - ttl)}` : sql`FALSE`}
341
470
  OR (
342
471
  COALESCE(
@@ -351,7 +480,7 @@ export const make = (
351
480
  ELSE ${jobs.keep}->>'ageMs' END
352
481
  )::double precision) / 1000.0))
353
482
  )
354
- `)
483
+ `))
355
484
  }
356
485
  // Dead dedup rows: expired windows, or pointers at vanished jobs.
357
486
  yield* db.execute(sql`
@@ -359,7 +488,7 @@ export const make = (
359
488
  WHERE (${dedupe.windowExpiresAt} IS NOT NULL AND ${dedupe.windowExpiresAt} <= ${now})
360
489
  OR (${dedupe.windowExpiresAt} IS NULL AND NOT EXISTS (
361
490
  SELECT 1 FROM ${jobs} WHERE ${jobs.id} = ${dedupe.jobId}
362
- AND ${jobs.state} IN ('waiting', 'delayed', 'active')
491
+ AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
363
492
  ))
364
493
  `)
365
494
  }).pipe(
@@ -433,23 +562,23 @@ export const make = (
433
562
  const keep = keepPolicyFor(row.keep, row.state)
434
563
  if (keep === undefined) return
435
564
  if (keep.ageMs !== undefined) {
436
- yield* tx.execute(sql`
437
- DELETE FROM ${jobs}
438
- WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
565
+ yield* tx.execute(purgeJobsWhere(sql`
566
+ ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
439
567
  AND ${jobs.finishedAt} <= ${new Date(now.getTime() - keep.ageMs)}
440
- `)
568
+ AND NOT ${owesCascades}
569
+ `))
441
570
  }
442
571
  if (keep.count !== undefined) {
443
- yield* tx.execute(sql`
444
- DELETE FROM ${jobs}
445
- WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
572
+ yield* tx.execute(purgeJobsWhere(sql`
573
+ ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
446
574
  AND ${jobs.id} NOT IN (
447
575
  SELECT ${jobs.id} FROM ${jobs}
448
576
  WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
449
577
  ORDER BY ${jobs.finishedAt} DESC, ${jobs.seq} DESC
450
578
  LIMIT ${keep.count}
451
579
  )
452
- `)
580
+ AND NOT ${owesCascades}
581
+ `))
453
582
  }
454
583
  })
455
584
 
@@ -495,7 +624,7 @@ export const make = (
495
624
  : sql`'j-' || ${seqExpr}::text`
496
625
  const rows = rowsOf(yield* exec.execute<{ id: string }>(sql`
497
626
  INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
498
- attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
627
+ attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
499
628
  VALUES (${idExpr}, ${request.name}, ${request.queue}, ${state}, ${request.priority},
500
629
  ${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
501
630
  ${request.attemptsMax},
@@ -503,6 +632,7 @@ export const make = (
503
632
  ${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
504
633
  ${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
505
634
  ${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
635
+ ${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
506
636
  ${runAt}, ${now}${extraColumnValues(request)})
507
637
  ON CONFLICT (id) DO NOTHING
508
638
  RETURNING ${jobs.id} AS id
@@ -633,7 +763,7 @@ export const make = (
633
763
  // killed and safe to retry.
634
764
  Effect.retry({
635
765
  times: 3,
636
- while: (error) => String(error).includes("40P01") || String(error).includes("deadlock detected")
766
+ while: isDeadlockError
637
767
  }),
638
768
  Effect.mapError((error) =>
639
769
  error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)
@@ -658,59 +788,139 @@ export const make = (
658
788
  AND (${dedupe.windowExpiresAt} IS NULL OR ${dedupe.windowExpiresAt} <= ${now})
659
789
  `).pipe(Effect.asVoid)
660
790
 
791
+ // The outbox invariant: every operation that moves a job carrying a
792
+ // `parent` envelope INTO a terminal state appends its report here in the
793
+ // same transaction (see `JobStore.OutboxEntry`). `exit`/`failedReason`
794
+ // mirror the job row AFTER the transition; JSON.stringify drops
795
+ // undefined fields so absent values read back as undefined.
796
+ const appendOutbox = (
797
+ exec: Pick<Db, "execute">,
798
+ parent: JobStore.ParentEnvelope | null | undefined,
799
+ outcome: JobStore.FlowChildReport["outcome"],
800
+ exit: JobStore.FlowChildReport["exit"],
801
+ failedReason: string | null | undefined
802
+ ) =>
803
+ parent === null || parent === undefined
804
+ ? Effect.void
805
+ : exec.execute(sql`
806
+ INSERT INTO ${flowOutbox} (flow_name, parent_store_key, report)
807
+ VALUES (${parent.flowName}, ${parent.parentStoreKey}, ${
808
+ JSON.stringify({
809
+ flowId: parent.flowId,
810
+ childKey: parent.childKey,
811
+ outcome,
812
+ exit,
813
+ failedReason: failedReason ?? undefined
814
+ })
815
+ }::jsonb)
816
+ `).pipe(Effect.asVoid)
817
+
818
+ // Flip a flow's still-pending dependency rows to `cancelled` and NOT
819
+ // `cascaded` (the sweeper still owes the child stores real cancels).
820
+ // Embedded as a CTE body so the caller can count the marked rows into
821
+ // the parent's `cancelled` counter in the same statement.
822
+ const cancelPendingChildren = (flowId: string) =>
823
+ sql`
824
+ UPDATE ${flowChildren} SET status = 'cancelled', cascaded = FALSE
825
+ WHERE ${flowChildren.flowId} = ${flowId} AND ${flowChildren.status} = 'pending'
826
+ RETURNING 1
827
+ `
828
+
661
829
  // Shared by cancel and cancelByDedupe.
662
830
  const cancelJob = (id: JobStore.JobId) =>
663
831
  db.transaction((tx) =>
664
- Effect.gen(function*() {
665
- const now = yield* nowDate
666
- // One guarded statement: waiting/delayed become terminal, active
667
- // gets the cancel-request flag; anything else is reported by state.
668
- const rows = rowsOf(yield* tx.execute<
669
- {
670
- id: string
671
- state: string
672
- processedAt: Date | null
673
- name: string
674
- keep: JobStore.KeepPolicy | null
675
- dedupeKey: string | null
676
- }
677
- >(sql`
678
- UPDATE ${jobs} SET
679
- state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN 'cancelled' ELSE ${jobs.state} END,
680
- finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
681
- cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
682
- WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active')
683
- RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
684
- ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
685
- ${jobs.dedupeKey} AS "dedupeKey"
832
+ Effect.gen(function*() {
833
+ const now = yield* nowDate
834
+ // Contract lock order (dependency rows first, parent second): mark
835
+ // a waiting-children parent's remaining pending rows cancelled.
836
+ // Pending rows exist only while the parent is `waiting-children`,
837
+ // so this is a no-op for every other state; a non-cancellable
838
+ // parent rolls the transaction back anyway.
839
+ const firstPass = rowsOf(yield* tx.execute<{ marked: number }>(sql`
840
+ WITH marked AS (${cancelPendingChildren(id)})
841
+ SELECT count(*)::int AS marked FROM marked
842
+ `))
843
+ const preMarked = firstPass[0]?.marked ?? 0
844
+ // One guarded statement: waiting/delayed/waiting-children become
845
+ // terminal, active gets the cancel-request flag; anything else is
846
+ // reported by state.
847
+ const rows = rowsOf(yield* tx.execute<
848
+ {
849
+ id: string
850
+ state: string
851
+ processedAt: Date | null
852
+ name: string
853
+ keep: JobStore.KeepPolicy | null
854
+ dedupeKey: string | null
855
+ hasFlow: boolean
856
+ parent: JobStore.ParentEnvelope | null
857
+ exit: unknown
858
+ failedReason: string | null
859
+ }
860
+ >(sql`
861
+ UPDATE ${jobs} SET
862
+ state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN 'cancelled' ELSE ${jobs.state} END,
863
+ finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
864
+ flow_pending = CASE WHEN ${jobs.state} = 'waiting-children' THEN 0 ELSE ${jobs.flowPending} END,
865
+ cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
866
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
867
+ RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
868
+ ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
869
+ ${jobs.dedupeKey} AS "dedupeKey", (${jobs.flowPending} IS NOT NULL) AS "hasFlow",
870
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
871
+ `))
872
+ const row = rows[0]
873
+ if (row === undefined) {
874
+ const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
875
+ SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
686
876
  `))
687
- const row = rows[0]
688
- if (row === undefined) {
689
- const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
690
- SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
691
- `))
692
- const found = existing[0]
693
- if (found === undefined) {
694
- return yield* new JobStore.JobNotFoundError({ jobId: id })
695
- }
696
- return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
877
+ const found = existing[0]
878
+ if (found === undefined) {
879
+ return yield* new JobStore.JobNotFoundError({ jobId: id })
697
880
  }
698
- if (row.state === "cancelled") {
699
- yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
700
- yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
701
- yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
881
+ return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
882
+ }
883
+ if (row.state === "cancelled") {
884
+ if (row.hasFlow) {
885
+ // Re-run the marking now that the parent lock is held: the
886
+ // first UPDATE above races a concurrent FanOut — its
887
+ // uncommitted dependency-row INSERTs are invisible, while
888
+ // this UPDATE's own EPQ re-check can still see the parent as
889
+ // 'waiting-children' after the FanOut commits. Without this
890
+ // pass those rows would stay 'pending' forever, invisible to
891
+ // both sweep classes. Every row either pass marked lands in
892
+ // the `cancelled` counter.
893
+ yield* tx.execute(sql`
894
+ WITH marked AS (${cancelPendingChildren(id)})
895
+ UPDATE ${jobs} SET flow_cancelled = ${jobs.flowCancelled} + ${preMarked}::int
896
+ + (SELECT count(*)::int FROM marked)
897
+ WHERE ${jobs.id} = ${id}
898
+ `)
702
899
  }
703
- })
704
- ).pipe(
705
- Effect.mapError((error) =>
706
- error instanceof JobStore.JobNotFoundError ||
707
- error instanceof JobStore.JobNotCancellableError ||
708
- error instanceof JobStore.JobStoreError
709
- ? error
710
- : storeError("cancel failed")(error)
711
- ),
712
- Effect.asVoid
713
- )
900
+ yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
901
+ // A cancelled child reports upward through the outbox.
902
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
903
+ yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
904
+ yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
905
+ }
906
+ })
907
+ ).pipe(
908
+ // Rare lock-order inversion (this row-then-parent flip vs a
909
+ // fail-fast settle's parent-then-rows marking) surfaces as a
910
+ // Postgres deadlock (40P01); one side is killed and safe to retry.
911
+ Effect.retry({
912
+ times: 3,
913
+ while: isDeadlockError
914
+ }),
915
+ Effect.mapError((error) =>
916
+ error instanceof JobStore.JobNotFoundError ||
917
+ error instanceof JobStore.JobNotCancellableError ||
918
+ error instanceof JobStore.JobStoreError
919
+ ? error
920
+ : storeError("cancel failed")(error)
921
+ ),
922
+ Effect.asVoid
923
+ )
714
924
 
715
925
  const enqueueOne = (request: JobStore.EnqueueRequest) =>
716
926
  Effect.gen(function*() {
@@ -800,11 +1010,12 @@ export const make = (
800
1010
  ${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
801
1011
  ${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
802
1012
  ${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
1013
+ ${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
803
1014
  ${new Date(now.getTime() + Math.max(0, request.delayMs))}, ${now}${extraColumnValues(request)})`
804
1015
  )
805
1016
  const rows = rowsOf(yield* db.execute<{ id: string }>(sql`
806
1017
  INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
807
- attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
1018
+ attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
808
1019
  VALUES ${sql.join(values, sql`, `)}
809
1020
  ON CONFLICT (id) DO NOTHING
810
1021
  RETURNING ${jobs.id} AS id
@@ -845,6 +1056,123 @@ export const make = (
845
1056
  return resolved
846
1057
  })
847
1058
 
1059
+ // The FanOut ack: land the manifest (flow columns + dependency rows) and
1060
+ // park the parent, all lock-token-guarded in one transaction. A fan-out
1061
+ // is a phase transition, not a completed run — `attempts_made` is not
1062
+ // incremented; the ledger records `fanned-out` with no exit.
1063
+ const ackFanOut = (
1064
+ id: JobStore.JobId,
1065
+ token: string,
1066
+ outcome: Extract<JobStore.AckOutcome, { _tag: "FanOut" }>
1067
+ ): Effect.Effect<void, JobStore.JobStoreError | JobStore.JobNotFoundError | JobStore.LockLostError> =>
1068
+ Effect.gen(function*() {
1069
+ // Validate BEFORE any mutation, so a bad spec cannot leave the job
1070
+ // half-acked (lock cleared, ledger written, still active).
1071
+ if (outcome.children.some((child) => child.request.id === undefined)) {
1072
+ return yield* new JobStore.JobStoreError({
1073
+ message: "FanOut child specs require an explicit request.id"
1074
+ })
1075
+ }
1076
+ const wakeQueue = yield* db.transaction((tx) =>
1077
+ Effect.gen(function*() {
1078
+ const now = yield* nowDate
1079
+ const rows = rowsOf(yield* tx.execute<
1080
+ {
1081
+ processedAt: Date | null
1082
+ name: string
1083
+ keep: JobStore.KeepPolicy | null
1084
+ dedupeKey: string | null
1085
+ queue: string
1086
+ cancelRequested: boolean
1087
+ flowPending: number | null
1088
+ parent: JobStore.ParentEnvelope | null
1089
+ exit: unknown
1090
+ failedReason: string | null
1091
+ }
1092
+ >(sql`
1093
+ UPDATE ${jobs} SET lock_token = NULL, lock_expires_at = NULL
1094
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
1095
+ RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
1096
+ ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1097
+ ${jobs.cancelRequested} AS "cancelRequested", ${jobs.flowPending} AS "flowPending",
1098
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
1099
+ `))
1100
+ const row = rows[0]
1101
+ if (row === undefined) {
1102
+ return yield* explainMiss(id)
1103
+ }
1104
+ yield* insertAttempt(tx, id, "fanned-out", row.processedAt, now, undefined)
1105
+ let pending: number
1106
+ if (row.flowPending === null || row.flowPending === undefined) {
1107
+ pending = outcome.children.length
1108
+ yield* tx.execute(sql`
1109
+ UPDATE ${jobs} SET flow_fail_fast = ${outcome.failFast}, flow_pending = ${pending},
1110
+ flow_completed = 0, flow_failed = 0, flow_cancelled = 0
1111
+ WHERE ${jobs.id} = ${id}
1112
+ `)
1113
+ // Chunked multi-row VALUES, like enqueueMany's insertBatch.
1114
+ for (let start = 0; start < outcome.children.length; start += 500) {
1115
+ const chunk = outcome.children.slice(start, start + 500)
1116
+ const values = chunk.map((child) =>
1117
+ sql`(${id}, ${child.childKey}, ${child.request.name}, ${child.storeKey},
1118
+ ${JSON.stringify(child.request)}::jsonb, 'pending', NULL, NULL, FALSE, ${now})`
1119
+ )
1120
+ yield* tx.execute(sql`
1121
+ INSERT INTO ${flowChildren} (flow_id, child_key, name, store_key, spec,
1122
+ status, exit, failed_reason, cascaded, pending_since)
1123
+ VALUES ${sql.join(values, sql`, `)}
1124
+ `)
1125
+ }
1126
+ } else {
1127
+ // A manifest that was already present is kept untouched (double
1128
+ // fan-out converges on the persisted children); the state
1129
+ // transition follows the persisted pending count either way.
1130
+ pending = Number(row.flowPending)
1131
+ }
1132
+ if (row.cancelRequested) {
1133
+ // A cancel raced the fan-out: cancellation wins. The rows exist
1134
+ // and get marked (into the `cancelled` counter), so the sweeper
1135
+ // cascades (mostly no-op cancels for never-enqueued children).
1136
+ yield* tx.execute(sql`
1137
+ WITH marked AS (${cancelPendingChildren(id)})
1138
+ UPDATE ${jobs} SET state = 'cancelled', finished_at = ${now},
1139
+ cancel_requested = FALSE, flow_pending = 0,
1140
+ flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
1141
+ WHERE ${jobs.id} = ${id}
1142
+ `)
1143
+ yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
1144
+ // A cancelled NESTED parent reports upward through the outbox.
1145
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
1146
+ yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
1147
+ yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
1148
+ return undefined
1149
+ }
1150
+ if (pending > 0) {
1151
+ yield* tx.execute(sql`
1152
+ UPDATE ${jobs} SET state = 'waiting-children' WHERE ${jobs.id} = ${id}
1153
+ `)
1154
+ return undefined
1155
+ }
1156
+ // Empty (or fully recorded) manifest: straight to runnable collect.
1157
+ yield* tx.execute(sql`
1158
+ UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
1159
+ WHERE ${jobs.id} = ${id}
1160
+ `)
1161
+ return JobStore.QueueName(row.queue)
1162
+ })
1163
+ ).pipe(
1164
+ Effect.mapError((error) =>
1165
+ error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
1166
+ error instanceof JobStore.JobStoreError
1167
+ ? error
1168
+ : storeError("ack failed")(error)
1169
+ )
1170
+ )
1171
+ if (wakeQueue !== undefined) {
1172
+ yield* wakeUp(wakeQueue)
1173
+ }
1174
+ })
1175
+
848
1176
  const store: JobStore.Service = {
849
1177
  enqueue: enqueueOne,
850
1178
 
@@ -919,7 +1247,10 @@ export const make = (
919
1247
  ${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
920
1248
  ${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
921
1249
  ${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
922
- ${jobs.trace} AS "trace",
1250
+ ${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
1251
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
1252
+ ${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
1253
+ ${jobs.flowCancelled} AS "flowCancelled",
923
1254
  ${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
924
1255
  ${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
925
1256
  ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
@@ -946,8 +1277,11 @@ export const make = (
946
1277
  error instanceof JobStore.JobStoreError ? error : storeError("claim failed")(error)
947
1278
  )),
948
1279
 
949
- ack: (id, token, outcome) =>
950
- db.transaction((tx) =>
1280
+ ack: (id, token, outcome) => {
1281
+ if (outcome._tag === "FanOut") {
1282
+ return ackFanOut(id, token, outcome)
1283
+ }
1284
+ return db.transaction((tx) =>
951
1285
  Effect.gen(function*() {
952
1286
  const now = yield* nowDate
953
1287
  const update = outcome._tag === "Complete"
@@ -973,6 +1307,9 @@ export const make = (
973
1307
  keep: JobStore.KeepPolicy | null
974
1308
  dedupeKey: string | null
975
1309
  queue: string
1310
+ parent: JobStore.ParentEnvelope | null
1311
+ exit: unknown
1312
+ failedReason: string | null
976
1313
  }
977
1314
  >(sql`
978
1315
  UPDATE ${jobs} SET ${update},
@@ -980,7 +1317,8 @@ export const make = (
980
1317
  WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
981
1318
  RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
982
1319
  ${jobs.state} AS "state", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
983
- ${jobs.queue} AS "queue"
1320
+ ${jobs.queue} AS "queue", ${jobs.parent} AS "parent", ${jobs.exit} AS "exit",
1321
+ ${jobs.failedReason} AS "failedReason"
984
1322
  `))
985
1323
  const row = rows[0]
986
1324
  if (row === undefined) {
@@ -1003,6 +1341,15 @@ export const make = (
1003
1341
  outcome._tag === "Cancelled" || cancelledRetry ? undefined : outcome.exit
1004
1342
  )
1005
1343
  if (outcome._tag !== "Retry" || cancelledRetry) {
1344
+ // A terminal transition of an envelope-carrying child reports
1345
+ // upward through the outbox (exit/failedReason as persisted).
1346
+ yield* appendOutbox(
1347
+ tx,
1348
+ row.parent,
1349
+ ledgerOutcome === "retried" ? "cancelled" : ledgerOutcome,
1350
+ row.exit ?? undefined,
1351
+ row.failedReason
1352
+ )
1006
1353
  yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
1007
1354
  yield* applyKeep(tx, row, now)
1008
1355
  }
@@ -1019,7 +1366,8 @@ export const make = (
1019
1366
  ),
1020
1367
  Effect.tap((queue) => queue !== undefined ? wakeUp(queue) : Effect.void),
1021
1368
  Effect.asVoid
1022
- ),
1369
+ )
1370
+ },
1023
1371
 
1024
1372
  release: (id, token) =>
1025
1373
  Effect.gen(function*() {
@@ -1037,6 +1385,9 @@ export const make = (
1037
1385
  keep: JobStore.KeepPolicy | null
1038
1386
  dedupeKey: string | null
1039
1387
  queue: string
1388
+ parent: JobStore.ParentEnvelope | null
1389
+ exit: unknown
1390
+ failedReason: string | null
1040
1391
  }
1041
1392
  >(sql`
1042
1393
  UPDATE ${jobs} SET
@@ -1047,12 +1398,15 @@ export const make = (
1047
1398
  WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
1048
1399
  RETURNING ${jobs.id} AS id, (${jobs.state} = 'cancelled') AS cancelled,
1049
1400
  ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
1050
- ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue"
1401
+ ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1402
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
1051
1403
  `))
1052
1404
  const row = rows[0]
1053
1405
  if (row === undefined) return undefined
1054
1406
  if (row.cancelled) {
1055
1407
  yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
1408
+ // A cancel honoured at release is a terminal transition too.
1409
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
1056
1410
  yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
1057
1411
  yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
1058
1412
  }
@@ -1119,6 +1473,9 @@ export const make = (
1119
1473
  name: string
1120
1474
  keep: JobStore.KeepPolicy | null
1121
1475
  dedupeKey: string | null
1476
+ parent: JobStore.ParentEnvelope | null
1477
+ exit: unknown
1478
+ failedReason: string | null
1122
1479
  }
1123
1480
  >(sql`
1124
1481
  UPDATE ${jobs} SET
@@ -1139,7 +1496,8 @@ export const make = (
1139
1496
  cancel_requested = FALSE
1140
1497
  WHERE ${jobs.state} = 'active' AND ${jobs.lockExpiresAt} <= ${now}::timestamptz
1141
1498
  RETURNING ${jobs.id} AS "id", ${jobs.state} AS "state", ${jobs.processedAt} AS "processedAt",
1142
- ${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey"
1499
+ ${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
1500
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
1143
1501
  `))
1144
1502
  const recovered: Array<{ id: JobStore.JobId; failed: boolean }> = []
1145
1503
  for (const row of rows) {
@@ -1152,6 +1510,15 @@ export const make = (
1152
1510
  undefined
1153
1511
  )
1154
1512
  if (row.state === "cancelled" || row.state === "failed") {
1513
+ // Honoured cancels and stall exhaustion are terminal
1514
+ // transitions: envelope-carrying children report upward.
1515
+ yield* appendOutbox(
1516
+ tx,
1517
+ row.parent,
1518
+ row.state === "cancelled" ? "cancelled" : "failed",
1519
+ row.exit ?? undefined,
1520
+ row.failedReason
1521
+ )
1155
1522
  yield* releaseDedupe(tx, row.name, row.dedupeKey, row.id, now)
1156
1523
  }
1157
1524
  if (row.state === "cancelled") {
@@ -1233,7 +1600,11 @@ export const make = (
1233
1600
  ${jobs.attemptsMax} AS "attemptsMax", ${jobs.attemptsMade} AS "attemptsMade",
1234
1601
  ${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
1235
1602
  ${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
1236
- ${jobs.cancelRequested} AS "cancelRequested",
1603
+ ${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
1604
+ ${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
1605
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
1606
+ ${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
1607
+ ${jobs.flowCancelled} AS "flowCancelled",
1237
1608
  ${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
1238
1609
  ${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
1239
1610
  ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
@@ -1464,6 +1835,365 @@ export const make = (
1464
1835
  return fired
1465
1836
  }),
1466
1837
 
1838
+ recordChildResults: (reports) =>
1839
+ Effect.gen(function*() {
1840
+ if (reports.length === 0) {
1841
+ const none: Array<{ applied: boolean; parentSettled: boolean }> = []
1842
+ return none
1843
+ }
1844
+ const batch = yield* db.transaction((tx) =>
1845
+ Effect.gen(function*() {
1846
+ const now = yield* nowDate
1847
+ const results = reports.map(() => ({ applied: false, parentSettled: false }))
1848
+
1849
+ // Phase 1a — apply every row update (contract lock order:
1850
+ // dependency rows FIRST, parents second). Only a (flow, key)'s
1851
+ // first occurrence in the batch can apply — later duplicates
1852
+ // would find the row non-pending anyway, and UPDATE ... FROM
1853
+ // must never see two source rows for one target.
1854
+ const candidates: Array<{ readonly index: number; readonly report: JobStore.FlowChildReport }> = []
1855
+ const seen = new Set<string>()
1856
+ for (const [index, report] of reports.entries()) {
1857
+ const key = `${report.flowId}\u0000${report.childKey}`
1858
+ if (seen.has(key)) continue
1859
+ seen.add(key)
1860
+ candidates.push({ index, report })
1861
+ }
1862
+ for (let start = 0; start < candidates.length; start += 200) {
1863
+ const chunk = candidates.slice(start, start + 200)
1864
+ const values = chunk.map(({ index, report }) =>
1865
+ sql`(${index}::int, ${report.flowId}::text, ${report.childKey}::text,
1866
+ ${report.outcome}::text,
1867
+ ${report.exit === undefined ? null : JSON.stringify(report.exit)}::jsonb,
1868
+ ${report.failedReason ?? null}::text)`
1869
+ )
1870
+ const appliedRows = rowsOf(yield* tx.execute<{ ord: number }>(sql`
1871
+ UPDATE ${flowChildren} SET status = v.outcome, exit = v.exit,
1872
+ failed_reason = v.failed_reason, cascaded = TRUE
1873
+ FROM (VALUES ${sql.join(values, sql`, `)})
1874
+ AS v(ord, flow_id, child_key, outcome, exit, failed_reason)
1875
+ WHERE ${flowChildren.flowId} = v.flow_id AND ${flowChildren.childKey} = v.child_key
1876
+ AND ${flowChildren.status} = 'pending'
1877
+ RETURNING v.ord AS "ord"
1878
+ `))
1879
+ for (const row of appliedRows) {
1880
+ const result = results[Number(row.ord)]
1881
+ if (result !== undefined) {
1882
+ result.applied = true
1883
+ }
1884
+ }
1885
+ }
1886
+
1887
+ // Tally the applied reports per touched flow, in batch order.
1888
+ interface Touch {
1889
+ appliedCount: number
1890
+ completed: number
1891
+ failed: number
1892
+ cancelled: number
1893
+ firstAppliedFailed: number | undefined
1894
+ lastApplied: number
1895
+ }
1896
+ const touched = new Map<string, Touch>()
1897
+ for (const [index, report] of reports.entries()) {
1898
+ if (results[index]?.applied !== true) continue
1899
+ const touch = touched.get(report.flowId) ?? {
1900
+ appliedCount: 0,
1901
+ completed: 0,
1902
+ failed: 0,
1903
+ cancelled: 0,
1904
+ firstAppliedFailed: undefined,
1905
+ lastApplied: index
1906
+ }
1907
+ touch.appliedCount += 1
1908
+ touch.lastApplied = index
1909
+ if (report.outcome === "completed") touch.completed += 1
1910
+ if (report.outcome === "cancelled") touch.cancelled += 1
1911
+ if (report.outcome === "failed") {
1912
+ touch.failed += 1
1913
+ touch.firstAppliedFailed ??= index
1914
+ }
1915
+ touched.set(report.flowId, touch)
1916
+ }
1917
+
1918
+ // Phase 1b + 2 — per touched flow (sorted, so concurrent
1919
+ // batches take parent locks in one order): move the applied
1920
+ // children from `pending` to their outcome counters, then make
1921
+ // at most one settle decision. Fail-fast wins the tie.
1922
+ const wakeQueues: Array<JobStore.QueueName> = []
1923
+ for (const flowId of [...touched.keys()].toSorted()) {
1924
+ const touch = touched.get(flowId)
1925
+ if (touch === undefined) continue
1926
+ const parents = rowsOf(yield* tx.execute<
1927
+ {
1928
+ state: JobStore.JobState
1929
+ flowPending: number
1930
+ flowFailFast: boolean | null
1931
+ processedAt: Date | null
1932
+ name: string
1933
+ keep: JobStore.KeepPolicy | null
1934
+ dedupeKey: string | null
1935
+ queue: string
1936
+ parent: JobStore.ParentEnvelope | null
1937
+ }
1938
+ >(sql`
1939
+ UPDATE ${jobs} SET
1940
+ flow_pending = GREATEST(${jobs.flowPending} - ${touch.appliedCount}::int, 0),
1941
+ flow_completed = ${jobs.flowCompleted} + ${touch.completed}::int,
1942
+ flow_failed = ${jobs.flowFailed} + ${touch.failed}::int,
1943
+ flow_cancelled = ${jobs.flowCancelled} + ${touch.cancelled}::int
1944
+ WHERE ${jobs.id} = ${flowId} AND ${jobs.flowPending} IS NOT NULL
1945
+ RETURNING ${jobs.state} AS "state", ${jobs.flowPending} AS "flowPending",
1946
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.processedAt} AS "processedAt",
1947
+ ${jobs.name} AS "name", ${jobs.keep} AS "keep",
1948
+ ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1949
+ ${jobs.parent} AS "parent"
1950
+ `))
1951
+ const parent = parents[0]
1952
+ if (parent === undefined || parent.state !== "waiting-children") continue
1953
+ const failedIndex = parent.flowFailFast === true ? touch.firstAppliedFailed : undefined
1954
+ const failedReport = failedIndex !== undefined ? reports[failedIndex] : undefined
1955
+ if (failedIndex !== undefined && failedReport !== undefined) {
1956
+ // The first applied failure settles the parent terminally
1957
+ // (store-side, like stall exhaustion: failedReason, no
1958
+ // exit) and marks the remaining rows in the same
1959
+ // transaction. A nested parent's own report goes to the
1960
+ // outbox here — this settle IS its terminal transition.
1961
+ const reason = `effect-mq: flow child "${failedReport.childKey}" failed`
1962
+ yield* tx.execute(sql`
1963
+ WITH marked AS (${cancelPendingChildren(flowId)})
1964
+ UPDATE ${jobs} SET state = 'failed', finished_at = ${now},
1965
+ failed_reason = ${reason}, cancel_requested = FALSE, flow_pending = 0,
1966
+ flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
1967
+ WHERE ${jobs.id} = ${flowId}
1968
+ `)
1969
+ yield* insertAttempt(tx, flowId, "failed", parent.processedAt, now, undefined)
1970
+ yield* appendOutbox(tx, parent.parent, "failed", undefined, reason)
1971
+ yield* releaseDedupe(tx, parent.name, parent.dedupeKey, flowId, now)
1972
+ yield* applyKeep(tx, { name: parent.name, state: "failed", keep: parent.keep }, now)
1973
+ const decided = results[failedIndex]
1974
+ if (decided !== undefined) {
1975
+ decided.parentSettled = true
1976
+ }
1977
+ continue
1978
+ }
1979
+ if (Number(parent.flowPending) === 0) {
1980
+ // All children settled: the parent resumes runnable, phase
1981
+ // collect, settled at the flow's last applied report.
1982
+ yield* tx.execute(sql`
1983
+ UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
1984
+ WHERE ${jobs.id} = ${flowId}
1985
+ `)
1986
+ wakeQueues.push(JobStore.QueueName(parent.queue))
1987
+ const decided = results[touch.lastApplied]
1988
+ if (decided !== undefined) {
1989
+ decided.parentSettled = true
1990
+ }
1991
+ }
1992
+ }
1993
+ return { results, wakeQueues }
1994
+ })
1995
+ ).pipe(
1996
+ // Rare lock-order inversion (a concurrent cancel/settle marking
1997
+ // rows) surfaces as a Postgres deadlock (40P01); the killed side
1998
+ // is safe to retry — the row-state guard keeps it idempotent.
1999
+ Effect.retry({
2000
+ times: 3,
2001
+ while: isDeadlockError
2002
+ }),
2003
+ Effect.mapError((error) =>
2004
+ error instanceof JobStore.JobStoreError ? error : storeError("recordChildResults failed")(error)
2005
+ )
2006
+ )
2007
+ for (const queue of batch.wakeQueues) {
2008
+ yield* wakeUp(queue)
2009
+ }
2010
+ return batch.results
2011
+ }),
2012
+
2013
+ peekOutbox: (peekOptions) =>
2014
+ Effect.gen(function*() {
2015
+ const limit = Math.max(0, peekOptions.limit)
2016
+ if (limit === 0) {
2017
+ const none: Array<JobStore.OutboxEntry> = []
2018
+ return none
2019
+ }
2020
+ // `after` pages past a previously returned id (exclusive), whether
2021
+ // or not that entry still exists. Anything that is not a canonical
2022
+ // id this store issued is treated as unset.
2023
+ const after = peekOptions.after !== undefined && CANONICAL_BIGSERIAL.test(peekOptions.after)
2024
+ ? peekOptions.after
2025
+ : undefined
2026
+ const rows = rowsOf(yield* db.execute<
2027
+ {
2028
+ id: string
2029
+ flowName: string
2030
+ parentStoreKey: string
2031
+ report: JobStore.FlowChildReport
2032
+ }
2033
+ >(sql`
2034
+ SELECT ${flowOutbox.id}::text AS "id", ${flowOutbox.flowName} AS "flowName",
2035
+ ${flowOutbox.parentStoreKey} AS "parentStoreKey", ${flowOutbox.report} AS "report"
2036
+ FROM ${flowOutbox}
2037
+ ${after === undefined ? sql`` : sql`WHERE ${flowOutbox.id} > ${after}::bigint`}
2038
+ ORDER BY ${flowOutbox.id} ASC
2039
+ LIMIT ${limit}
2040
+ `).pipe(Effect.mapError(storeError("peekOutbox failed"))))
2041
+ return rows.map((row): JobStore.OutboxEntry => ({
2042
+ id: row.id,
2043
+ flowName: row.flowName,
2044
+ parentStoreKey: row.parentStoreKey,
2045
+ report: row.report
2046
+ }))
2047
+ }),
2048
+
2049
+ deleteOutbox: (ids) =>
2050
+ Effect.suspend(() => {
2051
+ // Ids are opaque strings to callers; only CANONICAL ids this store
2052
+ // could have issued can match. The strictness matters twice: a
2053
+ // foreign/garbled id must not blow up the ::bigint cast, and a
2054
+ // non-canonical spelling ("007") must stay an unknown-id no-op
2055
+ // rather than cast to 7 and delete a live entry.
2056
+ const numeric = ids.filter((id) => CANONICAL_BIGSERIAL.test(id))
2057
+ if (numeric.length === 0) return Effect.void
2058
+ return db.execute(sql`
2059
+ DELETE FROM ${flowOutbox}
2060
+ WHERE ${flowOutbox.id} = ANY(${sql.param(numeric)}::bigint[])
2061
+ `).pipe(
2062
+ Effect.mapError(storeError("deleteOutbox failed")),
2063
+ Effect.asVoid
2064
+ )
2065
+ }),
2066
+
2067
+ listChildResults: (flowId, listOptions) =>
2068
+ Effect.gen(function*() {
2069
+ const limit = Math.max(1, listOptions?.limit ?? 1000)
2070
+ const cursor = listOptions?.cursor
2071
+ // `spec->>'id'` instead of the whole spec: at 10k children the full
2072
+ // payloads would transfer on every collect.
2073
+ const rows = rowsOf(yield* db.execute<FlowChildRow>(sql`
2074
+ SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
2075
+ ${flowChildren.name} AS "name", ${flowChildren.storeKey} AS "storeKey",
2076
+ ${flowChildren.spec}->>'id' AS "childJobId", ${flowChildren.status} AS "status",
2077
+ ${flowChildren.exit} AS "exit", ${flowChildren.failedReason} AS "failedReason",
2078
+ ${flowChildren.cascaded} AS "cascaded"
2079
+ FROM ${flowChildren}
2080
+ WHERE ${flowChildren.flowId} = ${flowId}
2081
+ ${cursor === undefined ? sql`` : sql`AND ${flowChildren.childKey} > ${cursor}`}
2082
+ ORDER BY ${flowChildren.childKey} ASC
2083
+ LIMIT ${limit + 1}
2084
+ `).pipe(Effect.mapError(storeError("listChildResults failed"))))
2085
+ const page = rows.slice(0, limit)
2086
+ const items = page.map(toFlowChildRecord)
2087
+ const last = items[items.length - 1]
2088
+ return {
2089
+ items,
2090
+ cursor: rows.length > limit && last !== undefined ? last.childKey : undefined
2091
+ }
2092
+ }),
2093
+
2094
+ flowSweepWork: (sweepOptions) =>
2095
+ Effect.gen(function*() {
2096
+ const now = yield* nowDate
2097
+ const limit = Math.max(1, sweepOptions.limit ?? 1000)
2098
+ const threshold = new Date(now.getTime() - sweepOptions.pendingAgeMs)
2099
+ type SweepRow = {
2100
+ readonly flowId: string
2101
+ readonly childKey: string
2102
+ readonly storeKey: string
2103
+ readonly spec: JobStore.EnqueueRequest
2104
+ }
2105
+ // Reconcile: pending rows past the eligibility threshold whose
2106
+ // parent is still parked (a settled flow never re-drives work).
2107
+ // Returning a row re-arms `pending_since` in the same statement, so
2108
+ // a full page rotates across sweeps instead of pinning its head.
2109
+ // SKIP LOCKED: rows a concurrent report/settle holds are its
2110
+ // business, and never waiting means this statement cannot deadlock.
2111
+ // The raw column names are safe: only table names vary across
2112
+ // factory instances. RETURNING order is unspecified — sorted below.
2113
+ const reconcileRows = rowsOf(yield* db.execute<SweepRow>(sql`
2114
+ WITH due AS (
2115
+ SELECT c.flow_id, c.child_key
2116
+ FROM ${flowChildren} c
2117
+ JOIN ${jobs} j ON j.id = c.flow_id
2118
+ WHERE c.status = 'pending' AND c.pending_since <= ${threshold}
2119
+ AND j.state = 'waiting-children'
2120
+ ORDER BY c.flow_id, c.child_key
2121
+ LIMIT ${limit}
2122
+ FOR UPDATE OF c SKIP LOCKED
2123
+ )
2124
+ UPDATE ${flowChildren} SET pending_since = ${now}
2125
+ FROM due
2126
+ WHERE ${flowChildren.flowId} = due.flow_id AND ${flowChildren.childKey} = due.child_key
2127
+ RETURNING ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
2128
+ ${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
2129
+ `).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
2130
+ .toSorted((a, b) =>
2131
+ a.flowId !== b.flowId
2132
+ ? (a.flowId < b.flowId ? -1 : 1)
2133
+ : a.childKey < b.childKey
2134
+ ? -1
2135
+ : a.childKey > b.childKey
2136
+ ? 1
2137
+ : 0
2138
+ )
2139
+ // Cascade: cancelled rows whose cancel has not been delivered into
2140
+ // the child's store yet (any parent state).
2141
+ const cascadeRows = rowsOf(yield* db.execute<SweepRow>(sql`
2142
+ SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
2143
+ ${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
2144
+ FROM ${flowChildren}
2145
+ WHERE ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
2146
+ ORDER BY ${flowChildren.flowId}, ${flowChildren.childKey}
2147
+ LIMIT ${limit}
2148
+ `).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
2149
+ const reconcile: Array<{
2150
+ flowId: JobStore.JobId
2151
+ children: Array<JobStore.FlowChildSpec>
2152
+ }> = []
2153
+ for (const row of reconcileRows) {
2154
+ const flowId = JobId(row.flowId)
2155
+ let group = reconcile[reconcile.length - 1]
2156
+ if (group === undefined || group.flowId !== flowId) {
2157
+ group = { flowId, children: [] }
2158
+ reconcile.push(group)
2159
+ }
2160
+ group.children.push({ childKey: row.childKey, storeKey: row.storeKey, request: row.spec })
2161
+ }
2162
+ const cascade: Array<{
2163
+ flowId: JobStore.JobId
2164
+ children: Array<{ childKey: string; storeKey: string; childJobId: JobStore.JobId }>
2165
+ }> = []
2166
+ for (const row of cascadeRows) {
2167
+ const flowId = JobId(row.flowId)
2168
+ let group = cascade[cascade.length - 1]
2169
+ if (group === undefined || group.flowId !== flowId) {
2170
+ group = { flowId, children: [] }
2171
+ cascade.push(group)
2172
+ }
2173
+ group.children.push({
2174
+ childKey: row.childKey,
2175
+ storeKey: row.storeKey,
2176
+ // SAFETY: the FanOut ack validated every spec id before
2177
+ // persisting it.
2178
+ childJobId: row.spec.id as JobStore.JobId
2179
+ })
2180
+ }
2181
+ const work: JobStore.FlowSweepWork = { reconcile, cascade }
2182
+ return work
2183
+ }),
2184
+
2185
+ markChildrenCascaded: (flowId, childKeys) =>
2186
+ childKeys.length === 0
2187
+ ? Effect.void
2188
+ : db.execute(sql`
2189
+ UPDATE ${flowChildren} SET cascaded = TRUE
2190
+ WHERE ${flowChildren.flowId} = ${flowId}
2191
+ AND ${flowChildren.childKey} = ANY(${sql.param([...childKeys])})
2192
+ `).pipe(
2193
+ Effect.mapError(storeError("markChildrenCascaded failed")),
2194
+ Effect.asVoid
2195
+ ),
2196
+
1467
2197
  counts: (queue) =>
1468
2198
  db.execute<{ state: JobStore.JobState; count: number }>(sql`
1469
2199
  SELECT ${jobs.state} AS "state", count(*)::int AS "count" FROM ${jobs}
@@ -1477,6 +2207,7 @@ export const make = (
1477
2207
  waiting: 0,
1478
2208
  delayed: 0,
1479
2209
  active: 0,
2210
+ "waiting-children": 0,
1480
2211
  completed: 0,
1481
2212
  failed: 0,
1482
2213
  cancelled: 0
@@ -1487,10 +2218,18 @@ export const make = (
1487
2218
  ),
1488
2219
 
1489
2220
  remove: (id) =>
2221
+ // The purge CTE takes a removed flow parent's dependency rows with it
2222
+ // in the same statement.
1490
2223
  db.execute(sql`
1491
- DELETE FROM ${jobs}
1492
- WHERE ${jobs.id} = ${id} AND ${jobs.state} <> 'active'
1493
- RETURNING ${jobs.id} AS id
2224
+ WITH deleted AS (
2225
+ DELETE FROM ${jobs}
2226
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} NOT IN ('active', 'waiting-children')
2227
+ RETURNING ${jobs.id} AS id
2228
+ ), purged AS (
2229
+ DELETE FROM ${flowChildren}
2230
+ WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
2231
+ )
2232
+ SELECT id FROM deleted
1494
2233
  `).pipe(
1495
2234
  Effect.mapError(storeError("remove failed")),
1496
2235
  Effect.map((result) => rowsOf(result).length > 0)