effect-mq 0.4.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +85 -17
  2. package/dist/Flow.d.ts +381 -0
  3. package/dist/Flow.d.ts.map +1 -0
  4. package/dist/Flow.js +340 -0
  5. package/dist/Flow.js.map +1 -0
  6. package/dist/Job.d.ts +37 -6
  7. package/dist/Job.d.ts.map +1 -1
  8. package/dist/Job.js +17 -2
  9. package/dist/Job.js.map +1 -1
  10. package/dist/JobSchedules.d.ts +112 -0
  11. package/dist/JobSchedules.d.ts.map +1 -0
  12. package/dist/JobSchedules.js +106 -0
  13. package/dist/JobSchedules.js.map +1 -0
  14. package/dist/JobStore.d.ts +320 -10
  15. package/dist/JobStore.d.ts.map +1 -1
  16. package/dist/JobStore.js.map +1 -1
  17. package/dist/MemoryJobStore.d.ts.map +1 -1
  18. package/dist/MemoryJobStore.js +336 -8
  19. package/dist/MemoryJobStore.js.map +1 -1
  20. package/dist/Metrics.d.ts +31 -0
  21. package/dist/Metrics.d.ts.map +1 -1
  22. package/dist/Metrics.js +39 -0
  23. package/dist/Metrics.js.map +1 -1
  24. package/dist/Worker.d.ts +120 -11
  25. package/dist/Worker.d.ts.map +1 -1
  26. package/dist/Worker.js +452 -26
  27. package/dist/Worker.js.map +1 -1
  28. package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
  29. package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
  30. package/dist/drizzle-postgres/DrizzleJobStore.js +662 -81
  31. package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
  32. package/dist/drizzle-postgres/schema.d.ts +310 -3
  33. package/dist/drizzle-postgres/schema.d.ts.map +1 -1
  34. package/dist/drizzle-postgres/schema.js +68 -1
  35. package/dist/drizzle-postgres/schema.js.map +1 -1
  36. package/dist/index.d.ts +14 -0
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +14 -0
  39. package/dist/index.js.map +1 -1
  40. package/dist/redis/RedisJobStore.d.ts.map +1 -1
  41. package/dist/redis/RedisJobStore.js +221 -19
  42. package/dist/redis/RedisJobStore.js.map +1 -1
  43. package/dist/redis/scripts.d.ts +118 -11
  44. package/dist/redis/scripts.d.ts.map +1 -1
  45. package/dist/redis/scripts.js +497 -28
  46. package/dist/redis/scripts.js.map +1 -1
  47. package/dist/testing/conformance.d.ts +6 -0
  48. package/dist/testing/conformance.d.ts.map +1 -1
  49. package/dist/testing/conformance.js +765 -1
  50. package/dist/testing/conformance.js.map +1 -1
  51. package/package.json +1 -1
  52. package/src/Flow.ts +778 -0
  53. package/src/Job.ts +42 -11
  54. package/src/JobSchedules.ts +223 -0
  55. package/src/JobStore.ts +347 -9
  56. package/src/MemoryJobStore.ts +372 -8
  57. package/src/Metrics.ts +43 -0
  58. package/src/Worker.ts +726 -37
  59. package/src/drizzle-postgres/DrizzleJobStore.ts +827 -82
  60. package/src/drizzle-postgres/schema.ts +94 -0
  61. package/src/index.ts +16 -0
  62. package/src/redis/RedisJobStore.ts +291 -8
  63. package/src/redis/scripts.ts +529 -26
  64. package/src/testing/conformance.ts +989 -1
@@ -18,11 +18,19 @@
18
18
  */
19
19
  import * as JobStore from "../JobStore.ts"
20
20
  import type { PgClient } from "@effect/sql-pg"
21
- import { asc, eq, getTableColumns, sql } from "drizzle-orm"
21
+ import { asc, eq, getTableColumns, type SQL, sql } from "drizzle-orm"
22
22
  import * as PgDrizzle from "drizzle-orm/effect-postgres"
23
23
  import { getTableConfig } from "drizzle-orm/pg-core"
24
- import { Clock, type Context, Deferred, Duration, Effect, Layer, Option, type Scope, Stream } from "effect"
25
- import type { MqDedupeTable, MqJobAttemptsTable, MqJobsTable, MqQueueControlTable, MqSchedulesTable } from "./schema.ts"
24
+ import { Cause, Clock, type Context, Deferred, Duration, Effect, Layer, Option, Predicate, type Scope, Stream } from "effect"
25
+ import type {
26
+ MqDedupeTable,
27
+ MqFlowChildrenTable,
28
+ MqFlowOutboxTable,
29
+ MqJobAttemptsTable,
30
+ MqJobsTable,
31
+ MqQueueControlTable,
32
+ MqSchedulesTable
33
+ } from "./schema.ts"
26
34
 
27
35
  const { JobId } = JobStore
28
36
 
@@ -40,6 +48,10 @@ export interface DrizzleJobStoreOptions<StoreId = JobStore.JobStore> {
40
48
  readonly queues: MqQueueControlTable
41
49
  /** The dedup-key registry table (from `mqDedupe`). */
42
50
  readonly dedupe: MqDedupeTable
51
+ /** The flow dependency-rows table (from `mqFlowChildren`). */
52
+ readonly flowChildren: MqFlowChildrenTable
53
+ /** The child-report outbox table (from `mqFlowOutbox`). */
54
+ readonly flowOutbox: MqFlowOutboxTable
43
55
  /**
44
56
  * Values for columns added via `mqJobs({ extend })`, evaluated at enqueue
45
57
  * (and on dedupe `replace`). Keys are the extended columns' TS names.
@@ -84,6 +96,50 @@ export type ExtraColumnValue = string | number | boolean | Date | null
84
96
  const storeError = (message: string) => (cause: unknown) =>
85
97
  new JobStore.JobStoreError({ message, cause })
86
98
 
99
+ // The exact spellings a bigserial ever produces: no leading zeros, no signs,
100
+ // no whitespace — what outbox ids look like and nothing Postgres would
101
+ // silently normalize into one.
102
+ const CANONICAL_BIGSERIAL = /^[1-9]\d*$/
103
+
104
+ /**
105
+ * Whether an error surfaced by drizzle's Effect driver is a Postgres
106
+ * deadlock (40P01), i.e. safe to retry.
107
+ *
108
+ * The deadlock travels wrapped: drizzle's session fails with an
109
+ * `EffectDrizzleQueryError` whose `cause` field is `Cause.fail(SqlError)`
110
+ * (drizzle-orm `pg-core/effect/session.ts`), and `@effect/sql-pg` classifies
111
+ * pg code 40P01 into the `SqlError`'s `reason: DeadlockError` (PgClient.ts
112
+ * `classifyError`). Neither layer's `toString` renders the pg code, so this
113
+ * unwraps structurally instead of string-matching the rendered error.
114
+ *
115
+ * @internal
116
+ */
117
+ // oxlint-disable-next-line anti-slop/no-unknown-parameters -- a retry-predicate classifier over whatever the driver threw IS the boundary parser
118
+ export const isDeadlockError = (error: unknown): boolean => {
119
+ // Unwrap drizzle's envelope: its `cause` is an Effect Cause holding the
120
+ // original failure. A bare SqlError (non-drizzle path) passes through.
121
+ const unwrapped = Predicate.hasProperty(error, "cause") && Cause.isCause(error.cause)
122
+ ? Option.getOrUndefined(Cause.findErrorOption(error.cause))
123
+ : error
124
+ if (
125
+ !Predicate.hasProperty(unwrapped, "_tag") || unwrapped._tag !== "SqlError" ||
126
+ !Predicate.hasProperty(unwrapped, "reason")
127
+ ) {
128
+ return false
129
+ }
130
+ const reason = unwrapped.reason
131
+ if (Predicate.hasProperty(reason, "_tag") && reason._tag === "DeadlockError") {
132
+ return true
133
+ }
134
+ // Fallback for classifiers that missed the code: the reason keeps the raw
135
+ // pg error, whose message for 40P01 is "deadlock detected".
136
+ if (Predicate.hasProperty(reason, "cause")) {
137
+ const raw = String(reason.cause)
138
+ return raw.includes("40P01") || raw.includes("deadlock detected")
139
+ }
140
+ return false
141
+ }
142
+
87
143
  /**
88
144
  * drizzle's Effect driver types raw `execute` results as row arrays, but at
89
145
  * runtime the raw path yields the node-postgres `QueryResult` envelope
@@ -114,6 +170,12 @@ type JobRow = {
114
170
  readonly cancelRequested: boolean
115
171
  readonly dedupeKey: string | null
116
172
  readonly trace: JobStore.TraceContext | null
173
+ readonly parent: JobStore.ParentEnvelope | null
174
+ readonly flowFailFast: boolean | null
175
+ readonly flowPending: number | null
176
+ readonly flowCompleted: number | null
177
+ readonly flowFailed: number | null
178
+ readonly flowCancelled: number | null
117
179
  readonly runAt: Date
118
180
  readonly enqueuedAt: Date
119
181
  readonly processedAt: Date | null
@@ -136,6 +198,7 @@ type ScheduleRow = {
136
198
  readonly backoff: JobStore.BackoffPolicy | null
137
199
  readonly keep: JobStore.KeepPolicy | null
138
200
  readonly timeoutMs: number | string | null
201
+ readonly group: string | null
139
202
  readonly nextRunAt: Date
140
203
  }
141
204
 
@@ -153,6 +216,7 @@ const toSchedule = (row: ScheduleRow): JobStore.ScheduleRecord => ({
153
216
  backoff: row.backoff ?? undefined,
154
217
  keep: row.keep ?? undefined,
155
218
  timeoutMs: row.timeoutMs === null ? undefined : Number(row.timeoutMs),
219
+ group: row.group ?? undefined,
156
220
  nextRunAt: row.nextRunAt.getTime()
157
221
  })
158
222
 
@@ -173,6 +237,17 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
173
237
  cancelRequested: row.cancelRequested,
174
238
  dedupeKey: row.dedupeKey ?? undefined,
175
239
  trace: row.trace ?? undefined,
240
+ parent: row.parent ?? undefined,
241
+ // The flow columns are NULL together; `flowPending` is the presence marker.
242
+ flow: row.flowPending === null || row.flowPending === undefined
243
+ ? undefined
244
+ : {
245
+ failFast: row.flowFailFast === true,
246
+ pending: Number(row.flowPending),
247
+ completed: Number(row.flowCompleted ?? 0),
248
+ failed: Number(row.flowFailed ?? 0),
249
+ cancelled: Number(row.flowCancelled ?? 0)
250
+ },
176
251
  runAt: row.runAt.getTime(),
177
252
  enqueuedAt: row.enqueuedAt.getTime(),
178
253
  processedAt: row.processedAt?.getTime(),
@@ -181,6 +256,31 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
181
256
  failedReason: row.failedReason ?? undefined
182
257
  })
183
258
 
259
+ type FlowChildRow = {
260
+ readonly flowId: string
261
+ readonly childKey: string
262
+ readonly name: string
263
+ readonly storeKey: string
264
+ /** Projected from `spec->>'id'` (never NULL: the FanOut ack validated it). */
265
+ readonly childJobId: string
266
+ readonly status: JobStore.FlowChildRecord["status"]
267
+ readonly exit: unknown
268
+ readonly failedReason: string | null
269
+ readonly cascaded: boolean
270
+ }
271
+
272
+ const toFlowChildRecord = (row: FlowChildRow): JobStore.FlowChildRecord => ({
273
+ flowId: JobId(row.flowId),
274
+ childKey: row.childKey,
275
+ name: row.name,
276
+ storeKey: row.storeKey,
277
+ childJobId: JobId(row.childJobId),
278
+ status: row.status,
279
+ exit: row.exit ?? undefined,
280
+ failedReason: row.failedReason ?? undefined,
281
+ cascaded: row.cascaded
282
+ })
283
+
184
284
  /**
185
285
  * Build the store implementation. Requires `PgClient` and a `Scope` (for the
186
286
  * LISTEN subscription).
@@ -198,6 +298,8 @@ export const make = (
198
298
  const schedules = options.schedules
199
299
  const queues = options.queues
200
300
  const dedupe = options.dedupe
301
+ const flowChildren = options.flowChildren
302
+ const flowOutbox = options.flowOutbox
201
303
  const jobsName = getTableConfig(jobs).name
202
304
  const attemptsName = getTableConfig(attempts).name
203
305
  const wakeChannel = `effect_mq_wake_${jobsName}`
@@ -223,6 +325,12 @@ export const make = (
223
325
  "cancelRequested",
224
326
  "dedupeKey",
225
327
  "trace",
328
+ "parent",
329
+ "flowFailFast",
330
+ "flowPending",
331
+ "flowCompleted",
332
+ "flowFailed",
333
+ "flowCancelled",
226
334
  "runAt",
227
335
  "enqueuedAt",
228
336
  "processedAt",
@@ -261,7 +369,9 @@ export const make = (
261
369
  db.select({ jobId: attempts.jobId }).from(attempts).limit(0),
262
370
  db.select({ key: schedules.key }).from(schedules).limit(0),
263
371
  db.select({ queue: queues.queue }).from(queues).limit(0),
264
- db.select({ key: dedupe.key }).from(dedupe).limit(0)
372
+ db.select({ key: dedupe.key }).from(dedupe).limit(0),
373
+ db.select({ flowId: flowChildren.flowId }).from(flowChildren).limit(0),
374
+ db.select({ id: flowOutbox.id }).from(flowOutbox).limit(0)
265
375
  ]).pipe(
266
376
  Effect.mapError(storeError(
267
377
  `effect-mq: tables "${jobsName}"/"${attemptsName}" are missing or mismatched — ` +
@@ -322,6 +432,28 @@ export const make = (
322
432
  Effect.forever,
323
433
  Effect.forkScoped
324
434
  )
435
+ // Automatic retention (the store-level sweep and per-job `keep`) never
436
+ // prunes a flow parent that still owes cascade cancels: its dependency
437
+ // rows marked `cancelled` and not `cascaded` are the only record that
438
+ // real cancels are still due in the child stores. The explicit `remove`
439
+ // verb is not exempted.
440
+ const owesCascades = sql`EXISTS (
441
+ SELECT 1 FROM ${flowChildren}
442
+ WHERE ${flowChildren.flowId} = ${jobs.id}
443
+ AND ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
444
+ )`
445
+ // A pruned flow parent's dependency rows go with it in the same statement.
446
+ const purgeJobsWhere = (predicate: SQL) =>
447
+ sql`
448
+ WITH deleted AS (
449
+ DELETE FROM ${jobs}
450
+ WHERE ${predicate}
451
+ RETURNING ${jobs.id} AS id
452
+ )
453
+ DELETE FROM ${flowChildren}
454
+ WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
455
+ `
456
+
325
457
  if (options.historyTtl !== undefined) {
326
458
  const ttlByState = JobStore.normalizeHistoryTtl(options.historyTtl)
327
459
  const sweepMs = Duration.toMillis(options.historySweepInterval ?? "1 minute")
@@ -332,9 +464,8 @@ export const make = (
332
464
  // job name is pruned on the timer, not only when its group is acked.
333
465
  for (const state of ["completed", "failed", "cancelled"] as const) {
334
466
  const ttl = ttlByState[state]
335
- yield* db.execute(sql`
336
- DELETE FROM ${jobs}
337
- WHERE ${jobs.state} = ${state} AND (
467
+ yield* db.execute(purgeJobsWhere(sql`
468
+ ${jobs.state} = ${state} AND NOT ${owesCascades} AND (
338
469
  ${ttl !== undefined ? sql`${jobs.finishedAt} <= ${new Date(now.getTime() - ttl)}` : sql`FALSE`}
339
470
  OR (
340
471
  COALESCE(
@@ -349,7 +480,7 @@ export const make = (
349
480
  ELSE ${jobs.keep}->>'ageMs' END
350
481
  )::double precision) / 1000.0))
351
482
  )
352
- `)
483
+ `))
353
484
  }
354
485
  // Dead dedup rows: expired windows, or pointers at vanished jobs.
355
486
  yield* db.execute(sql`
@@ -357,7 +488,7 @@ export const make = (
357
488
  WHERE (${dedupe.windowExpiresAt} IS NOT NULL AND ${dedupe.windowExpiresAt} <= ${now})
358
489
  OR (${dedupe.windowExpiresAt} IS NULL AND NOT EXISTS (
359
490
  SELECT 1 FROM ${jobs} WHERE ${jobs.id} = ${dedupe.jobId}
360
- AND ${jobs.state} IN ('waiting', 'delayed', 'active')
491
+ AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
361
492
  ))
362
493
  `)
363
494
  }).pipe(
@@ -431,23 +562,23 @@ export const make = (
431
562
  const keep = keepPolicyFor(row.keep, row.state)
432
563
  if (keep === undefined) return
433
564
  if (keep.ageMs !== undefined) {
434
- yield* tx.execute(sql`
435
- DELETE FROM ${jobs}
436
- WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
565
+ yield* tx.execute(purgeJobsWhere(sql`
566
+ ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
437
567
  AND ${jobs.finishedAt} <= ${new Date(now.getTime() - keep.ageMs)}
438
- `)
568
+ AND NOT ${owesCascades}
569
+ `))
439
570
  }
440
571
  if (keep.count !== undefined) {
441
- yield* tx.execute(sql`
442
- DELETE FROM ${jobs}
443
- WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
572
+ yield* tx.execute(purgeJobsWhere(sql`
573
+ ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
444
574
  AND ${jobs.id} NOT IN (
445
575
  SELECT ${jobs.id} FROM ${jobs}
446
576
  WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
447
577
  ORDER BY ${jobs.finishedAt} DESC, ${jobs.seq} DESC
448
578
  LIMIT ${keep.count}
449
579
  )
450
- `)
580
+ AND NOT ${owesCascades}
581
+ `))
451
582
  }
452
583
  })
453
584
 
@@ -493,7 +624,7 @@ export const make = (
493
624
  : sql`'j-' || ${seqExpr}::text`
494
625
  const rows = rowsOf(yield* exec.execute<{ id: string }>(sql`
495
626
  INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
496
- attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
627
+ attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
497
628
  VALUES (${idExpr}, ${request.name}, ${request.queue}, ${state}, ${request.priority},
498
629
  ${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
499
630
  ${request.attemptsMax},
@@ -501,6 +632,7 @@ export const make = (
501
632
  ${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
502
633
  ${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
503
634
  ${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
635
+ ${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
504
636
  ${runAt}, ${now}${extraColumnValues(request)})
505
637
  ON CONFLICT (id) DO NOTHING
506
638
  RETURNING ${jobs.id} AS id
@@ -631,7 +763,7 @@ export const make = (
631
763
  // killed and safe to retry.
632
764
  Effect.retry({
633
765
  times: 3,
634
- while: (error) => String(error).includes("40P01") || String(error).includes("deadlock detected")
766
+ while: isDeadlockError
635
767
  }),
636
768
  Effect.mapError((error) =>
637
769
  error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)
@@ -656,59 +788,139 @@ export const make = (
656
788
  AND (${dedupe.windowExpiresAt} IS NULL OR ${dedupe.windowExpiresAt} <= ${now})
657
789
  `).pipe(Effect.asVoid)
658
790
 
791
+ // The outbox invariant: every operation that moves a job carrying a
792
+ // `parent` envelope INTO a terminal state appends its report here in the
793
+ // same transaction (see `JobStore.OutboxEntry`). `exit`/`failedReason`
794
+ // mirror the job row AFTER the transition; JSON.stringify drops
795
+ // undefined fields so absent values read back as undefined.
796
+ const appendOutbox = (
797
+ exec: Pick<Db, "execute">,
798
+ parent: JobStore.ParentEnvelope | null | undefined,
799
+ outcome: JobStore.FlowChildReport["outcome"],
800
+ exit: JobStore.FlowChildReport["exit"],
801
+ failedReason: string | null | undefined
802
+ ) =>
803
+ parent === null || parent === undefined
804
+ ? Effect.void
805
+ : exec.execute(sql`
806
+ INSERT INTO ${flowOutbox} (flow_name, parent_store_key, report)
807
+ VALUES (${parent.flowName}, ${parent.parentStoreKey}, ${
808
+ JSON.stringify({
809
+ flowId: parent.flowId,
810
+ childKey: parent.childKey,
811
+ outcome,
812
+ exit,
813
+ failedReason: failedReason ?? undefined
814
+ })
815
+ }::jsonb)
816
+ `).pipe(Effect.asVoid)
817
+
818
+ // Flip a flow's still-pending dependency rows to `cancelled` and NOT
819
+ // `cascaded` (the sweeper still owes the child stores real cancels).
820
+ // Embedded as a CTE body so the caller can count the marked rows into
821
+ // the parent's `cancelled` counter in the same statement.
822
+ const cancelPendingChildren = (flowId: string) =>
823
+ sql`
824
+ UPDATE ${flowChildren} SET status = 'cancelled', cascaded = FALSE
825
+ WHERE ${flowChildren.flowId} = ${flowId} AND ${flowChildren.status} = 'pending'
826
+ RETURNING 1
827
+ `
828
+
659
829
  // Shared by cancel and cancelByDedupe.
660
830
  const cancelJob = (id: JobStore.JobId) =>
661
831
  db.transaction((tx) =>
662
- Effect.gen(function*() {
663
- const now = yield* nowDate
664
- // One guarded statement: waiting/delayed become terminal, active
665
- // gets the cancel-request flag; anything else is reported by state.
666
- const rows = rowsOf(yield* tx.execute<
667
- {
668
- id: string
669
- state: string
670
- processedAt: Date | null
671
- name: string
672
- keep: JobStore.KeepPolicy | null
673
- dedupeKey: string | null
674
- }
675
- >(sql`
676
- UPDATE ${jobs} SET
677
- state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN 'cancelled' ELSE ${jobs.state} END,
678
- finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
679
- cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
680
- WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active')
681
- RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
682
- ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
683
- ${jobs.dedupeKey} AS "dedupeKey"
832
+ Effect.gen(function*() {
833
+ const now = yield* nowDate
834
+ // Contract lock order (dependency rows first, parent second): mark
835
+ // a waiting-children parent's remaining pending rows cancelled.
836
+ // Pending rows exist only while the parent is `waiting-children`,
837
+ // so this is a no-op for every other state; a non-cancellable
838
+ // parent rolls the transaction back anyway.
839
+ const firstPass = rowsOf(yield* tx.execute<{ marked: number }>(sql`
840
+ WITH marked AS (${cancelPendingChildren(id)})
841
+ SELECT count(*)::int AS marked FROM marked
842
+ `))
843
+ const preMarked = firstPass[0]?.marked ?? 0
844
+ // One guarded statement: waiting/delayed/waiting-children become
845
+ // terminal, active gets the cancel-request flag; anything else is
846
+ // reported by state.
847
+ const rows = rowsOf(yield* tx.execute<
848
+ {
849
+ id: string
850
+ state: string
851
+ processedAt: Date | null
852
+ name: string
853
+ keep: JobStore.KeepPolicy | null
854
+ dedupeKey: string | null
855
+ hasFlow: boolean
856
+ parent: JobStore.ParentEnvelope | null
857
+ exit: unknown
858
+ failedReason: string | null
859
+ }
860
+ >(sql`
861
+ UPDATE ${jobs} SET
862
+ state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN 'cancelled' ELSE ${jobs.state} END,
863
+ finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
864
+ flow_pending = CASE WHEN ${jobs.state} = 'waiting-children' THEN 0 ELSE ${jobs.flowPending} END,
865
+ cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
866
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
867
+ RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
868
+ ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
869
+ ${jobs.dedupeKey} AS "dedupeKey", (${jobs.flowPending} IS NOT NULL) AS "hasFlow",
870
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
871
+ `))
872
+ const row = rows[0]
873
+ if (row === undefined) {
874
+ const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
875
+ SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
684
876
  `))
685
- const row = rows[0]
686
- if (row === undefined) {
687
- const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
688
- SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
689
- `))
690
- const found = existing[0]
691
- if (found === undefined) {
692
- return yield* new JobStore.JobNotFoundError({ jobId: id })
693
- }
694
- return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
877
+ const found = existing[0]
878
+ if (found === undefined) {
879
+ return yield* new JobStore.JobNotFoundError({ jobId: id })
695
880
  }
696
- if (row.state === "cancelled") {
697
- yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
698
- yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
699
- yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
881
+ return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
882
+ }
883
+ if (row.state === "cancelled") {
884
+ if (row.hasFlow) {
885
+ // Re-run the marking now that the parent lock is held: the
886
+ // first UPDATE above races a concurrent FanOut — its
887
+ // uncommitted dependency-row INSERTs are invisible, while
888
+ // this UPDATE's own EPQ re-check can still see the parent as
889
+ // 'waiting-children' after the FanOut commits. Without this
890
+ // pass those rows would stay 'pending' forever, invisible to
891
+ // both sweep classes. Every row either pass marked lands in
892
+ // the `cancelled` counter.
893
+ yield* tx.execute(sql`
894
+ WITH marked AS (${cancelPendingChildren(id)})
895
+ UPDATE ${jobs} SET flow_cancelled = ${jobs.flowCancelled} + ${preMarked}::int
896
+ + (SELECT count(*)::int FROM marked)
897
+ WHERE ${jobs.id} = ${id}
898
+ `)
700
899
  }
701
- })
702
- ).pipe(
703
- Effect.mapError((error) =>
704
- error instanceof JobStore.JobNotFoundError ||
705
- error instanceof JobStore.JobNotCancellableError ||
706
- error instanceof JobStore.JobStoreError
707
- ? error
708
- : storeError("cancel failed")(error)
709
- ),
710
- Effect.asVoid
711
- )
900
+ yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
901
+ // A cancelled child reports upward through the outbox.
902
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
903
+ yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
904
+ yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
905
+ }
906
+ })
907
+ ).pipe(
908
+ // Rare lock-order inversion (this row-then-parent flip vs a
909
+ // fail-fast settle's parent-then-rows marking) surfaces as a
910
+ // Postgres deadlock (40P01); one side is killed and safe to retry.
911
+ Effect.retry({
912
+ times: 3,
913
+ while: isDeadlockError
914
+ }),
915
+ Effect.mapError((error) =>
916
+ error instanceof JobStore.JobNotFoundError ||
917
+ error instanceof JobStore.JobNotCancellableError ||
918
+ error instanceof JobStore.JobStoreError
919
+ ? error
920
+ : storeError("cancel failed")(error)
921
+ ),
922
+ Effect.asVoid
923
+ )
712
924
 
713
925
  const enqueueOne = (request: JobStore.EnqueueRequest) =>
714
926
  Effect.gen(function*() {
@@ -798,11 +1010,12 @@ export const make = (
798
1010
  ${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
799
1011
  ${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
800
1012
  ${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
1013
+ ${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
801
1014
  ${new Date(now.getTime() + Math.max(0, request.delayMs))}, ${now}${extraColumnValues(request)})`
802
1015
  )
803
1016
  const rows = rowsOf(yield* db.execute<{ id: string }>(sql`
804
1017
  INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
805
- attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
1018
+ attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
806
1019
  VALUES ${sql.join(values, sql`, `)}
807
1020
  ON CONFLICT (id) DO NOTHING
808
1021
  RETURNING ${jobs.id} AS id
@@ -843,6 +1056,123 @@ export const make = (
843
1056
  return resolved
844
1057
  })
845
1058
 
1059
+ // The FanOut ack: land the manifest (flow columns + dependency rows) and
1060
+ // park the parent, all lock-token-guarded in one transaction. A fan-out
1061
+ // is a phase transition, not a completed run — `attempts_made` is not
1062
+ // incremented; the ledger records `fanned-out` with no exit.
1063
+ const ackFanOut = (
1064
+ id: JobStore.JobId,
1065
+ token: string,
1066
+ outcome: Extract<JobStore.AckOutcome, { _tag: "FanOut" }>
1067
+ ): Effect.Effect<void, JobStore.JobStoreError | JobStore.JobNotFoundError | JobStore.LockLostError> =>
1068
+ Effect.gen(function*() {
1069
+ // Validate BEFORE any mutation, so a bad spec cannot leave the job
1070
+ // half-acked (lock cleared, ledger written, still active).
1071
+ if (outcome.children.some((child) => child.request.id === undefined)) {
1072
+ return yield* new JobStore.JobStoreError({
1073
+ message: "FanOut child specs require an explicit request.id"
1074
+ })
1075
+ }
1076
+ const wakeQueue = yield* db.transaction((tx) =>
1077
+ Effect.gen(function*() {
1078
+ const now = yield* nowDate
1079
+ const rows = rowsOf(yield* tx.execute<
1080
+ {
1081
+ processedAt: Date | null
1082
+ name: string
1083
+ keep: JobStore.KeepPolicy | null
1084
+ dedupeKey: string | null
1085
+ queue: string
1086
+ cancelRequested: boolean
1087
+ flowPending: number | null
1088
+ parent: JobStore.ParentEnvelope | null
1089
+ exit: unknown
1090
+ failedReason: string | null
1091
+ }
1092
+ >(sql`
1093
+ UPDATE ${jobs} SET lock_token = NULL, lock_expires_at = NULL
1094
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
1095
+ RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
1096
+ ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1097
+ ${jobs.cancelRequested} AS "cancelRequested", ${jobs.flowPending} AS "flowPending",
1098
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
1099
+ `))
1100
+ const row = rows[0]
1101
+ if (row === undefined) {
1102
+ return yield* explainMiss(id)
1103
+ }
1104
+ yield* insertAttempt(tx, id, "fanned-out", row.processedAt, now, undefined)
1105
+ let pending: number
1106
+ if (row.flowPending === null || row.flowPending === undefined) {
1107
+ pending = outcome.children.length
1108
+ yield* tx.execute(sql`
1109
+ UPDATE ${jobs} SET flow_fail_fast = ${outcome.failFast}, flow_pending = ${pending},
1110
+ flow_completed = 0, flow_failed = 0, flow_cancelled = 0
1111
+ WHERE ${jobs.id} = ${id}
1112
+ `)
1113
+ // Chunked multi-row VALUES, like enqueueMany's insertBatch.
1114
+ for (let start = 0; start < outcome.children.length; start += 500) {
1115
+ const chunk = outcome.children.slice(start, start + 500)
1116
+ const values = chunk.map((child) =>
1117
+ sql`(${id}, ${child.childKey}, ${child.request.name}, ${child.storeKey},
1118
+ ${JSON.stringify(child.request)}::jsonb, 'pending', NULL, NULL, FALSE, ${now})`
1119
+ )
1120
+ yield* tx.execute(sql`
1121
+ INSERT INTO ${flowChildren} (flow_id, child_key, name, store_key, spec,
1122
+ status, exit, failed_reason, cascaded, pending_since)
1123
+ VALUES ${sql.join(values, sql`, `)}
1124
+ `)
1125
+ }
1126
+ } else {
1127
+ // A manifest that was already present is kept untouched (double
1128
+ // fan-out converges on the persisted children); the state
1129
+ // transition follows the persisted pending count either way.
1130
+ pending = Number(row.flowPending)
1131
+ }
1132
+ if (row.cancelRequested) {
1133
+ // A cancel raced the fan-out: cancellation wins. The rows exist
1134
+ // and get marked (into the `cancelled` counter), so the sweeper
1135
+ // cascades (mostly no-op cancels for never-enqueued children).
1136
+ yield* tx.execute(sql`
1137
+ WITH marked AS (${cancelPendingChildren(id)})
1138
+ UPDATE ${jobs} SET state = 'cancelled', finished_at = ${now},
1139
+ cancel_requested = FALSE, flow_pending = 0,
1140
+ flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
1141
+ WHERE ${jobs.id} = ${id}
1142
+ `)
1143
+ yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
1144
+ // A cancelled NESTED parent reports upward through the outbox.
1145
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
1146
+ yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
1147
+ yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
1148
+ return undefined
1149
+ }
1150
+ if (pending > 0) {
1151
+ yield* tx.execute(sql`
1152
+ UPDATE ${jobs} SET state = 'waiting-children' WHERE ${jobs.id} = ${id}
1153
+ `)
1154
+ return undefined
1155
+ }
1156
+ // Empty (or fully recorded) manifest: straight to runnable collect.
1157
+ yield* tx.execute(sql`
1158
+ UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
1159
+ WHERE ${jobs.id} = ${id}
1160
+ `)
1161
+ return JobStore.QueueName(row.queue)
1162
+ })
1163
+ ).pipe(
1164
+ Effect.mapError((error) =>
1165
+ error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
1166
+ error instanceof JobStore.JobStoreError
1167
+ ? error
1168
+ : storeError("ack failed")(error)
1169
+ )
1170
+ )
1171
+ if (wakeQueue !== undefined) {
1172
+ yield* wakeUp(wakeQueue)
1173
+ }
1174
+ })
1175
+
846
1176
  const store: JobStore.Service = {
847
1177
  enqueue: enqueueOne,
848
1178
 
@@ -917,7 +1247,10 @@ export const make = (
917
1247
  ${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
918
1248
  ${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
919
1249
  ${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
920
- ${jobs.trace} AS "trace",
1250
+ ${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
1251
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
1252
+ ${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
1253
+ ${jobs.flowCancelled} AS "flowCancelled",
921
1254
  ${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
922
1255
  ${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
923
1256
  ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
@@ -944,8 +1277,11 @@ export const make = (
944
1277
  error instanceof JobStore.JobStoreError ? error : storeError("claim failed")(error)
945
1278
  )),
946
1279
 
947
- ack: (id, token, outcome) =>
948
- db.transaction((tx) =>
1280
+ ack: (id, token, outcome) => {
1281
+ if (outcome._tag === "FanOut") {
1282
+ return ackFanOut(id, token, outcome)
1283
+ }
1284
+ return db.transaction((tx) =>
949
1285
  Effect.gen(function*() {
950
1286
  const now = yield* nowDate
951
1287
  const update = outcome._tag === "Complete"
@@ -971,6 +1307,9 @@ export const make = (
971
1307
  keep: JobStore.KeepPolicy | null
972
1308
  dedupeKey: string | null
973
1309
  queue: string
1310
+ parent: JobStore.ParentEnvelope | null
1311
+ exit: unknown
1312
+ failedReason: string | null
974
1313
  }
975
1314
  >(sql`
976
1315
  UPDATE ${jobs} SET ${update},
@@ -978,7 +1317,8 @@ export const make = (
978
1317
  WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
979
1318
  RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
980
1319
  ${jobs.state} AS "state", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
981
- ${jobs.queue} AS "queue"
1320
+ ${jobs.queue} AS "queue", ${jobs.parent} AS "parent", ${jobs.exit} AS "exit",
1321
+ ${jobs.failedReason} AS "failedReason"
982
1322
  `))
983
1323
  const row = rows[0]
984
1324
  if (row === undefined) {
@@ -1001,6 +1341,15 @@ export const make = (
1001
1341
  outcome._tag === "Cancelled" || cancelledRetry ? undefined : outcome.exit
1002
1342
  )
1003
1343
  if (outcome._tag !== "Retry" || cancelledRetry) {
1344
+ // A terminal transition of an envelope-carrying child reports
1345
+ // upward through the outbox (exit/failedReason as persisted).
1346
+ yield* appendOutbox(
1347
+ tx,
1348
+ row.parent,
1349
+ ledgerOutcome === "retried" ? "cancelled" : ledgerOutcome,
1350
+ row.exit ?? undefined,
1351
+ row.failedReason
1352
+ )
1004
1353
  yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
1005
1354
  yield* applyKeep(tx, row, now)
1006
1355
  }
@@ -1017,7 +1366,8 @@ export const make = (
1017
1366
  ),
1018
1367
  Effect.tap((queue) => queue !== undefined ? wakeUp(queue) : Effect.void),
1019
1368
  Effect.asVoid
1020
- ),
1369
+ )
1370
+ },
1021
1371
 
1022
1372
  release: (id, token) =>
1023
1373
  Effect.gen(function*() {
@@ -1035,6 +1385,9 @@ export const make = (
1035
1385
  keep: JobStore.KeepPolicy | null
1036
1386
  dedupeKey: string | null
1037
1387
  queue: string
1388
+ parent: JobStore.ParentEnvelope | null
1389
+ exit: unknown
1390
+ failedReason: string | null
1038
1391
  }
1039
1392
  >(sql`
1040
1393
  UPDATE ${jobs} SET
@@ -1045,12 +1398,15 @@ export const make = (
1045
1398
  WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
1046
1399
  RETURNING ${jobs.id} AS id, (${jobs.state} = 'cancelled') AS cancelled,
1047
1400
  ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
1048
- ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue"
1401
+ ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1402
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
1049
1403
  `))
1050
1404
  const row = rows[0]
1051
1405
  if (row === undefined) return undefined
1052
1406
  if (row.cancelled) {
1053
1407
  yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
1408
+ // A cancel honoured at release is a terminal transition too.
1409
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
1054
1410
  yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
1055
1411
  yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
1056
1412
  }
@@ -1117,6 +1473,9 @@ export const make = (
1117
1473
  name: string
1118
1474
  keep: JobStore.KeepPolicy | null
1119
1475
  dedupeKey: string | null
1476
+ parent: JobStore.ParentEnvelope | null
1477
+ exit: unknown
1478
+ failedReason: string | null
1120
1479
  }
1121
1480
  >(sql`
1122
1481
  UPDATE ${jobs} SET
@@ -1137,7 +1496,8 @@ export const make = (
1137
1496
  cancel_requested = FALSE
1138
1497
  WHERE ${jobs.state} = 'active' AND ${jobs.lockExpiresAt} <= ${now}::timestamptz
1139
1498
  RETURNING ${jobs.id} AS "id", ${jobs.state} AS "state", ${jobs.processedAt} AS "processedAt",
1140
- ${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey"
1499
+ ${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
1500
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
1141
1501
  `))
1142
1502
  const recovered: Array<{ id: JobStore.JobId; failed: boolean }> = []
1143
1503
  for (const row of rows) {
@@ -1150,6 +1510,15 @@ export const make = (
1150
1510
  undefined
1151
1511
  )
1152
1512
  if (row.state === "cancelled" || row.state === "failed") {
1513
+ // Honoured cancels and stall exhaustion are terminal
1514
+ // transitions: envelope-carrying children report upward.
1515
+ yield* appendOutbox(
1516
+ tx,
1517
+ row.parent,
1518
+ row.state === "cancelled" ? "cancelled" : "failed",
1519
+ row.exit ?? undefined,
1520
+ row.failedReason
1521
+ )
1153
1522
  yield* releaseDedupe(tx, row.name, row.dedupeKey, row.id, now)
1154
1523
  }
1155
1524
  if (row.state === "cancelled") {
@@ -1231,7 +1600,11 @@ export const make = (
1231
1600
  ${jobs.attemptsMax} AS "attemptsMax", ${jobs.attemptsMade} AS "attemptsMade",
1232
1601
  ${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
1233
1602
  ${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
1234
- ${jobs.cancelRequested} AS "cancelRequested",
1603
+ ${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
1604
+ ${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
1605
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
1606
+ ${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
1607
+ ${jobs.flowCancelled} AS "flowCancelled",
1235
1608
  ${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
1236
1609
  ${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
1237
1610
  ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
@@ -1340,20 +1713,21 @@ export const make = (
1340
1713
  upsertSchedule: (schedule) =>
1341
1714
  db.execute(sql`
1342
1715
  INSERT INTO ${schedules} (key, job_name, queue, cron, tz, every_ms, payload, metadata,
1343
- priority, attempts_max, backoff, keep, timeout_ms, next_run_at)
1716
+ priority, attempts_max, backoff, keep, timeout_ms, group_name, next_run_at)
1344
1717
  VALUES (${schedule.key}, ${schedule.jobName}, ${schedule.queue},
1345
1718
  ${schedule.cron ?? null}, ${schedule.tz ?? null}, ${schedule.everyMs ?? null},
1346
1719
  ${JSON.stringify(schedule.payload ?? null)}::jsonb, ${JSON.stringify(schedule.metadata)}::jsonb,
1347
1720
  ${schedule.priority}, ${schedule.attemptsMax},
1348
1721
  ${schedule.backoff === undefined ? null : JSON.stringify(schedule.backoff)}::jsonb,
1349
1722
  ${schedule.keep === undefined ? null : JSON.stringify(schedule.keep)}::jsonb,
1350
- ${schedule.timeoutMs ?? null}, ${new Date(schedule.nextRunAt)})
1723
+ ${schedule.timeoutMs ?? null}, ${schedule.group ?? null}, ${new Date(schedule.nextRunAt)})
1351
1724
  ON CONFLICT (key) DO UPDATE SET
1352
1725
  job_name = EXCLUDED.job_name, queue = EXCLUDED.queue, cron = EXCLUDED.cron,
1353
1726
  tz = EXCLUDED.tz, every_ms = EXCLUDED.every_ms, payload = EXCLUDED.payload,
1354
1727
  metadata = EXCLUDED.metadata, priority = EXCLUDED.priority,
1355
1728
  attempts_max = EXCLUDED.attempts_max, backoff = EXCLUDED.backoff,
1356
1729
  keep = EXCLUDED.keep, timeout_ms = EXCLUDED.timeout_ms,
1730
+ group_name = EXCLUDED.group_name,
1357
1731
  next_run_at = CASE
1358
1732
  WHEN ${schedules.cron} IS NOT DISTINCT FROM EXCLUDED.cron
1359
1733
  AND ${schedules.tz} IS NOT DISTINCT FROM EXCLUDED.tz
@@ -1383,6 +1757,9 @@ export const make = (
1383
1757
  if (listOptions?.queue !== undefined) {
1384
1758
  conditions.push(sql`${schedules.queue} = ${listOptions.queue}`)
1385
1759
  }
1760
+ if (listOptions?.group !== undefined) {
1761
+ conditions.push(sql`${schedules.group} = ${listOptions.group}`)
1762
+ }
1386
1763
  const rows = rowsOf(yield* db.execute<ScheduleRow>(sql`
1387
1764
  SELECT ${schedules.key} AS "key", ${schedules.jobName} AS "jobName",
1388
1765
  ${schedules.queue} AS "queue", ${schedules.cron} AS "cron", ${schedules.tz} AS "tz",
@@ -1390,7 +1767,7 @@ export const make = (
1390
1767
  ${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
1391
1768
  ${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
1392
1769
  ${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
1393
- ${schedules.nextRunAt} AS "nextRunAt"
1770
+ ${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
1394
1771
  FROM ${schedules}
1395
1772
  WHERE ${sql.join(conditions, sql` AND `)}
1396
1773
  ORDER BY ${schedules.key}
@@ -1408,7 +1785,7 @@ export const make = (
1408
1785
  ${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
1409
1786
  ${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
1410
1787
  ${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
1411
- ${schedules.nextRunAt} AS "nextRunAt"
1788
+ ${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
1412
1789
  FROM ${schedules}
1413
1790
  WHERE ${schedules.nextRunAt} <= ${now}
1414
1791
  ORDER BY ${schedules.nextRunAt} ASC
@@ -1458,6 +1835,365 @@ export const make = (
1458
1835
  return fired
1459
1836
  }),
1460
1837
 
1838
+ recordChildResults: (reports) =>
1839
+ Effect.gen(function*() {
1840
+ if (reports.length === 0) {
1841
+ const none: Array<{ applied: boolean; parentSettled: boolean }> = []
1842
+ return none
1843
+ }
1844
+ const batch = yield* db.transaction((tx) =>
1845
+ Effect.gen(function*() {
1846
+ const now = yield* nowDate
1847
+ const results = reports.map(() => ({ applied: false, parentSettled: false }))
1848
+
1849
+ // Phase 1a — apply every row update (contract lock order:
1850
+ // dependency rows FIRST, parents second). Only a (flow, key)'s
1851
+ // first occurrence in the batch can apply — later duplicates
1852
+ // would find the row non-pending anyway, and UPDATE ... FROM
1853
+ // must never see two source rows for one target.
1854
+ const candidates: Array<{ readonly index: number; readonly report: JobStore.FlowChildReport }> = []
1855
+ const seen = new Set<string>()
1856
+ for (const [index, report] of reports.entries()) {
1857
+ const key = `${report.flowId}\u0000${report.childKey}`
1858
+ if (seen.has(key)) continue
1859
+ seen.add(key)
1860
+ candidates.push({ index, report })
1861
+ }
1862
+ for (let start = 0; start < candidates.length; start += 200) {
1863
+ const chunk = candidates.slice(start, start + 200)
1864
+ const values = chunk.map(({ index, report }) =>
1865
+ sql`(${index}::int, ${report.flowId}::text, ${report.childKey}::text,
1866
+ ${report.outcome}::text,
1867
+ ${report.exit === undefined ? null : JSON.stringify(report.exit)}::jsonb,
1868
+ ${report.failedReason ?? null}::text)`
1869
+ )
1870
+ const appliedRows = rowsOf(yield* tx.execute<{ ord: number }>(sql`
1871
+ UPDATE ${flowChildren} SET status = v.outcome, exit = v.exit,
1872
+ failed_reason = v.failed_reason, cascaded = TRUE
1873
+ FROM (VALUES ${sql.join(values, sql`, `)})
1874
+ AS v(ord, flow_id, child_key, outcome, exit, failed_reason)
1875
+ WHERE ${flowChildren.flowId} = v.flow_id AND ${flowChildren.childKey} = v.child_key
1876
+ AND ${flowChildren.status} = 'pending'
1877
+ RETURNING v.ord AS "ord"
1878
+ `))
1879
+ for (const row of appliedRows) {
1880
+ const result = results[Number(row.ord)]
1881
+ if (result !== undefined) {
1882
+ result.applied = true
1883
+ }
1884
+ }
1885
+ }
1886
+
1887
+ // Tally the applied reports per touched flow, in batch order.
1888
+ interface Touch {
1889
+ appliedCount: number
1890
+ completed: number
1891
+ failed: number
1892
+ cancelled: number
1893
+ firstAppliedFailed: number | undefined
1894
+ lastApplied: number
1895
+ }
1896
+ const touched = new Map<string, Touch>()
1897
+ for (const [index, report] of reports.entries()) {
1898
+ if (results[index]?.applied !== true) continue
1899
+ const touch = touched.get(report.flowId) ?? {
1900
+ appliedCount: 0,
1901
+ completed: 0,
1902
+ failed: 0,
1903
+ cancelled: 0,
1904
+ firstAppliedFailed: undefined,
1905
+ lastApplied: index
1906
+ }
1907
+ touch.appliedCount += 1
1908
+ touch.lastApplied = index
1909
+ if (report.outcome === "completed") touch.completed += 1
1910
+ if (report.outcome === "cancelled") touch.cancelled += 1
1911
+ if (report.outcome === "failed") {
1912
+ touch.failed += 1
1913
+ touch.firstAppliedFailed ??= index
1914
+ }
1915
+ touched.set(report.flowId, touch)
1916
+ }
1917
+
1918
+ // Phase 1b + 2 — per touched flow (sorted, so concurrent
1919
+ // batches take parent locks in one order): move the applied
1920
+ // children from `pending` to their outcome counters, then make
1921
+ // at most one settle decision. Fail-fast wins the tie.
1922
+ const wakeQueues: Array<JobStore.QueueName> = []
1923
+ for (const flowId of [...touched.keys()].toSorted()) {
1924
+ const touch = touched.get(flowId)
1925
+ if (touch === undefined) continue
1926
+ const parents = rowsOf(yield* tx.execute<
1927
+ {
1928
+ state: JobStore.JobState
1929
+ flowPending: number
1930
+ flowFailFast: boolean | null
1931
+ processedAt: Date | null
1932
+ name: string
1933
+ keep: JobStore.KeepPolicy | null
1934
+ dedupeKey: string | null
1935
+ queue: string
1936
+ parent: JobStore.ParentEnvelope | null
1937
+ }
1938
+ >(sql`
1939
+ UPDATE ${jobs} SET
1940
+ flow_pending = GREATEST(${jobs.flowPending} - ${touch.appliedCount}::int, 0),
1941
+ flow_completed = ${jobs.flowCompleted} + ${touch.completed}::int,
1942
+ flow_failed = ${jobs.flowFailed} + ${touch.failed}::int,
1943
+ flow_cancelled = ${jobs.flowCancelled} + ${touch.cancelled}::int
1944
+ WHERE ${jobs.id} = ${flowId} AND ${jobs.flowPending} IS NOT NULL
1945
+ RETURNING ${jobs.state} AS "state", ${jobs.flowPending} AS "flowPending",
1946
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.processedAt} AS "processedAt",
1947
+ ${jobs.name} AS "name", ${jobs.keep} AS "keep",
1948
+ ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1949
+ ${jobs.parent} AS "parent"
1950
+ `))
1951
+ const parent = parents[0]
1952
+ if (parent === undefined || parent.state !== "waiting-children") continue
1953
+ const failedIndex = parent.flowFailFast === true ? touch.firstAppliedFailed : undefined
1954
+ const failedReport = failedIndex !== undefined ? reports[failedIndex] : undefined
1955
+ if (failedIndex !== undefined && failedReport !== undefined) {
1956
+ // The first applied failure settles the parent terminally
1957
+ // (store-side, like stall exhaustion: failedReason, no
1958
+ // exit) and marks the remaining rows in the same
1959
+ // transaction. A nested parent's own report goes to the
1960
+ // outbox here — this settle IS its terminal transition.
1961
+ const reason = `effect-mq: flow child "${failedReport.childKey}" failed`
1962
+ yield* tx.execute(sql`
1963
+ WITH marked AS (${cancelPendingChildren(flowId)})
1964
+ UPDATE ${jobs} SET state = 'failed', finished_at = ${now},
1965
+ failed_reason = ${reason}, cancel_requested = FALSE, flow_pending = 0,
1966
+ flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
1967
+ WHERE ${jobs.id} = ${flowId}
1968
+ `)
1969
+ yield* insertAttempt(tx, flowId, "failed", parent.processedAt, now, undefined)
1970
+ yield* appendOutbox(tx, parent.parent, "failed", undefined, reason)
1971
+ yield* releaseDedupe(tx, parent.name, parent.dedupeKey, flowId, now)
1972
+ yield* applyKeep(tx, { name: parent.name, state: "failed", keep: parent.keep }, now)
1973
+ const decided = results[failedIndex]
1974
+ if (decided !== undefined) {
1975
+ decided.parentSettled = true
1976
+ }
1977
+ continue
1978
+ }
1979
+ if (Number(parent.flowPending) === 0) {
1980
+ // All children settled: the parent resumes runnable, phase
1981
+ // collect, settled at the flow's last applied report.
1982
+ yield* tx.execute(sql`
1983
+ UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
1984
+ WHERE ${jobs.id} = ${flowId}
1985
+ `)
1986
+ wakeQueues.push(JobStore.QueueName(parent.queue))
1987
+ const decided = results[touch.lastApplied]
1988
+ if (decided !== undefined) {
1989
+ decided.parentSettled = true
1990
+ }
1991
+ }
1992
+ }
1993
+ return { results, wakeQueues }
1994
+ })
1995
+ ).pipe(
1996
+ // Rare lock-order inversion (a concurrent cancel/settle marking
1997
+ // rows) surfaces as a Postgres deadlock (40P01); the killed side
1998
+ // is safe to retry — the row-state guard keeps it idempotent.
1999
+ Effect.retry({
2000
+ times: 3,
2001
+ while: isDeadlockError
2002
+ }),
2003
+ Effect.mapError((error) =>
2004
+ error instanceof JobStore.JobStoreError ? error : storeError("recordChildResults failed")(error)
2005
+ )
2006
+ )
2007
+ for (const queue of batch.wakeQueues) {
2008
+ yield* wakeUp(queue)
2009
+ }
2010
+ return batch.results
2011
+ }),
2012
+
2013
+ peekOutbox: (peekOptions) =>
2014
+ Effect.gen(function*() {
2015
+ const limit = Math.max(0, peekOptions.limit)
2016
+ if (limit === 0) {
2017
+ const none: Array<JobStore.OutboxEntry> = []
2018
+ return none
2019
+ }
2020
+ // `after` pages past a previously returned id (exclusive), whether
2021
+ // or not that entry still exists. Anything that is not a canonical
2022
+ // id this store issued is treated as unset.
2023
+ const after = peekOptions.after !== undefined && CANONICAL_BIGSERIAL.test(peekOptions.after)
2024
+ ? peekOptions.after
2025
+ : undefined
2026
+ const rows = rowsOf(yield* db.execute<
2027
+ {
2028
+ id: string
2029
+ flowName: string
2030
+ parentStoreKey: string
2031
+ report: JobStore.FlowChildReport
2032
+ }
2033
+ >(sql`
2034
+ SELECT ${flowOutbox.id}::text AS "id", ${flowOutbox.flowName} AS "flowName",
2035
+ ${flowOutbox.parentStoreKey} AS "parentStoreKey", ${flowOutbox.report} AS "report"
2036
+ FROM ${flowOutbox}
2037
+ ${after === undefined ? sql`` : sql`WHERE ${flowOutbox.id} > ${after}::bigint`}
2038
+ ORDER BY ${flowOutbox.id} ASC
2039
+ LIMIT ${limit}
2040
+ `).pipe(Effect.mapError(storeError("peekOutbox failed"))))
2041
+ return rows.map((row): JobStore.OutboxEntry => ({
2042
+ id: row.id,
2043
+ flowName: row.flowName,
2044
+ parentStoreKey: row.parentStoreKey,
2045
+ report: row.report
2046
+ }))
2047
+ }),
2048
+
2049
+ deleteOutbox: (ids) =>
2050
+ Effect.suspend(() => {
2051
+ // Ids are opaque strings to callers; only CANONICAL ids this store
2052
+ // could have issued can match. The strictness matters twice: a
2053
+ // foreign/garbled id must not blow up the ::bigint cast, and a
2054
+ // non-canonical spelling ("007") must stay an unknown-id no-op
2055
+ // rather than cast to 7 and delete a live entry.
2056
+ const numeric = ids.filter((id) => CANONICAL_BIGSERIAL.test(id))
2057
+ if (numeric.length === 0) return Effect.void
2058
+ return db.execute(sql`
2059
+ DELETE FROM ${flowOutbox}
2060
+ WHERE ${flowOutbox.id} = ANY(${sql.param(numeric)}::bigint[])
2061
+ `).pipe(
2062
+ Effect.mapError(storeError("deleteOutbox failed")),
2063
+ Effect.asVoid
2064
+ )
2065
+ }),
2066
+
2067
+ listChildResults: (flowId, listOptions) =>
2068
+ Effect.gen(function*() {
2069
+ const limit = Math.max(1, listOptions?.limit ?? 1000)
2070
+ const cursor = listOptions?.cursor
2071
+ // `spec->>'id'` instead of the whole spec: at 10k children the full
2072
+ // payloads would transfer on every collect.
2073
+ const rows = rowsOf(yield* db.execute<FlowChildRow>(sql`
2074
+ SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
2075
+ ${flowChildren.name} AS "name", ${flowChildren.storeKey} AS "storeKey",
2076
+ ${flowChildren.spec}->>'id' AS "childJobId", ${flowChildren.status} AS "status",
2077
+ ${flowChildren.exit} AS "exit", ${flowChildren.failedReason} AS "failedReason",
2078
+ ${flowChildren.cascaded} AS "cascaded"
2079
+ FROM ${flowChildren}
2080
+ WHERE ${flowChildren.flowId} = ${flowId}
2081
+ ${cursor === undefined ? sql`` : sql`AND ${flowChildren.childKey} > ${cursor}`}
2082
+ ORDER BY ${flowChildren.childKey} ASC
2083
+ LIMIT ${limit + 1}
2084
+ `).pipe(Effect.mapError(storeError("listChildResults failed"))))
2085
+ const page = rows.slice(0, limit)
2086
+ const items = page.map(toFlowChildRecord)
2087
+ const last = items[items.length - 1]
2088
+ return {
2089
+ items,
2090
+ cursor: rows.length > limit && last !== undefined ? last.childKey : undefined
2091
+ }
2092
+ }),
2093
+
2094
+ flowSweepWork: (sweepOptions) =>
2095
+ Effect.gen(function*() {
2096
+ const now = yield* nowDate
2097
+ const limit = Math.max(1, sweepOptions.limit ?? 1000)
2098
+ const threshold = new Date(now.getTime() - sweepOptions.pendingAgeMs)
2099
+ type SweepRow = {
2100
+ readonly flowId: string
2101
+ readonly childKey: string
2102
+ readonly storeKey: string
2103
+ readonly spec: JobStore.EnqueueRequest
2104
+ }
2105
+ // Reconcile: pending rows past the eligibility threshold whose
2106
+ // parent is still parked (a settled flow never re-drives work).
2107
+ // Returning a row re-arms `pending_since` in the same statement, so
2108
+ // a full page rotates across sweeps instead of pinning its head.
2109
+ // SKIP LOCKED: rows a concurrent report/settle holds are its
2110
+ // business, and never waiting means this statement cannot deadlock.
2111
+ // The raw column names are safe: only table names vary across
2112
+ // factory instances. RETURNING order is unspecified — sorted below.
2113
+ const reconcileRows = rowsOf(yield* db.execute<SweepRow>(sql`
2114
+ WITH due AS (
2115
+ SELECT c.flow_id, c.child_key
2116
+ FROM ${flowChildren} c
2117
+ JOIN ${jobs} j ON j.id = c.flow_id
2118
+ WHERE c.status = 'pending' AND c.pending_since <= ${threshold}
2119
+ AND j.state = 'waiting-children'
2120
+ ORDER BY c.flow_id, c.child_key
2121
+ LIMIT ${limit}
2122
+ FOR UPDATE OF c SKIP LOCKED
2123
+ )
2124
+ UPDATE ${flowChildren} SET pending_since = ${now}
2125
+ FROM due
2126
+ WHERE ${flowChildren.flowId} = due.flow_id AND ${flowChildren.childKey} = due.child_key
2127
+ RETURNING ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
2128
+ ${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
2129
+ `).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
2130
+ .toSorted((a, b) =>
2131
+ a.flowId !== b.flowId
2132
+ ? (a.flowId < b.flowId ? -1 : 1)
2133
+ : a.childKey < b.childKey
2134
+ ? -1
2135
+ : a.childKey > b.childKey
2136
+ ? 1
2137
+ : 0
2138
+ )
2139
+ // Cascade: cancelled rows whose cancel has not been delivered into
2140
+ // the child's store yet (any parent state).
2141
+ const cascadeRows = rowsOf(yield* db.execute<SweepRow>(sql`
2142
+ SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
2143
+ ${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
2144
+ FROM ${flowChildren}
2145
+ WHERE ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
2146
+ ORDER BY ${flowChildren.flowId}, ${flowChildren.childKey}
2147
+ LIMIT ${limit}
2148
+ `).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
2149
+ const reconcile: Array<{
2150
+ flowId: JobStore.JobId
2151
+ children: Array<JobStore.FlowChildSpec>
2152
+ }> = []
2153
+ for (const row of reconcileRows) {
2154
+ const flowId = JobId(row.flowId)
2155
+ let group = reconcile[reconcile.length - 1]
2156
+ if (group === undefined || group.flowId !== flowId) {
2157
+ group = { flowId, children: [] }
2158
+ reconcile.push(group)
2159
+ }
2160
+ group.children.push({ childKey: row.childKey, storeKey: row.storeKey, request: row.spec })
2161
+ }
2162
+ const cascade: Array<{
2163
+ flowId: JobStore.JobId
2164
+ children: Array<{ childKey: string; storeKey: string; childJobId: JobStore.JobId }>
2165
+ }> = []
2166
+ for (const row of cascadeRows) {
2167
+ const flowId = JobId(row.flowId)
2168
+ let group = cascade[cascade.length - 1]
2169
+ if (group === undefined || group.flowId !== flowId) {
2170
+ group = { flowId, children: [] }
2171
+ cascade.push(group)
2172
+ }
2173
+ group.children.push({
2174
+ childKey: row.childKey,
2175
+ storeKey: row.storeKey,
2176
+ // SAFETY: the FanOut ack validated every spec id before
2177
+ // persisting it.
2178
+ childJobId: row.spec.id as JobStore.JobId
2179
+ })
2180
+ }
2181
+ const work: JobStore.FlowSweepWork = { reconcile, cascade }
2182
+ return work
2183
+ }),
2184
+
2185
+ markChildrenCascaded: (flowId, childKeys) =>
2186
+ childKeys.length === 0
2187
+ ? Effect.void
2188
+ : db.execute(sql`
2189
+ UPDATE ${flowChildren} SET cascaded = TRUE
2190
+ WHERE ${flowChildren.flowId} = ${flowId}
2191
+ AND ${flowChildren.childKey} = ANY(${sql.param([...childKeys])})
2192
+ `).pipe(
2193
+ Effect.mapError(storeError("markChildrenCascaded failed")),
2194
+ Effect.asVoid
2195
+ ),
2196
+
1461
2197
  counts: (queue) =>
1462
2198
  db.execute<{ state: JobStore.JobState; count: number }>(sql`
1463
2199
  SELECT ${jobs.state} AS "state", count(*)::int AS "count" FROM ${jobs}
@@ -1471,6 +2207,7 @@ export const make = (
1471
2207
  waiting: 0,
1472
2208
  delayed: 0,
1473
2209
  active: 0,
2210
+ "waiting-children": 0,
1474
2211
  completed: 0,
1475
2212
  failed: 0,
1476
2213
  cancelled: 0
@@ -1481,10 +2218,18 @@ export const make = (
1481
2218
  ),
1482
2219
 
1483
2220
  remove: (id) =>
2221
+ // The purge CTE takes a removed flow parent's dependency rows with it
2222
+ // in the same statement.
1484
2223
  db.execute(sql`
1485
- DELETE FROM ${jobs}
1486
- WHERE ${jobs.id} = ${id} AND ${jobs.state} <> 'active'
1487
- RETURNING ${jobs.id} AS id
2224
+ WITH deleted AS (
2225
+ DELETE FROM ${jobs}
2226
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} NOT IN ('active', 'waiting-children')
2227
+ RETURNING ${jobs.id} AS id
2228
+ ), purged AS (
2229
+ DELETE FROM ${flowChildren}
2230
+ WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
2231
+ )
2232
+ SELECT id FROM deleted
1488
2233
  `).pipe(
1489
2234
  Effect.mapError(storeError("remove failed")),
1490
2235
  Effect.map((result) => rowsOf(result).length > 0)