effect-mq 0.4.2 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/README.md +85 -17
  2. package/dist/Flow.d.ts +381 -0
  3. package/dist/Flow.d.ts.map +1 -0
  4. package/dist/Flow.js +340 -0
  5. package/dist/Flow.js.map +1 -0
  6. package/dist/Job.d.ts +37 -6
  7. package/dist/Job.d.ts.map +1 -1
  8. package/dist/Job.js +17 -2
  9. package/dist/Job.js.map +1 -1
  10. package/dist/JobSchedules.d.ts +112 -0
  11. package/dist/JobSchedules.d.ts.map +1 -0
  12. package/dist/JobSchedules.js +106 -0
  13. package/dist/JobSchedules.js.map +1 -0
  14. package/dist/JobStore.d.ts +320 -10
  15. package/dist/JobStore.d.ts.map +1 -1
  16. package/dist/JobStore.js.map +1 -1
  17. package/dist/MemoryJobStore.d.ts.map +1 -1
  18. package/dist/MemoryJobStore.js +336 -8
  19. package/dist/MemoryJobStore.js.map +1 -1
  20. package/dist/Metrics.d.ts +31 -0
  21. package/dist/Metrics.d.ts.map +1 -1
  22. package/dist/Metrics.js +39 -0
  23. package/dist/Metrics.js.map +1 -1
  24. package/dist/Worker.d.ts +120 -11
  25. package/dist/Worker.d.ts.map +1 -1
  26. package/dist/Worker.js +452 -26
  27. package/dist/Worker.js.map +1 -1
  28. package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
  29. package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
  30. package/dist/drizzle-postgres/DrizzleJobStore.js +662 -81
  31. package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
  32. package/dist/drizzle-postgres/schema.d.ts +310 -3
  33. package/dist/drizzle-postgres/schema.d.ts.map +1 -1
  34. package/dist/drizzle-postgres/schema.js +68 -1
  35. package/dist/drizzle-postgres/schema.js.map +1 -1
  36. package/dist/index.d.ts +14 -0
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +14 -0
  39. package/dist/index.js.map +1 -1
  40. package/dist/redis/RedisJobStore.d.ts.map +1 -1
  41. package/dist/redis/RedisJobStore.js +221 -19
  42. package/dist/redis/RedisJobStore.js.map +1 -1
  43. package/dist/redis/scripts.d.ts +118 -11
  44. package/dist/redis/scripts.d.ts.map +1 -1
  45. package/dist/redis/scripts.js +497 -28
  46. package/dist/redis/scripts.js.map +1 -1
  47. package/dist/testing/conformance.d.ts +6 -0
  48. package/dist/testing/conformance.d.ts.map +1 -1
  49. package/dist/testing/conformance.js +765 -1
  50. package/dist/testing/conformance.js.map +1 -1
  51. package/package.json +1 -1
  52. package/src/Flow.ts +778 -0
  53. package/src/Job.ts +42 -11
  54. package/src/JobSchedules.ts +223 -0
  55. package/src/JobStore.ts +347 -9
  56. package/src/MemoryJobStore.ts +372 -8
  57. package/src/Metrics.ts +43 -0
  58. package/src/Worker.ts +726 -37
  59. package/src/drizzle-postgres/DrizzleJobStore.ts +827 -82
  60. package/src/drizzle-postgres/schema.ts +94 -0
  61. package/src/index.ts +16 -0
  62. package/src/redis/RedisJobStore.ts +291 -8
  63. package/src/redis/scripts.ts +529 -26
  64. package/src/testing/conformance.ts +989 -1
@@ -20,9 +20,49 @@ import * as JobStore from "../JobStore.js";
20
20
  import { asc, eq, getTableColumns, sql } from "drizzle-orm";
21
21
  import * as PgDrizzle from "drizzle-orm/effect-postgres";
22
22
  import { getTableConfig } from "drizzle-orm/pg-core";
23
- import { Clock, Deferred, Duration, Effect, Layer, Option, Stream } from "effect";
23
+ import { Cause, Clock, Deferred, Duration, Effect, Layer, Option, Predicate, Stream } from "effect";
24
24
  const { JobId } = JobStore;
25
25
  const storeError = (message) => (cause) => new JobStore.JobStoreError({ message, cause });
26
+ // The exact spellings a bigserial ever produces: no leading zeros, no signs,
27
+ // no whitespace — what outbox ids look like and nothing Postgres would
28
+ // silently normalize into one.
29
+ const CANONICAL_BIGSERIAL = /^[1-9]\d*$/;
30
+ /**
31
+ * Whether an error surfaced by drizzle's Effect driver is a Postgres
32
+ * deadlock (40P01), i.e. safe to retry.
33
+ *
34
+ * The deadlock travels wrapped: drizzle's session fails with an
35
+ * `EffectDrizzleQueryError` whose `cause` field is `Cause.fail(SqlError)`
36
+ * (drizzle-orm `pg-core/effect/session.ts`), and `@effect/sql-pg` classifies
37
+ * pg code 40P01 into the `SqlError`'s `reason: DeadlockError` (PgClient.ts
38
+ * `classifyError`). Neither layer's `toString` renders the pg code, so this
39
+ * unwraps structurally instead of string-matching the rendered error.
40
+ *
41
+ * @internal
42
+ */
43
+ // oxlint-disable-next-line anti-slop/no-unknown-parameters -- a retry-predicate classifier over whatever the driver threw IS the boundary parser
44
+ export const isDeadlockError = (error) => {
45
+ // Unwrap drizzle's envelope: its `cause` is an Effect Cause holding the
46
+ // original failure. A bare SqlError (non-drizzle path) passes through.
47
+ const unwrapped = Predicate.hasProperty(error, "cause") && Cause.isCause(error.cause)
48
+ ? Option.getOrUndefined(Cause.findErrorOption(error.cause))
49
+ : error;
50
+ if (!Predicate.hasProperty(unwrapped, "_tag") || unwrapped._tag !== "SqlError" ||
51
+ !Predicate.hasProperty(unwrapped, "reason")) {
52
+ return false;
53
+ }
54
+ const reason = unwrapped.reason;
55
+ if (Predicate.hasProperty(reason, "_tag") && reason._tag === "DeadlockError") {
56
+ return true;
57
+ }
58
+ // Fallback for classifiers that missed the code: the reason keeps the raw
59
+ // pg error, whose message for 40P01 is "deadlock detected".
60
+ if (Predicate.hasProperty(reason, "cause")) {
61
+ const raw = String(reason.cause);
62
+ return raw.includes("40P01") || raw.includes("deadlock detected");
63
+ }
64
+ return false;
65
+ };
26
66
  const rowsOf = (result) => "rows" in result ? result.rows : result;
27
67
  const toSchedule = (row) => ({
28
68
  key: JobStore.ScheduleKey(row.key),
@@ -38,6 +78,7 @@ const toSchedule = (row) => ({
38
78
  backoff: row.backoff ?? undefined,
39
79
  keep: row.keep ?? undefined,
40
80
  timeoutMs: row.timeoutMs === null ? undefined : Number(row.timeoutMs),
81
+ group: row.group ?? undefined,
41
82
  nextRunAt: row.nextRunAt.getTime()
42
83
  });
43
84
  const toRecord = (row) => ({
@@ -57,6 +98,17 @@ const toRecord = (row) => ({
57
98
  cancelRequested: row.cancelRequested,
58
99
  dedupeKey: row.dedupeKey ?? undefined,
59
100
  trace: row.trace ?? undefined,
101
+ parent: row.parent ?? undefined,
102
+ // The flow columns are NULL together; `flowPending` is the presence marker.
103
+ flow: row.flowPending === null || row.flowPending === undefined
104
+ ? undefined
105
+ : {
106
+ failFast: row.flowFailFast === true,
107
+ pending: Number(row.flowPending),
108
+ completed: Number(row.flowCompleted ?? 0),
109
+ failed: Number(row.flowFailed ?? 0),
110
+ cancelled: Number(row.flowCancelled ?? 0)
111
+ },
60
112
  runAt: row.runAt.getTime(),
61
113
  enqueuedAt: row.enqueuedAt.getTime(),
62
114
  processedAt: row.processedAt?.getTime(),
@@ -64,6 +116,17 @@ const toRecord = (row) => ({
64
116
  exit: row.exit ?? undefined,
65
117
  failedReason: row.failedReason ?? undefined
66
118
  });
119
+ const toFlowChildRecord = (row) => ({
120
+ flowId: JobId(row.flowId),
121
+ childKey: row.childKey,
122
+ name: row.name,
123
+ storeKey: row.storeKey,
124
+ childJobId: JobId(row.childJobId),
125
+ status: row.status,
126
+ exit: row.exit ?? undefined,
127
+ failedReason: row.failedReason ?? undefined,
128
+ cascaded: row.cascaded
129
+ });
67
130
  /**
68
131
  * Build the store implementation. Requires `PgClient` and a `Scope` (for the
69
132
  * LISTEN subscription).
@@ -78,6 +141,8 @@ export const make = (options) => Effect.gen(function* () {
78
141
  const schedules = options.schedules;
79
142
  const queues = options.queues;
80
143
  const dedupe = options.dedupe;
144
+ const flowChildren = options.flowChildren;
145
+ const flowOutbox = options.flowOutbox;
81
146
  const jobsName = getTableConfig(jobs).name;
82
147
  const attemptsName = getTableConfig(attempts).name;
83
148
  const wakeChannel = `effect_mq_wake_${jobsName}`;
@@ -102,6 +167,12 @@ export const make = (options) => Effect.gen(function* () {
102
167
  "cancelRequested",
103
168
  "dedupeKey",
104
169
  "trace",
170
+ "parent",
171
+ "flowFailFast",
172
+ "flowPending",
173
+ "flowCompleted",
174
+ "flowFailed",
175
+ "flowCancelled",
105
176
  "runAt",
106
177
  "enqueuedAt",
107
178
  "processedAt",
@@ -135,7 +206,9 @@ export const make = (options) => Effect.gen(function* () {
135
206
  db.select({ jobId: attempts.jobId }).from(attempts).limit(0),
136
207
  db.select({ key: schedules.key }).from(schedules).limit(0),
137
208
  db.select({ queue: queues.queue }).from(queues).limit(0),
138
- db.select({ key: dedupe.key }).from(dedupe).limit(0)
209
+ db.select({ key: dedupe.key }).from(dedupe).limit(0),
210
+ db.select({ flowId: flowChildren.flowId }).from(flowChildren).limit(0),
211
+ db.select({ id: flowOutbox.id }).from(flowOutbox).limit(0)
139
212
  ]).pipe(Effect.mapError(storeError(`effect-mq: tables "${jobsName}"/"${attemptsName}" are missing or mismatched — ` +
140
213
  `re-export the effect-mq/drizzle schema factories from your drizzle schema and run your migrations (drizzle-kit generate)`)));
141
214
  }
@@ -175,6 +248,26 @@ export const make = (options) => Effect.gen(function* () {
175
248
  // Resubscribe forever: if the LISTEN stream ends or fails, wake-ups
176
249
  // degrade to the worker's pollInterval until the next attempt succeeds.
177
250
  yield* client.listen(wakeChannel).pipe(Stream.runForEach((payload) => Effect.sync(() => signalWake(payload === "*" ? undefined : JobStore.QueueName(payload)))), Effect.catchCause((cause) => Effect.logWarning("effect-mq: LISTEN subscription failed; wake-ups degraded to polling until resubscribe", cause)), Effect.andThen(Effect.sleep("1 second")), Effect.forever, Effect.forkScoped);
251
+ // Automatic retention (the store-level sweep and per-job `keep`) never
252
+ // prunes a flow parent that still owes cascade cancels: its dependency
253
+ // rows marked `cancelled` and not `cascaded` are the only record that
254
+ // real cancels are still due in the child stores. The explicit `remove`
255
+ // verb is not exempted.
256
+ const owesCascades = sql `EXISTS (
257
+ SELECT 1 FROM ${flowChildren}
258
+ WHERE ${flowChildren.flowId} = ${jobs.id}
259
+ AND ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
260
+ )`;
261
+ // A pruned flow parent's dependency rows go with it in the same statement.
262
+ const purgeJobsWhere = (predicate) => sql `
263
+ WITH deleted AS (
264
+ DELETE FROM ${jobs}
265
+ WHERE ${predicate}
266
+ RETURNING ${jobs.id} AS id
267
+ )
268
+ DELETE FROM ${flowChildren}
269
+ WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
270
+ `;
178
271
  if (options.historyTtl !== undefined) {
179
272
  const ttlByState = JobStore.normalizeHistoryTtl(options.historyTtl);
180
273
  const sweepMs = Duration.toMillis(options.historySweepInterval ?? "1 minute");
@@ -185,9 +278,8 @@ export const make = (options) => Effect.gen(function* () {
185
278
  // job name is pruned on the timer, not only when its group is acked.
186
279
  for (const state of ["completed", "failed", "cancelled"]) {
187
280
  const ttl = ttlByState[state];
188
- yield* db.execute(sql `
189
- DELETE FROM ${jobs}
190
- WHERE ${jobs.state} = ${state} AND (
281
+ yield* db.execute(purgeJobsWhere(sql `
282
+ ${jobs.state} = ${state} AND NOT ${owesCascades} AND (
191
283
  ${ttl !== undefined ? sql `${jobs.finishedAt} <= ${new Date(now.getTime() - ttl)}` : sql `FALSE`}
192
284
  OR (
193
285
  COALESCE(
@@ -202,7 +294,7 @@ export const make = (options) => Effect.gen(function* () {
202
294
  ELSE ${jobs.keep}->>'ageMs' END
203
295
  )::double precision) / 1000.0))
204
296
  )
205
- `);
297
+ `));
206
298
  }
207
299
  // Dead dedup rows: expired windows, or pointers at vanished jobs.
208
300
  yield* db.execute(sql `
@@ -210,7 +302,7 @@ export const make = (options) => Effect.gen(function* () {
210
302
  WHERE (${dedupe.windowExpiresAt} IS NOT NULL AND ${dedupe.windowExpiresAt} <= ${now})
211
303
  OR (${dedupe.windowExpiresAt} IS NULL AND NOT EXISTS (
212
304
  SELECT 1 FROM ${jobs} WHERE ${jobs.id} = ${dedupe.jobId}
213
- AND ${jobs.state} IN ('waiting', 'delayed', 'active')
305
+ AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
214
306
  ))
215
307
  `);
216
308
  }).pipe(Effect.catchCause((cause) => Effect.logWarning("effect-mq: history sweep failed", cause)), Effect.forever, Effect.forkScoped);
@@ -258,23 +350,23 @@ export const make = (options) => Effect.gen(function* () {
258
350
  if (keep === undefined)
259
351
  return;
260
352
  if (keep.ageMs !== undefined) {
261
- yield* tx.execute(sql `
262
- DELETE FROM ${jobs}
263
- WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
353
+ yield* tx.execute(purgeJobsWhere(sql `
354
+ ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
264
355
  AND ${jobs.finishedAt} <= ${new Date(now.getTime() - keep.ageMs)}
265
- `);
356
+ AND NOT ${owesCascades}
357
+ `));
266
358
  }
267
359
  if (keep.count !== undefined) {
268
- yield* tx.execute(sql `
269
- DELETE FROM ${jobs}
270
- WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
360
+ yield* tx.execute(purgeJobsWhere(sql `
361
+ ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
271
362
  AND ${jobs.id} NOT IN (
272
363
  SELECT ${jobs.id} FROM ${jobs}
273
364
  WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
274
365
  ORDER BY ${jobs.finishedAt} DESC, ${jobs.seq} DESC
275
366
  LIMIT ${keep.count}
276
367
  )
277
- `);
368
+ AND NOT ${owesCascades}
369
+ `));
278
370
  }
279
371
  });
280
372
  // Distinguish JobNotFound vs LockLost after a guarded UPDATE hit 0 rows.
@@ -303,7 +395,7 @@ export const make = (options) => Effect.gen(function* () {
303
395
  : sql `'j-' || ${seqExpr}::text`;
304
396
  const rows = rowsOf(yield* exec.execute(sql `
305
397
  INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
306
- attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
398
+ attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
307
399
  VALUES (${idExpr}, ${request.name}, ${request.queue}, ${state}, ${request.priority},
308
400
  ${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
309
401
  ${request.attemptsMax},
@@ -311,6 +403,7 @@ export const make = (options) => Effect.gen(function* () {
311
403
  ${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
312
404
  ${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
313
405
  ${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
406
+ ${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
314
407
  ${runAt}, ${now}${extraColumnValues(request)})
315
408
  ON CONFLICT (id) DO NOTHING
316
409
  RETURNING ${jobs.id} AS id
@@ -437,7 +530,7 @@ export const make = (options) => Effect.gen(function* () {
437
530
  // killed and safe to retry.
438
531
  Effect.retry({
439
532
  times: 3,
440
- while: (error) => String(error).includes("40P01") || String(error).includes("deadlock detected")
533
+ while: isDeadlockError
441
534
  }), Effect.mapError((error) => error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)));
442
535
  // A job leaving the pending states frees its pending-mode dedup row; live
443
536
  // throttle windows deliberately outlast the job.
@@ -449,26 +542,65 @@ export const make = (options) => Effect.gen(function* () {
449
542
  AND ${dedupe.jobId} = ${jobId}
450
543
  AND (${dedupe.windowExpiresAt} IS NULL OR ${dedupe.windowExpiresAt} <= ${now})
451
544
  `).pipe(Effect.asVoid);
545
+ // The outbox invariant: every operation that moves a job carrying a
546
+ // `parent` envelope INTO a terminal state appends its report here in the
547
+ // same transaction (see `JobStore.OutboxEntry`). `exit`/`failedReason`
548
+ // mirror the job row AFTER the transition; JSON.stringify drops
549
+ // undefined fields so absent values read back as undefined.
550
+ const appendOutbox = (exec, parent, outcome, exit, failedReason) => parent === null || parent === undefined
551
+ ? Effect.void
552
+ : exec.execute(sql `
553
+ INSERT INTO ${flowOutbox} (flow_name, parent_store_key, report)
554
+ VALUES (${parent.flowName}, ${parent.parentStoreKey}, ${JSON.stringify({
555
+ flowId: parent.flowId,
556
+ childKey: parent.childKey,
557
+ outcome,
558
+ exit,
559
+ failedReason: failedReason ?? undefined
560
+ })}::jsonb)
561
+ `).pipe(Effect.asVoid);
562
+ // Flip a flow's still-pending dependency rows to `cancelled` and NOT
563
+ // `cascaded` (the sweeper still owes the child stores real cancels).
564
+ // Embedded as a CTE body so the caller can count the marked rows into
565
+ // the parent's `cancelled` counter in the same statement.
566
+ const cancelPendingChildren = (flowId) => sql `
567
+ UPDATE ${flowChildren} SET status = 'cancelled', cascaded = FALSE
568
+ WHERE ${flowChildren.flowId} = ${flowId} AND ${flowChildren.status} = 'pending'
569
+ RETURNING 1
570
+ `;
452
571
  // Shared by cancel and cancelByDedupe.
453
572
  const cancelJob = (id) => db.transaction((tx) => Effect.gen(function* () {
454
573
  const now = yield* nowDate;
455
- // One guarded statement: waiting/delayed become terminal, active
456
- // gets the cancel-request flag; anything else is reported by state.
574
+ // Contract lock order (dependency rows first, parent second): mark
575
+ // a waiting-children parent's remaining pending rows cancelled.
576
+ // Pending rows exist only while the parent is `waiting-children`,
577
+ // so this is a no-op for every other state; a non-cancellable
578
+ // parent rolls the transaction back anyway.
579
+ const firstPass = rowsOf(yield* tx.execute(sql `
580
+ WITH marked AS (${cancelPendingChildren(id)})
581
+ SELECT count(*)::int AS marked FROM marked
582
+ `));
583
+ const preMarked = firstPass[0]?.marked ?? 0;
584
+ // One guarded statement: waiting/delayed/waiting-children become
585
+ // terminal, active gets the cancel-request flag; anything else is
586
+ // reported by state.
457
587
  const rows = rowsOf(yield* tx.execute(sql `
458
- UPDATE ${jobs} SET
459
- state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN 'cancelled' ELSE ${jobs.state} END,
460
- finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
461
- cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
462
- WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active')
463
- RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
464
- ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
465
- ${jobs.dedupeKey} AS "dedupeKey"
466
- `));
588
+ UPDATE ${jobs} SET
589
+ state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN 'cancelled' ELSE ${jobs.state} END,
590
+ finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
591
+ flow_pending = CASE WHEN ${jobs.state} = 'waiting-children' THEN 0 ELSE ${jobs.flowPending} END,
592
+ cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
593
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
594
+ RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
595
+ ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
596
+ ${jobs.dedupeKey} AS "dedupeKey", (${jobs.flowPending} IS NOT NULL) AS "hasFlow",
597
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
598
+ `));
467
599
  const row = rows[0];
468
600
  if (row === undefined) {
469
601
  const existing = rowsOf(yield* tx.execute(sql `
470
- SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
471
- `));
602
+ SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
603
+ `));
472
604
  const found = existing[0];
473
605
  if (found === undefined) {
474
606
  return yield* new JobStore.JobNotFoundError({ jobId: id });
@@ -476,11 +608,36 @@ export const make = (options) => Effect.gen(function* () {
476
608
  return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state });
477
609
  }
478
610
  if (row.state === "cancelled") {
611
+ if (row.hasFlow) {
612
+ // Re-run the marking now that the parent lock is held: the
613
+ // first UPDATE above races a concurrent FanOut — its
614
+ // uncommitted dependency-row INSERTs are invisible, while
615
+ // this UPDATE's own EPQ re-check can still see the parent as
616
+ // 'waiting-children' after the FanOut commits. Without this
617
+ // pass those rows would stay 'pending' forever, invisible to
618
+ // both sweep classes. Every row either pass marked lands in
619
+ // the `cancelled` counter.
620
+ yield* tx.execute(sql `
621
+ WITH marked AS (${cancelPendingChildren(id)})
622
+ UPDATE ${jobs} SET flow_cancelled = ${jobs.flowCancelled} + ${preMarked}::int
623
+ + (SELECT count(*)::int FROM marked)
624
+ WHERE ${jobs.id} = ${id}
625
+ `);
626
+ }
479
627
  yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined);
628
+ // A cancelled child reports upward through the outbox.
629
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason);
480
630
  yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
481
631
  yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now);
482
632
  }
483
- })).pipe(Effect.mapError((error) => error instanceof JobStore.JobNotFoundError ||
633
+ })).pipe(
634
+ // Rare lock-order inversion (this row-then-parent flip vs a
635
+ // fail-fast settle's parent-then-rows marking) surfaces as a
636
+ // Postgres deadlock (40P01); one side is killed and safe to retry.
637
+ Effect.retry({
638
+ times: 3,
639
+ while: isDeadlockError
640
+ }), Effect.mapError((error) => error instanceof JobStore.JobNotFoundError ||
484
641
  error instanceof JobStore.JobNotCancellableError ||
485
642
  error instanceof JobStore.JobStoreError
486
643
  ? error
@@ -565,10 +722,11 @@ export const make = (options) => Effect.gen(function* () {
565
722
  ${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
566
723
  ${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
567
724
  ${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
725
+ ${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
568
726
  ${new Date(now.getTime() + Math.max(0, request.delayMs))}, ${now}${extraColumnValues(request)})`);
569
727
  const rows = rowsOf(yield* db.execute(sql `
570
728
  INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
571
- attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
729
+ attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
572
730
  VALUES ${sql.join(values, sql `, `)}
573
731
  ON CONFLICT (id) DO NOTHING
574
732
  RETURNING ${jobs.id} AS id
@@ -610,6 +768,97 @@ export const make = (options) => Effect.gen(function* () {
610
768
  }
611
769
  return resolved;
612
770
  });
771
+ // The FanOut ack: land the manifest (flow columns + dependency rows) and
772
+ // park the parent, all lock-token-guarded in one transaction. A fan-out
773
+ // is a phase transition, not a completed run — `attempts_made` is not
774
+ // incremented; the ledger records `fanned-out` with no exit.
775
+ const ackFanOut = (id, token, outcome) => Effect.gen(function* () {
776
+ // Validate BEFORE any mutation, so a bad spec cannot leave the job
777
+ // half-acked (lock cleared, ledger written, still active).
778
+ if (outcome.children.some((child) => child.request.id === undefined)) {
779
+ return yield* new JobStore.JobStoreError({
780
+ message: "FanOut child specs require an explicit request.id"
781
+ });
782
+ }
783
+ const wakeQueue = yield* db.transaction((tx) => Effect.gen(function* () {
784
+ const now = yield* nowDate;
785
+ const rows = rowsOf(yield* tx.execute(sql `
786
+ UPDATE ${jobs} SET lock_token = NULL, lock_expires_at = NULL
787
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
788
+ RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
789
+ ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
790
+ ${jobs.cancelRequested} AS "cancelRequested", ${jobs.flowPending} AS "flowPending",
791
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
792
+ `));
793
+ const row = rows[0];
794
+ if (row === undefined) {
795
+ return yield* explainMiss(id);
796
+ }
797
+ yield* insertAttempt(tx, id, "fanned-out", row.processedAt, now, undefined);
798
+ let pending;
799
+ if (row.flowPending === null || row.flowPending === undefined) {
800
+ pending = outcome.children.length;
801
+ yield* tx.execute(sql `
802
+ UPDATE ${jobs} SET flow_fail_fast = ${outcome.failFast}, flow_pending = ${pending},
803
+ flow_completed = 0, flow_failed = 0, flow_cancelled = 0
804
+ WHERE ${jobs.id} = ${id}
805
+ `);
806
+ // Chunked multi-row VALUES, like enqueueMany's insertBatch.
807
+ for (let start = 0; start < outcome.children.length; start += 500) {
808
+ const chunk = outcome.children.slice(start, start + 500);
809
+ const values = chunk.map((child) => sql `(${id}, ${child.childKey}, ${child.request.name}, ${child.storeKey},
810
+ ${JSON.stringify(child.request)}::jsonb, 'pending', NULL, NULL, FALSE, ${now})`);
811
+ yield* tx.execute(sql `
812
+ INSERT INTO ${flowChildren} (flow_id, child_key, name, store_key, spec,
813
+ status, exit, failed_reason, cascaded, pending_since)
814
+ VALUES ${sql.join(values, sql `, `)}
815
+ `);
816
+ }
817
+ }
818
+ else {
819
+ // A manifest that was already present is kept untouched (double
820
+ // fan-out converges on the persisted children); the state
821
+ // transition follows the persisted pending count either way.
822
+ pending = Number(row.flowPending);
823
+ }
824
+ if (row.cancelRequested) {
825
+ // A cancel raced the fan-out: cancellation wins. The rows exist
826
+ // and get marked (into the `cancelled` counter), so the sweeper
827
+ // cascades (mostly no-op cancels for never-enqueued children).
828
+ yield* tx.execute(sql `
829
+ WITH marked AS (${cancelPendingChildren(id)})
830
+ UPDATE ${jobs} SET state = 'cancelled', finished_at = ${now},
831
+ cancel_requested = FALSE, flow_pending = 0,
832
+ flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
833
+ WHERE ${jobs.id} = ${id}
834
+ `);
835
+ yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined);
836
+ // A cancelled NESTED parent reports upward through the outbox.
837
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason);
838
+ yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
839
+ yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now);
840
+ return undefined;
841
+ }
842
+ if (pending > 0) {
843
+ yield* tx.execute(sql `
844
+ UPDATE ${jobs} SET state = 'waiting-children' WHERE ${jobs.id} = ${id}
845
+ `);
846
+ return undefined;
847
+ }
848
+ // Empty (or fully recorded) manifest: straight to runnable collect.
849
+ yield* tx.execute(sql `
850
+ UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
851
+ WHERE ${jobs.id} = ${id}
852
+ `);
853
+ return JobStore.QueueName(row.queue);
854
+ })).pipe(Effect.mapError((error) => error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
855
+ error instanceof JobStore.JobStoreError
856
+ ? error
857
+ : storeError("ack failed")(error)));
858
+ if (wakeQueue !== undefined) {
859
+ yield* wakeUp(wakeQueue);
860
+ }
861
+ });
613
862
  const store = {
614
863
  enqueue: enqueueOne,
615
864
  enqueueMany: (requests) => Effect.gen(function* () {
@@ -680,7 +929,10 @@ export const make = (options) => Effect.gen(function* () {
680
929
  ${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
681
930
  ${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
682
931
  ${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
683
- ${jobs.trace} AS "trace",
932
+ ${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
933
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
934
+ ${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
935
+ ${jobs.flowCancelled} AS "flowCancelled",
684
936
  ${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
685
937
  ${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
686
938
  ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
@@ -703,55 +955,64 @@ export const make = (options) => Effect.gen(function* () {
703
955
  return empty;
704
956
  }));
705
957
  }).pipe(Effect.mapError((error) => error instanceof JobStore.JobStoreError ? error : storeError("claim failed")(error))),
706
- ack: (id, token, outcome) => db.transaction((tx) => Effect.gen(function* () {
707
- const now = yield* nowDate;
708
- const update = outcome._tag === "Complete"
709
- ? sql `state = 'completed', cancel_requested = FALSE, exit = ${JSON.stringify(outcome.exit ?? null)}::jsonb, finished_at = ${now}`
710
- : outcome._tag === "Fail"
711
- ? sql `state = 'failed', cancel_requested = FALSE, exit = ${JSON.stringify(outcome.exit ?? null)}::jsonb, finished_at = ${now}`
712
- : outcome._tag === "Cancelled"
713
- ? sql `state = 'cancelled', cancel_requested = FALSE, finished_at = ${now}`
714
- // A cancel that raced this natural failure wins over revival
715
- // (mirrors release/recoverStalled).
716
- : sql `state = CASE WHEN ${jobs.cancelRequested} THEN 'cancelled'
958
+ ack: (id, token, outcome) => {
959
+ if (outcome._tag === "FanOut") {
960
+ return ackFanOut(id, token, outcome);
961
+ }
962
+ return db.transaction((tx) => Effect.gen(function* () {
963
+ const now = yield* nowDate;
964
+ const update = outcome._tag === "Complete"
965
+ ? sql `state = 'completed', cancel_requested = FALSE, exit = ${JSON.stringify(outcome.exit ?? null)}::jsonb, finished_at = ${now}`
966
+ : outcome._tag === "Fail"
967
+ ? sql `state = 'failed', cancel_requested = FALSE, exit = ${JSON.stringify(outcome.exit ?? null)}::jsonb, finished_at = ${now}`
968
+ : outcome._tag === "Cancelled"
969
+ ? sql `state = 'cancelled', cancel_requested = FALSE, finished_at = ${now}`
970
+ // A cancel that raced this natural failure wins over revival
971
+ // (mirrors release/recoverStalled).
972
+ : sql `state = CASE WHEN ${jobs.cancelRequested} THEN 'cancelled'
717
973
  ELSE ${outcome.delayMs > 0 ? "delayed" : "waiting"} END,
718
974
  finished_at = CASE WHEN ${jobs.cancelRequested} THEN ${now}::timestamptz ELSE NULL END,
719
975
  run_at = CASE WHEN ${jobs.cancelRequested} THEN ${jobs.runAt}
720
976
  ELSE ${new Date(now.getTime() + Math.max(0, outcome.delayMs))}::timestamptz END,
721
977
  seq = CASE WHEN ${jobs.cancelRequested} THEN ${jobs.seq} ELSE ${seqExpr} END,
722
978
  cancel_requested = FALSE`;
723
- const rows = rowsOf(yield* tx.execute(sql `
979
+ const rows = rowsOf(yield* tx.execute(sql `
724
980
  UPDATE ${jobs} SET ${update},
725
981
  attempts_made = ${jobs.attemptsMade} + 1, lock_token = NULL, lock_expires_at = NULL
726
982
  WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
727
983
  RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
728
984
  ${jobs.state} AS "state", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
729
- ${jobs.queue} AS "queue"
985
+ ${jobs.queue} AS "queue", ${jobs.parent} AS "parent", ${jobs.exit} AS "exit",
986
+ ${jobs.failedReason} AS "failedReason"
730
987
  `));
731
- const row = rows[0];
732
- if (row === undefined) {
733
- return yield* explainMiss(id);
734
- }
735
- const cancelledRetry = outcome._tag === "Retry" && row.state === "cancelled";
736
- const ledgerOutcome = outcome._tag === "Complete"
737
- ? "completed"
738
- : outcome._tag === "Fail"
739
- ? "failed"
740
- : outcome._tag === "Cancelled" || cancelledRetry
741
- ? "cancelled"
742
- : "retried";
743
- yield* insertAttempt(tx, id, ledgerOutcome, row.processedAt, now, outcome._tag === "Cancelled" || cancelledRetry ? undefined : outcome.exit);
744
- if (outcome._tag !== "Retry" || cancelledRetry) {
745
- yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
746
- yield* applyKeep(tx, row, now);
747
- }
748
- return outcome._tag === "Retry" && !cancelledRetry
749
- ? JobStore.QueueName(row.queue)
750
- : undefined;
751
- })).pipe(Effect.mapError((error) => error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
752
- error instanceof JobStore.JobStoreError
753
- ? error
754
- : storeError("ack failed")(error)), Effect.tap((queue) => queue !== undefined ? wakeUp(queue) : Effect.void), Effect.asVoid),
988
+ const row = rows[0];
989
+ if (row === undefined) {
990
+ return yield* explainMiss(id);
991
+ }
992
+ const cancelledRetry = outcome._tag === "Retry" && row.state === "cancelled";
993
+ const ledgerOutcome = outcome._tag === "Complete"
994
+ ? "completed"
995
+ : outcome._tag === "Fail"
996
+ ? "failed"
997
+ : outcome._tag === "Cancelled" || cancelledRetry
998
+ ? "cancelled"
999
+ : "retried";
1000
+ yield* insertAttempt(tx, id, ledgerOutcome, row.processedAt, now, outcome._tag === "Cancelled" || cancelledRetry ? undefined : outcome.exit);
1001
+ if (outcome._tag !== "Retry" || cancelledRetry) {
1002
+ // A terminal transition of an envelope-carrying child reports
1003
+ // upward through the outbox (exit/failedReason as persisted).
1004
+ yield* appendOutbox(tx, row.parent, ledgerOutcome === "retried" ? "cancelled" : ledgerOutcome, row.exit ?? undefined, row.failedReason);
1005
+ yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
1006
+ yield* applyKeep(tx, row, now);
1007
+ }
1008
+ return outcome._tag === "Retry" && !cancelledRetry
1009
+ ? JobStore.QueueName(row.queue)
1010
+ : undefined;
1011
+ })).pipe(Effect.mapError((error) => error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
1012
+ error instanceof JobStore.JobStoreError
1013
+ ? error
1014
+ : storeError("ack failed")(error)), Effect.tap((queue) => queue !== undefined ? wakeUp(queue) : Effect.void), Effect.asVoid);
1015
+ },
755
1016
  release: (id, token) => Effect.gen(function* () {
756
1017
  const released = yield* db.transaction((tx) => Effect.gen(function* () {
757
1018
  const now = yield* nowDate;
@@ -766,13 +1027,16 @@ export const make = (options) => Effect.gen(function* () {
766
1027
  WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
767
1028
  RETURNING ${jobs.id} AS id, (${jobs.state} = 'cancelled') AS cancelled,
768
1029
  ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
769
- ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue"
1030
+ ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1031
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
770
1032
  `));
771
1033
  const row = rows[0];
772
1034
  if (row === undefined)
773
1035
  return undefined;
774
1036
  if (row.cancelled) {
775
1037
  yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined);
1038
+ // A cancel honoured at release is a terminal transition too.
1039
+ yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason);
776
1040
  yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
777
1041
  yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now);
778
1042
  }
@@ -842,12 +1106,16 @@ export const make = (options) => Effect.gen(function* () {
842
1106
  cancel_requested = FALSE
843
1107
  WHERE ${jobs.state} = 'active' AND ${jobs.lockExpiresAt} <= ${now}::timestamptz
844
1108
  RETURNING ${jobs.id} AS "id", ${jobs.state} AS "state", ${jobs.processedAt} AS "processedAt",
845
- ${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey"
1109
+ ${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
1110
+ ${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
846
1111
  `));
847
1112
  const recovered = [];
848
1113
  for (const row of rows) {
849
1114
  yield* insertAttempt(tx, row.id, row.state === "cancelled" ? "cancelled" : "stalled", row.processedAt, now, undefined);
850
1115
  if (row.state === "cancelled" || row.state === "failed") {
1116
+ // Honoured cancels and stall exhaustion are terminal
1117
+ // transitions: envelope-carrying children report upward.
1118
+ yield* appendOutbox(tx, row.parent, row.state === "cancelled" ? "cancelled" : "failed", row.exit ?? undefined, row.failedReason);
851
1119
  yield* releaseDedupe(tx, row.name, row.dedupeKey, row.id, now);
852
1120
  }
853
1121
  if (row.state === "cancelled") {
@@ -905,7 +1173,11 @@ export const make = (options) => Effect.gen(function* () {
905
1173
  ${jobs.attemptsMax} AS "attemptsMax", ${jobs.attemptsMade} AS "attemptsMade",
906
1174
  ${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
907
1175
  ${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
908
- ${jobs.cancelRequested} AS "cancelRequested",
1176
+ ${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
1177
+ ${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
1178
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
1179
+ ${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
1180
+ ${jobs.flowCancelled} AS "flowCancelled",
909
1181
  ${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
910
1182
  ${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
911
1183
  ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
@@ -990,20 +1262,21 @@ export const make = (options) => Effect.gen(function* () {
990
1262
  `).pipe(Effect.mapError(storeError("pausedQueues failed")), Effect.map((result) => rowsOf(result).map((row) => JobStore.QueueName(row.queue)))),
991
1263
  upsertSchedule: (schedule) => db.execute(sql `
992
1264
  INSERT INTO ${schedules} (key, job_name, queue, cron, tz, every_ms, payload, metadata,
993
- priority, attempts_max, backoff, keep, timeout_ms, next_run_at)
1265
+ priority, attempts_max, backoff, keep, timeout_ms, group_name, next_run_at)
994
1266
  VALUES (${schedule.key}, ${schedule.jobName}, ${schedule.queue},
995
1267
  ${schedule.cron ?? null}, ${schedule.tz ?? null}, ${schedule.everyMs ?? null},
996
1268
  ${JSON.stringify(schedule.payload ?? null)}::jsonb, ${JSON.stringify(schedule.metadata)}::jsonb,
997
1269
  ${schedule.priority}, ${schedule.attemptsMax},
998
1270
  ${schedule.backoff === undefined ? null : JSON.stringify(schedule.backoff)}::jsonb,
999
1271
  ${schedule.keep === undefined ? null : JSON.stringify(schedule.keep)}::jsonb,
1000
- ${schedule.timeoutMs ?? null}, ${new Date(schedule.nextRunAt)})
1272
+ ${schedule.timeoutMs ?? null}, ${schedule.group ?? null}, ${new Date(schedule.nextRunAt)})
1001
1273
  ON CONFLICT (key) DO UPDATE SET
1002
1274
  job_name = EXCLUDED.job_name, queue = EXCLUDED.queue, cron = EXCLUDED.cron,
1003
1275
  tz = EXCLUDED.tz, every_ms = EXCLUDED.every_ms, payload = EXCLUDED.payload,
1004
1276
  metadata = EXCLUDED.metadata, priority = EXCLUDED.priority,
1005
1277
  attempts_max = EXCLUDED.attempts_max, backoff = EXCLUDED.backoff,
1006
1278
  keep = EXCLUDED.keep, timeout_ms = EXCLUDED.timeout_ms,
1279
+ group_name = EXCLUDED.group_name,
1007
1280
  next_run_at = CASE
1008
1281
  WHEN ${schedules.cron} IS NOT DISTINCT FROM EXCLUDED.cron
1009
1282
  AND ${schedules.tz} IS NOT DISTINCT FROM EXCLUDED.tz
@@ -1023,6 +1296,9 @@ export const make = (options) => Effect.gen(function* () {
1023
1296
  if (listOptions?.queue !== undefined) {
1024
1297
  conditions.push(sql `${schedules.queue} = ${listOptions.queue}`);
1025
1298
  }
1299
+ if (listOptions?.group !== undefined) {
1300
+ conditions.push(sql `${schedules.group} = ${listOptions.group}`);
1301
+ }
1026
1302
  const rows = rowsOf(yield* db.execute(sql `
1027
1303
  SELECT ${schedules.key} AS "key", ${schedules.jobName} AS "jobName",
1028
1304
  ${schedules.queue} AS "queue", ${schedules.cron} AS "cron", ${schedules.tz} AS "tz",
@@ -1030,7 +1306,7 @@ export const make = (options) => Effect.gen(function* () {
1030
1306
  ${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
1031
1307
  ${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
1032
1308
  ${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
1033
- ${schedules.nextRunAt} AS "nextRunAt"
1309
+ ${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
1034
1310
  FROM ${schedules}
1035
1311
  WHERE ${sql.join(conditions, sql ` AND `)}
1036
1312
  ORDER BY ${schedules.key}
@@ -1046,7 +1322,7 @@ export const make = (options) => Effect.gen(function* () {
1046
1322
  ${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
1047
1323
  ${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
1048
1324
  ${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
1049
- ${schedules.nextRunAt} AS "nextRunAt"
1325
+ ${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
1050
1326
  FROM ${schedules}
1051
1327
  WHERE ${schedules.nextRunAt} <= ${now}
1052
1328
  ORDER BY ${schedules.nextRunAt} ASC
@@ -1085,6 +1361,301 @@ export const make = (options) => Effect.gen(function* () {
1085
1361
  }
1086
1362
  return fired;
1087
1363
  }),
1364
+ recordChildResults: (reports) => Effect.gen(function* () {
1365
+ if (reports.length === 0) {
1366
+ const none = [];
1367
+ return none;
1368
+ }
1369
+ const batch = yield* db.transaction((tx) => Effect.gen(function* () {
1370
+ const now = yield* nowDate;
1371
+ const results = reports.map(() => ({ applied: false, parentSettled: false }));
1372
+ // Phase 1a — apply every row update (contract lock order:
1373
+ // dependency rows FIRST, parents second). Only a (flow, key)'s
1374
+ // first occurrence in the batch can apply — later duplicates
1375
+ // would find the row non-pending anyway, and UPDATE ... FROM
1376
+ // must never see two source rows for one target.
1377
+ const candidates = [];
1378
+ const seen = new Set();
1379
+ for (const [index, report] of reports.entries()) {
1380
+ const key = `${report.flowId}\u0000${report.childKey}`;
1381
+ if (seen.has(key))
1382
+ continue;
1383
+ seen.add(key);
1384
+ candidates.push({ index, report });
1385
+ }
1386
+ for (let start = 0; start < candidates.length; start += 200) {
1387
+ const chunk = candidates.slice(start, start + 200);
1388
+ const values = chunk.map(({ index, report }) => sql `(${index}::int, ${report.flowId}::text, ${report.childKey}::text,
1389
+ ${report.outcome}::text,
1390
+ ${report.exit === undefined ? null : JSON.stringify(report.exit)}::jsonb,
1391
+ ${report.failedReason ?? null}::text)`);
1392
+ const appliedRows = rowsOf(yield* tx.execute(sql `
1393
+ UPDATE ${flowChildren} SET status = v.outcome, exit = v.exit,
1394
+ failed_reason = v.failed_reason, cascaded = TRUE
1395
+ FROM (VALUES ${sql.join(values, sql `, `)})
1396
+ AS v(ord, flow_id, child_key, outcome, exit, failed_reason)
1397
+ WHERE ${flowChildren.flowId} = v.flow_id AND ${flowChildren.childKey} = v.child_key
1398
+ AND ${flowChildren.status} = 'pending'
1399
+ RETURNING v.ord AS "ord"
1400
+ `));
1401
+ for (const row of appliedRows) {
1402
+ const result = results[Number(row.ord)];
1403
+ if (result !== undefined) {
1404
+ result.applied = true;
1405
+ }
1406
+ }
1407
+ }
1408
+ const touched = new Map();
1409
+ for (const [index, report] of reports.entries()) {
1410
+ if (results[index]?.applied !== true)
1411
+ continue;
1412
+ const touch = touched.get(report.flowId) ?? {
1413
+ appliedCount: 0,
1414
+ completed: 0,
1415
+ failed: 0,
1416
+ cancelled: 0,
1417
+ firstAppliedFailed: undefined,
1418
+ lastApplied: index
1419
+ };
1420
+ touch.appliedCount += 1;
1421
+ touch.lastApplied = index;
1422
+ if (report.outcome === "completed")
1423
+ touch.completed += 1;
1424
+ if (report.outcome === "cancelled")
1425
+ touch.cancelled += 1;
1426
+ if (report.outcome === "failed") {
1427
+ touch.failed += 1;
1428
+ touch.firstAppliedFailed ??= index;
1429
+ }
1430
+ touched.set(report.flowId, touch);
1431
+ }
1432
+ // Phase 1b + 2 — per touched flow (sorted, so concurrent
1433
+ // batches take parent locks in one order): move the applied
1434
+ // children from `pending` to their outcome counters, then make
1435
+ // at most one settle decision. Fail-fast wins the tie.
1436
+ const wakeQueues = [];
1437
+ for (const flowId of [...touched.keys()].toSorted()) {
1438
+ const touch = touched.get(flowId);
1439
+ if (touch === undefined)
1440
+ continue;
1441
+ const parents = rowsOf(yield* tx.execute(sql `
1442
+ UPDATE ${jobs} SET
1443
+ flow_pending = GREATEST(${jobs.flowPending} - ${touch.appliedCount}::int, 0),
1444
+ flow_completed = ${jobs.flowCompleted} + ${touch.completed}::int,
1445
+ flow_failed = ${jobs.flowFailed} + ${touch.failed}::int,
1446
+ flow_cancelled = ${jobs.flowCancelled} + ${touch.cancelled}::int
1447
+ WHERE ${jobs.id} = ${flowId} AND ${jobs.flowPending} IS NOT NULL
1448
+ RETURNING ${jobs.state} AS "state", ${jobs.flowPending} AS "flowPending",
1449
+ ${jobs.flowFailFast} AS "flowFailFast", ${jobs.processedAt} AS "processedAt",
1450
+ ${jobs.name} AS "name", ${jobs.keep} AS "keep",
1451
+ ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
1452
+ ${jobs.parent} AS "parent"
1453
+ `));
1454
+ const parent = parents[0];
1455
+ if (parent === undefined || parent.state !== "waiting-children")
1456
+ continue;
1457
+ const failedIndex = parent.flowFailFast === true ? touch.firstAppliedFailed : undefined;
1458
+ const failedReport = failedIndex !== undefined ? reports[failedIndex] : undefined;
1459
+ if (failedIndex !== undefined && failedReport !== undefined) {
1460
+ // The first applied failure settles the parent terminally
1461
+ // (store-side, like stall exhaustion: failedReason, no
1462
+ // exit) and marks the remaining rows in the same
1463
+ // transaction. A nested parent's own report goes to the
1464
+ // outbox here — this settle IS its terminal transition.
1465
+ const reason = `effect-mq: flow child "${failedReport.childKey}" failed`;
1466
+ yield* tx.execute(sql `
1467
+ WITH marked AS (${cancelPendingChildren(flowId)})
1468
+ UPDATE ${jobs} SET state = 'failed', finished_at = ${now},
1469
+ failed_reason = ${reason}, cancel_requested = FALSE, flow_pending = 0,
1470
+ flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
1471
+ WHERE ${jobs.id} = ${flowId}
1472
+ `);
1473
+ yield* insertAttempt(tx, flowId, "failed", parent.processedAt, now, undefined);
1474
+ yield* appendOutbox(tx, parent.parent, "failed", undefined, reason);
1475
+ yield* releaseDedupe(tx, parent.name, parent.dedupeKey, flowId, now);
1476
+ yield* applyKeep(tx, { name: parent.name, state: "failed", keep: parent.keep }, now);
1477
+ const decided = results[failedIndex];
1478
+ if (decided !== undefined) {
1479
+ decided.parentSettled = true;
1480
+ }
1481
+ continue;
1482
+ }
1483
+ if (Number(parent.flowPending) === 0) {
1484
+ // All children settled: the parent resumes runnable, phase
1485
+ // collect, settled at the flow's last applied report.
1486
+ yield* tx.execute(sql `
1487
+ UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
1488
+ WHERE ${jobs.id} = ${flowId}
1489
+ `);
1490
+ wakeQueues.push(JobStore.QueueName(parent.queue));
1491
+ const decided = results[touch.lastApplied];
1492
+ if (decided !== undefined) {
1493
+ decided.parentSettled = true;
1494
+ }
1495
+ }
1496
+ }
1497
+ return { results, wakeQueues };
1498
+ })).pipe(
1499
+ // Rare lock-order inversion (a concurrent cancel/settle marking
1500
+ // rows) surfaces as a Postgres deadlock (40P01); the killed side
1501
+ // is safe to retry — the row-state guard keeps it idempotent.
1502
+ Effect.retry({
1503
+ times: 3,
1504
+ while: isDeadlockError
1505
+ }), Effect.mapError((error) => error instanceof JobStore.JobStoreError ? error : storeError("recordChildResults failed")(error)));
1506
+ for (const queue of batch.wakeQueues) {
1507
+ yield* wakeUp(queue);
1508
+ }
1509
+ return batch.results;
1510
+ }),
1511
+ peekOutbox: (peekOptions) => Effect.gen(function* () {
1512
+ const limit = Math.max(0, peekOptions.limit);
1513
+ if (limit === 0) {
1514
+ const none = [];
1515
+ return none;
1516
+ }
1517
+ // `after` pages past a previously returned id (exclusive), whether
1518
+ // or not that entry still exists. Anything that is not a canonical
1519
+ // id this store issued is treated as unset.
1520
+ const after = peekOptions.after !== undefined && CANONICAL_BIGSERIAL.test(peekOptions.after)
1521
+ ? peekOptions.after
1522
+ : undefined;
1523
+ const rows = rowsOf(yield* db.execute(sql `
1524
+ SELECT ${flowOutbox.id}::text AS "id", ${flowOutbox.flowName} AS "flowName",
1525
+ ${flowOutbox.parentStoreKey} AS "parentStoreKey", ${flowOutbox.report} AS "report"
1526
+ FROM ${flowOutbox}
1527
+ ${after === undefined ? sql `` : sql `WHERE ${flowOutbox.id} > ${after}::bigint`}
1528
+ ORDER BY ${flowOutbox.id} ASC
1529
+ LIMIT ${limit}
1530
+ `).pipe(Effect.mapError(storeError("peekOutbox failed"))));
1531
+ return rows.map((row) => ({
1532
+ id: row.id,
1533
+ flowName: row.flowName,
1534
+ parentStoreKey: row.parentStoreKey,
1535
+ report: row.report
1536
+ }));
1537
+ }),
1538
+ deleteOutbox: (ids) => Effect.suspend(() => {
1539
+ // Ids are opaque strings to callers; only CANONICAL ids this store
1540
+ // could have issued can match. The strictness matters twice: a
1541
+ // foreign/garbled id must not blow up the ::bigint cast, and a
1542
+ // non-canonical spelling ("007") must stay an unknown-id no-op
1543
+ // rather than cast to 7 and delete a live entry.
1544
+ const numeric = ids.filter((id) => CANONICAL_BIGSERIAL.test(id));
1545
+ if (numeric.length === 0)
1546
+ return Effect.void;
1547
+ return db.execute(sql `
1548
+ DELETE FROM ${flowOutbox}
1549
+ WHERE ${flowOutbox.id} = ANY(${sql.param(numeric)}::bigint[])
1550
+ `).pipe(Effect.mapError(storeError("deleteOutbox failed")), Effect.asVoid);
1551
+ }),
1552
+ listChildResults: (flowId, listOptions) => Effect.gen(function* () {
1553
+ const limit = Math.max(1, listOptions?.limit ?? 1000);
1554
+ const cursor = listOptions?.cursor;
1555
+ // `spec->>'id'` instead of the whole spec: at 10k children the full
1556
+ // payloads would transfer on every collect.
1557
+ const rows = rowsOf(yield* db.execute(sql `
1558
+ SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
1559
+ ${flowChildren.name} AS "name", ${flowChildren.storeKey} AS "storeKey",
1560
+ ${flowChildren.spec}->>'id' AS "childJobId", ${flowChildren.status} AS "status",
1561
+ ${flowChildren.exit} AS "exit", ${flowChildren.failedReason} AS "failedReason",
1562
+ ${flowChildren.cascaded} AS "cascaded"
1563
+ FROM ${flowChildren}
1564
+ WHERE ${flowChildren.flowId} = ${flowId}
1565
+ ${cursor === undefined ? sql `` : sql `AND ${flowChildren.childKey} > ${cursor}`}
1566
+ ORDER BY ${flowChildren.childKey} ASC
1567
+ LIMIT ${limit + 1}
1568
+ `).pipe(Effect.mapError(storeError("listChildResults failed"))));
1569
+ const page = rows.slice(0, limit);
1570
+ const items = page.map(toFlowChildRecord);
1571
+ const last = items[items.length - 1];
1572
+ return {
1573
+ items,
1574
+ cursor: rows.length > limit && last !== undefined ? last.childKey : undefined
1575
+ };
1576
+ }),
1577
+ flowSweepWork: (sweepOptions) => Effect.gen(function* () {
1578
+ const now = yield* nowDate;
1579
+ const limit = Math.max(1, sweepOptions.limit ?? 1000);
1580
+ const threshold = new Date(now.getTime() - sweepOptions.pendingAgeMs);
1581
+ // Reconcile: pending rows past the eligibility threshold whose
1582
+ // parent is still parked (a settled flow never re-drives work).
1583
+ // Returning a row re-arms `pending_since` in the same statement, so
1584
+ // a full page rotates across sweeps instead of pinning its head.
1585
+ // SKIP LOCKED: rows a concurrent report/settle holds are its
1586
+ // business, and never waiting means this statement cannot deadlock.
1587
+ // The raw column names are safe: only table names vary across
1588
+ // factory instances. RETURNING order is unspecified — sorted below.
1589
+ const reconcileRows = rowsOf(yield* db.execute(sql `
1590
+ WITH due AS (
1591
+ SELECT c.flow_id, c.child_key
1592
+ FROM ${flowChildren} c
1593
+ JOIN ${jobs} j ON j.id = c.flow_id
1594
+ WHERE c.status = 'pending' AND c.pending_since <= ${threshold}
1595
+ AND j.state = 'waiting-children'
1596
+ ORDER BY c.flow_id, c.child_key
1597
+ LIMIT ${limit}
1598
+ FOR UPDATE OF c SKIP LOCKED
1599
+ )
1600
+ UPDATE ${flowChildren} SET pending_since = ${now}
1601
+ FROM due
1602
+ WHERE ${flowChildren.flowId} = due.flow_id AND ${flowChildren.childKey} = due.child_key
1603
+ RETURNING ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
1604
+ ${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
1605
+ `).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
1606
+ .toSorted((a, b) => a.flowId !== b.flowId
1607
+ ? (a.flowId < b.flowId ? -1 : 1)
1608
+ : a.childKey < b.childKey
1609
+ ? -1
1610
+ : a.childKey > b.childKey
1611
+ ? 1
1612
+ : 0);
1613
+ // Cascade: cancelled rows whose cancel has not been delivered into
1614
+ // the child's store yet (any parent state).
1615
+ const cascadeRows = rowsOf(yield* db.execute(sql `
1616
+ SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
1617
+ ${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
1618
+ FROM ${flowChildren}
1619
+ WHERE ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
1620
+ ORDER BY ${flowChildren.flowId}, ${flowChildren.childKey}
1621
+ LIMIT ${limit}
1622
+ `).pipe(Effect.mapError(storeError("flowSweepWork failed"))));
1623
+ const reconcile = [];
1624
+ for (const row of reconcileRows) {
1625
+ const flowId = JobId(row.flowId);
1626
+ let group = reconcile[reconcile.length - 1];
1627
+ if (group === undefined || group.flowId !== flowId) {
1628
+ group = { flowId, children: [] };
1629
+ reconcile.push(group);
1630
+ }
1631
+ group.children.push({ childKey: row.childKey, storeKey: row.storeKey, request: row.spec });
1632
+ }
1633
+ const cascade = [];
1634
+ for (const row of cascadeRows) {
1635
+ const flowId = JobId(row.flowId);
1636
+ let group = cascade[cascade.length - 1];
1637
+ if (group === undefined || group.flowId !== flowId) {
1638
+ group = { flowId, children: [] };
1639
+ cascade.push(group);
1640
+ }
1641
+ group.children.push({
1642
+ childKey: row.childKey,
1643
+ storeKey: row.storeKey,
1644
+ // SAFETY: the FanOut ack validated every spec id before
1645
+ // persisting it.
1646
+ childJobId: row.spec.id
1647
+ });
1648
+ }
1649
+ const work = { reconcile, cascade };
1650
+ return work;
1651
+ }),
1652
+ markChildrenCascaded: (flowId, childKeys) => childKeys.length === 0
1653
+ ? Effect.void
1654
+ : db.execute(sql `
1655
+ UPDATE ${flowChildren} SET cascaded = TRUE
1656
+ WHERE ${flowChildren.flowId} = ${flowId}
1657
+ AND ${flowChildren.childKey} = ANY(${sql.param([...childKeys])})
1658
+ `).pipe(Effect.mapError(storeError("markChildrenCascaded failed")), Effect.asVoid),
1088
1659
  counts: (queue) => db.execute(sql `
1089
1660
  SELECT ${jobs.state} AS "state", count(*)::int AS "count" FROM ${jobs}
1090
1661
  ${queue === undefined ? sql `` : sql `WHERE ${jobs.queue} = ${queue}`}
@@ -1095,6 +1666,7 @@ export const make = (options) => Effect.gen(function* () {
1095
1666
  waiting: 0,
1096
1667
  delayed: 0,
1097
1668
  active: 0,
1669
+ "waiting-children": 0,
1098
1670
  completed: 0,
1099
1671
  failed: 0,
1100
1672
  cancelled: 0
@@ -1103,10 +1675,19 @@ export const make = (options) => Effect.gen(function* () {
1103
1675
  counts[row.state] = row.count;
1104
1676
  return counts;
1105
1677
  })),
1106
- remove: (id) => db.execute(sql `
1107
- DELETE FROM ${jobs}
1108
- WHERE ${jobs.id} = ${id} AND ${jobs.state} <> 'active'
1109
- RETURNING ${jobs.id} AS id
1678
+ remove: (id) =>
1679
+ // The purge CTE takes a removed flow parent's dependency rows with it
1680
+ // in the same statement.
1681
+ db.execute(sql `
1682
+ WITH deleted AS (
1683
+ DELETE FROM ${jobs}
1684
+ WHERE ${jobs.id} = ${id} AND ${jobs.state} NOT IN ('active', 'waiting-children')
1685
+ RETURNING ${jobs.id} AS id
1686
+ ), purged AS (
1687
+ DELETE FROM ${flowChildren}
1688
+ WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
1689
+ )
1690
+ SELECT id FROM deleted
1110
1691
  `).pipe(Effect.mapError(storeError("remove failed")), Effect.map((result) => rowsOf(result).length > 0))
1111
1692
  };
1112
1693
  return store;