effect-mq 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/README.md +85 -17
  2. package/dist/Flow.d.ts +381 -0
  3. package/dist/Flow.d.ts.map +1 -0
  4. package/dist/Flow.js +340 -0
  5. package/dist/Flow.js.map +1 -0
  6. package/dist/Job.d.ts +31 -6
  7. package/dist/Job.d.ts.map +1 -1
  8. package/dist/Job.js +16 -2
  9. package/dist/Job.js.map +1 -1
  10. package/dist/JobStore.d.ts +353 -13
  11. package/dist/JobStore.d.ts.map +1 -1
  12. package/dist/JobStore.js +10 -0
  13. package/dist/JobStore.js.map +1 -1
  14. package/dist/MemoryJobStore.d.ts.map +1 -1
  15. package/dist/MemoryJobStore.js +361 -21
  16. package/dist/MemoryJobStore.js.map +1 -1
  17. package/dist/Metrics.d.ts +31 -0
  18. package/dist/Metrics.d.ts.map +1 -1
  19. package/dist/Metrics.js +39 -0
  20. package/dist/Metrics.js.map +1 -1
  21. package/dist/Worker.d.ts +120 -11
  22. package/dist/Worker.d.ts.map +1 -1
  23. package/dist/Worker.js +452 -26
  24. package/dist/Worker.js.map +1 -1
  25. package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
  26. package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
  27. package/dist/drizzle-postgres/DrizzleJobStore.js +678 -80
  28. package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
  29. package/dist/drizzle-postgres/schema.d.ts +293 -3
  30. package/dist/drizzle-postgres/schema.d.ts.map +1 -1
  31. package/dist/drizzle-postgres/schema.js +66 -1
  32. package/dist/drizzle-postgres/schema.js.map +1 -1
  33. package/dist/index.d.ts +7 -0
  34. package/dist/index.d.ts.map +1 -1
  35. package/dist/index.js +7 -0
  36. package/dist/index.js.map +1 -1
  37. package/dist/redis/RedisJobStore.d.ts +53 -0
  38. package/dist/redis/RedisJobStore.d.ts.map +1 -1
  39. package/dist/redis/RedisJobStore.js +402 -50
  40. package/dist/redis/RedisJobStore.js.map +1 -1
  41. package/dist/redis/scripts.d.ts +213 -35
  42. package/dist/redis/scripts.d.ts.map +1 -1
  43. package/dist/redis/scripts.js +689 -73
  44. package/dist/redis/scripts.js.map +1 -1
  45. package/dist/testing/conformance.d.ts +6 -0
  46. package/dist/testing/conformance.d.ts.map +1 -1
  47. package/dist/testing/conformance.js +855 -12
  48. package/dist/testing/conformance.js.map +1 -1
  49. package/package.json +1 -1
  50. package/src/Flow.ts +778 -0
  51. package/src/Job.ts +35 -11
  52. package/src/JobStore.ts +377 -12
  53. package/src/MemoryJobStore.ts +396 -25
  54. package/src/Metrics.ts +43 -0
  55. package/src/Worker.ts +726 -37
  56. package/src/drizzle-postgres/DrizzleJobStore.ts +844 -81
  57. package/src/drizzle-postgres/schema.ts +92 -0
  58. package/src/index.ts +8 -0
  59. package/src/redis/RedisJobStore.ts +540 -39
  60. package/src/redis/scripts.ts +751 -78
  61. package/src/testing/conformance.ts +1088 -12
@@ -12,10 +12,22 @@
12
12
  *
13
13
  * @since 0.2.0
14
14
  */
15
- import { Clock, Deferred, Duration, Effect, Exit, Layer, Option, Queue, Schedule } from "effect";
15
+ import { Clock, Data, Deferred, Duration, Effect, Exit, Layer, Option, Queue, Schedule } from "effect";
16
16
  import { Redis } from "effect/unstable/persistence";
17
17
  import * as JobStore from "../JobStore.js";
18
18
  import * as scripts from "./scripts.js";
19
+ /**
20
+ * A `list` query routed to a list index this store is configured not to
21
+ * maintain (`RedisJobStoreOptions.indexes`). Delivered as a defect, not a
22
+ * typed failure: the configuration said "we never list this way", so the
23
+ * query contradicting it is a programming mistake, matching the library's
24
+ * die-on-config-mistake idiom.
25
+ *
26
+ * @since 0.7.0
27
+ */
28
+ export class ListIndexDisabledError extends Data.TaggedError("ListIndexDisabledError") {
29
+ }
30
+ const TERMINAL_STATES = new Set(["completed", "failed", "cancelled"]);
19
31
  const storeError = (message) => (cause) => new JobStore.JobStoreError({ message, cause });
20
32
  /** Fold a Lua `HGETALL` reply (flat `[field, value, ...]`) into a map. */
21
33
  const foldPairs = (flat) => {
@@ -56,6 +68,18 @@ const toRecord = (hash) => ({
56
68
  cancelRequested: hash.get("cancelRequested") === "1",
57
69
  dedupeKey: optionalString(hash.get("dedupeKey")),
58
70
  trace: optionalJson(hash.get("trace")),
71
+ parent: optionalJson(hash.get("parent")),
72
+ // Manifest presence IS the phase marker: hashes written before flows (or
73
+ // parents that never fanned out) simply lack the flow fields.
74
+ flow: optionalString(hash.get("flowPending")) === undefined
75
+ ? undefined
76
+ : {
77
+ failFast: hash.get("flowFailFast") === "1",
78
+ pending: Number(hash.get("flowPending")),
79
+ completed: Number(hash.get("flowCompleted") ?? 0),
80
+ failed: Number(hash.get("flowFailed") ?? 0),
81
+ cancelled: Number(hash.get("flowCancelled") ?? 0)
82
+ },
59
83
  runAt: Number(hash.get("runAt") ?? 0),
60
84
  enqueuedAt: Number(hash.get("enqueuedAt") ?? 0),
61
85
  processedAt: optionalNumber(hash.get("processedAt")),
@@ -82,14 +106,57 @@ const toSchedule = (hash) => ({
82
106
  });
83
107
  // cjson encodes an empty Lua table as {}, not [] — normalize.
84
108
  const asArray = (value) => Array.isArray(value) ? value : [];
109
+ /**
110
+ * Decode one outbox zset member: `<seq>\0<json>` where the json carries the
111
+ * verbatim parent envelope plus the terminal outcome. The full member string
112
+ * is the opaque entry id (deleteOutbox is then a plain ZREM).
113
+ */
114
+ const toOutboxEntry = (member) => {
115
+ const sep = member.indexOf("\u0000");
116
+ const body = JSON.parse(member.slice(sep + 1));
117
+ return {
118
+ id: member,
119
+ flowName: body.parent.flowName,
120
+ parentStoreKey: body.parent.parentStoreKey,
121
+ report: {
122
+ flowId: body.parent.flowId,
123
+ childKey: body.parent.childKey,
124
+ outcome: body.outcome,
125
+ // The exit key is omitted entirely when absent (ledger convention),
126
+ // so a legitimate encoded null exit survives the round trip.
127
+ exit: Object.hasOwn(body, "exit") ? body.exit : undefined,
128
+ failedReason: body.failedReason
129
+ }
130
+ };
131
+ };
85
132
  const JOB_STATES = [
86
133
  "waiting",
87
134
  "delayed",
88
135
  "active",
136
+ "waiting-children",
89
137
  "completed",
90
138
  "failed",
91
139
  "cancelled"
92
140
  ];
141
+ /**
142
+ * Fold one positional dependency-row tuple into a `FlowChildRecord`. The
143
+ * order must stay in lockstep with the HMGET field list in the
144
+ * `listChildResults` script:
145
+ * childKey, storeKey, childJobId, name, status, exit, failedReason, cascaded.
146
+ */
147
+ const toChildRecord = (flowId, row) => ({
148
+ flowId,
149
+ childKey: row[0] ?? "",
150
+ storeKey: row[1] ?? "",
151
+ childJobId: JobStore.JobId(row[2] ?? ""),
152
+ name: row[3] ?? "",
153
+ // SAFETY: the status field is only ever written with FlowChildRecord
154
+ // status members ("pending" at insert, a report/settle outcome after).
155
+ status: (row[4] ?? "pending"),
156
+ exit: optionalJson(row[5]),
157
+ failedReason: optionalString(row[6]),
158
+ cascaded: row[7] === "1"
159
+ });
93
160
  /**
94
161
  * Build a `RedisJobStore` service. Needs the `Redis` service and a `Scope`
95
162
  * (the wake-up subscription and the optional history sweeper live in it).
@@ -100,28 +167,105 @@ export const make = (options) => Effect.gen(function* () {
100
167
  const redis = yield* Redis.Redis;
101
168
  const prefix = options?.prefix ?? "effect-mq";
102
169
  const wakeChannel = `${prefix}:wake`;
103
- const evalEnqueue = redis.eval(scripts.enqueue);
104
- const evalClaim = redis.eval(scripts.claim);
105
- const evalAck = redis.eval(scripts.ack);
106
- const evalRelease = redis.eval(scripts.release);
107
- const evalExtendLocks = redis.eval(scripts.extendLocks);
108
- const evalRecoverStalled = redis.eval(scripts.recoverStalled);
109
- const evalGetJob = redis.eval(scripts.getJob);
110
- const evalList = redis.eval(scripts.list);
111
- const evalCounts = redis.eval(scripts.counts);
112
- const evalRemove = redis.eval(scripts.remove);
113
- const evalRetry = redis.eval(scripts.retry);
114
- const evalCancel = redis.eval(scripts.cancel);
115
- const evalPromote = redis.eval(scripts.promote);
116
- const evalUpsertSchedule = redis.eval(scripts.upsertSchedule);
117
- const evalRemoveSchedule = redis.eval(scripts.removeSchedule);
118
- const evalListSchedules = redis.eval(scripts.listSchedules);
119
- const evalDueSchedules = redis.eval(scripts.dueSchedules);
120
- const evalAdvanceSchedule = redis.eval(scripts.advanceSchedule);
121
- const evalTickSchedule = redis.eval(scripts.tickSchedule);
122
- const evalEnqueueMany = redis.eval(scripts.enqueueMany);
123
- const evalSweepState = redis.eval(scripts.sweepState);
124
- const evalSweepDedupes = redis.eval(scripts.sweepDedupes);
170
+ const indexOptions = options?.indexes;
171
+ const indexes = indexOptions === false
172
+ ? { name: false, queue: false }
173
+ : { name: indexOptions?.name ?? true, queue: indexOptions?.queue ?? true };
174
+ const HELPERS = scripts.helpers(indexes);
175
+ const evalEnqueue = redis.eval(scripts.enqueue(HELPERS));
176
+ const evalClaim = redis.eval(scripts.claim(HELPERS));
177
+ const evalAck = redis.eval(scripts.ack(HELPERS));
178
+ const evalRelease = redis.eval(scripts.release(HELPERS));
179
+ const evalExtendLocks = redis.eval(scripts.extendLocks(HELPERS));
180
+ const evalRecoverStalled = redis.eval(scripts.recoverStalled(HELPERS));
181
+ const evalGetJob = redis.eval(scripts.getJob(HELPERS));
182
+ const evalList = redis.eval(scripts.list(HELPERS));
183
+ const evalIndexMembers = redis.eval(scripts.indexMembers(HELPERS));
184
+ const evalIndexTailPage = redis.eval(scripts.indexTailPage(HELPERS));
185
+ const evalCounts = redis.eval(scripts.counts(HELPERS));
186
+ const evalRemove = redis.eval(scripts.remove(HELPERS));
187
+ const evalRetry = redis.eval(scripts.retry(HELPERS));
188
+ const evalCancel = redis.eval(scripts.cancel(HELPERS));
189
+ const evalPromote = redis.eval(scripts.promote(HELPERS));
190
+ const evalUpsertSchedule = redis.eval(scripts.upsertSchedule(HELPERS));
191
+ const evalRemoveSchedule = redis.eval(scripts.removeSchedule(HELPERS));
192
+ const evalListSchedules = redis.eval(scripts.listSchedules(HELPERS));
193
+ const evalDueSchedules = redis.eval(scripts.dueSchedules(HELPERS));
194
+ const evalAdvanceSchedule = redis.eval(scripts.advanceSchedule(HELPERS));
195
+ const evalTickSchedule = redis.eval(scripts.tickSchedule(HELPERS));
196
+ const evalEnqueueMany = redis.eval(scripts.enqueueMany(HELPERS));
197
+ const evalSweepState = redis.eval(scripts.sweepState(HELPERS));
198
+ const evalSweepDedupes = redis.eval(scripts.sweepDedupes(HELPERS));
199
+ const evalFanOut = redis.eval(scripts.fanOut(HELPERS));
200
+ const evalRecordChildResults = redis.eval(scripts.recordChildResults(HELPERS));
201
+ const evalListChildResults = redis.eval(scripts.listChildResults(HELPERS));
202
+ const evalFlowSweepWork = redis.eval(scripts.flowSweepWork(HELPERS));
203
+ const evalMarkChildrenCascaded = redis.eval(scripts.markChildrenCascaded(HELPERS));
204
+ // List-index reconcile, once per boot per index. All work is driver-paged
205
+ // — never one giant Lua call — so the single-threaded server is never
206
+ // held. There is no build lock: concurrent boots duplicate idempotent
207
+ // ZADDs, a crash before the marker stamp makes the next boot redo the
208
+ // work, and rows inserted meanwhile are indexed live by insertJobRow.
209
+ //
210
+ // - enabled, no marker: full rebuild via ZSCAN over `all` (cursor-based
211
+ // and linear — immune to score ties and rank shifts; every member
212
+ // present for the whole scan is guaranteed returned). Rows deleted
213
+ // mid-scan leave at most stale members the read path self-heals.
214
+ // - enabled, marker present: heal the tail. Index writes are per-process,
215
+ // so writers without them (an older version mid-rolling-deploy, or a
216
+ // misconfigured indexes-off store) may have inserted unindexed rows
217
+ // AFTER the marker landed. Re-indexing everything enqueued since the
218
+ // marker minus a 60s margin closes that window: the last enabled boot
219
+ // after such writers stop covers everything they wrote before it.
220
+ // - disabled: delete ONLY the marker (a later re-enable must not trust
221
+ // stale zsets). The zsets stay — an enabled sibling may be reading
222
+ // them, and this store cannot know.
223
+ //
224
+ // Both paths re-stamp the marker with this boot's start time. Init-time
225
+ // infra failures die — the store never starts half-configured.
226
+ yield* Effect.gen(function* () {
227
+ const bootAt = yield* Clock.currentTimeMillis;
228
+ for (const kind of ["name", "queue"]) {
229
+ const marker = `${prefix}:index:${kind}:ready`;
230
+ if (!indexes[kind]) {
231
+ yield* redis.send("DEL", marker);
232
+ continue;
233
+ }
234
+ // SAFETY: GET always replies with a bulk string or null.
235
+ const stamped = (yield* redis.send("GET", marker));
236
+ const markerAt = stamped === null || stamped === "" ? Number.NaN : Number(stamped);
237
+ if (Number.isNaN(markerAt)) {
238
+ let cursor = "0";
239
+ do {
240
+ const reply = yield* redis.send("ZSCAN", `${prefix}:all`, cursor, "COUNT", "500");
241
+ // SAFETY: ZSCAN always replies [nextCursor, member/score pairs].
242
+ const [next, flat] = reply;
243
+ cursor = next;
244
+ const ids = [];
245
+ for (let i = 0; i < flat.length; i += 2) {
246
+ const id = flat[i];
247
+ if (id !== undefined)
248
+ ids.push(id);
249
+ }
250
+ // COUNT is only a hint — chunk what actually came back.
251
+ for (let start = 0; start < ids.length; start += 500) {
252
+ yield* evalIndexMembers(prefix, kind, JSON.stringify(ids.slice(start, start + 500)));
253
+ }
254
+ } while (cursor !== "0");
255
+ }
256
+ else {
257
+ const min = String(markerAt - 60_000);
258
+ let offset = 0;
259
+ while (true) {
260
+ const scanned = Number(yield* evalIndexTailPage(prefix, kind, min, offset, 500));
261
+ if (scanned < 500)
262
+ break;
263
+ offset += scanned;
264
+ }
265
+ }
266
+ yield* redis.send("SET", marker, String(bootAt));
267
+ }
268
+ }).pipe(Effect.orDie);
125
269
  // Wake protocol: a queue-filtered waiter registry (same-process wake-ups
126
270
  // never depend on the pub/sub round trip), with the channel carrying
127
271
  // cross-process wake-ups — the message names the queue ("*" broadcasts).
@@ -196,7 +340,46 @@ export const make = (options) => Effect.gen(function* () {
196
340
  const raw = generate(request);
197
341
  return Effect.isEffect(raw) ? raw : Effect.succeed(raw);
198
342
  });
199
- const enqueueOnce = (request, idMode, id, now) => evalEnqueue(prefix, idMode, id, request.name, request.queue, JSON.stringify(request.payload ?? null), JSON.stringify(request.metadata), request.priority, request.attemptsMax, request.backoff === undefined ? "" : JSON.stringify(request.backoff), request.keep === undefined ? "" : JSON.stringify(request.keep), request.timeoutMs === undefined ? "" : String(request.timeoutMs), Math.max(0, request.delayMs), now, request.dedupe?.key ?? "", request.dedupe?.ttlMs === undefined ? "" : String(request.dedupe.ttlMs), request.dedupe?.extend === true ? "1" : "0", request.dedupe?.replace === true ? "1" : "0", request.trace === undefined ? "" : JSON.stringify(request.trace));
343
+ const enqueueOnce = (request, idMode, id, now) => evalEnqueue(prefix, idMode, id, request.name, request.queue, JSON.stringify(request.payload ?? null), JSON.stringify(request.metadata), request.priority, request.attemptsMax, request.backoff === undefined ? "" : JSON.stringify(request.backoff), request.keep === undefined ? "" : JSON.stringify(request.keep), request.timeoutMs === undefined ? "" : String(request.timeoutMs), Math.max(0, request.delayMs), now, request.dedupe?.key ?? "", request.dedupe?.ttlMs === undefined ? "" : String(request.dedupe.ttlMs), request.dedupe?.extend === true ? "1" : "0", request.dedupe?.replace === true ? "1" : "0", request.trace === undefined ? "" : JSON.stringify(request.trace), request.parent === undefined ? "" : JSON.stringify(request.parent));
344
+ // The FanOut ack. Large manifests chunk their dependency rows across
345
+ // several lock-token-guarded script calls (ARGV strides, like
346
+ // enqueueMany); only the FINAL chunk writes the manifest and flips the
347
+ // state, so a crash mid-staging leaves the job active and recoverable —
348
+ // the next attempt's first chunk clears the orphaned staged rows.
349
+ const fanOutAck = (id, token, failFast, children) => Effect.gen(function* () {
350
+ if (children.some((child) => child.request.id === undefined)) {
351
+ // Validate BEFORE any script call, so a bad spec cannot leave the
352
+ // job half-acked (rows staged, ledger written, still active).
353
+ return yield* new JobStore.JobStoreError({
354
+ message: "FanOut child specs require an explicit request.id"
355
+ });
356
+ }
357
+ const now = yield* Clock.currentTimeMillis;
358
+ const chunks = [];
359
+ for (let start = 0; start < children.length; start += 500) {
360
+ chunks.push(children.slice(start, start + 500));
361
+ }
362
+ // An empty manifest still needs the final (state-flipping) call.
363
+ if (chunks.length === 0)
364
+ chunks.push([]);
365
+ for (let i = 0; i < chunks.length; i++) {
366
+ const chunk = chunks[i];
367
+ if (chunk === undefined)
368
+ continue;
369
+ const items = [];
370
+ for (const child of chunk) {
371
+ items.push(child.childKey, child.storeKey, child.request.id ?? "", child.request.name, JSON.stringify(child.request));
372
+ }
373
+ const reply = JSON.parse(yield* evalFanOut(prefix, id, token, i === chunks.length - 1 ? "1" : "0", i === 0 ? "1" : "0", failFast ? "1" : "0", children.length, now, chunk.length, items).pipe(Effect.mapError(storeError("ack failed"))));
374
+ if (reply.error === "notfound")
375
+ return yield* new JobStore.JobNotFoundError({ jobId: id });
376
+ if (reply.error === "locklost")
377
+ return yield* new JobStore.LockLostError({ jobId: id });
378
+ if (reply.wake === true) {
379
+ yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined);
380
+ }
381
+ }
382
+ });
200
383
  // Shared by cancel and cancelByDedupe.
201
384
  const cancelJob = (id) => Effect.gen(function* () {
202
385
  const now = yield* Clock.currentTimeMillis;
@@ -263,7 +446,7 @@ export const make = (options) => Effect.gen(function* () {
263
446
  : generate !== undefined
264
447
  ? yield* generateCandidate(request, generate)
265
448
  : "";
266
- itemArgs.push(mode, candidate, request.name, request.queue, JSON.stringify(request.payload ?? null), JSON.stringify(request.metadata), String(request.priority), String(request.attemptsMax), request.backoff === undefined ? "" : JSON.stringify(request.backoff), request.keep === undefined ? "" : JSON.stringify(request.keep), request.timeoutMs === undefined ? "" : String(request.timeoutMs), request.trace === undefined ? "" : JSON.stringify(request.trace), String(Math.max(0, request.delayMs)));
449
+ itemArgs.push(mode, candidate, request.name, request.queue, JSON.stringify(request.payload ?? null), JSON.stringify(request.metadata), String(request.priority), String(request.attemptsMax), request.backoff === undefined ? "" : JSON.stringify(request.backoff), request.keep === undefined ? "" : JSON.stringify(request.keep), request.timeoutMs === undefined ? "" : String(request.timeoutMs), request.trace === undefined ? "" : JSON.stringify(request.trace), request.parent === undefined ? "" : JSON.stringify(request.parent), String(Math.max(0, request.delayMs)));
267
450
  }
268
451
  const replies = JSON
269
452
  .parse(yield* evalEnqueueMany(prefix, now, chunk.length, itemArgs));
@@ -370,21 +553,28 @@ export const make = (options) => Effect.gen(function* () {
370
553
  return empty;
371
554
  });
372
555
  }).pipe(Effect.mapError(storeError("claim failed"))),
373
- ack: (id, token, outcome) => Effect.gen(function* () {
374
- const now = yield* Clock.currentTimeMillis;
375
- const exitJson = outcome._tag === "Cancelled" || outcome.exit === undefined
376
- ? ""
377
- : JSON.stringify(outcome.exit);
378
- const delayMs = outcome._tag === "Retry" ? Math.max(0, outcome.delayMs) : 0;
379
- const reply = JSON.parse(yield* evalAck(prefix, id, token, outcome._tag, exitJson, delayMs, now).pipe(Effect.mapError(storeError("ack failed"))));
380
- if (reply.error === "notfound")
381
- return yield* new JobStore.JobNotFoundError({ jobId: id });
382
- if (reply.error === "locklost")
383
- return yield* new JobStore.LockLostError({ jobId: id });
384
- if (reply.wake === true) {
385
- yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined);
556
+ ack: (id, token, outcome) => {
557
+ if (outcome._tag === "FanOut") {
558
+ return fanOutAck(id, token, outcome.failFast, outcome.children);
386
559
  }
387
- }),
560
+ // Narrowed binding: the closure below must see the FanOut-free union.
561
+ const settled = outcome;
562
+ return Effect.gen(function* () {
563
+ const now = yield* Clock.currentTimeMillis;
564
+ const exitJson = settled._tag === "Cancelled" || settled.exit === undefined
565
+ ? ""
566
+ : JSON.stringify(settled.exit);
567
+ const delayMs = settled._tag === "Retry" ? Math.max(0, settled.delayMs) : 0;
568
+ const reply = JSON.parse(yield* evalAck(prefix, id, token, settled._tag, exitJson, delayMs, now).pipe(Effect.mapError(storeError("ack failed"))));
569
+ if (reply.error === "notfound")
570
+ return yield* new JobStore.JobNotFoundError({ jobId: id });
571
+ if (reply.error === "locklost")
572
+ return yield* new JobStore.LockLostError({ jobId: id });
573
+ if (reply.wake === true) {
574
+ yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined);
575
+ }
576
+ });
577
+ },
388
578
  release: (id, token) => Effect.gen(function* () {
389
579
  const now = yield* Clock.currentTimeMillis;
390
580
  const reply = JSON.parse(yield* evalRelease(prefix, id, token, now).pipe(Effect.mapError(storeError("release failed"))));
@@ -448,22 +638,89 @@ export const make = (options) => Effect.gen(function* () {
448
638
  })),
449
639
  list: (listOptions) => Effect.gen(function* () {
450
640
  const limit = Math.max(1, listOptions.limit ?? 50);
451
- const filters = {
452
- queue: listOptions.queue,
453
- name: listOptions.name,
454
- states: listOptions.states,
455
- metadata: listOptions.metadata
456
- };
641
+ const { metadata, name, queue, states } = listOptions;
642
+ const orderBy = listOptions.orderBy ?? "enqueuedAt";
643
+ const order = listOptions.order ?? "desc";
644
+ // Routing: the narrowest structure whose zset score IS the
645
+ // requested order value; everything it does not pin stays a
646
+ // residual predicate applied in-script. Routing runs BEFORE the
647
+ // disabled-index check — a query another structure serves (e.g.
648
+ // name + terminal states ordered by finishedAt) must never die.
649
+ let sources;
650
+ let residual;
651
+ if (orderBy === "finishedAt") {
652
+ if (states === undefined || states.some((state) => !TERMINAL_STATES.has(state))) {
653
+ return yield* Effect.die(new JobStore.ListOrderUnsupportedError({
654
+ orderBy,
655
+ message: "effect-mq: the Redis store serves orderBy \"finishedAt\" only with " +
656
+ "states ⊆ {completed, failed, cancelled} (the finished/terminal zsets carry that order)"
657
+ }));
658
+ }
659
+ // ≤3 per-state sources, merged by (finishedAt, id) in-script.
660
+ const uniqueStates = [...new Set(states)];
661
+ sources = name === undefined
662
+ ? uniqueStates.map((state) => `${prefix}:finished:${state}`)
663
+ : uniqueStates.map((state) => `${prefix}:terminal:${name}:${state}`);
664
+ residual = { queue, metadata };
665
+ }
666
+ else if (orderBy === "runAt") {
667
+ const onlyDelayed = states !== undefined && states.length > 0 &&
668
+ states.every((state) => state === "delayed");
669
+ if (queue === undefined || !onlyDelayed) {
670
+ return yield* Effect.die(new JobStore.ListOrderUnsupportedError({
671
+ orderBy,
672
+ message: "effect-mq: the Redis store serves orderBy \"runAt\" only with " +
673
+ "states: [\"delayed\"] and a queue (the delayed:<queue> zset carries that order)"
674
+ }));
675
+ }
676
+ sources = [`${prefix}:delayed:${queue}`];
677
+ residual = { name, metadata };
678
+ }
679
+ else if (name !== undefined) {
680
+ if (!indexes.name) {
681
+ return yield* Effect.die(new ListIndexDisabledError({
682
+ index: "name",
683
+ message: "effect-mq: list({ name }) routes to the byname index, " +
684
+ "but RedisJobStoreOptions.indexes.name is disabled for this store"
685
+ }));
686
+ }
687
+ sources = [`${prefix}:byname:${name}`];
688
+ residual = { queue, states, metadata };
689
+ }
690
+ else if (queue !== undefined) {
691
+ if (!indexes.queue) {
692
+ return yield* Effect.die(new ListIndexDisabledError({
693
+ index: "queue",
694
+ message: "effect-mq: list({ queue }) routes to the byqueue index, " +
695
+ "but RedisJobStoreOptions.indexes.queue is disabled for this store"
696
+ }));
697
+ }
698
+ sources = [`${prefix}:byqueue:${queue}`];
699
+ residual = { states, metadata };
700
+ }
701
+ else {
702
+ sources = [`${prefix}:all`];
703
+ residual = { states, metadata };
704
+ }
457
705
  const reply = JSON
458
- .parse(yield* evalList(prefix, JSON.stringify(filters), listOptions.cursor ?? "", limit));
706
+ .parse(yield* evalList(prefix, JSON.stringify(sources), order, JSON.stringify(residual), listOptions.cursor ?? "", limit).pipe(Effect.mapError(storeError("list failed"))));
459
707
  const items = asArray(reply.items).map((flat) => toRecord(foldPairs(flat)));
460
708
  const last = items[items.length - 1];
709
+ // The cursor value is the order field, which equals the routed
710
+ // zset's score for every row the script returned.
711
+ const orderValue = last === undefined
712
+ ? 0
713
+ : orderBy === "enqueuedAt"
714
+ ? last.enqueuedAt
715
+ : orderBy === "runAt"
716
+ ? last.runAt
717
+ : last.finishedAt ?? 0;
461
718
  const result = {
462
719
  items,
463
- cursor: reply.more && last !== undefined ? `${last.enqueuedAt}:${last.id}` : undefined
720
+ cursor: reply.more && last !== undefined ? `${orderValue}:${last.id}` : undefined
464
721
  };
465
722
  return result;
466
- }).pipe(Effect.mapError(storeError("list failed"))),
723
+ }),
467
724
  retry: (id) => Effect.gen(function* () {
468
725
  const now = yield* Clock.currentTimeMillis;
469
726
  const reply = JSON.parse(yield* evalRetry(prefix, id, now).pipe(Effect.mapError(storeError("retry failed"))));
@@ -537,13 +794,108 @@ export const make = (options) => Effect.gen(function* () {
537
794
  });
538
795
  }
539
796
  const now = yield* Clock.currentTimeMillis;
540
- const fired = (yield* evalTickSchedule(prefix, key, expectedRunAt, nextRunAt, request.id, request.name, request.queue, JSON.stringify(request.payload ?? null), JSON.stringify(request.metadata), request.priority, request.attemptsMax, request.backoff === undefined ? "" : JSON.stringify(request.backoff), request.keep === undefined ? "" : JSON.stringify(request.keep), request.timeoutMs === undefined ? "" : String(request.timeoutMs), request.trace === undefined ? "" : JSON.stringify(request.trace), Math.max(0, request.delayMs), now)) === "1";
797
+ const fired = (yield* evalTickSchedule(prefix, key, expectedRunAt, nextRunAt, request.id, request.name, request.queue, JSON.stringify(request.payload ?? null), JSON.stringify(request.metadata), request.priority, request.attemptsMax, request.backoff === undefined ? "" : JSON.stringify(request.backoff), request.keep === undefined ? "" : JSON.stringify(request.keep), request.timeoutMs === undefined ? "" : String(request.timeoutMs), request.trace === undefined ? "" : JSON.stringify(request.trace), request.parent === undefined ? "" : JSON.stringify(request.parent), Math.max(0, request.delayMs), now)) === "1";
541
798
  if (fired) {
542
799
  yield* wakeUp(request.queue);
543
800
  }
544
801
  return fired;
545
802
  }).pipe(Effect.mapError((error) => error instanceof JobStore.JobStoreError ? error : storeError("tickSchedule failed")(error))),
546
- advanceSchedule: (key, expectedRunAt, nextRunAt) => evalAdvanceSchedule(prefix, key, expectedRunAt, nextRunAt).pipe(Effect.mapError(storeError("advanceSchedule failed")), Effect.asVoid)
803
+ advanceSchedule: (key, expectedRunAt, nextRunAt) => evalAdvanceSchedule(prefix, key, expectedRunAt, nextRunAt).pipe(Effect.mapError(storeError("advanceSchedule failed")), Effect.asVoid),
804
+ recordChildResults: (reports) => Effect.gen(function* () {
805
+ if (reports.length === 0) {
806
+ const none = [];
807
+ return none;
808
+ }
809
+ const now = yield* Clock.currentTimeMillis;
810
+ const all = [];
811
+ // One atomic batch per chunk of 500 (ARGV headroom); the
812
+ // contract's per-batch settle semantics then apply per chunk.
813
+ for (let start = 0; start < reports.length; start += 500) {
814
+ const chunk = reports.slice(start, start + 500);
815
+ const items = [];
816
+ for (const report of chunk) {
817
+ items.push(report.flowId, report.childKey, report.outcome, report.exit === undefined ? "" : JSON.stringify(report.exit), report.failedReason ?? "");
818
+ }
819
+ const reply = JSON.parse(yield* evalRecordChildResults(prefix, now, chunk.length, items));
820
+ for (const queue of asArray(reply.wakes)) {
821
+ // A parent settled to runnable collect: wake its queue.
822
+ yield* wakeUp(JobStore.QueueName(queue));
823
+ }
824
+ for (const result of reply.results) {
825
+ all.push(result);
826
+ }
827
+ }
828
+ return all;
829
+ }).pipe(Effect.mapError(storeError("recordChildResults failed"))),
830
+ peekOutbox: (peekOptions) => Effect.gen(function* () {
831
+ const limit = Math.floor(peekOptions.limit);
832
+ if (limit <= 0) {
833
+ const none = [];
834
+ return none;
835
+ }
836
+ // The `after` cursor compares by the id's embedded score (the seq
837
+ // prefix before the NUL), so the walk moves past the named entry
838
+ // whether or not it still exists. Unparseable input reads as unset.
839
+ let afterSeq = undefined;
840
+ if (peekOptions.after !== undefined) {
841
+ const nul = peekOptions.after.indexOf("\u0000");
842
+ const seq = nul > 0 ? Number(peekOptions.after.slice(0, nul)) : Number.NaN;
843
+ if (Number.isFinite(seq))
844
+ afterSeq = seq;
845
+ }
846
+ const raw = afterSeq === undefined
847
+ ? yield* redis.send("ZRANGE", `${prefix}:flowoutbox`, "0", String(limit - 1))
848
+ : yield* redis.send("ZRANGEBYSCORE", `${prefix}:flowoutbox`, `(${afterSeq}`, "+inf", "LIMIT", "0", String(limit));
849
+ // SAFETY: ZRANGE/ZRANGEBYSCORE always reply with arrays of bulk
850
+ // strings.
851
+ const members = raw;
852
+ return members.map(toOutboxEntry);
853
+ }).pipe(Effect.mapError(storeError("peekOutbox failed"))),
854
+ deleteOutbox: (ids) => Effect.gen(function* () {
855
+ // Chunked ZREMs keep the variadic argument count bounded; each
856
+ // chunk is idempotent, so a partial failure just redelivers.
857
+ for (let start = 0; start < ids.length; start += 500) {
858
+ yield* redis.send("ZREM", `${prefix}:flowoutbox`, ...ids.slice(start, start + 500));
859
+ }
860
+ }).pipe(Effect.mapError(storeError("deleteOutbox failed"))),
861
+ listChildResults: (flowId, listOptions) => Effect.gen(function* () {
862
+ const limit = Math.max(1, listOptions?.limit ?? 1000);
863
+ const reply = JSON
864
+ .parse(yield* evalListChildResults(prefix, flowId, listOptions?.cursor ?? "", limit));
865
+ const items = asArray(reply.items).map((row) => toChildRecord(flowId, row));
866
+ const last = items[items.length - 1];
867
+ return {
868
+ items,
869
+ cursor: reply.more && last !== undefined ? last.childKey : undefined
870
+ };
871
+ }).pipe(Effect.mapError(storeError("listChildResults failed"))),
872
+ flowSweepWork: (sweepOptions) => Effect.gen(function* () {
873
+ const now = yield* Clock.currentTimeMillis;
874
+ const limit = Math.max(1, sweepOptions.limit ?? 1000);
875
+ const reply = JSON.parse(yield* evalFlowSweepWork(prefix, sweepOptions.pendingAgeMs, limit, now));
876
+ const work = {
877
+ reconcile: asArray(reply.reconcile).map((group) => ({
878
+ flowId: JobStore.JobId(group.flowId),
879
+ children: group.children.map((child) => {
880
+ // The stored spec is the verbatim JSON this driver wrote at
881
+ // fan-out time (never routed through cjson), so it re-parses
882
+ // to the original EnqueueRequest.
883
+ const request = JSON.parse(child.spec);
884
+ return { childKey: child.childKey, storeKey: child.storeKey, request };
885
+ })
886
+ })),
887
+ cascade: asArray(reply.cascade).map((group) => ({
888
+ flowId: JobStore.JobId(group.flowId),
889
+ children: group.children.map((child) => ({
890
+ childKey: child.childKey,
891
+ storeKey: child.storeKey,
892
+ childJobId: JobStore.JobId(child.childJobId)
893
+ }))
894
+ }))
895
+ };
896
+ return work;
897
+ }).pipe(Effect.mapError(storeError("flowSweepWork failed"))),
898
+ markChildrenCascaded: (flowId, childKeys) => evalMarkChildrenCascaded(prefix, flowId, JSON.stringify(childKeys)).pipe(Effect.mapError(storeError("markChildrenCascaded failed")), Effect.asVoid)
547
899
  };
548
900
  return store;
549
901
  });