effect-mq 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +305 -25
  2. package/dist/Job.d.ts +118 -5
  3. package/dist/Job.d.ts.map +1 -1
  4. package/dist/Job.js +119 -4
  5. package/dist/Job.js.map +1 -1
  6. package/dist/JobStore.d.ts +260 -9
  7. package/dist/JobStore.d.ts.map +1 -1
  8. package/dist/JobStore.js +115 -1
  9. package/dist/JobStore.js.map +1 -1
  10. package/dist/MemoryJobStore.d.ts +38 -6
  11. package/dist/MemoryJobStore.d.ts.map +1 -1
  12. package/dist/MemoryJobStore.js +351 -47
  13. package/dist/MemoryJobStore.js.map +1 -1
  14. package/dist/Worker.d.ts +5 -1
  15. package/dist/Worker.d.ts.map +1 -1
  16. package/dist/Worker.js +117 -10
  17. package/dist/Worker.js.map +1 -1
  18. package/dist/{drizzle → drizzle-postgres}/DrizzleJobStore.d.ts +35 -2
  19. package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -0
  20. package/dist/drizzle-postgres/DrizzleJobStore.js +941 -0
  21. package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -0
  22. package/dist/drizzle-postgres/index.d.ts.map +1 -0
  23. package/dist/drizzle-postgres/index.js.map +1 -0
  24. package/dist/drizzle-postgres/schema.d.ts +670 -0
  25. package/dist/drizzle-postgres/schema.d.ts.map +1 -0
  26. package/dist/drizzle-postgres/schema.js +150 -0
  27. package/dist/drizzle-postgres/schema.js.map +1 -0
  28. package/dist/redis/RedisJobStore.d.ts +58 -0
  29. package/dist/redis/RedisJobStore.d.ts.map +1 -0
  30. package/dist/redis/RedisJobStore.js +424 -0
  31. package/dist/redis/RedisJobStore.js.map +1 -0
  32. package/dist/redis/index.d.ts +9 -0
  33. package/dist/redis/index.d.ts.map +1 -0
  34. package/dist/redis/index.js +9 -0
  35. package/dist/redis/index.js.map +1 -0
  36. package/dist/redis/scripts.d.ts +181 -0
  37. package/dist/redis/scripts.d.ts.map +1 -0
  38. package/dist/redis/scripts.js +940 -0
  39. package/dist/redis/scripts.js.map +1 -0
  40. package/dist/testing/conformance.d.ts.map +1 -1
  41. package/dist/testing/conformance.js +502 -8
  42. package/dist/testing/conformance.js.map +1 -1
  43. package/package.json +8 -4
  44. package/src/Job.ts +301 -10
  45. package/src/JobStore.ts +373 -9
  46. package/src/MemoryJobStore.ts +440 -53
  47. package/src/Worker.ts +153 -10
  48. package/src/drizzle-postgres/DrizzleJobStore.ts +1311 -0
  49. package/src/drizzle-postgres/schema.ts +279 -0
  50. package/src/redis/RedisJobStore.ts +652 -0
  51. package/src/redis/index.ts +8 -0
  52. package/src/redis/scripts.ts +1055 -0
  53. package/src/testing/conformance.ts +665 -8
  54. package/dist/drizzle/DrizzleJobStore.d.ts.map +0 -1
  55. package/dist/drizzle/DrizzleJobStore.js +0 -426
  56. package/dist/drizzle/DrizzleJobStore.js.map +0 -1
  57. package/dist/drizzle/index.d.ts.map +0 -1
  58. package/dist/drizzle/index.js.map +0 -1
  59. package/dist/drizzle/schema.d.ts +0 -464
  60. package/dist/drizzle/schema.d.ts.map +0 -1
  61. package/dist/drizzle/schema.js +0 -68
  62. package/dist/drizzle/schema.js.map +0 -1
  63. package/src/drizzle/DrizzleJobStore.ts +0 -599
  64. package/src/drizzle/schema.ts +0 -116
  65. /package/dist/{drizzle → drizzle-postgres}/index.d.ts +0 -0
  66. /package/dist/{drizzle → drizzle-postgres}/index.js +0 -0
  67. /package/src/{drizzle → drizzle-postgres}/index.ts +0 -0
@@ -0,0 +1,652 @@
1
+ /**
2
+ * A Redis-backed `JobStore` built on `effect/unstable/persistence`'s `Redis`
3
+ * service. Every mutation is a single Lua script (see `scripts.ts`), so all
4
+ * `JobStore` operations are atomic on the server and safe across processes.
5
+ *
6
+ * Provide the `Redis` service from your platform package — `NodeRedis.layer`
7
+ * (`@effect/platform-node`, node-redis), `BunRedis.layer`
8
+ * (`@effect/platform-bun`, `Bun.redis`) — or `Redis.make` over any client.
9
+ *
10
+ * Wake-ups ride the client's pub/sub channel, so workers in other processes
11
+ * pick jobs up promptly; the worker's `pollInterval` is the fallback.
12
+ *
13
+ * @since 0.2.0
14
+ */
15
+ import { Clock, type Context, Deferred, Duration, Effect, Exit, Layer, Option, Queue, Schedule, type Scope } from "effect"
16
+ import { Redis } from "effect/unstable/persistence"
17
+ import * as JobStore from "../JobStore.ts"
18
+ import * as scripts from "./scripts.ts"
19
+
20
+ /**
21
+ * @since 0.2.0
22
+ */
23
+ export interface RedisJobStoreOptions {
24
+ /** Key prefix for everything this store writes (default `effect-mq`). */
25
+ readonly prefix?: string | undefined
26
+ /**
27
+ * Store-level retention ceiling: terminal records older than this are
28
+ * removed by a periodic sweep — one duration for all terminal states or a
29
+ * per-state split (`{ completed: "1 day", failed: "30 days" }`). The sweep
30
+ * also honours stricter per-job `keep.age` rules.
31
+ */
32
+ readonly historyTtl?: JobStore.HistoryTtlInput | undefined
33
+ /** History sweep cadence (default 1 minute). */
34
+ readonly historySweepInterval?: Duration.Input | undefined
35
+ /**
36
+ * Generator for store-assigned job ids (e.g. `() => \`job_${ulid()}\``).
37
+ * Default: `j-<n>` from the store's counter. See `JobStore.IdGenerator`.
38
+ */
39
+ readonly idGenerator?: JobStore.IdGenerator | undefined
40
+ }
41
+
42
+ const storeError = (message: string) => (cause: unknown) => new JobStore.JobStoreError({ message, cause })
43
+
44
+ /** Fold a Lua `HGETALL` reply (flat `[field, value, ...]`) into a map. */
45
+ const foldPairs = (flat: ReadonlyArray<string>): ReadonlyMap<string, string> => {
46
+ const out = new Map<string, string>()
47
+ for (let i = 0; i + 1 < flat.length; i += 2) {
48
+ const field = flat[i]
49
+ const value = flat[i + 1]
50
+ if (field !== undefined && value !== undefined) {
51
+ out.set(field, value)
52
+ }
53
+ }
54
+ return out
55
+ }
56
+
57
+ // Hash fields use "" for absent optional values.
58
+ const optionalString = (value: string | undefined): string | undefined =>
59
+ value === undefined || value === "" ? undefined : value
60
+
61
+ const optionalNumber = (value: string | undefined): number | undefined =>
62
+ value === undefined || value === "" ? undefined : Number(value)
63
+
64
+ const optionalJson = <A = unknown>(value: string | undefined): A | undefined => {
65
+ if (value === undefined || value === "") return undefined
66
+ // SAFETY: the value round-trips JSON this driver itself wrote for the field.
67
+ return JSON.parse(value) as A
68
+ }
69
+
70
+ const toRecord = (hash: ReadonlyMap<string, string>): JobStore.JobRecord => ({
71
+ id: JobStore.JobId(hash.get("id") ?? ""),
72
+ name: hash.get("name") ?? "",
73
+ queue: JobStore.QueueName(hash.get("queue") ?? ""),
74
+ payload: optionalJson(hash.get("payload")) ?? null,
75
+ metadata: optionalJson<Readonly<Record<string, string>>>(hash.get("metadata")) ?? {},
76
+ // SAFETY: the state field is only ever written with JobState members.
77
+ state: (hash.get("state") ?? "waiting") as JobStore.JobState,
78
+ priority: Number(hash.get("priority") ?? 0),
79
+ attemptsMax: Number(hash.get("attemptsMax") ?? 1),
80
+ attemptsMade: Number(hash.get("attemptsMade") ?? 0),
81
+ stalledCount: Number(hash.get("stalledCount") ?? 0),
82
+ backoff: optionalJson<JobStore.BackoffPolicy>(hash.get("backoff")),
83
+ keep: optionalJson<JobStore.KeepPolicy>(hash.get("keep")),
84
+ timeoutMs: optionalNumber(hash.get("timeoutMs")),
85
+ cancelRequested: hash.get("cancelRequested") === "1",
86
+ dedupeKey: optionalString(hash.get("dedupeKey")),
87
+ runAt: Number(hash.get("runAt") ?? 0),
88
+ enqueuedAt: Number(hash.get("enqueuedAt") ?? 0),
89
+ processedAt: optionalNumber(hash.get("processedAt")),
90
+ finishedAt: optionalNumber(hash.get("finishedAt")),
91
+ exit: optionalJson(hash.get("exit")),
92
+ failedReason: optionalString(hash.get("failedReason"))
93
+ })
94
+
95
+ const toSchedule = (hash: ReadonlyMap<string, string>): JobStore.ScheduleRecord => ({
96
+ key: JobStore.ScheduleKey(hash.get("key") ?? ""),
97
+ jobName: hash.get("jobName") ?? "",
98
+ queue: JobStore.QueueName(hash.get("queue") ?? ""),
99
+ cron: optionalString(hash.get("cron")),
100
+ tz: optionalString(hash.get("tz")),
101
+ everyMs: optionalNumber(hash.get("everyMs")),
102
+ payload: optionalJson(hash.get("payload")),
103
+ metadata: optionalJson<Readonly<Record<string, string>>>(hash.get("metadata")) ?? {},
104
+ priority: Number(hash.get("priority") ?? 0),
105
+ attemptsMax: Number(hash.get("attemptsMax") ?? 1),
106
+ backoff: optionalJson<JobStore.BackoffPolicy>(hash.get("backoff")),
107
+ keep: optionalJson<JobStore.KeepPolicy>(hash.get("keep")),
108
+ timeoutMs: optionalNumber(hash.get("timeoutMs")),
109
+ nextRunAt: Number(hash.get("nextRunAt") ?? 0)
110
+ })
111
+
112
+ // cjson encodes an empty Lua table as {}, not [] — normalize.
113
+ const asArray = <A>(value: ReadonlyArray<A> | Record<string, never>): ReadonlyArray<A> =>
114
+ Array.isArray(value) ? value : []
115
+
116
+ const JOB_STATES: ReadonlyArray<JobStore.JobState> = [
117
+ "waiting",
118
+ "delayed",
119
+ "active",
120
+ "completed",
121
+ "failed",
122
+ "cancelled"
123
+ ]
124
+
125
+ /**
126
+ * Build a `RedisJobStore` service. Needs the `Redis` service and a `Scope`
127
+ * (the wake-up subscription and the optional history sweeper live in it).
128
+ *
129
+ * @since 0.2.0
130
+ */
131
+ export const make = (
132
+ options?: RedisJobStoreOptions | undefined
133
+ ): Effect.Effect<JobStore.Service, never, Redis.Redis | Scope.Scope> =>
134
+ Effect.gen(function*() {
135
+ const redis = yield* Redis.Redis
136
+ const prefix = options?.prefix ?? "effect-mq"
137
+ const wakeChannel = `${prefix}:wake`
138
+
139
+ const evalEnqueue = redis.eval(scripts.enqueue)
140
+ const evalClaim = redis.eval(scripts.claim)
141
+ const evalAck = redis.eval(scripts.ack)
142
+ const evalRelease = redis.eval(scripts.release)
143
+ const evalExtendLocks = redis.eval(scripts.extendLocks)
144
+ const evalRecoverStalled = redis.eval(scripts.recoverStalled)
145
+ const evalGetJob = redis.eval(scripts.getJob)
146
+ const evalList = redis.eval(scripts.list)
147
+ const evalCounts = redis.eval(scripts.counts)
148
+ const evalRemove = redis.eval(scripts.remove)
149
+ const evalRetry = redis.eval(scripts.retry)
150
+ const evalCancel = redis.eval(scripts.cancel)
151
+ const evalPromote = redis.eval(scripts.promote)
152
+ const evalUpsertSchedule = redis.eval(scripts.upsertSchedule)
153
+ const evalRemoveSchedule = redis.eval(scripts.removeSchedule)
154
+ const evalListSchedules = redis.eval(scripts.listSchedules)
155
+ const evalDueSchedules = redis.eval(scripts.dueSchedules)
156
+ const evalAdvanceSchedule = redis.eval(scripts.advanceSchedule)
157
+ const evalSweepState = redis.eval(scripts.sweepState)
158
+ const evalSweepDedupes = redis.eval(scripts.sweepDedupes)
159
+
160
+ // Wake protocol: a queue-filtered waiter registry (same-process wake-ups
161
+ // never depend on the pub/sub round trip), with the channel carrying
162
+ // cross-process wake-ups — the message names the queue ("*" broadcasts).
163
+ // Filtering matters at scale: without it every enqueue wakes every idle
164
+ // taker of every queue on the store.
165
+ let wakeVersion = 0
166
+ let lastBroadcast = 0
167
+ const lastWake = new Map<JobStore.QueueName, number>()
168
+ interface Waiter {
169
+ readonly queues: ReadonlySet<JobStore.QueueName>
170
+ readonly deferred: Deferred.Deferred<void>
171
+ }
172
+ const waiters = new Set<Waiter>()
173
+ const lastWakeFor = (queue: JobStore.QueueName) => Math.max(lastWake.get(queue) ?? 0, lastBroadcast)
174
+ const signalWakeLocal = (queue?: JobStore.QueueName) => {
175
+ wakeVersion += 1
176
+ if (queue === undefined) {
177
+ lastBroadcast = wakeVersion
178
+ } else {
179
+ lastWake.set(queue, wakeVersion)
180
+ }
181
+ // Snapshot-and-clear BEFORE resolving: doneUnsafe resumes waiting
182
+ // fibers synchronously, and a woken taker that re-parks registers a
183
+ // NEW waiter — resolving inside the live Set iteration would visit it
184
+ // and livelock.
185
+ const toWake: Array<Waiter> = []
186
+ for (const waiter of waiters) {
187
+ if (queue === undefined || waiter.queues.has(queue)) {
188
+ waiters.delete(waiter)
189
+ toWake.push(waiter)
190
+ }
191
+ }
192
+ for (const waiter of toWake) {
193
+ Deferred.doneUnsafe(waiter.deferred, Exit.succeed<void>(void 0))
194
+ }
195
+ }
196
+ const wakeUp = (queue?: JobStore.QueueName): Effect.Effect<void> =>
197
+ Effect.suspend(() => {
198
+ signalWakeLocal(queue)
199
+ return redis.send("PUBLISH", wakeChannel, queue !== undefined && queue.length > 0 ? queue : "*").pipe(
200
+ Effect.ignore
201
+ )
202
+ })
203
+
204
+ // Cross-process wake-ups. The pump resubscribes on connection loss (Bun
205
+ // subscribers do not auto-reconnect); the retry delay only ever runs
206
+ // after a real failure, so TestClock runs are unaffected.
207
+ yield* Effect.scoped(
208
+ Effect.gen(function*() {
209
+ const messages = yield* redis.subscribe(wakeChannel)
210
+ while (true) {
211
+ const message = yield* Queue.take(messages)
212
+ signalWakeLocal(message.message === "*" ? undefined : JobStore.QueueName(message.message))
213
+ }
214
+ })
215
+ ).pipe(
216
+ Effect.retry(Schedule.spaced("1 second")),
217
+ Effect.catchCause((cause) => Effect.logWarning("effect-mq: redis wake subscription failed", cause)),
218
+ Effect.forkScoped
219
+ )
220
+
221
+ if (options?.historyTtl !== undefined) {
222
+ const ttlByState = JobStore.normalizeHistoryTtl(options.historyTtl)
223
+ const intervalMs = Duration.toMillis(options.historySweepInterval ?? "1 minute")
224
+ yield* Effect.gen(function*() {
225
+ yield* Effect.sleep(intervalMs)
226
+ const now = yield* Clock.currentTimeMillis
227
+ // Per-state ceilings refined by stricter per-row keep ages, paged by
228
+ // an offset cursor so young rows are visited once per sweep.
229
+ for (const state of ["completed", "failed", "cancelled"] as const) {
230
+ const ttl = ttlByState[state]
231
+ let offset = 0
232
+ while (true) {
233
+ const page: { scanned: number; deleted: number } = JSON.parse(
234
+ yield* evalSweepState(prefix, state, ttl === undefined ? "" : String(ttl), 200, offset, now)
235
+ )
236
+ if (page.scanned < 200) break
237
+ offset += page.scanned - page.deleted
238
+ }
239
+ }
240
+ while ((yield* evalSweepDedupes(prefix, 200, now)) !== "0") {
241
+ // bounded batches until the dedup backlog is clean
242
+ }
243
+ }).pipe(
244
+ Effect.catchCause((cause) => Effect.logError("effect-mq: redis history sweep failed", cause)),
245
+ Effect.forever,
246
+ Effect.forkScoped
247
+ )
248
+ }
249
+
250
+ const generateCandidate = (request: JobStore.EnqueueRequest, generate: JobStore.IdGenerator) =>
251
+ Effect.suspend(() => {
252
+ const raw = generate(request)
253
+ return Effect.isEffect(raw) ? raw : Effect.succeed(raw)
254
+ })
255
+
256
+ const enqueueOnce = (
257
+ request: JobStore.EnqueueRequest,
258
+ idMode: "user" | "generated" | "auto",
259
+ id: string,
260
+ now: number
261
+ ) =>
262
+ evalEnqueue(
263
+ prefix,
264
+ idMode,
265
+ id,
266
+ request.name,
267
+ request.queue,
268
+ JSON.stringify(request.payload ?? null),
269
+ JSON.stringify(request.metadata),
270
+ request.priority,
271
+ request.attemptsMax,
272
+ request.backoff === undefined ? "" : JSON.stringify(request.backoff),
273
+ request.keep === undefined ? "" : JSON.stringify(request.keep),
274
+ request.timeoutMs === undefined ? "" : String(request.timeoutMs),
275
+ Math.max(0, request.delayMs),
276
+ now,
277
+ request.dedupe?.key ?? "",
278
+ request.dedupe?.ttlMs === undefined ? "" : String(request.dedupe.ttlMs),
279
+ request.dedupe?.extend === true ? "1" : "0",
280
+ request.dedupe?.replace === true ? "1" : "0"
281
+ )
282
+
283
+ const store: JobStore.Service = {
284
+ enqueue: (request) =>
285
+ Effect.gen(function*() {
286
+ const now = yield* Clock.currentTimeMillis
287
+ const generate = options?.idGenerator
288
+ for (let i = 0; i < 5; i++) {
289
+ const mode = request.id !== undefined
290
+ ? "user" as const
291
+ : generate !== undefined
292
+ ? "generated" as const
293
+ : "auto" as const
294
+ const candidate = mode === "user"
295
+ ? request.id ?? ""
296
+ : mode === "generated" && generate !== undefined
297
+ ? yield* generateCandidate(request, generate)
298
+ : ""
299
+ const reply: {
300
+ id?: string
301
+ duplicate?: boolean
302
+ wake?: boolean
303
+ collision?: boolean
304
+ error?: string
305
+ queue?: string
306
+ } = JSON.parse(yield* enqueueOnce(request, mode, candidate, now))
307
+ if (reply.collision === true) continue
308
+ if (reply.error !== undefined || reply.id === undefined) {
309
+ return yield* new JobStore.JobStoreError({
310
+ message: "enqueue failed: could not generate a unique job id"
311
+ })
312
+ }
313
+ if (reply.wake === true) {
314
+ // A replace-while-delayed reply names the keyed job's queue.
315
+ yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : request.queue)
316
+ }
317
+ return { id: JobStore.JobId(reply.id), duplicate: reply.duplicate === true }
318
+ }
319
+ return yield* new JobStore.JobStoreError({
320
+ message: "enqueue failed: could not generate a unique job id"
321
+ })
322
+ }).pipe(
323
+ Effect.mapError((error) =>
324
+ error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)
325
+ )
326
+ ),
327
+
328
+ claim: (claimOptions) =>
329
+ // Snapshot BEFORE the script runs: a wake that fires while the claim
330
+ // executes must make awaitWake(token) return immediately.
331
+ Effect.suspend(() => {
332
+ const observedWake = wakeVersion
333
+ return Effect.gen(function*() {
334
+ const now = yield* Clock.currentTimeMillis
335
+ const reply: { job?: ReadonlyArray<string>; empty?: boolean; nextRunAt?: number } = JSON.parse(
336
+ yield* evalClaim(
337
+ prefix,
338
+ claimOptions.queue,
339
+ JSON.stringify(claimOptions.names),
340
+ claimOptions.token,
341
+ claimOptions.lockDurationMs,
342
+ now
343
+ )
344
+ )
345
+ if (reply.job !== undefined) {
346
+ const claimed: JobStore.ClaimResult = { _tag: "Claimed", job: toRecord(foldPairs(reply.job)) }
347
+ return claimed
348
+ }
349
+ const empty: JobStore.ClaimResult = {
350
+ _tag: "Empty",
351
+ nextRunAt: reply.nextRunAt,
352
+ wakeToken: observedWake
353
+ }
354
+ return empty
355
+ })
356
+ }).pipe(Effect.mapError(storeError("claim failed"))),
357
+
358
+ ack: (id, token, outcome) =>
359
+ Effect.gen(function*() {
360
+ const now = yield* Clock.currentTimeMillis
361
+ const exitJson = outcome._tag === "Cancelled" || outcome.exit === undefined
362
+ ? ""
363
+ : JSON.stringify(outcome.exit)
364
+ const delayMs = outcome._tag === "Retry" ? Math.max(0, outcome.delayMs) : 0
365
+ const reply: { error?: string; wake?: boolean; queue?: string } = JSON.parse(
366
+ yield* evalAck(prefix, id, token, outcome._tag, exitJson, delayMs, now).pipe(
367
+ Effect.mapError(storeError("ack failed"))
368
+ )
369
+ )
370
+ if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
371
+ if (reply.error === "locklost") return yield* new JobStore.LockLostError({ jobId: id })
372
+ if (reply.wake === true) {
373
+ yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined)
374
+ }
375
+ }),
376
+
377
+ release: (id, token) =>
378
+ Effect.gen(function*() {
379
+ const now = yield* Clock.currentTimeMillis
380
+ const reply: { error?: string; wake?: boolean; queue?: string } = JSON.parse(
381
+ yield* evalRelease(prefix, id, token, now).pipe(Effect.mapError(storeError("release failed")))
382
+ )
383
+ if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
384
+ if (reply.error === "locklost") return yield* new JobStore.LockLostError({ jobId: id })
385
+ if (reply.wake === true) {
386
+ yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined)
387
+ }
388
+ }),
389
+
390
+ extendLocks: (locks, durationMs) =>
391
+ Effect.gen(function*() {
392
+ if (locks.length === 0) {
393
+ const empty: JobStore.ExtendLocksResult = { lost: [], cancelRequested: [] }
394
+ return empty
395
+ }
396
+ const now = yield* Clock.currentTimeMillis
397
+ const reply: {
398
+ lost: ReadonlyArray<string> | Record<string, never>
399
+ cancel: ReadonlyArray<string> | Record<string, never>
400
+ } = JSON.parse(yield* evalExtendLocks(prefix, JSON.stringify(locks), durationMs, now))
401
+ const result: JobStore.ExtendLocksResult = {
402
+ lost: asArray(reply.lost).map(JobStore.JobId),
403
+ cancelRequested: asArray(reply.cancel).map(JobStore.JobId)
404
+ }
405
+ return result
406
+ }).pipe(Effect.mapError(storeError("extendLocks failed"))),
407
+
408
+ recoverStalled: (recoverOptions) =>
409
+ Effect.gen(function*() {
410
+ const now = yield* Clock.currentTimeMillis
411
+ const recovered: ReadonlyArray<{ id: string; failed: boolean }> = JSON.parse(
412
+ yield* evalRecoverStalled(prefix, recoverOptions.maxStalledCount, now)
413
+ )
414
+ const result = recovered.map((entry) => ({ id: JobStore.JobId(entry.id), failed: entry.failed }))
415
+ if (result.some((entry) => !entry.failed)) {
416
+ yield* wakeUp()
417
+ }
418
+ return result
419
+ }).pipe(Effect.mapError(storeError("recoverStalled failed"))),
420
+
421
+ awaitWake: (queues, wakeToken) =>
422
+ Effect.suspend(() => {
423
+ if (queues.some((queue) => lastWakeFor(queue) > wakeToken)) return Effect.void
424
+ const waiter: Waiter = { queues: new Set(queues), deferred: Deferred.makeUnsafe<void>() }
425
+ waiters.add(waiter)
426
+ return Deferred.await(waiter.deferred).pipe(
427
+ Effect.ensuring(Effect.sync(() => waiters.delete(waiter)))
428
+ )
429
+ }),
430
+
431
+ getJob: (id) =>
432
+ evalGetJob(prefix, id).pipe(
433
+ Effect.mapError(storeError("getJob failed")),
434
+ Effect.map((raw) => {
435
+ const flat: ReadonlyArray<string> = JSON.parse(raw)
436
+ return flat.length === 0 ? Option.none() : Option.some(toRecord(foldPairs(flat)))
437
+ })
438
+ ),
439
+
440
+ getAttempts: (id) =>
441
+ redis.send("LRANGE", `${prefix}:attempts:${id}`, "0", "-1").pipe(
442
+ Effect.mapError(storeError("getAttempts failed")),
443
+ Effect.map((raw) => {
444
+ // SAFETY: LRANGE always replies with an array of bulk strings.
445
+ const entries = raw as ReadonlyArray<string>
446
+ return entries.map((entry) => {
447
+ const parsed: {
448
+ attempt: number
449
+ startedAt: number | null
450
+ finishedAt: number
451
+ outcome: JobStore.AttemptRecord["outcome"]
452
+ exit?: unknown
453
+ } = JSON.parse(entry)
454
+ const record: JobStore.AttemptRecord = {
455
+ attempt: parsed.attempt,
456
+ startedAt: parsed.startedAt ?? undefined,
457
+ finishedAt: parsed.finishedAt,
458
+ outcome: parsed.outcome,
459
+ // The ledger omits the key entirely for absent exits, so a
460
+ // legitimate encoded null exit survives the round trip.
461
+ exit: Object.hasOwn(parsed, "exit") ? parsed.exit : undefined
462
+ }
463
+ return record
464
+ })
465
+ })
466
+ ),
467
+
468
+ list: (listOptions) =>
469
+ Effect.gen(function*() {
470
+ const limit = Math.max(1, listOptions.limit ?? 50)
471
+ const filters = {
472
+ queue: listOptions.queue,
473
+ name: listOptions.name,
474
+ states: listOptions.states,
475
+ metadata: listOptions.metadata
476
+ }
477
+ const reply: { items: ReadonlyArray<ReadonlyArray<string>> | Record<string, never>; more: boolean } = JSON
478
+ .parse(
479
+ yield* evalList(prefix, JSON.stringify(filters), listOptions.cursor ?? "", limit)
480
+ )
481
+ const items = asArray(reply.items).map((flat) => toRecord(foldPairs(flat)))
482
+ const last = items[items.length - 1]
483
+ const result: JobStore.ListResult = {
484
+ items,
485
+ cursor: reply.more && last !== undefined ? `${last.enqueuedAt}:${last.id}` : undefined
486
+ }
487
+ return result
488
+ }).pipe(Effect.mapError(storeError("list failed"))),
489
+
490
+ retry: (id) =>
491
+ Effect.gen(function*() {
492
+ const now = yield* Clock.currentTimeMillis
493
+ const reply: { error?: string; state?: JobStore.JobState; queue?: string } = JSON.parse(
494
+ yield* evalRetry(prefix, id, now).pipe(Effect.mapError(storeError("retry failed")))
495
+ )
496
+ if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
497
+ if (reply.error === "state") {
498
+ return yield* new JobStore.JobNotRetryableError({ jobId: id, state: reply.state ?? "failed" })
499
+ }
500
+ yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined)
501
+ }),
502
+
503
+ counts: (queue) =>
504
+ evalCounts(prefix).pipe(
505
+ Effect.mapError(storeError("counts failed")),
506
+ Effect.map((raw) => {
507
+ const flat: ReadonlyArray<string> = JSON.parse(raw)
508
+ const byField = foldPairs(flat)
509
+ // SAFETY: fromEntries over the exhaustive JOB_STATES list yields
510
+ // exactly one zeroed entry per JobState member.
511
+ const totals = Object.fromEntries(JOB_STATES.map((state) => [state, 0])) as Record<
512
+ JobStore.JobState,
513
+ number
514
+ >
515
+ for (const [field, count] of byField) {
516
+ const split = field.lastIndexOf("|")
517
+ const fieldQueue = field.slice(0, split)
518
+ // SAFETY: counts fields are written as `<queue>|<JobState>`.
519
+ const state = field.slice(split + 1) as JobStore.JobState
520
+ if (queue === undefined || fieldQueue === queue) {
521
+ totals[state] += Number(count)
522
+ }
523
+ }
524
+ return totals
525
+ })
526
+ ),
527
+
528
+ remove: (id) =>
529
+ evalRemove(prefix, id).pipe(
530
+ Effect.mapError(storeError("remove failed")),
531
+ Effect.map((raw) => raw !== "0")
532
+ ),
533
+
534
+ cancel: (id) =>
535
+ Effect.gen(function*() {
536
+ const now = yield* Clock.currentTimeMillis
537
+ const reply: { error?: string; state?: JobStore.JobState } = JSON.parse(
538
+ yield* evalCancel(prefix, id, now).pipe(Effect.mapError(storeError("cancel failed")))
539
+ )
540
+ if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
541
+ if (reply.error === "state") {
542
+ return yield* new JobStore.JobNotCancellableError({ jobId: id, state: reply.state ?? "completed" })
543
+ }
544
+ }),
545
+
546
+ promote: (id) =>
547
+ Effect.gen(function*() {
548
+ const now = yield* Clock.currentTimeMillis
549
+ const reply: { error?: string; state?: JobStore.JobState; queue?: string } = JSON.parse(
550
+ yield* evalPromote(prefix, id, now).pipe(Effect.mapError(storeError("promote failed")))
551
+ )
552
+ if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
553
+ if (reply.error === "state") {
554
+ return yield* new JobStore.JobNotPromotableError({ jobId: id, state: reply.state ?? "completed" })
555
+ }
556
+ yield* wakeUp(reply.queue !== undefined ? JobStore.QueueName(reply.queue) : undefined)
557
+ }),
558
+
559
+ pause: (queue) =>
560
+ redis.send("SADD", `${prefix}:paused`, queue).pipe(
561
+ Effect.mapError(storeError("pause failed")),
562
+ Effect.asVoid
563
+ ),
564
+
565
+ resume: (queue) =>
566
+ redis.send("SREM", `${prefix}:paused`, queue).pipe(
567
+ Effect.mapError(storeError("resume failed")),
568
+ Effect.flatMap((removed) => Number(removed) > 0 ? wakeUp(queue) : Effect.void)
569
+ ),
570
+
571
+ pausedQueues: () =>
572
+ redis.send("SMEMBERS", `${prefix}:paused`).pipe(
573
+ Effect.mapError(storeError("pausedQueues failed")),
574
+ Effect.map((raw) => {
575
+ // SAFETY: SMEMBERS always replies with an array of bulk strings.
576
+ const members = raw as ReadonlyArray<string>
577
+ return members.map(JobStore.QueueName)
578
+ })
579
+ ),
580
+
581
+ upsertSchedule: (schedule) =>
582
+ evalUpsertSchedule(
583
+ prefix,
584
+ schedule.key,
585
+ schedule.jobName,
586
+ schedule.queue,
587
+ schedule.cron ?? "",
588
+ schedule.tz ?? "",
589
+ schedule.everyMs === undefined ? "" : String(schedule.everyMs),
590
+ schedule.payload === undefined ? "" : JSON.stringify(schedule.payload),
591
+ JSON.stringify(schedule.metadata),
592
+ String(schedule.priority),
593
+ String(schedule.attemptsMax),
594
+ schedule.backoff === undefined ? "" : JSON.stringify(schedule.backoff),
595
+ schedule.keep === undefined ? "" : JSON.stringify(schedule.keep),
596
+ schedule.timeoutMs === undefined ? "" : String(schedule.timeoutMs),
597
+ schedule.nextRunAt
598
+ ).pipe(
599
+ Effect.mapError(storeError("upsertSchedule failed")),
600
+ Effect.andThen(wakeUp(schedule.queue))
601
+ ),
602
+
603
+ removeSchedule: (key) =>
604
+ evalRemoveSchedule(prefix, key).pipe(
605
+ Effect.mapError(storeError("removeSchedule failed")),
606
+ Effect.map((raw) => raw !== "0")
607
+ ),
608
+
609
+ listSchedules: (listOptions) =>
610
+ evalListSchedules(prefix, JSON.stringify(listOptions ?? {})).pipe(
611
+ Effect.mapError(storeError("listSchedules failed")),
612
+ Effect.map((raw) => {
613
+ const flat: ReadonlyArray<ReadonlyArray<string>> = JSON.parse(raw)
614
+ return flat.map((pairs) => toSchedule(foldPairs(pairs)))
615
+ })
616
+ ),
617
+
618
+ dueSchedules: () =>
619
+ Effect.gen(function*() {
620
+ const now = yield* Clock.currentTimeMillis
621
+ const flat: ReadonlyArray<ReadonlyArray<string>> = JSON.parse(yield* evalDueSchedules(prefix, now))
622
+ return flat.map((pairs) => toSchedule(foldPairs(pairs)))
623
+ }).pipe(Effect.mapError(storeError("dueSchedules failed"))),
624
+
625
+ advanceSchedule: (key, expectedRunAt, nextRunAt) =>
626
+ evalAdvanceSchedule(prefix, key, expectedRunAt, nextRunAt).pipe(
627
+ Effect.mapError(storeError("advanceSchedule failed")),
628
+ Effect.asVoid
629
+ )
630
+ }
631
+
632
+ return store
633
+ })
634
+
635
+ /**
636
+ * A Redis-backed layer for the default `JobStore`.
637
+ *
638
+ * @since 0.2.0
639
+ */
640
+ export const layer = (
641
+ options?: RedisJobStoreOptions | undefined
642
+ ): Layer.Layer<JobStore.JobStore, never, Redis.Redis> => Layer.effect(JobStore.JobStore, make(options))
643
+
644
+ /**
645
+ * A Redis-backed layer for a specific named store key.
646
+ *
647
+ * @since 0.2.0
648
+ */
649
+ export const layerFor = <StoreId>(
650
+ store: Context.Key<StoreId, JobStore.Service>,
651
+ options?: RedisJobStoreOptions | undefined
652
+ ): Layer.Layer<StoreId, never, Redis.Redis> => Layer.effect(store, make(options))
@@ -0,0 +1,8 @@
1
+ /**
2
+ * The Redis-backed `JobStore`, built on `effect/unstable/persistence`'s
3
+ * `Redis` service (provide it via `NodeRedis.layer`, `BunRedis.layer`, or
4
+ * `Redis.make` over any client).
5
+ *
6
+ * @since 0.2.0
7
+ */
8
+ export * as RedisJobStore from "./RedisJobStore.ts"