effect-mq 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +167 -22
- package/dist/Job.d.ts +67 -4
- package/dist/Job.d.ts.map +1 -1
- package/dist/Job.js +76 -3
- package/dist/Job.js.map +1 -1
- package/dist/JobStore.d.ts +174 -3
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js +89 -1
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts +37 -6
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +201 -25
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/Worker.d.ts +5 -1
- package/dist/Worker.d.ts.map +1 -1
- package/dist/Worker.js +116 -10
- package/dist/Worker.js.map +1 -1
- package/dist/{drizzle → drizzle-postgres}/DrizzleJobStore.d.ts +18 -2
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -0
- package/dist/{drizzle → drizzle-postgres}/DrizzleJobStore.js +287 -40
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -0
- package/dist/drizzle-postgres/index.d.ts.map +1 -0
- package/dist/drizzle-postgres/index.js.map +1 -0
- package/dist/{drizzle → drizzle-postgres}/schema.d.ts +306 -4
- package/dist/drizzle-postgres/schema.d.ts.map +1 -0
- package/dist/{drizzle → drizzle-postgres}/schema.js +36 -2
- package/dist/drizzle-postgres/schema.js.map +1 -0
- package/dist/redis/RedisJobStore.d.ts +56 -0
- package/dist/redis/RedisJobStore.d.ts.map +1 -0
- package/dist/redis/RedisJobStore.js +385 -0
- package/dist/redis/RedisJobStore.js.map +1 -0
- package/dist/redis/index.d.ts +9 -0
- package/dist/redis/index.d.ts.map +1 -0
- package/dist/redis/index.js +9 -0
- package/dist/redis/index.js.map +1 -0
- package/dist/redis/scripts.d.ts +168 -0
- package/dist/redis/scripts.d.ts.map +1 -0
- package/dist/redis/scripts.js +755 -0
- package/dist/redis/scripts.js.map +1 -0
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +332 -3
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +8 -4
- package/src/Job.ts +189 -5
- package/src/JobStore.ts +252 -4
- package/src/MemoryJobStore.ts +273 -26
- package/src/Worker.ts +152 -10
- package/src/{drizzle → drizzle-postgres}/DrizzleJobStore.ts +412 -40
- package/src/{drizzle → drizzle-postgres}/schema.ts +52 -3
- package/src/redis/RedisJobStore.ts +597 -0
- package/src/redis/index.ts +8 -0
- package/src/redis/scripts.ts +862 -0
- package/src/testing/conformance.ts +421 -3
- package/dist/drizzle/DrizzleJobStore.d.ts.map +0 -1
- package/dist/drizzle/DrizzleJobStore.js.map +0 -1
- package/dist/drizzle/index.d.ts.map +0 -1
- package/dist/drizzle/index.js.map +0 -1
- package/dist/drizzle/schema.d.ts.map +0 -1
- package/dist/drizzle/schema.js.map +0 -1
- /package/dist/{drizzle → drizzle-postgres}/index.d.ts +0 -0
- /package/dist/{drizzle → drizzle-postgres}/index.js +0 -0
- /package/src/{drizzle → drizzle-postgres}/index.ts +0 -0
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
* // schema.ts
|
|
10
10
|
* import { mqJobAttempts, mqJobs } from "effect-mq/drizzle"
|
|
11
11
|
*
|
|
12
|
-
* type DurableJobs = typeof
|
|
12
|
+
* type DurableJobs = typeof GenerateInvoice._tag | typeof GenerateReport._tag
|
|
13
13
|
* export const jobs = mqJobs<DurableJobs>()
|
|
14
14
|
* export const jobAttempts = mqJobAttempts(jobs)
|
|
15
15
|
* ```
|
|
@@ -23,17 +23,18 @@
|
|
|
23
23
|
*/
|
|
24
24
|
import type * as JobStore from "../JobStore.ts"
|
|
25
25
|
import { sql } from "drizzle-orm"
|
|
26
|
-
import { bigint, index, integer, jsonb, pgTable, primaryKey, text, timestamp } from "drizzle-orm/pg-core"
|
|
26
|
+
import { bigint, boolean, index, integer, jsonb, pgTable, primaryKey, text, timestamp } from "drizzle-orm/pg-core"
|
|
27
27
|
|
|
28
28
|
type JobId = JobStore.JobId
|
|
29
29
|
type QueueName = JobStore.QueueName
|
|
30
|
+
type ScheduleKey = JobStore.ScheduleKey
|
|
30
31
|
type JobState = JobStore.JobState
|
|
31
32
|
type BackoffPolicy = JobStore.BackoffPolicy
|
|
32
33
|
type KeepPolicy = JobStore.KeepPolicy
|
|
33
34
|
type AttemptOutcome = JobStore.AttemptRecord["outcome"]
|
|
34
35
|
/**
|
|
35
36
|
* The jobs table factory. `JobName` types the `name` column — derive it from
|
|
36
|
-
* your job definitions: `mqJobs<typeof
|
|
37
|
+
* your job definitions: `mqJobs<typeof GenerateInvoice._tag | typeof Report._tag>()`.
|
|
37
38
|
*
|
|
38
39
|
* @since 0.1.0
|
|
39
40
|
*/
|
|
@@ -55,6 +56,8 @@ export const mqJobs = <JobName extends string = string>(
|
|
|
55
56
|
stalledCount: integer("stalled_count").notNull().default(0),
|
|
56
57
|
backoff: jsonb("backoff").$type<BackoffPolicy>(),
|
|
57
58
|
keep: jsonb("keep").$type<KeepPolicy>(),
|
|
59
|
+
timeoutMs: bigint("timeout_ms", { mode: "number" }),
|
|
60
|
+
cancelRequested: boolean("cancel_requested").notNull().default(false),
|
|
58
61
|
runAt: timestamp("run_at", { withTimezone: true, mode: "date" }).notNull(),
|
|
59
62
|
enqueuedAt: timestamp("enqueued_at", { withTimezone: true, mode: "date" }).notNull(),
|
|
60
63
|
processedAt: timestamp("processed_at", { withTimezone: true, mode: "date" }),
|
|
@@ -105,6 +108,42 @@ export const mqJobAttempts = (
|
|
|
105
108
|
primaryKey({ columns: [table.jobId, table.attempt] })
|
|
106
109
|
])
|
|
107
110
|
|
|
111
|
+
/**
|
|
112
|
+
* Repeatable-job schedules (one row per `Job.schedule` key).
|
|
113
|
+
*
|
|
114
|
+
* @since 0.2.0
|
|
115
|
+
*/
|
|
116
|
+
export const mqSchedules = (tableName = "effect_mq_schedules") =>
|
|
117
|
+
pgTable(tableName, {
|
|
118
|
+
key: text("key").primaryKey().$type<ScheduleKey>(),
|
|
119
|
+
jobName: text("job_name").notNull(),
|
|
120
|
+
queue: text("queue").notNull().$type<QueueName>(),
|
|
121
|
+
cron: text("cron"),
|
|
122
|
+
tz: text("tz"),
|
|
123
|
+
everyMs: bigint("every_ms", { mode: "number" }),
|
|
124
|
+
payload: jsonb("payload"),
|
|
125
|
+
metadata: jsonb("metadata").notNull().default({}).$type<Record<string, string>>(),
|
|
126
|
+
priority: integer("priority").notNull().default(0),
|
|
127
|
+
attemptsMax: integer("attempts_max").notNull(),
|
|
128
|
+
backoff: jsonb("backoff").$type<BackoffPolicy>(),
|
|
129
|
+
keep: jsonb("keep").$type<KeepPolicy>(),
|
|
130
|
+
timeoutMs: bigint("timeout_ms", { mode: "number" }),
|
|
131
|
+
nextRunAt: timestamp("next_run_at", { withTimezone: true, mode: "date" }).notNull()
|
|
132
|
+
}, (table) => [
|
|
133
|
+
index(`${tableName}_due_idx`).on(table.nextRunAt)
|
|
134
|
+
])
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Durable queue control flags (pause/resume).
|
|
138
|
+
*
|
|
139
|
+
* @since 0.2.0
|
|
140
|
+
*/
|
|
141
|
+
export const mqQueueControl = (tableName = "effect_mq_queue_control") =>
|
|
142
|
+
pgTable(tableName, {
|
|
143
|
+
queue: text("queue").primaryKey().$type<QueueName>(),
|
|
144
|
+
paused: boolean("paused").notNull().default(false)
|
|
145
|
+
})
|
|
146
|
+
|
|
108
147
|
/**
|
|
109
148
|
* @since 0.1.0
|
|
110
149
|
*/
|
|
@@ -114,3 +153,13 @@ export type MqJobsTable = ReturnType<typeof mqJobs<any>>
|
|
|
114
153
|
* @since 0.1.0
|
|
115
154
|
*/
|
|
116
155
|
export type MqJobAttemptsTable = ReturnType<typeof mqJobAttempts>
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* @since 0.2.0
|
|
159
|
+
*/
|
|
160
|
+
export type MqSchedulesTable = ReturnType<typeof mqSchedules>
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* @since 0.2.0
|
|
164
|
+
*/
|
|
165
|
+
export type MqQueueControlTable = ReturnType<typeof mqQueueControl>
|
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A Redis-backed `JobStore` built on `effect/unstable/persistence`'s `Redis`
|
|
3
|
+
* service. Every mutation is a single Lua script (see `scripts.ts`), so all
|
|
4
|
+
* `JobStore` operations are atomic on the server and safe across processes.
|
|
5
|
+
*
|
|
6
|
+
* Provide the `Redis` service from your platform package — `NodeRedis.layer`
|
|
7
|
+
* (`@effect/platform-node`, node-redis), `BunRedis.layer`
|
|
8
|
+
* (`@effect/platform-bun`, `Bun.redis`) — or `Redis.make` over any client.
|
|
9
|
+
*
|
|
10
|
+
* Wake-ups ride the client's pub/sub channel, so workers in other processes
|
|
11
|
+
* pick jobs up promptly; the worker's `pollInterval` is the fallback.
|
|
12
|
+
*
|
|
13
|
+
* @since 0.2.0
|
|
14
|
+
*/
|
|
15
|
+
import { Clock, type Context, Deferred, Duration, Effect, Exit, Layer, Option, Queue, Schedule, type Scope } from "effect"
|
|
16
|
+
import { Redis } from "effect/unstable/persistence"
|
|
17
|
+
import * as JobStore from "../JobStore.ts"
|
|
18
|
+
import * as scripts from "./scripts.ts"
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* @since 0.2.0
|
|
22
|
+
*/
|
|
23
|
+
export interface RedisJobStoreOptions {
|
|
24
|
+
/** Key prefix for everything this store writes (default `effect-mq`). */
|
|
25
|
+
readonly prefix?: string | undefined
|
|
26
|
+
/**
|
|
27
|
+
* Store-level retention ceiling: terminal records older than this are
|
|
28
|
+
* removed by a periodic sweep. Per-job `keep` may only be stricter.
|
|
29
|
+
*/
|
|
30
|
+
readonly historyTtl?: Duration.Input | undefined
|
|
31
|
+
/** History sweep cadence (default 1 minute). */
|
|
32
|
+
readonly historySweepInterval?: Duration.Input | undefined
|
|
33
|
+
/**
|
|
34
|
+
* Generator for store-assigned job ids (e.g. `() => \`job_${ulid()}\``).
|
|
35
|
+
* Default: `j-<n>` from the store's counter. See `JobStore.IdGenerator`.
|
|
36
|
+
*/
|
|
37
|
+
readonly idGenerator?: JobStore.IdGenerator | undefined
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
const storeError = (message: string) => (cause: unknown) => new JobStore.JobStoreError({ message, cause })
|
|
41
|
+
|
|
42
|
+
/** Fold a Lua `HGETALL` reply (flat `[field, value, ...]`) into a map. */
|
|
43
|
+
const foldPairs = (flat: ReadonlyArray<string>): ReadonlyMap<string, string> => {
|
|
44
|
+
const out = new Map<string, string>()
|
|
45
|
+
for (let i = 0; i + 1 < flat.length; i += 2) {
|
|
46
|
+
const field = flat[i]
|
|
47
|
+
const value = flat[i + 1]
|
|
48
|
+
if (field !== undefined && value !== undefined) {
|
|
49
|
+
out.set(field, value)
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return out
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// Hash fields use "" for absent optional values.
|
|
56
|
+
const optionalString = (value: string | undefined): string | undefined =>
|
|
57
|
+
value === undefined || value === "" ? undefined : value
|
|
58
|
+
|
|
59
|
+
const optionalNumber = (value: string | undefined): number | undefined =>
|
|
60
|
+
value === undefined || value === "" ? undefined : Number(value)
|
|
61
|
+
|
|
62
|
+
const optionalJson = <A = unknown>(value: string | undefined): A | undefined => {
|
|
63
|
+
if (value === undefined || value === "") return undefined
|
|
64
|
+
// SAFETY: the value round-trips JSON this driver itself wrote for the field.
|
|
65
|
+
return JSON.parse(value) as A
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const toRecord = (hash: ReadonlyMap<string, string>): JobStore.JobRecord => ({
|
|
69
|
+
id: JobStore.JobId(hash.get("id") ?? ""),
|
|
70
|
+
name: hash.get("name") ?? "",
|
|
71
|
+
queue: JobStore.QueueName(hash.get("queue") ?? ""),
|
|
72
|
+
payload: optionalJson(hash.get("payload")) ?? null,
|
|
73
|
+
metadata: optionalJson<Readonly<Record<string, string>>>(hash.get("metadata")) ?? {},
|
|
74
|
+
// SAFETY: the state field is only ever written with JobState members.
|
|
75
|
+
state: (hash.get("state") ?? "waiting") as JobStore.JobState,
|
|
76
|
+
priority: Number(hash.get("priority") ?? 0),
|
|
77
|
+
attemptsMax: Number(hash.get("attemptsMax") ?? 1),
|
|
78
|
+
attemptsMade: Number(hash.get("attemptsMade") ?? 0),
|
|
79
|
+
stalledCount: Number(hash.get("stalledCount") ?? 0),
|
|
80
|
+
backoff: optionalJson<JobStore.BackoffPolicy>(hash.get("backoff")),
|
|
81
|
+
keep: optionalJson<JobStore.KeepPolicy>(hash.get("keep")),
|
|
82
|
+
timeoutMs: optionalNumber(hash.get("timeoutMs")),
|
|
83
|
+
cancelRequested: hash.get("cancelRequested") === "1",
|
|
84
|
+
runAt: Number(hash.get("runAt") ?? 0),
|
|
85
|
+
enqueuedAt: Number(hash.get("enqueuedAt") ?? 0),
|
|
86
|
+
processedAt: optionalNumber(hash.get("processedAt")),
|
|
87
|
+
finishedAt: optionalNumber(hash.get("finishedAt")),
|
|
88
|
+
exit: optionalJson(hash.get("exit")),
|
|
89
|
+
failedReason: optionalString(hash.get("failedReason"))
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
const toSchedule = (hash: ReadonlyMap<string, string>): JobStore.ScheduleRecord => ({
|
|
93
|
+
key: JobStore.ScheduleKey(hash.get("key") ?? ""),
|
|
94
|
+
jobName: hash.get("jobName") ?? "",
|
|
95
|
+
queue: JobStore.QueueName(hash.get("queue") ?? ""),
|
|
96
|
+
cron: optionalString(hash.get("cron")),
|
|
97
|
+
tz: optionalString(hash.get("tz")),
|
|
98
|
+
everyMs: optionalNumber(hash.get("everyMs")),
|
|
99
|
+
payload: optionalJson(hash.get("payload")),
|
|
100
|
+
metadata: optionalJson<Readonly<Record<string, string>>>(hash.get("metadata")) ?? {},
|
|
101
|
+
priority: Number(hash.get("priority") ?? 0),
|
|
102
|
+
attemptsMax: Number(hash.get("attemptsMax") ?? 1),
|
|
103
|
+
backoff: optionalJson<JobStore.BackoffPolicy>(hash.get("backoff")),
|
|
104
|
+
keep: optionalJson<JobStore.KeepPolicy>(hash.get("keep")),
|
|
105
|
+
timeoutMs: optionalNumber(hash.get("timeoutMs")),
|
|
106
|
+
nextRunAt: Number(hash.get("nextRunAt") ?? 0)
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
// cjson encodes an empty Lua table as {}, not [] — normalize.
|
|
110
|
+
const asArray = <A>(value: ReadonlyArray<A> | Record<string, never>): ReadonlyArray<A> =>
|
|
111
|
+
Array.isArray(value) ? value : []
|
|
112
|
+
|
|
113
|
+
const JOB_STATES: ReadonlyArray<JobStore.JobState> = [
|
|
114
|
+
"waiting",
|
|
115
|
+
"delayed",
|
|
116
|
+
"active",
|
|
117
|
+
"completed",
|
|
118
|
+
"failed",
|
|
119
|
+
"cancelled"
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Build a `RedisJobStore` service. Needs the `Redis` service and a `Scope`
|
|
124
|
+
* (the wake-up subscription and the optional history sweeper live in it).
|
|
125
|
+
*
|
|
126
|
+
* @since 0.2.0
|
|
127
|
+
*/
|
|
128
|
+
export const make = (
|
|
129
|
+
options?: RedisJobStoreOptions | undefined
|
|
130
|
+
): Effect.Effect<JobStore.Service, never, Redis.Redis | Scope.Scope> =>
|
|
131
|
+
Effect.gen(function*() {
|
|
132
|
+
const redis = yield* Redis.Redis
|
|
133
|
+
const prefix = options?.prefix ?? "effect-mq"
|
|
134
|
+
const wakeChannel = `${prefix}:wake`
|
|
135
|
+
|
|
136
|
+
const evalEnqueue = redis.eval(scripts.enqueue)
|
|
137
|
+
const evalClaim = redis.eval(scripts.claim)
|
|
138
|
+
const evalAck = redis.eval(scripts.ack)
|
|
139
|
+
const evalRelease = redis.eval(scripts.release)
|
|
140
|
+
const evalExtendLocks = redis.eval(scripts.extendLocks)
|
|
141
|
+
const evalRecoverStalled = redis.eval(scripts.recoverStalled)
|
|
142
|
+
const evalGetJob = redis.eval(scripts.getJob)
|
|
143
|
+
const evalList = redis.eval(scripts.list)
|
|
144
|
+
const evalCounts = redis.eval(scripts.counts)
|
|
145
|
+
const evalRemove = redis.eval(scripts.remove)
|
|
146
|
+
const evalRetry = redis.eval(scripts.retry)
|
|
147
|
+
const evalCancel = redis.eval(scripts.cancel)
|
|
148
|
+
const evalPromote = redis.eval(scripts.promote)
|
|
149
|
+
const evalUpsertSchedule = redis.eval(scripts.upsertSchedule)
|
|
150
|
+
const evalRemoveSchedule = redis.eval(scripts.removeSchedule)
|
|
151
|
+
const evalListSchedules = redis.eval(scripts.listSchedules)
|
|
152
|
+
const evalDueSchedules = redis.eval(scripts.dueSchedules)
|
|
153
|
+
const evalAdvanceSchedule = redis.eval(scripts.advanceSchedule)
|
|
154
|
+
const evalSweepHistory = redis.eval(scripts.sweepHistory)
|
|
155
|
+
|
|
156
|
+
// Wake protocol: a local version + Deferred chain (same-process wake-ups
|
|
157
|
+
// never depend on the pub/sub round trip), with the channel carrying
|
|
158
|
+
// cross-process wake-ups. Same shape as the Postgres driver's NOTIFY.
|
|
159
|
+
let wakeVersion = 0
|
|
160
|
+
let wake = Deferred.makeUnsafe<void>()
|
|
161
|
+
const signalWakeLocal = () => {
|
|
162
|
+
wakeVersion += 1
|
|
163
|
+
const current = wake
|
|
164
|
+
wake = Deferred.makeUnsafe<void>()
|
|
165
|
+
Deferred.doneUnsafe(current, Exit.succeed<void>(void 0))
|
|
166
|
+
}
|
|
167
|
+
const wakeUp = Effect.suspend(() => {
|
|
168
|
+
signalWakeLocal()
|
|
169
|
+
return redis.send("PUBLISH", wakeChannel, "1").pipe(Effect.ignore)
|
|
170
|
+
})
|
|
171
|
+
|
|
172
|
+
// Cross-process wake-ups. The pump resubscribes on connection loss (Bun
|
|
173
|
+
// subscribers do not auto-reconnect); the retry delay only ever runs
|
|
174
|
+
// after a real failure, so TestClock runs are unaffected.
|
|
175
|
+
yield* Effect.scoped(
|
|
176
|
+
Effect.gen(function*() {
|
|
177
|
+
const messages = yield* redis.subscribe(wakeChannel)
|
|
178
|
+
while (true) {
|
|
179
|
+
yield* Queue.take(messages)
|
|
180
|
+
signalWakeLocal()
|
|
181
|
+
}
|
|
182
|
+
})
|
|
183
|
+
).pipe(
|
|
184
|
+
Effect.retry(Schedule.spaced("1 second")),
|
|
185
|
+
Effect.catchCause((cause) => Effect.logWarning("effect-mq: redis wake subscription failed", cause)),
|
|
186
|
+
Effect.forkScoped
|
|
187
|
+
)
|
|
188
|
+
|
|
189
|
+
if (options?.historyTtl !== undefined) {
|
|
190
|
+
const ttlMs = Duration.toMillis(options.historyTtl)
|
|
191
|
+
const intervalMs = Duration.toMillis(options.historySweepInterval ?? "1 minute")
|
|
192
|
+
yield* Effect.gen(function*() {
|
|
193
|
+
yield* Effect.sleep(intervalMs)
|
|
194
|
+
const now = yield* Clock.currentTimeMillis
|
|
195
|
+
while ((yield* evalSweepHistory(prefix, now - ttlMs, 500)) !== "0") {
|
|
196
|
+
// bounded batches until the window is clean
|
|
197
|
+
}
|
|
198
|
+
}).pipe(
|
|
199
|
+
Effect.catchCause((cause) => Effect.logError("effect-mq: redis history sweep failed", cause)),
|
|
200
|
+
Effect.forever,
|
|
201
|
+
Effect.forkScoped
|
|
202
|
+
)
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const generateCandidate = (request: JobStore.EnqueueRequest, generate: JobStore.IdGenerator) =>
|
|
206
|
+
Effect.suspend(() => {
|
|
207
|
+
const raw = generate(request)
|
|
208
|
+
return Effect.isEffect(raw) ? raw : Effect.succeed(raw)
|
|
209
|
+
})
|
|
210
|
+
|
|
211
|
+
const enqueueOnce = (
|
|
212
|
+
request: JobStore.EnqueueRequest,
|
|
213
|
+
idMode: "user" | "generated" | "auto",
|
|
214
|
+
id: string,
|
|
215
|
+
now: number
|
|
216
|
+
) =>
|
|
217
|
+
evalEnqueue(
|
|
218
|
+
prefix,
|
|
219
|
+
idMode,
|
|
220
|
+
id,
|
|
221
|
+
request.name,
|
|
222
|
+
request.queue,
|
|
223
|
+
JSON.stringify(request.payload ?? null),
|
|
224
|
+
JSON.stringify(request.metadata),
|
|
225
|
+
request.priority,
|
|
226
|
+
request.attemptsMax,
|
|
227
|
+
request.backoff === undefined ? "" : JSON.stringify(request.backoff),
|
|
228
|
+
request.keep === undefined ? "" : JSON.stringify(request.keep),
|
|
229
|
+
request.timeoutMs === undefined ? "" : String(request.timeoutMs),
|
|
230
|
+
Math.max(0, request.delayMs),
|
|
231
|
+
now
|
|
232
|
+
)
|
|
233
|
+
|
|
234
|
+
const store: JobStore.Service = {
|
|
235
|
+
enqueue: (request) =>
|
|
236
|
+
Effect.gen(function*() {
|
|
237
|
+
const now = yield* Clock.currentTimeMillis
|
|
238
|
+
const generate = options?.idGenerator
|
|
239
|
+
for (let i = 0; i < 5; i++) {
|
|
240
|
+
const mode = request.id !== undefined
|
|
241
|
+
? "user" as const
|
|
242
|
+
: generate !== undefined
|
|
243
|
+
? "generated" as const
|
|
244
|
+
: "auto" as const
|
|
245
|
+
const candidate = mode === "user"
|
|
246
|
+
? request.id ?? ""
|
|
247
|
+
: mode === "generated" && generate !== undefined
|
|
248
|
+
? yield* generateCandidate(request, generate)
|
|
249
|
+
: ""
|
|
250
|
+
const reply: {
|
|
251
|
+
id?: string
|
|
252
|
+
duplicate?: boolean
|
|
253
|
+
wake?: boolean
|
|
254
|
+
collision?: boolean
|
|
255
|
+
error?: string
|
|
256
|
+
} = JSON.parse(yield* enqueueOnce(request, mode, candidate, now))
|
|
257
|
+
if (reply.collision === true) continue
|
|
258
|
+
if (reply.error !== undefined || reply.id === undefined) {
|
|
259
|
+
return yield* new JobStore.JobStoreError({
|
|
260
|
+
message: "enqueue failed: could not generate a unique job id"
|
|
261
|
+
})
|
|
262
|
+
}
|
|
263
|
+
if (reply.wake === true) {
|
|
264
|
+
yield* wakeUp
|
|
265
|
+
}
|
|
266
|
+
return { id: JobStore.JobId(reply.id), duplicate: reply.duplicate === true }
|
|
267
|
+
}
|
|
268
|
+
return yield* new JobStore.JobStoreError({
|
|
269
|
+
message: "enqueue failed: could not generate a unique job id"
|
|
270
|
+
})
|
|
271
|
+
}).pipe(
|
|
272
|
+
Effect.mapError((error) =>
|
|
273
|
+
error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)
|
|
274
|
+
)
|
|
275
|
+
),
|
|
276
|
+
|
|
277
|
+
claim: (claimOptions) =>
|
|
278
|
+
// Snapshot BEFORE the script runs: a wake that fires while the claim
|
|
279
|
+
// executes must make awaitWake(token) return immediately.
|
|
280
|
+
Effect.suspend(() => {
|
|
281
|
+
const observedWake = wakeVersion
|
|
282
|
+
return Effect.gen(function*() {
|
|
283
|
+
const now = yield* Clock.currentTimeMillis
|
|
284
|
+
const reply: { job?: ReadonlyArray<string>; empty?: boolean; nextRunAt?: number } = JSON.parse(
|
|
285
|
+
yield* evalClaim(
|
|
286
|
+
prefix,
|
|
287
|
+
claimOptions.queue,
|
|
288
|
+
JSON.stringify(claimOptions.names),
|
|
289
|
+
claimOptions.token,
|
|
290
|
+
claimOptions.lockDurationMs,
|
|
291
|
+
now
|
|
292
|
+
)
|
|
293
|
+
)
|
|
294
|
+
if (reply.job !== undefined) {
|
|
295
|
+
const claimed: JobStore.ClaimResult = { _tag: "Claimed", job: toRecord(foldPairs(reply.job)) }
|
|
296
|
+
return claimed
|
|
297
|
+
}
|
|
298
|
+
const empty: JobStore.ClaimResult = {
|
|
299
|
+
_tag: "Empty",
|
|
300
|
+
nextRunAt: reply.nextRunAt,
|
|
301
|
+
wakeToken: observedWake
|
|
302
|
+
}
|
|
303
|
+
return empty
|
|
304
|
+
})
|
|
305
|
+
}).pipe(Effect.mapError(storeError("claim failed"))),
|
|
306
|
+
|
|
307
|
+
ack: (id, token, outcome) =>
|
|
308
|
+
Effect.gen(function*() {
|
|
309
|
+
const now = yield* Clock.currentTimeMillis
|
|
310
|
+
const exitJson = outcome._tag === "Cancelled" || outcome.exit === undefined
|
|
311
|
+
? ""
|
|
312
|
+
: JSON.stringify(outcome.exit)
|
|
313
|
+
const delayMs = outcome._tag === "Retry" ? Math.max(0, outcome.delayMs) : 0
|
|
314
|
+
const reply: { error?: string; wake?: boolean } = JSON.parse(
|
|
315
|
+
yield* evalAck(prefix, id, token, outcome._tag, exitJson, delayMs, now).pipe(
|
|
316
|
+
Effect.mapError(storeError("ack failed"))
|
|
317
|
+
)
|
|
318
|
+
)
|
|
319
|
+
if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
320
|
+
if (reply.error === "locklost") return yield* new JobStore.LockLostError({ jobId: id })
|
|
321
|
+
if (reply.wake === true) {
|
|
322
|
+
yield* wakeUp
|
|
323
|
+
}
|
|
324
|
+
}),
|
|
325
|
+
|
|
326
|
+
release: (id, token) =>
|
|
327
|
+
Effect.gen(function*() {
|
|
328
|
+
const now = yield* Clock.currentTimeMillis
|
|
329
|
+
const reply: { error?: string; wake?: boolean } = JSON.parse(
|
|
330
|
+
yield* evalRelease(prefix, id, token, now).pipe(Effect.mapError(storeError("release failed")))
|
|
331
|
+
)
|
|
332
|
+
if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
333
|
+
if (reply.error === "locklost") return yield* new JobStore.LockLostError({ jobId: id })
|
|
334
|
+
if (reply.wake === true) {
|
|
335
|
+
yield* wakeUp
|
|
336
|
+
}
|
|
337
|
+
}),
|
|
338
|
+
|
|
339
|
+
extendLocks: (locks, durationMs) =>
|
|
340
|
+
Effect.gen(function*() {
|
|
341
|
+
if (locks.length === 0) {
|
|
342
|
+
const empty: JobStore.ExtendLocksResult = { lost: [], cancelRequested: [] }
|
|
343
|
+
return empty
|
|
344
|
+
}
|
|
345
|
+
const now = yield* Clock.currentTimeMillis
|
|
346
|
+
const reply: {
|
|
347
|
+
lost: ReadonlyArray<string> | Record<string, never>
|
|
348
|
+
cancel: ReadonlyArray<string> | Record<string, never>
|
|
349
|
+
} = JSON.parse(yield* evalExtendLocks(prefix, JSON.stringify(locks), durationMs, now))
|
|
350
|
+
const result: JobStore.ExtendLocksResult = {
|
|
351
|
+
lost: asArray(reply.lost).map(JobStore.JobId),
|
|
352
|
+
cancelRequested: asArray(reply.cancel).map(JobStore.JobId)
|
|
353
|
+
}
|
|
354
|
+
return result
|
|
355
|
+
}).pipe(Effect.mapError(storeError("extendLocks failed"))),
|
|
356
|
+
|
|
357
|
+
recoverStalled: (recoverOptions) =>
|
|
358
|
+
Effect.gen(function*() {
|
|
359
|
+
const now = yield* Clock.currentTimeMillis
|
|
360
|
+
const recovered: ReadonlyArray<{ id: string; failed: boolean }> = JSON.parse(
|
|
361
|
+
yield* evalRecoverStalled(prefix, recoverOptions.maxStalledCount, now)
|
|
362
|
+
)
|
|
363
|
+
const result = recovered.map((entry) => ({ id: JobStore.JobId(entry.id), failed: entry.failed }))
|
|
364
|
+
if (result.some((entry) => !entry.failed)) {
|
|
365
|
+
yield* wakeUp
|
|
366
|
+
}
|
|
367
|
+
return result
|
|
368
|
+
}).pipe(Effect.mapError(storeError("recoverStalled failed"))),
|
|
369
|
+
|
|
370
|
+
awaitWake: (_queues, wakeToken) =>
|
|
371
|
+
Effect.suspend(() => {
|
|
372
|
+
if (wakeVersion > wakeToken) return Effect.void
|
|
373
|
+
return Deferred.await(wake)
|
|
374
|
+
}),
|
|
375
|
+
|
|
376
|
+
getJob: (id) =>
|
|
377
|
+
evalGetJob(prefix, id).pipe(
|
|
378
|
+
Effect.mapError(storeError("getJob failed")),
|
|
379
|
+
Effect.map((raw) => {
|
|
380
|
+
const flat: ReadonlyArray<string> = JSON.parse(raw)
|
|
381
|
+
return flat.length === 0 ? Option.none() : Option.some(toRecord(foldPairs(flat)))
|
|
382
|
+
})
|
|
383
|
+
),
|
|
384
|
+
|
|
385
|
+
getAttempts: (id) =>
|
|
386
|
+
redis.send("LRANGE", `${prefix}:attempts:${id}`, "0", "-1").pipe(
|
|
387
|
+
Effect.mapError(storeError("getAttempts failed")),
|
|
388
|
+
Effect.map((raw) => {
|
|
389
|
+
// SAFETY: LRANGE always replies with an array of bulk strings.
|
|
390
|
+
const entries = raw as ReadonlyArray<string>
|
|
391
|
+
return entries.map((entry) => {
|
|
392
|
+
const parsed: {
|
|
393
|
+
attempt: number
|
|
394
|
+
startedAt: number | null
|
|
395
|
+
finishedAt: number
|
|
396
|
+
outcome: JobStore.AttemptRecord["outcome"]
|
|
397
|
+
exit?: unknown
|
|
398
|
+
} = JSON.parse(entry)
|
|
399
|
+
const record: JobStore.AttemptRecord = {
|
|
400
|
+
attempt: parsed.attempt,
|
|
401
|
+
startedAt: parsed.startedAt ?? undefined,
|
|
402
|
+
finishedAt: parsed.finishedAt,
|
|
403
|
+
outcome: parsed.outcome,
|
|
404
|
+
// The ledger omits the key entirely for absent exits, so a
|
|
405
|
+
// legitimate encoded null exit survives the round trip.
|
|
406
|
+
exit: Object.hasOwn(parsed, "exit") ? parsed.exit : undefined
|
|
407
|
+
}
|
|
408
|
+
return record
|
|
409
|
+
})
|
|
410
|
+
})
|
|
411
|
+
),
|
|
412
|
+
|
|
413
|
+
list: (listOptions) =>
|
|
414
|
+
Effect.gen(function*() {
|
|
415
|
+
const limit = Math.max(1, listOptions.limit ?? 50)
|
|
416
|
+
const filters = {
|
|
417
|
+
queue: listOptions.queue,
|
|
418
|
+
name: listOptions.name,
|
|
419
|
+
states: listOptions.states,
|
|
420
|
+
metadata: listOptions.metadata
|
|
421
|
+
}
|
|
422
|
+
const reply: { items: ReadonlyArray<ReadonlyArray<string>> | Record<string, never>; more: boolean } = JSON
|
|
423
|
+
.parse(
|
|
424
|
+
yield* evalList(prefix, JSON.stringify(filters), listOptions.cursor ?? "", limit)
|
|
425
|
+
)
|
|
426
|
+
const items = asArray(reply.items).map((flat) => toRecord(foldPairs(flat)))
|
|
427
|
+
const last = items[items.length - 1]
|
|
428
|
+
const result: JobStore.ListResult = {
|
|
429
|
+
items,
|
|
430
|
+
cursor: reply.more && last !== undefined ? `${last.enqueuedAt}:${last.id}` : undefined
|
|
431
|
+
}
|
|
432
|
+
return result
|
|
433
|
+
}).pipe(Effect.mapError(storeError("list failed"))),
|
|
434
|
+
|
|
435
|
+
retry: (id) =>
|
|
436
|
+
Effect.gen(function*() {
|
|
437
|
+
const now = yield* Clock.currentTimeMillis
|
|
438
|
+
const reply: { error?: string; state?: JobStore.JobState } = JSON.parse(
|
|
439
|
+
yield* evalRetry(prefix, id, now).pipe(Effect.mapError(storeError("retry failed")))
|
|
440
|
+
)
|
|
441
|
+
if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
442
|
+
if (reply.error === "state") {
|
|
443
|
+
return yield* new JobStore.JobNotRetryableError({ jobId: id, state: reply.state ?? "failed" })
|
|
444
|
+
}
|
|
445
|
+
yield* wakeUp
|
|
446
|
+
}),
|
|
447
|
+
|
|
448
|
+
counts: (queue) =>
|
|
449
|
+
evalCounts(prefix).pipe(
|
|
450
|
+
Effect.mapError(storeError("counts failed")),
|
|
451
|
+
Effect.map((raw) => {
|
|
452
|
+
const flat: ReadonlyArray<string> = JSON.parse(raw)
|
|
453
|
+
const byField = foldPairs(flat)
|
|
454
|
+
// SAFETY: fromEntries over the exhaustive JOB_STATES list yields
|
|
455
|
+
// exactly one zeroed entry per JobState member.
|
|
456
|
+
const totals = Object.fromEntries(JOB_STATES.map((state) => [state, 0])) as Record<
|
|
457
|
+
JobStore.JobState,
|
|
458
|
+
number
|
|
459
|
+
>
|
|
460
|
+
for (const [field, count] of byField) {
|
|
461
|
+
const split = field.lastIndexOf("|")
|
|
462
|
+
const fieldQueue = field.slice(0, split)
|
|
463
|
+
// SAFETY: counts fields are written as `<queue>|<JobState>`.
|
|
464
|
+
const state = field.slice(split + 1) as JobStore.JobState
|
|
465
|
+
if (queue === undefined || fieldQueue === queue) {
|
|
466
|
+
totals[state] += Number(count)
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
return totals
|
|
470
|
+
})
|
|
471
|
+
),
|
|
472
|
+
|
|
473
|
+
remove: (id) =>
|
|
474
|
+
evalRemove(prefix, id).pipe(
|
|
475
|
+
Effect.mapError(storeError("remove failed")),
|
|
476
|
+
Effect.map((raw) => raw !== "0")
|
|
477
|
+
),
|
|
478
|
+
|
|
479
|
+
cancel: (id) =>
|
|
480
|
+
Effect.gen(function*() {
|
|
481
|
+
const now = yield* Clock.currentTimeMillis
|
|
482
|
+
const reply: { error?: string; state?: JobStore.JobState } = JSON.parse(
|
|
483
|
+
yield* evalCancel(prefix, id, now).pipe(Effect.mapError(storeError("cancel failed")))
|
|
484
|
+
)
|
|
485
|
+
if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
486
|
+
if (reply.error === "state") {
|
|
487
|
+
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: reply.state ?? "completed" })
|
|
488
|
+
}
|
|
489
|
+
}),
|
|
490
|
+
|
|
491
|
+
promote: (id) =>
|
|
492
|
+
Effect.gen(function*() {
|
|
493
|
+
const now = yield* Clock.currentTimeMillis
|
|
494
|
+
const reply: { error?: string; state?: JobStore.JobState } = JSON.parse(
|
|
495
|
+
yield* evalPromote(prefix, id, now).pipe(Effect.mapError(storeError("promote failed")))
|
|
496
|
+
)
|
|
497
|
+
if (reply.error === "notfound") return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
498
|
+
if (reply.error === "state") {
|
|
499
|
+
return yield* new JobStore.JobNotPromotableError({ jobId: id, state: reply.state ?? "completed" })
|
|
500
|
+
}
|
|
501
|
+
yield* wakeUp
|
|
502
|
+
}),
|
|
503
|
+
|
|
504
|
+
pause: (queue) =>
|
|
505
|
+
redis.send("SADD", `${prefix}:paused`, queue).pipe(
|
|
506
|
+
Effect.mapError(storeError("pause failed")),
|
|
507
|
+
Effect.asVoid
|
|
508
|
+
),
|
|
509
|
+
|
|
510
|
+
resume: (queue) =>
|
|
511
|
+
redis.send("SREM", `${prefix}:paused`, queue).pipe(
|
|
512
|
+
Effect.mapError(storeError("resume failed")),
|
|
513
|
+
Effect.flatMap((removed) => Number(removed) > 0 ? wakeUp : Effect.void)
|
|
514
|
+
),
|
|
515
|
+
|
|
516
|
+
pausedQueues: () =>
|
|
517
|
+
redis.send("SMEMBERS", `${prefix}:paused`).pipe(
|
|
518
|
+
Effect.mapError(storeError("pausedQueues failed")),
|
|
519
|
+
Effect.map((raw) => {
|
|
520
|
+
// SAFETY: SMEMBERS always replies with an array of bulk strings.
|
|
521
|
+
const members = raw as ReadonlyArray<string>
|
|
522
|
+
return members.map(JobStore.QueueName)
|
|
523
|
+
})
|
|
524
|
+
),
|
|
525
|
+
|
|
526
|
+
upsertSchedule: (schedule) =>
|
|
527
|
+
evalUpsertSchedule(
|
|
528
|
+
prefix,
|
|
529
|
+
schedule.key,
|
|
530
|
+
schedule.jobName,
|
|
531
|
+
schedule.queue,
|
|
532
|
+
schedule.cron ?? "",
|
|
533
|
+
schedule.tz ?? "",
|
|
534
|
+
schedule.everyMs === undefined ? "" : String(schedule.everyMs),
|
|
535
|
+
schedule.payload === undefined ? "" : JSON.stringify(schedule.payload),
|
|
536
|
+
JSON.stringify(schedule.metadata),
|
|
537
|
+
String(schedule.priority),
|
|
538
|
+
String(schedule.attemptsMax),
|
|
539
|
+
schedule.backoff === undefined ? "" : JSON.stringify(schedule.backoff),
|
|
540
|
+
schedule.keep === undefined ? "" : JSON.stringify(schedule.keep),
|
|
541
|
+
schedule.timeoutMs === undefined ? "" : String(schedule.timeoutMs),
|
|
542
|
+
schedule.nextRunAt
|
|
543
|
+
).pipe(
|
|
544
|
+
Effect.mapError(storeError("upsertSchedule failed")),
|
|
545
|
+
Effect.andThen(wakeUp)
|
|
546
|
+
),
|
|
547
|
+
|
|
548
|
+
removeSchedule: (key) =>
|
|
549
|
+
evalRemoveSchedule(prefix, key).pipe(
|
|
550
|
+
Effect.mapError(storeError("removeSchedule failed")),
|
|
551
|
+
Effect.map((raw) => raw !== "0")
|
|
552
|
+
),
|
|
553
|
+
|
|
554
|
+
listSchedules: (listOptions) =>
|
|
555
|
+
evalListSchedules(prefix, JSON.stringify(listOptions ?? {})).pipe(
|
|
556
|
+
Effect.mapError(storeError("listSchedules failed")),
|
|
557
|
+
Effect.map((raw) => {
|
|
558
|
+
const flat: ReadonlyArray<ReadonlyArray<string>> = JSON.parse(raw)
|
|
559
|
+
return flat.map((pairs) => toSchedule(foldPairs(pairs)))
|
|
560
|
+
})
|
|
561
|
+
),
|
|
562
|
+
|
|
563
|
+
dueSchedules: () =>
|
|
564
|
+
Effect.gen(function*() {
|
|
565
|
+
const now = yield* Clock.currentTimeMillis
|
|
566
|
+
const flat: ReadonlyArray<ReadonlyArray<string>> = JSON.parse(yield* evalDueSchedules(prefix, now))
|
|
567
|
+
return flat.map((pairs) => toSchedule(foldPairs(pairs)))
|
|
568
|
+
}).pipe(Effect.mapError(storeError("dueSchedules failed"))),
|
|
569
|
+
|
|
570
|
+
advanceSchedule: (key, expectedRunAt, nextRunAt) =>
|
|
571
|
+
evalAdvanceSchedule(prefix, key, expectedRunAt, nextRunAt).pipe(
|
|
572
|
+
Effect.mapError(storeError("advanceSchedule failed")),
|
|
573
|
+
Effect.asVoid
|
|
574
|
+
)
|
|
575
|
+
}
|
|
576
|
+
|
|
577
|
+
return store
|
|
578
|
+
})
|
|
579
|
+
|
|
580
|
+
/**
|
|
581
|
+
* A Redis-backed layer for the default `JobStore`.
|
|
582
|
+
*
|
|
583
|
+
* @since 0.2.0
|
|
584
|
+
*/
|
|
585
|
+
export const layer = (
|
|
586
|
+
options?: RedisJobStoreOptions | undefined
|
|
587
|
+
): Layer.Layer<JobStore.JobStore, never, Redis.Redis> => Layer.effect(JobStore.JobStore, make(options))
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* A Redis-backed layer for a specific named store key.
|
|
591
|
+
*
|
|
592
|
+
* @since 0.2.0
|
|
593
|
+
*/
|
|
594
|
+
export const layerFor = <StoreId>(
|
|
595
|
+
store: Context.Key<StoreId, JobStore.Service>,
|
|
596
|
+
options?: RedisJobStoreOptions | undefined
|
|
597
|
+
): Layer.Layer<StoreId, never, Redis.Redis> => Layer.effect(store, make(options))
|