effect-mq 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -17
- package/dist/Flow.d.ts +381 -0
- package/dist/Flow.d.ts.map +1 -0
- package/dist/Flow.js +340 -0
- package/dist/Flow.js.map +1 -0
- package/dist/Job.d.ts +37 -6
- package/dist/Job.d.ts.map +1 -1
- package/dist/Job.js +17 -2
- package/dist/Job.js.map +1 -1
- package/dist/JobSchedules.d.ts +112 -0
- package/dist/JobSchedules.d.ts.map +1 -0
- package/dist/JobSchedules.js +106 -0
- package/dist/JobSchedules.js.map +1 -0
- package/dist/JobStore.d.ts +320 -10
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +336 -8
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/Metrics.d.ts +31 -0
- package/dist/Metrics.d.ts.map +1 -1
- package/dist/Metrics.js +39 -0
- package/dist/Metrics.js.map +1 -1
- package/dist/Worker.d.ts +120 -11
- package/dist/Worker.d.ts.map +1 -1
- package/dist/Worker.js +452 -26
- package/dist/Worker.js.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.js +662 -81
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
- package/dist/drizzle-postgres/schema.d.ts +310 -3
- package/dist/drizzle-postgres/schema.d.ts.map +1 -1
- package/dist/drizzle-postgres/schema.js +68 -1
- package/dist/drizzle-postgres/schema.js.map +1 -1
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -1
- package/dist/redis/RedisJobStore.d.ts.map +1 -1
- package/dist/redis/RedisJobStore.js +221 -19
- package/dist/redis/RedisJobStore.js.map +1 -1
- package/dist/redis/scripts.d.ts +118 -11
- package/dist/redis/scripts.d.ts.map +1 -1
- package/dist/redis/scripts.js +497 -28
- package/dist/redis/scripts.js.map +1 -1
- package/dist/testing/conformance.d.ts +6 -0
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +765 -1
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +1 -1
- package/src/Flow.ts +778 -0
- package/src/Job.ts +42 -11
- package/src/JobSchedules.ts +223 -0
- package/src/JobStore.ts +347 -9
- package/src/MemoryJobStore.ts +372 -8
- package/src/Metrics.ts +43 -0
- package/src/Worker.ts +726 -37
- package/src/drizzle-postgres/DrizzleJobStore.ts +827 -82
- package/src/drizzle-postgres/schema.ts +94 -0
- package/src/index.ts +16 -0
- package/src/redis/RedisJobStore.ts +291 -8
- package/src/redis/scripts.ts +529 -26
- package/src/testing/conformance.ts +989 -1
|
@@ -18,11 +18,19 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import * as JobStore from "../JobStore.ts"
|
|
20
20
|
import type { PgClient } from "@effect/sql-pg"
|
|
21
|
-
import { asc, eq, getTableColumns, sql } from "drizzle-orm"
|
|
21
|
+
import { asc, eq, getTableColumns, type SQL, sql } from "drizzle-orm"
|
|
22
22
|
import * as PgDrizzle from "drizzle-orm/effect-postgres"
|
|
23
23
|
import { getTableConfig } from "drizzle-orm/pg-core"
|
|
24
|
-
import { Clock, type Context, Deferred, Duration, Effect, Layer, Option, type Scope, Stream } from "effect"
|
|
25
|
-
import type {
|
|
24
|
+
import { Cause, Clock, type Context, Deferred, Duration, Effect, Layer, Option, Predicate, type Scope, Stream } from "effect"
|
|
25
|
+
import type {
|
|
26
|
+
MqDedupeTable,
|
|
27
|
+
MqFlowChildrenTable,
|
|
28
|
+
MqFlowOutboxTable,
|
|
29
|
+
MqJobAttemptsTable,
|
|
30
|
+
MqJobsTable,
|
|
31
|
+
MqQueueControlTable,
|
|
32
|
+
MqSchedulesTable
|
|
33
|
+
} from "./schema.ts"
|
|
26
34
|
|
|
27
35
|
const { JobId } = JobStore
|
|
28
36
|
|
|
@@ -40,6 +48,10 @@ export interface DrizzleJobStoreOptions<StoreId = JobStore.JobStore> {
|
|
|
40
48
|
readonly queues: MqQueueControlTable
|
|
41
49
|
/** The dedup-key registry table (from `mqDedupe`). */
|
|
42
50
|
readonly dedupe: MqDedupeTable
|
|
51
|
+
/** The flow dependency-rows table (from `mqFlowChildren`). */
|
|
52
|
+
readonly flowChildren: MqFlowChildrenTable
|
|
53
|
+
/** The child-report outbox table (from `mqFlowOutbox`). */
|
|
54
|
+
readonly flowOutbox: MqFlowOutboxTable
|
|
43
55
|
/**
|
|
44
56
|
* Values for columns added via `mqJobs({ extend })`, evaluated at enqueue
|
|
45
57
|
* (and on dedupe `replace`). Keys are the extended columns' TS names.
|
|
@@ -84,6 +96,50 @@ export type ExtraColumnValue = string | number | boolean | Date | null
|
|
|
84
96
|
const storeError = (message: string) => (cause: unknown) =>
|
|
85
97
|
new JobStore.JobStoreError({ message, cause })
|
|
86
98
|
|
|
99
|
+
// The exact spellings a bigserial ever produces: no leading zeros, no signs,
|
|
100
|
+
// no whitespace — what outbox ids look like and nothing Postgres would
|
|
101
|
+
// silently normalize into one.
|
|
102
|
+
const CANONICAL_BIGSERIAL = /^[1-9]\d*$/
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Whether an error surfaced by drizzle's Effect driver is a Postgres
|
|
106
|
+
* deadlock (40P01), i.e. safe to retry.
|
|
107
|
+
*
|
|
108
|
+
* The deadlock travels wrapped: drizzle's session fails with an
|
|
109
|
+
* `EffectDrizzleQueryError` whose `cause` field is `Cause.fail(SqlError)`
|
|
110
|
+
* (drizzle-orm `pg-core/effect/session.ts`), and `@effect/sql-pg` classifies
|
|
111
|
+
* pg code 40P01 into the `SqlError`'s `reason: DeadlockError` (PgClient.ts
|
|
112
|
+
* `classifyError`). Neither layer's `toString` renders the pg code, so this
|
|
113
|
+
* unwraps structurally instead of string-matching the rendered error.
|
|
114
|
+
*
|
|
115
|
+
* @internal
|
|
116
|
+
*/
|
|
117
|
+
// oxlint-disable-next-line anti-slop/no-unknown-parameters -- a retry-predicate classifier over whatever the driver threw IS the boundary parser
|
|
118
|
+
export const isDeadlockError = (error: unknown): boolean => {
|
|
119
|
+
// Unwrap drizzle's envelope: its `cause` is an Effect Cause holding the
|
|
120
|
+
// original failure. A bare SqlError (non-drizzle path) passes through.
|
|
121
|
+
const unwrapped = Predicate.hasProperty(error, "cause") && Cause.isCause(error.cause)
|
|
122
|
+
? Option.getOrUndefined(Cause.findErrorOption(error.cause))
|
|
123
|
+
: error
|
|
124
|
+
if (
|
|
125
|
+
!Predicate.hasProperty(unwrapped, "_tag") || unwrapped._tag !== "SqlError" ||
|
|
126
|
+
!Predicate.hasProperty(unwrapped, "reason")
|
|
127
|
+
) {
|
|
128
|
+
return false
|
|
129
|
+
}
|
|
130
|
+
const reason = unwrapped.reason
|
|
131
|
+
if (Predicate.hasProperty(reason, "_tag") && reason._tag === "DeadlockError") {
|
|
132
|
+
return true
|
|
133
|
+
}
|
|
134
|
+
// Fallback for classifiers that missed the code: the reason keeps the raw
|
|
135
|
+
// pg error, whose message for 40P01 is "deadlock detected".
|
|
136
|
+
if (Predicate.hasProperty(reason, "cause")) {
|
|
137
|
+
const raw = String(reason.cause)
|
|
138
|
+
return raw.includes("40P01") || raw.includes("deadlock detected")
|
|
139
|
+
}
|
|
140
|
+
return false
|
|
141
|
+
}
|
|
142
|
+
|
|
87
143
|
/**
|
|
88
144
|
* drizzle's Effect driver types raw `execute` results as row arrays, but at
|
|
89
145
|
* runtime the raw path yields the node-postgres `QueryResult` envelope
|
|
@@ -114,6 +170,12 @@ type JobRow = {
|
|
|
114
170
|
readonly cancelRequested: boolean
|
|
115
171
|
readonly dedupeKey: string | null
|
|
116
172
|
readonly trace: JobStore.TraceContext | null
|
|
173
|
+
readonly parent: JobStore.ParentEnvelope | null
|
|
174
|
+
readonly flowFailFast: boolean | null
|
|
175
|
+
readonly flowPending: number | null
|
|
176
|
+
readonly flowCompleted: number | null
|
|
177
|
+
readonly flowFailed: number | null
|
|
178
|
+
readonly flowCancelled: number | null
|
|
117
179
|
readonly runAt: Date
|
|
118
180
|
readonly enqueuedAt: Date
|
|
119
181
|
readonly processedAt: Date | null
|
|
@@ -136,6 +198,7 @@ type ScheduleRow = {
|
|
|
136
198
|
readonly backoff: JobStore.BackoffPolicy | null
|
|
137
199
|
readonly keep: JobStore.KeepPolicy | null
|
|
138
200
|
readonly timeoutMs: number | string | null
|
|
201
|
+
readonly group: string | null
|
|
139
202
|
readonly nextRunAt: Date
|
|
140
203
|
}
|
|
141
204
|
|
|
@@ -153,6 +216,7 @@ const toSchedule = (row: ScheduleRow): JobStore.ScheduleRecord => ({
|
|
|
153
216
|
backoff: row.backoff ?? undefined,
|
|
154
217
|
keep: row.keep ?? undefined,
|
|
155
218
|
timeoutMs: row.timeoutMs === null ? undefined : Number(row.timeoutMs),
|
|
219
|
+
group: row.group ?? undefined,
|
|
156
220
|
nextRunAt: row.nextRunAt.getTime()
|
|
157
221
|
})
|
|
158
222
|
|
|
@@ -173,6 +237,17 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
|
|
|
173
237
|
cancelRequested: row.cancelRequested,
|
|
174
238
|
dedupeKey: row.dedupeKey ?? undefined,
|
|
175
239
|
trace: row.trace ?? undefined,
|
|
240
|
+
parent: row.parent ?? undefined,
|
|
241
|
+
// The flow columns are NULL together; `flowPending` is the presence marker.
|
|
242
|
+
flow: row.flowPending === null || row.flowPending === undefined
|
|
243
|
+
? undefined
|
|
244
|
+
: {
|
|
245
|
+
failFast: row.flowFailFast === true,
|
|
246
|
+
pending: Number(row.flowPending),
|
|
247
|
+
completed: Number(row.flowCompleted ?? 0),
|
|
248
|
+
failed: Number(row.flowFailed ?? 0),
|
|
249
|
+
cancelled: Number(row.flowCancelled ?? 0)
|
|
250
|
+
},
|
|
176
251
|
runAt: row.runAt.getTime(),
|
|
177
252
|
enqueuedAt: row.enqueuedAt.getTime(),
|
|
178
253
|
processedAt: row.processedAt?.getTime(),
|
|
@@ -181,6 +256,31 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
|
|
|
181
256
|
failedReason: row.failedReason ?? undefined
|
|
182
257
|
})
|
|
183
258
|
|
|
259
|
+
type FlowChildRow = {
|
|
260
|
+
readonly flowId: string
|
|
261
|
+
readonly childKey: string
|
|
262
|
+
readonly name: string
|
|
263
|
+
readonly storeKey: string
|
|
264
|
+
/** Projected from `spec->>'id'` (never NULL: the FanOut ack validated it). */
|
|
265
|
+
readonly childJobId: string
|
|
266
|
+
readonly status: JobStore.FlowChildRecord["status"]
|
|
267
|
+
readonly exit: unknown
|
|
268
|
+
readonly failedReason: string | null
|
|
269
|
+
readonly cascaded: boolean
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const toFlowChildRecord = (row: FlowChildRow): JobStore.FlowChildRecord => ({
|
|
273
|
+
flowId: JobId(row.flowId),
|
|
274
|
+
childKey: row.childKey,
|
|
275
|
+
name: row.name,
|
|
276
|
+
storeKey: row.storeKey,
|
|
277
|
+
childJobId: JobId(row.childJobId),
|
|
278
|
+
status: row.status,
|
|
279
|
+
exit: row.exit ?? undefined,
|
|
280
|
+
failedReason: row.failedReason ?? undefined,
|
|
281
|
+
cascaded: row.cascaded
|
|
282
|
+
})
|
|
283
|
+
|
|
184
284
|
/**
|
|
185
285
|
* Build the store implementation. Requires `PgClient` and a `Scope` (for the
|
|
186
286
|
* LISTEN subscription).
|
|
@@ -198,6 +298,8 @@ export const make = (
|
|
|
198
298
|
const schedules = options.schedules
|
|
199
299
|
const queues = options.queues
|
|
200
300
|
const dedupe = options.dedupe
|
|
301
|
+
const flowChildren = options.flowChildren
|
|
302
|
+
const flowOutbox = options.flowOutbox
|
|
201
303
|
const jobsName = getTableConfig(jobs).name
|
|
202
304
|
const attemptsName = getTableConfig(attempts).name
|
|
203
305
|
const wakeChannel = `effect_mq_wake_${jobsName}`
|
|
@@ -223,6 +325,12 @@ export const make = (
|
|
|
223
325
|
"cancelRequested",
|
|
224
326
|
"dedupeKey",
|
|
225
327
|
"trace",
|
|
328
|
+
"parent",
|
|
329
|
+
"flowFailFast",
|
|
330
|
+
"flowPending",
|
|
331
|
+
"flowCompleted",
|
|
332
|
+
"flowFailed",
|
|
333
|
+
"flowCancelled",
|
|
226
334
|
"runAt",
|
|
227
335
|
"enqueuedAt",
|
|
228
336
|
"processedAt",
|
|
@@ -261,7 +369,9 @@ export const make = (
|
|
|
261
369
|
db.select({ jobId: attempts.jobId }).from(attempts).limit(0),
|
|
262
370
|
db.select({ key: schedules.key }).from(schedules).limit(0),
|
|
263
371
|
db.select({ queue: queues.queue }).from(queues).limit(0),
|
|
264
|
-
db.select({ key: dedupe.key }).from(dedupe).limit(0)
|
|
372
|
+
db.select({ key: dedupe.key }).from(dedupe).limit(0),
|
|
373
|
+
db.select({ flowId: flowChildren.flowId }).from(flowChildren).limit(0),
|
|
374
|
+
db.select({ id: flowOutbox.id }).from(flowOutbox).limit(0)
|
|
265
375
|
]).pipe(
|
|
266
376
|
Effect.mapError(storeError(
|
|
267
377
|
`effect-mq: tables "${jobsName}"/"${attemptsName}" are missing or mismatched — ` +
|
|
@@ -322,6 +432,28 @@ export const make = (
|
|
|
322
432
|
Effect.forever,
|
|
323
433
|
Effect.forkScoped
|
|
324
434
|
)
|
|
435
|
+
// Automatic retention (the store-level sweep and per-job `keep`) never
|
|
436
|
+
// prunes a flow parent that still owes cascade cancels: its dependency
|
|
437
|
+
// rows marked `cancelled` and not `cascaded` are the only record that
|
|
438
|
+
// real cancels are still due in the child stores. The explicit `remove`
|
|
439
|
+
// verb is not exempted.
|
|
440
|
+
const owesCascades = sql`EXISTS (
|
|
441
|
+
SELECT 1 FROM ${flowChildren}
|
|
442
|
+
WHERE ${flowChildren.flowId} = ${jobs.id}
|
|
443
|
+
AND ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
|
|
444
|
+
)`
|
|
445
|
+
// A pruned flow parent's dependency rows go with it in the same statement.
|
|
446
|
+
const purgeJobsWhere = (predicate: SQL) =>
|
|
447
|
+
sql`
|
|
448
|
+
WITH deleted AS (
|
|
449
|
+
DELETE FROM ${jobs}
|
|
450
|
+
WHERE ${predicate}
|
|
451
|
+
RETURNING ${jobs.id} AS id
|
|
452
|
+
)
|
|
453
|
+
DELETE FROM ${flowChildren}
|
|
454
|
+
WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
|
|
455
|
+
`
|
|
456
|
+
|
|
325
457
|
if (options.historyTtl !== undefined) {
|
|
326
458
|
const ttlByState = JobStore.normalizeHistoryTtl(options.historyTtl)
|
|
327
459
|
const sweepMs = Duration.toMillis(options.historySweepInterval ?? "1 minute")
|
|
@@ -332,9 +464,8 @@ export const make = (
|
|
|
332
464
|
// job name is pruned on the timer, not only when its group is acked.
|
|
333
465
|
for (const state of ["completed", "failed", "cancelled"] as const) {
|
|
334
466
|
const ttl = ttlByState[state]
|
|
335
|
-
yield* db.execute(sql`
|
|
336
|
-
|
|
337
|
-
WHERE ${jobs.state} = ${state} AND (
|
|
467
|
+
yield* db.execute(purgeJobsWhere(sql`
|
|
468
|
+
${jobs.state} = ${state} AND NOT ${owesCascades} AND (
|
|
338
469
|
${ttl !== undefined ? sql`${jobs.finishedAt} <= ${new Date(now.getTime() - ttl)}` : sql`FALSE`}
|
|
339
470
|
OR (
|
|
340
471
|
COALESCE(
|
|
@@ -349,7 +480,7 @@ export const make = (
|
|
|
349
480
|
ELSE ${jobs.keep}->>'ageMs' END
|
|
350
481
|
)::double precision) / 1000.0))
|
|
351
482
|
)
|
|
352
|
-
`)
|
|
483
|
+
`))
|
|
353
484
|
}
|
|
354
485
|
// Dead dedup rows: expired windows, or pointers at vanished jobs.
|
|
355
486
|
yield* db.execute(sql`
|
|
@@ -357,7 +488,7 @@ export const make = (
|
|
|
357
488
|
WHERE (${dedupe.windowExpiresAt} IS NOT NULL AND ${dedupe.windowExpiresAt} <= ${now})
|
|
358
489
|
OR (${dedupe.windowExpiresAt} IS NULL AND NOT EXISTS (
|
|
359
490
|
SELECT 1 FROM ${jobs} WHERE ${jobs.id} = ${dedupe.jobId}
|
|
360
|
-
AND ${jobs.state} IN ('waiting', 'delayed', 'active')
|
|
491
|
+
AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
|
|
361
492
|
))
|
|
362
493
|
`)
|
|
363
494
|
}).pipe(
|
|
@@ -431,23 +562,23 @@ export const make = (
|
|
|
431
562
|
const keep = keepPolicyFor(row.keep, row.state)
|
|
432
563
|
if (keep === undefined) return
|
|
433
564
|
if (keep.ageMs !== undefined) {
|
|
434
|
-
yield* tx.execute(sql`
|
|
435
|
-
|
|
436
|
-
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
565
|
+
yield* tx.execute(purgeJobsWhere(sql`
|
|
566
|
+
${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
437
567
|
AND ${jobs.finishedAt} <= ${new Date(now.getTime() - keep.ageMs)}
|
|
438
|
-
|
|
568
|
+
AND NOT ${owesCascades}
|
|
569
|
+
`))
|
|
439
570
|
}
|
|
440
571
|
if (keep.count !== undefined) {
|
|
441
|
-
yield* tx.execute(sql`
|
|
442
|
-
|
|
443
|
-
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
572
|
+
yield* tx.execute(purgeJobsWhere(sql`
|
|
573
|
+
${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
444
574
|
AND ${jobs.id} NOT IN (
|
|
445
575
|
SELECT ${jobs.id} FROM ${jobs}
|
|
446
576
|
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
447
577
|
ORDER BY ${jobs.finishedAt} DESC, ${jobs.seq} DESC
|
|
448
578
|
LIMIT ${keep.count}
|
|
449
579
|
)
|
|
450
|
-
|
|
580
|
+
AND NOT ${owesCascades}
|
|
581
|
+
`))
|
|
451
582
|
}
|
|
452
583
|
})
|
|
453
584
|
|
|
@@ -493,7 +624,7 @@ export const make = (
|
|
|
493
624
|
: sql`'j-' || ${seqExpr}::text`
|
|
494
625
|
const rows = rowsOf(yield* exec.execute<{ id: string }>(sql`
|
|
495
626
|
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
496
|
-
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
627
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
|
|
497
628
|
VALUES (${idExpr}, ${request.name}, ${request.queue}, ${state}, ${request.priority},
|
|
498
629
|
${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
|
|
499
630
|
${request.attemptsMax},
|
|
@@ -501,6 +632,7 @@ export const make = (
|
|
|
501
632
|
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
502
633
|
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
503
634
|
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
635
|
+
${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
|
|
504
636
|
${runAt}, ${now}${extraColumnValues(request)})
|
|
505
637
|
ON CONFLICT (id) DO NOTHING
|
|
506
638
|
RETURNING ${jobs.id} AS id
|
|
@@ -631,7 +763,7 @@ export const make = (
|
|
|
631
763
|
// killed and safe to retry.
|
|
632
764
|
Effect.retry({
|
|
633
765
|
times: 3,
|
|
634
|
-
while:
|
|
766
|
+
while: isDeadlockError
|
|
635
767
|
}),
|
|
636
768
|
Effect.mapError((error) =>
|
|
637
769
|
error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)
|
|
@@ -656,59 +788,139 @@ export const make = (
|
|
|
656
788
|
AND (${dedupe.windowExpiresAt} IS NULL OR ${dedupe.windowExpiresAt} <= ${now})
|
|
657
789
|
`).pipe(Effect.asVoid)
|
|
658
790
|
|
|
791
|
+
// The outbox invariant: every operation that moves a job carrying a
|
|
792
|
+
// `parent` envelope INTO a terminal state appends its report here in the
|
|
793
|
+
// same transaction (see `JobStore.OutboxEntry`). `exit`/`failedReason`
|
|
794
|
+
// mirror the job row AFTER the transition; JSON.stringify drops
|
|
795
|
+
// undefined fields so absent values read back as undefined.
|
|
796
|
+
const appendOutbox = (
|
|
797
|
+
exec: Pick<Db, "execute">,
|
|
798
|
+
parent: JobStore.ParentEnvelope | null | undefined,
|
|
799
|
+
outcome: JobStore.FlowChildReport["outcome"],
|
|
800
|
+
exit: JobStore.FlowChildReport["exit"],
|
|
801
|
+
failedReason: string | null | undefined
|
|
802
|
+
) =>
|
|
803
|
+
parent === null || parent === undefined
|
|
804
|
+
? Effect.void
|
|
805
|
+
: exec.execute(sql`
|
|
806
|
+
INSERT INTO ${flowOutbox} (flow_name, parent_store_key, report)
|
|
807
|
+
VALUES (${parent.flowName}, ${parent.parentStoreKey}, ${
|
|
808
|
+
JSON.stringify({
|
|
809
|
+
flowId: parent.flowId,
|
|
810
|
+
childKey: parent.childKey,
|
|
811
|
+
outcome,
|
|
812
|
+
exit,
|
|
813
|
+
failedReason: failedReason ?? undefined
|
|
814
|
+
})
|
|
815
|
+
}::jsonb)
|
|
816
|
+
`).pipe(Effect.asVoid)
|
|
817
|
+
|
|
818
|
+
// Flip a flow's still-pending dependency rows to `cancelled` and NOT
|
|
819
|
+
// `cascaded` (the sweeper still owes the child stores real cancels).
|
|
820
|
+
// Embedded as a CTE body so the caller can count the marked rows into
|
|
821
|
+
// the parent's `cancelled` counter in the same statement.
|
|
822
|
+
const cancelPendingChildren = (flowId: string) =>
|
|
823
|
+
sql`
|
|
824
|
+
UPDATE ${flowChildren} SET status = 'cancelled', cascaded = FALSE
|
|
825
|
+
WHERE ${flowChildren.flowId} = ${flowId} AND ${flowChildren.status} = 'pending'
|
|
826
|
+
RETURNING 1
|
|
827
|
+
`
|
|
828
|
+
|
|
659
829
|
// Shared by cancel and cancelByDedupe.
|
|
660
830
|
const cancelJob = (id: JobStore.JobId) =>
|
|
661
831
|
db.transaction((tx) =>
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
832
|
+
Effect.gen(function*() {
|
|
833
|
+
const now = yield* nowDate
|
|
834
|
+
// Contract lock order (dependency rows first, parent second): mark
|
|
835
|
+
// a waiting-children parent's remaining pending rows cancelled.
|
|
836
|
+
// Pending rows exist only while the parent is `waiting-children`,
|
|
837
|
+
// so this is a no-op for every other state; a non-cancellable
|
|
838
|
+
// parent rolls the transaction back anyway.
|
|
839
|
+
const firstPass = rowsOf(yield* tx.execute<{ marked: number }>(sql`
|
|
840
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
841
|
+
SELECT count(*)::int AS marked FROM marked
|
|
842
|
+
`))
|
|
843
|
+
const preMarked = firstPass[0]?.marked ?? 0
|
|
844
|
+
// One guarded statement: waiting/delayed/waiting-children become
|
|
845
|
+
// terminal, active gets the cancel-request flag; anything else is
|
|
846
|
+
// reported by state.
|
|
847
|
+
const rows = rowsOf(yield* tx.execute<
|
|
848
|
+
{
|
|
849
|
+
id: string
|
|
850
|
+
state: string
|
|
851
|
+
processedAt: Date | null
|
|
852
|
+
name: string
|
|
853
|
+
keep: JobStore.KeepPolicy | null
|
|
854
|
+
dedupeKey: string | null
|
|
855
|
+
hasFlow: boolean
|
|
856
|
+
parent: JobStore.ParentEnvelope | null
|
|
857
|
+
exit: unknown
|
|
858
|
+
failedReason: string | null
|
|
859
|
+
}
|
|
860
|
+
>(sql`
|
|
861
|
+
UPDATE ${jobs} SET
|
|
862
|
+
state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN 'cancelled' ELSE ${jobs.state} END,
|
|
863
|
+
finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
|
|
864
|
+
flow_pending = CASE WHEN ${jobs.state} = 'waiting-children' THEN 0 ELSE ${jobs.flowPending} END,
|
|
865
|
+
cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
|
|
866
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
|
|
867
|
+
RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
|
|
868
|
+
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
869
|
+
${jobs.dedupeKey} AS "dedupeKey", (${jobs.flowPending} IS NOT NULL) AS "hasFlow",
|
|
870
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
871
|
+
`))
|
|
872
|
+
const row = rows[0]
|
|
873
|
+
if (row === undefined) {
|
|
874
|
+
const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
|
|
875
|
+
SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
|
|
684
876
|
`))
|
|
685
|
-
const
|
|
686
|
-
if (
|
|
687
|
-
|
|
688
|
-
SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
|
|
689
|
-
`))
|
|
690
|
-
const found = existing[0]
|
|
691
|
-
if (found === undefined) {
|
|
692
|
-
return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
693
|
-
}
|
|
694
|
-
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
|
|
877
|
+
const found = existing[0]
|
|
878
|
+
if (found === undefined) {
|
|
879
|
+
return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
695
880
|
}
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
881
|
+
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
|
|
882
|
+
}
|
|
883
|
+
if (row.state === "cancelled") {
|
|
884
|
+
if (row.hasFlow) {
|
|
885
|
+
// Re-run the marking now that the parent lock is held: the
|
|
886
|
+
// first UPDATE above races a concurrent FanOut — its
|
|
887
|
+
// uncommitted dependency-row INSERTs are invisible, while
|
|
888
|
+
// this UPDATE's own EPQ re-check can still see the parent as
|
|
889
|
+
// 'waiting-children' after the FanOut commits. Without this
|
|
890
|
+
// pass those rows would stay 'pending' forever, invisible to
|
|
891
|
+
// both sweep classes. Every row either pass marked lands in
|
|
892
|
+
// the `cancelled` counter.
|
|
893
|
+
yield* tx.execute(sql`
|
|
894
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
895
|
+
UPDATE ${jobs} SET flow_cancelled = ${jobs.flowCancelled} + ${preMarked}::int
|
|
896
|
+
+ (SELECT count(*)::int FROM marked)
|
|
897
|
+
WHERE ${jobs.id} = ${id}
|
|
898
|
+
`)
|
|
700
899
|
}
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
)
|
|
900
|
+
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
901
|
+
// A cancelled child reports upward through the outbox.
|
|
902
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
|
|
903
|
+
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
904
|
+
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
905
|
+
}
|
|
906
|
+
})
|
|
907
|
+
).pipe(
|
|
908
|
+
// Rare lock-order inversion (this row-then-parent flip vs a
|
|
909
|
+
// fail-fast settle's parent-then-rows marking) surfaces as a
|
|
910
|
+
// Postgres deadlock (40P01); one side is killed and safe to retry.
|
|
911
|
+
Effect.retry({
|
|
912
|
+
times: 3,
|
|
913
|
+
while: isDeadlockError
|
|
914
|
+
}),
|
|
915
|
+
Effect.mapError((error) =>
|
|
916
|
+
error instanceof JobStore.JobNotFoundError ||
|
|
917
|
+
error instanceof JobStore.JobNotCancellableError ||
|
|
918
|
+
error instanceof JobStore.JobStoreError
|
|
919
|
+
? error
|
|
920
|
+
: storeError("cancel failed")(error)
|
|
921
|
+
),
|
|
922
|
+
Effect.asVoid
|
|
923
|
+
)
|
|
712
924
|
|
|
713
925
|
const enqueueOne = (request: JobStore.EnqueueRequest) =>
|
|
714
926
|
Effect.gen(function*() {
|
|
@@ -798,11 +1010,12 @@ export const make = (
|
|
|
798
1010
|
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
799
1011
|
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
800
1012
|
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
1013
|
+
${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
|
|
801
1014
|
${new Date(now.getTime() + Math.max(0, request.delayMs))}, ${now}${extraColumnValues(request)})`
|
|
802
1015
|
)
|
|
803
1016
|
const rows = rowsOf(yield* db.execute<{ id: string }>(sql`
|
|
804
1017
|
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
805
|
-
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
1018
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
|
|
806
1019
|
VALUES ${sql.join(values, sql`, `)}
|
|
807
1020
|
ON CONFLICT (id) DO NOTHING
|
|
808
1021
|
RETURNING ${jobs.id} AS id
|
|
@@ -843,6 +1056,123 @@ export const make = (
|
|
|
843
1056
|
return resolved
|
|
844
1057
|
})
|
|
845
1058
|
|
|
1059
|
+
// The FanOut ack: land the manifest (flow columns + dependency rows) and
|
|
1060
|
+
// park the parent, all lock-token-guarded in one transaction. A fan-out
|
|
1061
|
+
// is a phase transition, not a completed run — `attempts_made` is not
|
|
1062
|
+
// incremented; the ledger records `fanned-out` with no exit.
|
|
1063
|
+
const ackFanOut = (
|
|
1064
|
+
id: JobStore.JobId,
|
|
1065
|
+
token: string,
|
|
1066
|
+
outcome: Extract<JobStore.AckOutcome, { _tag: "FanOut" }>
|
|
1067
|
+
): Effect.Effect<void, JobStore.JobStoreError | JobStore.JobNotFoundError | JobStore.LockLostError> =>
|
|
1068
|
+
Effect.gen(function*() {
|
|
1069
|
+
// Validate BEFORE any mutation, so a bad spec cannot leave the job
|
|
1070
|
+
// half-acked (lock cleared, ledger written, still active).
|
|
1071
|
+
if (outcome.children.some((child) => child.request.id === undefined)) {
|
|
1072
|
+
return yield* new JobStore.JobStoreError({
|
|
1073
|
+
message: "FanOut child specs require an explicit request.id"
|
|
1074
|
+
})
|
|
1075
|
+
}
|
|
1076
|
+
const wakeQueue = yield* db.transaction((tx) =>
|
|
1077
|
+
Effect.gen(function*() {
|
|
1078
|
+
const now = yield* nowDate
|
|
1079
|
+
const rows = rowsOf(yield* tx.execute<
|
|
1080
|
+
{
|
|
1081
|
+
processedAt: Date | null
|
|
1082
|
+
name: string
|
|
1083
|
+
keep: JobStore.KeepPolicy | null
|
|
1084
|
+
dedupeKey: string | null
|
|
1085
|
+
queue: string
|
|
1086
|
+
cancelRequested: boolean
|
|
1087
|
+
flowPending: number | null
|
|
1088
|
+
parent: JobStore.ParentEnvelope | null
|
|
1089
|
+
exit: unknown
|
|
1090
|
+
failedReason: string | null
|
|
1091
|
+
}
|
|
1092
|
+
>(sql`
|
|
1093
|
+
UPDATE ${jobs} SET lock_token = NULL, lock_expires_at = NULL
|
|
1094
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
1095
|
+
RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
|
|
1096
|
+
${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1097
|
+
${jobs.cancelRequested} AS "cancelRequested", ${jobs.flowPending} AS "flowPending",
|
|
1098
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
1099
|
+
`))
|
|
1100
|
+
const row = rows[0]
|
|
1101
|
+
if (row === undefined) {
|
|
1102
|
+
return yield* explainMiss(id)
|
|
1103
|
+
}
|
|
1104
|
+
yield* insertAttempt(tx, id, "fanned-out", row.processedAt, now, undefined)
|
|
1105
|
+
let pending: number
|
|
1106
|
+
if (row.flowPending === null || row.flowPending === undefined) {
|
|
1107
|
+
pending = outcome.children.length
|
|
1108
|
+
yield* tx.execute(sql`
|
|
1109
|
+
UPDATE ${jobs} SET flow_fail_fast = ${outcome.failFast}, flow_pending = ${pending},
|
|
1110
|
+
flow_completed = 0, flow_failed = 0, flow_cancelled = 0
|
|
1111
|
+
WHERE ${jobs.id} = ${id}
|
|
1112
|
+
`)
|
|
1113
|
+
// Chunked multi-row VALUES, like enqueueMany's insertBatch.
|
|
1114
|
+
for (let start = 0; start < outcome.children.length; start += 500) {
|
|
1115
|
+
const chunk = outcome.children.slice(start, start + 500)
|
|
1116
|
+
const values = chunk.map((child) =>
|
|
1117
|
+
sql`(${id}, ${child.childKey}, ${child.request.name}, ${child.storeKey},
|
|
1118
|
+
${JSON.stringify(child.request)}::jsonb, 'pending', NULL, NULL, FALSE, ${now})`
|
|
1119
|
+
)
|
|
1120
|
+
yield* tx.execute(sql`
|
|
1121
|
+
INSERT INTO ${flowChildren} (flow_id, child_key, name, store_key, spec,
|
|
1122
|
+
status, exit, failed_reason, cascaded, pending_since)
|
|
1123
|
+
VALUES ${sql.join(values, sql`, `)}
|
|
1124
|
+
`)
|
|
1125
|
+
}
|
|
1126
|
+
} else {
|
|
1127
|
+
// A manifest that was already present is kept untouched (double
|
|
1128
|
+
// fan-out converges on the persisted children); the state
|
|
1129
|
+
// transition follows the persisted pending count either way.
|
|
1130
|
+
pending = Number(row.flowPending)
|
|
1131
|
+
}
|
|
1132
|
+
if (row.cancelRequested) {
|
|
1133
|
+
// A cancel raced the fan-out: cancellation wins. The rows exist
|
|
1134
|
+
// and get marked (into the `cancelled` counter), so the sweeper
|
|
1135
|
+
// cascades (mostly no-op cancels for never-enqueued children).
|
|
1136
|
+
yield* tx.execute(sql`
|
|
1137
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
1138
|
+
UPDATE ${jobs} SET state = 'cancelled', finished_at = ${now},
|
|
1139
|
+
cancel_requested = FALSE, flow_pending = 0,
|
|
1140
|
+
flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
|
|
1141
|
+
WHERE ${jobs.id} = ${id}
|
|
1142
|
+
`)
|
|
1143
|
+
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
1144
|
+
// A cancelled NESTED parent reports upward through the outbox.
|
|
1145
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
|
|
1146
|
+
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
1147
|
+
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
1148
|
+
return undefined
|
|
1149
|
+
}
|
|
1150
|
+
if (pending > 0) {
|
|
1151
|
+
yield* tx.execute(sql`
|
|
1152
|
+
UPDATE ${jobs} SET state = 'waiting-children' WHERE ${jobs.id} = ${id}
|
|
1153
|
+
`)
|
|
1154
|
+
return undefined
|
|
1155
|
+
}
|
|
1156
|
+
// Empty (or fully recorded) manifest: straight to runnable collect.
|
|
1157
|
+
yield* tx.execute(sql`
|
|
1158
|
+
UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
|
|
1159
|
+
WHERE ${jobs.id} = ${id}
|
|
1160
|
+
`)
|
|
1161
|
+
return JobStore.QueueName(row.queue)
|
|
1162
|
+
})
|
|
1163
|
+
).pipe(
|
|
1164
|
+
Effect.mapError((error) =>
|
|
1165
|
+
error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
|
|
1166
|
+
error instanceof JobStore.JobStoreError
|
|
1167
|
+
? error
|
|
1168
|
+
: storeError("ack failed")(error)
|
|
1169
|
+
)
|
|
1170
|
+
)
|
|
1171
|
+
if (wakeQueue !== undefined) {
|
|
1172
|
+
yield* wakeUp(wakeQueue)
|
|
1173
|
+
}
|
|
1174
|
+
})
|
|
1175
|
+
|
|
846
1176
|
const store: JobStore.Service = {
|
|
847
1177
|
enqueue: enqueueOne,
|
|
848
1178
|
|
|
@@ -917,7 +1247,10 @@ export const make = (
|
|
|
917
1247
|
${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
|
|
918
1248
|
${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
|
|
919
1249
|
${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
|
|
920
|
-
${jobs.trace} AS "trace",
|
|
1250
|
+
${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
|
|
1251
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
|
|
1252
|
+
${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
|
|
1253
|
+
${jobs.flowCancelled} AS "flowCancelled",
|
|
921
1254
|
${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
|
|
922
1255
|
${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
|
|
923
1256
|
${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
@@ -944,8 +1277,11 @@ export const make = (
|
|
|
944
1277
|
error instanceof JobStore.JobStoreError ? error : storeError("claim failed")(error)
|
|
945
1278
|
)),
|
|
946
1279
|
|
|
947
|
-
ack: (id, token, outcome) =>
|
|
948
|
-
|
|
1280
|
+
ack: (id, token, outcome) => {
|
|
1281
|
+
if (outcome._tag === "FanOut") {
|
|
1282
|
+
return ackFanOut(id, token, outcome)
|
|
1283
|
+
}
|
|
1284
|
+
return db.transaction((tx) =>
|
|
949
1285
|
Effect.gen(function*() {
|
|
950
1286
|
const now = yield* nowDate
|
|
951
1287
|
const update = outcome._tag === "Complete"
|
|
@@ -971,6 +1307,9 @@ export const make = (
|
|
|
971
1307
|
keep: JobStore.KeepPolicy | null
|
|
972
1308
|
dedupeKey: string | null
|
|
973
1309
|
queue: string
|
|
1310
|
+
parent: JobStore.ParentEnvelope | null
|
|
1311
|
+
exit: unknown
|
|
1312
|
+
failedReason: string | null
|
|
974
1313
|
}
|
|
975
1314
|
>(sql`
|
|
976
1315
|
UPDATE ${jobs} SET ${update},
|
|
@@ -978,7 +1317,8 @@ export const make = (
|
|
|
978
1317
|
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
979
1318
|
RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
|
|
980
1319
|
${jobs.state} AS "state", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
|
|
981
|
-
${jobs.queue} AS "queue"
|
|
1320
|
+
${jobs.queue} AS "queue", ${jobs.parent} AS "parent", ${jobs.exit} AS "exit",
|
|
1321
|
+
${jobs.failedReason} AS "failedReason"
|
|
982
1322
|
`))
|
|
983
1323
|
const row = rows[0]
|
|
984
1324
|
if (row === undefined) {
|
|
@@ -1001,6 +1341,15 @@ export const make = (
|
|
|
1001
1341
|
outcome._tag === "Cancelled" || cancelledRetry ? undefined : outcome.exit
|
|
1002
1342
|
)
|
|
1003
1343
|
if (outcome._tag !== "Retry" || cancelledRetry) {
|
|
1344
|
+
// A terminal transition of an envelope-carrying child reports
|
|
1345
|
+
// upward through the outbox (exit/failedReason as persisted).
|
|
1346
|
+
yield* appendOutbox(
|
|
1347
|
+
tx,
|
|
1348
|
+
row.parent,
|
|
1349
|
+
ledgerOutcome === "retried" ? "cancelled" : ledgerOutcome,
|
|
1350
|
+
row.exit ?? undefined,
|
|
1351
|
+
row.failedReason
|
|
1352
|
+
)
|
|
1004
1353
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
1005
1354
|
yield* applyKeep(tx, row, now)
|
|
1006
1355
|
}
|
|
@@ -1017,7 +1366,8 @@ export const make = (
|
|
|
1017
1366
|
),
|
|
1018
1367
|
Effect.tap((queue) => queue !== undefined ? wakeUp(queue) : Effect.void),
|
|
1019
1368
|
Effect.asVoid
|
|
1020
|
-
)
|
|
1369
|
+
)
|
|
1370
|
+
},
|
|
1021
1371
|
|
|
1022
1372
|
release: (id, token) =>
|
|
1023
1373
|
Effect.gen(function*() {
|
|
@@ -1035,6 +1385,9 @@ export const make = (
|
|
|
1035
1385
|
keep: JobStore.KeepPolicy | null
|
|
1036
1386
|
dedupeKey: string | null
|
|
1037
1387
|
queue: string
|
|
1388
|
+
parent: JobStore.ParentEnvelope | null
|
|
1389
|
+
exit: unknown
|
|
1390
|
+
failedReason: string | null
|
|
1038
1391
|
}
|
|
1039
1392
|
>(sql`
|
|
1040
1393
|
UPDATE ${jobs} SET
|
|
@@ -1045,12 +1398,15 @@ export const make = (
|
|
|
1045
1398
|
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
1046
1399
|
RETURNING ${jobs.id} AS id, (${jobs.state} = 'cancelled') AS cancelled,
|
|
1047
1400
|
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
1048
|
-
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue"
|
|
1401
|
+
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1402
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
1049
1403
|
`))
|
|
1050
1404
|
const row = rows[0]
|
|
1051
1405
|
if (row === undefined) return undefined
|
|
1052
1406
|
if (row.cancelled) {
|
|
1053
1407
|
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
1408
|
+
// A cancel honoured at release is a terminal transition too.
|
|
1409
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
|
|
1054
1410
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
1055
1411
|
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
1056
1412
|
}
|
|
@@ -1117,6 +1473,9 @@ export const make = (
|
|
|
1117
1473
|
name: string
|
|
1118
1474
|
keep: JobStore.KeepPolicy | null
|
|
1119
1475
|
dedupeKey: string | null
|
|
1476
|
+
parent: JobStore.ParentEnvelope | null
|
|
1477
|
+
exit: unknown
|
|
1478
|
+
failedReason: string | null
|
|
1120
1479
|
}
|
|
1121
1480
|
>(sql`
|
|
1122
1481
|
UPDATE ${jobs} SET
|
|
@@ -1137,7 +1496,8 @@ export const make = (
|
|
|
1137
1496
|
cancel_requested = FALSE
|
|
1138
1497
|
WHERE ${jobs.state} = 'active' AND ${jobs.lockExpiresAt} <= ${now}::timestamptz
|
|
1139
1498
|
RETURNING ${jobs.id} AS "id", ${jobs.state} AS "state", ${jobs.processedAt} AS "processedAt",
|
|
1140
|
-
${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey"
|
|
1499
|
+
${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
|
|
1500
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
1141
1501
|
`))
|
|
1142
1502
|
const recovered: Array<{ id: JobStore.JobId; failed: boolean }> = []
|
|
1143
1503
|
for (const row of rows) {
|
|
@@ -1150,6 +1510,15 @@ export const make = (
|
|
|
1150
1510
|
undefined
|
|
1151
1511
|
)
|
|
1152
1512
|
if (row.state === "cancelled" || row.state === "failed") {
|
|
1513
|
+
// Honoured cancels and stall exhaustion are terminal
|
|
1514
|
+
// transitions: envelope-carrying children report upward.
|
|
1515
|
+
yield* appendOutbox(
|
|
1516
|
+
tx,
|
|
1517
|
+
row.parent,
|
|
1518
|
+
row.state === "cancelled" ? "cancelled" : "failed",
|
|
1519
|
+
row.exit ?? undefined,
|
|
1520
|
+
row.failedReason
|
|
1521
|
+
)
|
|
1153
1522
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, row.id, now)
|
|
1154
1523
|
}
|
|
1155
1524
|
if (row.state === "cancelled") {
|
|
@@ -1231,7 +1600,11 @@ export const make = (
|
|
|
1231
1600
|
${jobs.attemptsMax} AS "attemptsMax", ${jobs.attemptsMade} AS "attemptsMade",
|
|
1232
1601
|
${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
|
|
1233
1602
|
${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
|
|
1234
|
-
${jobs.cancelRequested} AS "cancelRequested",
|
|
1603
|
+
${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
|
|
1604
|
+
${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
|
|
1605
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
|
|
1606
|
+
${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
|
|
1607
|
+
${jobs.flowCancelled} AS "flowCancelled",
|
|
1235
1608
|
${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
|
|
1236
1609
|
${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
|
|
1237
1610
|
${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
@@ -1340,20 +1713,21 @@ export const make = (
|
|
|
1340
1713
|
upsertSchedule: (schedule) =>
|
|
1341
1714
|
db.execute(sql`
|
|
1342
1715
|
INSERT INTO ${schedules} (key, job_name, queue, cron, tz, every_ms, payload, metadata,
|
|
1343
|
-
priority, attempts_max, backoff, keep, timeout_ms, next_run_at)
|
|
1716
|
+
priority, attempts_max, backoff, keep, timeout_ms, group_name, next_run_at)
|
|
1344
1717
|
VALUES (${schedule.key}, ${schedule.jobName}, ${schedule.queue},
|
|
1345
1718
|
${schedule.cron ?? null}, ${schedule.tz ?? null}, ${schedule.everyMs ?? null},
|
|
1346
1719
|
${JSON.stringify(schedule.payload ?? null)}::jsonb, ${JSON.stringify(schedule.metadata)}::jsonb,
|
|
1347
1720
|
${schedule.priority}, ${schedule.attemptsMax},
|
|
1348
1721
|
${schedule.backoff === undefined ? null : JSON.stringify(schedule.backoff)}::jsonb,
|
|
1349
1722
|
${schedule.keep === undefined ? null : JSON.stringify(schedule.keep)}::jsonb,
|
|
1350
|
-
${schedule.timeoutMs ?? null}, ${new Date(schedule.nextRunAt)})
|
|
1723
|
+
${schedule.timeoutMs ?? null}, ${schedule.group ?? null}, ${new Date(schedule.nextRunAt)})
|
|
1351
1724
|
ON CONFLICT (key) DO UPDATE SET
|
|
1352
1725
|
job_name = EXCLUDED.job_name, queue = EXCLUDED.queue, cron = EXCLUDED.cron,
|
|
1353
1726
|
tz = EXCLUDED.tz, every_ms = EXCLUDED.every_ms, payload = EXCLUDED.payload,
|
|
1354
1727
|
metadata = EXCLUDED.metadata, priority = EXCLUDED.priority,
|
|
1355
1728
|
attempts_max = EXCLUDED.attempts_max, backoff = EXCLUDED.backoff,
|
|
1356
1729
|
keep = EXCLUDED.keep, timeout_ms = EXCLUDED.timeout_ms,
|
|
1730
|
+
group_name = EXCLUDED.group_name,
|
|
1357
1731
|
next_run_at = CASE
|
|
1358
1732
|
WHEN ${schedules.cron} IS NOT DISTINCT FROM EXCLUDED.cron
|
|
1359
1733
|
AND ${schedules.tz} IS NOT DISTINCT FROM EXCLUDED.tz
|
|
@@ -1383,6 +1757,9 @@ export const make = (
|
|
|
1383
1757
|
if (listOptions?.queue !== undefined) {
|
|
1384
1758
|
conditions.push(sql`${schedules.queue} = ${listOptions.queue}`)
|
|
1385
1759
|
}
|
|
1760
|
+
if (listOptions?.group !== undefined) {
|
|
1761
|
+
conditions.push(sql`${schedules.group} = ${listOptions.group}`)
|
|
1762
|
+
}
|
|
1386
1763
|
const rows = rowsOf(yield* db.execute<ScheduleRow>(sql`
|
|
1387
1764
|
SELECT ${schedules.key} AS "key", ${schedules.jobName} AS "jobName",
|
|
1388
1765
|
${schedules.queue} AS "queue", ${schedules.cron} AS "cron", ${schedules.tz} AS "tz",
|
|
@@ -1390,7 +1767,7 @@ export const make = (
|
|
|
1390
1767
|
${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
|
|
1391
1768
|
${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
|
|
1392
1769
|
${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
|
|
1393
|
-
${schedules.nextRunAt} AS "nextRunAt"
|
|
1770
|
+
${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
|
|
1394
1771
|
FROM ${schedules}
|
|
1395
1772
|
WHERE ${sql.join(conditions, sql` AND `)}
|
|
1396
1773
|
ORDER BY ${schedules.key}
|
|
@@ -1408,7 +1785,7 @@ export const make = (
|
|
|
1408
1785
|
${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
|
|
1409
1786
|
${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
|
|
1410
1787
|
${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
|
|
1411
|
-
${schedules.nextRunAt} AS "nextRunAt"
|
|
1788
|
+
${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
|
|
1412
1789
|
FROM ${schedules}
|
|
1413
1790
|
WHERE ${schedules.nextRunAt} <= ${now}
|
|
1414
1791
|
ORDER BY ${schedules.nextRunAt} ASC
|
|
@@ -1458,6 +1835,365 @@ export const make = (
|
|
|
1458
1835
|
return fired
|
|
1459
1836
|
}),
|
|
1460
1837
|
|
|
1838
|
+
recordChildResults: (reports) =>
|
|
1839
|
+
Effect.gen(function*() {
|
|
1840
|
+
if (reports.length === 0) {
|
|
1841
|
+
const none: Array<{ applied: boolean; parentSettled: boolean }> = []
|
|
1842
|
+
return none
|
|
1843
|
+
}
|
|
1844
|
+
const batch = yield* db.transaction((tx) =>
|
|
1845
|
+
Effect.gen(function*() {
|
|
1846
|
+
const now = yield* nowDate
|
|
1847
|
+
const results = reports.map(() => ({ applied: false, parentSettled: false }))
|
|
1848
|
+
|
|
1849
|
+
// Phase 1a — apply every row update (contract lock order:
|
|
1850
|
+
// dependency rows FIRST, parents second). Only a (flow, key)'s
|
|
1851
|
+
// first occurrence in the batch can apply — later duplicates
|
|
1852
|
+
// would find the row non-pending anyway, and UPDATE ... FROM
|
|
1853
|
+
// must never see two source rows for one target.
|
|
1854
|
+
const candidates: Array<{ readonly index: number; readonly report: JobStore.FlowChildReport }> = []
|
|
1855
|
+
const seen = new Set<string>()
|
|
1856
|
+
for (const [index, report] of reports.entries()) {
|
|
1857
|
+
const key = `${report.flowId}\u0000${report.childKey}`
|
|
1858
|
+
if (seen.has(key)) continue
|
|
1859
|
+
seen.add(key)
|
|
1860
|
+
candidates.push({ index, report })
|
|
1861
|
+
}
|
|
1862
|
+
for (let start = 0; start < candidates.length; start += 200) {
|
|
1863
|
+
const chunk = candidates.slice(start, start + 200)
|
|
1864
|
+
const values = chunk.map(({ index, report }) =>
|
|
1865
|
+
sql`(${index}::int, ${report.flowId}::text, ${report.childKey}::text,
|
|
1866
|
+
${report.outcome}::text,
|
|
1867
|
+
${report.exit === undefined ? null : JSON.stringify(report.exit)}::jsonb,
|
|
1868
|
+
${report.failedReason ?? null}::text)`
|
|
1869
|
+
)
|
|
1870
|
+
const appliedRows = rowsOf(yield* tx.execute<{ ord: number }>(sql`
|
|
1871
|
+
UPDATE ${flowChildren} SET status = v.outcome, exit = v.exit,
|
|
1872
|
+
failed_reason = v.failed_reason, cascaded = TRUE
|
|
1873
|
+
FROM (VALUES ${sql.join(values, sql`, `)})
|
|
1874
|
+
AS v(ord, flow_id, child_key, outcome, exit, failed_reason)
|
|
1875
|
+
WHERE ${flowChildren.flowId} = v.flow_id AND ${flowChildren.childKey} = v.child_key
|
|
1876
|
+
AND ${flowChildren.status} = 'pending'
|
|
1877
|
+
RETURNING v.ord AS "ord"
|
|
1878
|
+
`))
|
|
1879
|
+
for (const row of appliedRows) {
|
|
1880
|
+
const result = results[Number(row.ord)]
|
|
1881
|
+
if (result !== undefined) {
|
|
1882
|
+
result.applied = true
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
// Tally the applied reports per touched flow, in batch order.
|
|
1888
|
+
interface Touch {
|
|
1889
|
+
appliedCount: number
|
|
1890
|
+
completed: number
|
|
1891
|
+
failed: number
|
|
1892
|
+
cancelled: number
|
|
1893
|
+
firstAppliedFailed: number | undefined
|
|
1894
|
+
lastApplied: number
|
|
1895
|
+
}
|
|
1896
|
+
const touched = new Map<string, Touch>()
|
|
1897
|
+
for (const [index, report] of reports.entries()) {
|
|
1898
|
+
if (results[index]?.applied !== true) continue
|
|
1899
|
+
const touch = touched.get(report.flowId) ?? {
|
|
1900
|
+
appliedCount: 0,
|
|
1901
|
+
completed: 0,
|
|
1902
|
+
failed: 0,
|
|
1903
|
+
cancelled: 0,
|
|
1904
|
+
firstAppliedFailed: undefined,
|
|
1905
|
+
lastApplied: index
|
|
1906
|
+
}
|
|
1907
|
+
touch.appliedCount += 1
|
|
1908
|
+
touch.lastApplied = index
|
|
1909
|
+
if (report.outcome === "completed") touch.completed += 1
|
|
1910
|
+
if (report.outcome === "cancelled") touch.cancelled += 1
|
|
1911
|
+
if (report.outcome === "failed") {
|
|
1912
|
+
touch.failed += 1
|
|
1913
|
+
touch.firstAppliedFailed ??= index
|
|
1914
|
+
}
|
|
1915
|
+
touched.set(report.flowId, touch)
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
// Phase 1b + 2 — per touched flow (sorted, so concurrent
|
|
1919
|
+
// batches take parent locks in one order): move the applied
|
|
1920
|
+
// children from `pending` to their outcome counters, then make
|
|
1921
|
+
// at most one settle decision. Fail-fast wins the tie.
|
|
1922
|
+
const wakeQueues: Array<JobStore.QueueName> = []
|
|
1923
|
+
for (const flowId of [...touched.keys()].toSorted()) {
|
|
1924
|
+
const touch = touched.get(flowId)
|
|
1925
|
+
if (touch === undefined) continue
|
|
1926
|
+
const parents = rowsOf(yield* tx.execute<
|
|
1927
|
+
{
|
|
1928
|
+
state: JobStore.JobState
|
|
1929
|
+
flowPending: number
|
|
1930
|
+
flowFailFast: boolean | null
|
|
1931
|
+
processedAt: Date | null
|
|
1932
|
+
name: string
|
|
1933
|
+
keep: JobStore.KeepPolicy | null
|
|
1934
|
+
dedupeKey: string | null
|
|
1935
|
+
queue: string
|
|
1936
|
+
parent: JobStore.ParentEnvelope | null
|
|
1937
|
+
}
|
|
1938
|
+
>(sql`
|
|
1939
|
+
UPDATE ${jobs} SET
|
|
1940
|
+
flow_pending = GREATEST(${jobs.flowPending} - ${touch.appliedCount}::int, 0),
|
|
1941
|
+
flow_completed = ${jobs.flowCompleted} + ${touch.completed}::int,
|
|
1942
|
+
flow_failed = ${jobs.flowFailed} + ${touch.failed}::int,
|
|
1943
|
+
flow_cancelled = ${jobs.flowCancelled} + ${touch.cancelled}::int
|
|
1944
|
+
WHERE ${jobs.id} = ${flowId} AND ${jobs.flowPending} IS NOT NULL
|
|
1945
|
+
RETURNING ${jobs.state} AS "state", ${jobs.flowPending} AS "flowPending",
|
|
1946
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.processedAt} AS "processedAt",
|
|
1947
|
+
${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
1948
|
+
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1949
|
+
${jobs.parent} AS "parent"
|
|
1950
|
+
`))
|
|
1951
|
+
const parent = parents[0]
|
|
1952
|
+
if (parent === undefined || parent.state !== "waiting-children") continue
|
|
1953
|
+
const failedIndex = parent.flowFailFast === true ? touch.firstAppliedFailed : undefined
|
|
1954
|
+
const failedReport = failedIndex !== undefined ? reports[failedIndex] : undefined
|
|
1955
|
+
if (failedIndex !== undefined && failedReport !== undefined) {
|
|
1956
|
+
// The first applied failure settles the parent terminally
|
|
1957
|
+
// (store-side, like stall exhaustion: failedReason, no
|
|
1958
|
+
// exit) and marks the remaining rows in the same
|
|
1959
|
+
// transaction. A nested parent's own report goes to the
|
|
1960
|
+
// outbox here — this settle IS its terminal transition.
|
|
1961
|
+
const reason = `effect-mq: flow child "${failedReport.childKey}" failed`
|
|
1962
|
+
yield* tx.execute(sql`
|
|
1963
|
+
WITH marked AS (${cancelPendingChildren(flowId)})
|
|
1964
|
+
UPDATE ${jobs} SET state = 'failed', finished_at = ${now},
|
|
1965
|
+
failed_reason = ${reason}, cancel_requested = FALSE, flow_pending = 0,
|
|
1966
|
+
flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
|
|
1967
|
+
WHERE ${jobs.id} = ${flowId}
|
|
1968
|
+
`)
|
|
1969
|
+
yield* insertAttempt(tx, flowId, "failed", parent.processedAt, now, undefined)
|
|
1970
|
+
yield* appendOutbox(tx, parent.parent, "failed", undefined, reason)
|
|
1971
|
+
yield* releaseDedupe(tx, parent.name, parent.dedupeKey, flowId, now)
|
|
1972
|
+
yield* applyKeep(tx, { name: parent.name, state: "failed", keep: parent.keep }, now)
|
|
1973
|
+
const decided = results[failedIndex]
|
|
1974
|
+
if (decided !== undefined) {
|
|
1975
|
+
decided.parentSettled = true
|
|
1976
|
+
}
|
|
1977
|
+
continue
|
|
1978
|
+
}
|
|
1979
|
+
if (Number(parent.flowPending) === 0) {
|
|
1980
|
+
// All children settled: the parent resumes runnable, phase
|
|
1981
|
+
// collect, settled at the flow's last applied report.
|
|
1982
|
+
yield* tx.execute(sql`
|
|
1983
|
+
UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
|
|
1984
|
+
WHERE ${jobs.id} = ${flowId}
|
|
1985
|
+
`)
|
|
1986
|
+
wakeQueues.push(JobStore.QueueName(parent.queue))
|
|
1987
|
+
const decided = results[touch.lastApplied]
|
|
1988
|
+
if (decided !== undefined) {
|
|
1989
|
+
decided.parentSettled = true
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
return { results, wakeQueues }
|
|
1994
|
+
})
|
|
1995
|
+
).pipe(
|
|
1996
|
+
// Rare lock-order inversion (a concurrent cancel/settle marking
|
|
1997
|
+
// rows) surfaces as a Postgres deadlock (40P01); the killed side
|
|
1998
|
+
// is safe to retry — the row-state guard keeps it idempotent.
|
|
1999
|
+
Effect.retry({
|
|
2000
|
+
times: 3,
|
|
2001
|
+
while: isDeadlockError
|
|
2002
|
+
}),
|
|
2003
|
+
Effect.mapError((error) =>
|
|
2004
|
+
error instanceof JobStore.JobStoreError ? error : storeError("recordChildResults failed")(error)
|
|
2005
|
+
)
|
|
2006
|
+
)
|
|
2007
|
+
for (const queue of batch.wakeQueues) {
|
|
2008
|
+
yield* wakeUp(queue)
|
|
2009
|
+
}
|
|
2010
|
+
return batch.results
|
|
2011
|
+
}),
|
|
2012
|
+
|
|
2013
|
+
peekOutbox: (peekOptions) =>
|
|
2014
|
+
Effect.gen(function*() {
|
|
2015
|
+
const limit = Math.max(0, peekOptions.limit)
|
|
2016
|
+
if (limit === 0) {
|
|
2017
|
+
const none: Array<JobStore.OutboxEntry> = []
|
|
2018
|
+
return none
|
|
2019
|
+
}
|
|
2020
|
+
// `after` pages past a previously returned id (exclusive), whether
|
|
2021
|
+
// or not that entry still exists. Anything that is not a canonical
|
|
2022
|
+
// id this store issued is treated as unset.
|
|
2023
|
+
const after = peekOptions.after !== undefined && CANONICAL_BIGSERIAL.test(peekOptions.after)
|
|
2024
|
+
? peekOptions.after
|
|
2025
|
+
: undefined
|
|
2026
|
+
const rows = rowsOf(yield* db.execute<
|
|
2027
|
+
{
|
|
2028
|
+
id: string
|
|
2029
|
+
flowName: string
|
|
2030
|
+
parentStoreKey: string
|
|
2031
|
+
report: JobStore.FlowChildReport
|
|
2032
|
+
}
|
|
2033
|
+
>(sql`
|
|
2034
|
+
SELECT ${flowOutbox.id}::text AS "id", ${flowOutbox.flowName} AS "flowName",
|
|
2035
|
+
${flowOutbox.parentStoreKey} AS "parentStoreKey", ${flowOutbox.report} AS "report"
|
|
2036
|
+
FROM ${flowOutbox}
|
|
2037
|
+
${after === undefined ? sql`` : sql`WHERE ${flowOutbox.id} > ${after}::bigint`}
|
|
2038
|
+
ORDER BY ${flowOutbox.id} ASC
|
|
2039
|
+
LIMIT ${limit}
|
|
2040
|
+
`).pipe(Effect.mapError(storeError("peekOutbox failed"))))
|
|
2041
|
+
return rows.map((row): JobStore.OutboxEntry => ({
|
|
2042
|
+
id: row.id,
|
|
2043
|
+
flowName: row.flowName,
|
|
2044
|
+
parentStoreKey: row.parentStoreKey,
|
|
2045
|
+
report: row.report
|
|
2046
|
+
}))
|
|
2047
|
+
}),
|
|
2048
|
+
|
|
2049
|
+
deleteOutbox: (ids) =>
|
|
2050
|
+
Effect.suspend(() => {
|
|
2051
|
+
// Ids are opaque strings to callers; only CANONICAL ids this store
|
|
2052
|
+
// could have issued can match. The strictness matters twice: a
|
|
2053
|
+
// foreign/garbled id must not blow up the ::bigint cast, and a
|
|
2054
|
+
// non-canonical spelling ("007") must stay an unknown-id no-op
|
|
2055
|
+
// rather than cast to 7 and delete a live entry.
|
|
2056
|
+
const numeric = ids.filter((id) => CANONICAL_BIGSERIAL.test(id))
|
|
2057
|
+
if (numeric.length === 0) return Effect.void
|
|
2058
|
+
return db.execute(sql`
|
|
2059
|
+
DELETE FROM ${flowOutbox}
|
|
2060
|
+
WHERE ${flowOutbox.id} = ANY(${sql.param(numeric)}::bigint[])
|
|
2061
|
+
`).pipe(
|
|
2062
|
+
Effect.mapError(storeError("deleteOutbox failed")),
|
|
2063
|
+
Effect.asVoid
|
|
2064
|
+
)
|
|
2065
|
+
}),
|
|
2066
|
+
|
|
2067
|
+
listChildResults: (flowId, listOptions) =>
|
|
2068
|
+
Effect.gen(function*() {
|
|
2069
|
+
const limit = Math.max(1, listOptions?.limit ?? 1000)
|
|
2070
|
+
const cursor = listOptions?.cursor
|
|
2071
|
+
// `spec->>'id'` instead of the whole spec: at 10k children the full
|
|
2072
|
+
// payloads would transfer on every collect.
|
|
2073
|
+
const rows = rowsOf(yield* db.execute<FlowChildRow>(sql`
|
|
2074
|
+
SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
2075
|
+
${flowChildren.name} AS "name", ${flowChildren.storeKey} AS "storeKey",
|
|
2076
|
+
${flowChildren.spec}->>'id' AS "childJobId", ${flowChildren.status} AS "status",
|
|
2077
|
+
${flowChildren.exit} AS "exit", ${flowChildren.failedReason} AS "failedReason",
|
|
2078
|
+
${flowChildren.cascaded} AS "cascaded"
|
|
2079
|
+
FROM ${flowChildren}
|
|
2080
|
+
WHERE ${flowChildren.flowId} = ${flowId}
|
|
2081
|
+
${cursor === undefined ? sql`` : sql`AND ${flowChildren.childKey} > ${cursor}`}
|
|
2082
|
+
ORDER BY ${flowChildren.childKey} ASC
|
|
2083
|
+
LIMIT ${limit + 1}
|
|
2084
|
+
`).pipe(Effect.mapError(storeError("listChildResults failed"))))
|
|
2085
|
+
const page = rows.slice(0, limit)
|
|
2086
|
+
const items = page.map(toFlowChildRecord)
|
|
2087
|
+
const last = items[items.length - 1]
|
|
2088
|
+
return {
|
|
2089
|
+
items,
|
|
2090
|
+
cursor: rows.length > limit && last !== undefined ? last.childKey : undefined
|
|
2091
|
+
}
|
|
2092
|
+
}),
|
|
2093
|
+
|
|
2094
|
+
flowSweepWork: (sweepOptions) =>
|
|
2095
|
+
Effect.gen(function*() {
|
|
2096
|
+
const now = yield* nowDate
|
|
2097
|
+
const limit = Math.max(1, sweepOptions.limit ?? 1000)
|
|
2098
|
+
const threshold = new Date(now.getTime() - sweepOptions.pendingAgeMs)
|
|
2099
|
+
type SweepRow = {
|
|
2100
|
+
readonly flowId: string
|
|
2101
|
+
readonly childKey: string
|
|
2102
|
+
readonly storeKey: string
|
|
2103
|
+
readonly spec: JobStore.EnqueueRequest
|
|
2104
|
+
}
|
|
2105
|
+
// Reconcile: pending rows past the eligibility threshold whose
|
|
2106
|
+
// parent is still parked (a settled flow never re-drives work).
|
|
2107
|
+
// Returning a row re-arms `pending_since` in the same statement, so
|
|
2108
|
+
// a full page rotates across sweeps instead of pinning its head.
|
|
2109
|
+
// SKIP LOCKED: rows a concurrent report/settle holds are its
|
|
2110
|
+
// business, and never waiting means this statement cannot deadlock.
|
|
2111
|
+
// The raw column names are safe: only table names vary across
|
|
2112
|
+
// factory instances. RETURNING order is unspecified — sorted below.
|
|
2113
|
+
const reconcileRows = rowsOf(yield* db.execute<SweepRow>(sql`
|
|
2114
|
+
WITH due AS (
|
|
2115
|
+
SELECT c.flow_id, c.child_key
|
|
2116
|
+
FROM ${flowChildren} c
|
|
2117
|
+
JOIN ${jobs} j ON j.id = c.flow_id
|
|
2118
|
+
WHERE c.status = 'pending' AND c.pending_since <= ${threshold}
|
|
2119
|
+
AND j.state = 'waiting-children'
|
|
2120
|
+
ORDER BY c.flow_id, c.child_key
|
|
2121
|
+
LIMIT ${limit}
|
|
2122
|
+
FOR UPDATE OF c SKIP LOCKED
|
|
2123
|
+
)
|
|
2124
|
+
UPDATE ${flowChildren} SET pending_since = ${now}
|
|
2125
|
+
FROM due
|
|
2126
|
+
WHERE ${flowChildren.flowId} = due.flow_id AND ${flowChildren.childKey} = due.child_key
|
|
2127
|
+
RETURNING ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
2128
|
+
${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
|
|
2129
|
+
`).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
|
|
2130
|
+
.toSorted((a, b) =>
|
|
2131
|
+
a.flowId !== b.flowId
|
|
2132
|
+
? (a.flowId < b.flowId ? -1 : 1)
|
|
2133
|
+
: a.childKey < b.childKey
|
|
2134
|
+
? -1
|
|
2135
|
+
: a.childKey > b.childKey
|
|
2136
|
+
? 1
|
|
2137
|
+
: 0
|
|
2138
|
+
)
|
|
2139
|
+
// Cascade: cancelled rows whose cancel has not been delivered into
|
|
2140
|
+
// the child's store yet (any parent state).
|
|
2141
|
+
const cascadeRows = rowsOf(yield* db.execute<SweepRow>(sql`
|
|
2142
|
+
SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
2143
|
+
${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
|
|
2144
|
+
FROM ${flowChildren}
|
|
2145
|
+
WHERE ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
|
|
2146
|
+
ORDER BY ${flowChildren.flowId}, ${flowChildren.childKey}
|
|
2147
|
+
LIMIT ${limit}
|
|
2148
|
+
`).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
|
|
2149
|
+
const reconcile: Array<{
|
|
2150
|
+
flowId: JobStore.JobId
|
|
2151
|
+
children: Array<JobStore.FlowChildSpec>
|
|
2152
|
+
}> = []
|
|
2153
|
+
for (const row of reconcileRows) {
|
|
2154
|
+
const flowId = JobId(row.flowId)
|
|
2155
|
+
let group = reconcile[reconcile.length - 1]
|
|
2156
|
+
if (group === undefined || group.flowId !== flowId) {
|
|
2157
|
+
group = { flowId, children: [] }
|
|
2158
|
+
reconcile.push(group)
|
|
2159
|
+
}
|
|
2160
|
+
group.children.push({ childKey: row.childKey, storeKey: row.storeKey, request: row.spec })
|
|
2161
|
+
}
|
|
2162
|
+
const cascade: Array<{
|
|
2163
|
+
flowId: JobStore.JobId
|
|
2164
|
+
children: Array<{ childKey: string; storeKey: string; childJobId: JobStore.JobId }>
|
|
2165
|
+
}> = []
|
|
2166
|
+
for (const row of cascadeRows) {
|
|
2167
|
+
const flowId = JobId(row.flowId)
|
|
2168
|
+
let group = cascade[cascade.length - 1]
|
|
2169
|
+
if (group === undefined || group.flowId !== flowId) {
|
|
2170
|
+
group = { flowId, children: [] }
|
|
2171
|
+
cascade.push(group)
|
|
2172
|
+
}
|
|
2173
|
+
group.children.push({
|
|
2174
|
+
childKey: row.childKey,
|
|
2175
|
+
storeKey: row.storeKey,
|
|
2176
|
+
// SAFETY: the FanOut ack validated every spec id before
|
|
2177
|
+
// persisting it.
|
|
2178
|
+
childJobId: row.spec.id as JobStore.JobId
|
|
2179
|
+
})
|
|
2180
|
+
}
|
|
2181
|
+
const work: JobStore.FlowSweepWork = { reconcile, cascade }
|
|
2182
|
+
return work
|
|
2183
|
+
}),
|
|
2184
|
+
|
|
2185
|
+
markChildrenCascaded: (flowId, childKeys) =>
|
|
2186
|
+
childKeys.length === 0
|
|
2187
|
+
? Effect.void
|
|
2188
|
+
: db.execute(sql`
|
|
2189
|
+
UPDATE ${flowChildren} SET cascaded = TRUE
|
|
2190
|
+
WHERE ${flowChildren.flowId} = ${flowId}
|
|
2191
|
+
AND ${flowChildren.childKey} = ANY(${sql.param([...childKeys])})
|
|
2192
|
+
`).pipe(
|
|
2193
|
+
Effect.mapError(storeError("markChildrenCascaded failed")),
|
|
2194
|
+
Effect.asVoid
|
|
2195
|
+
),
|
|
2196
|
+
|
|
1461
2197
|
counts: (queue) =>
|
|
1462
2198
|
db.execute<{ state: JobStore.JobState; count: number }>(sql`
|
|
1463
2199
|
SELECT ${jobs.state} AS "state", count(*)::int AS "count" FROM ${jobs}
|
|
@@ -1471,6 +2207,7 @@ export const make = (
|
|
|
1471
2207
|
waiting: 0,
|
|
1472
2208
|
delayed: 0,
|
|
1473
2209
|
active: 0,
|
|
2210
|
+
"waiting-children": 0,
|
|
1474
2211
|
completed: 0,
|
|
1475
2212
|
failed: 0,
|
|
1476
2213
|
cancelled: 0
|
|
@@ -1481,10 +2218,18 @@ export const make = (
|
|
|
1481
2218
|
),
|
|
1482
2219
|
|
|
1483
2220
|
remove: (id) =>
|
|
2221
|
+
// The purge CTE takes a removed flow parent's dependency rows with it
|
|
2222
|
+
// in the same statement.
|
|
1484
2223
|
db.execute(sql`
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
2224
|
+
WITH deleted AS (
|
|
2225
|
+
DELETE FROM ${jobs}
|
|
2226
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} NOT IN ('active', 'waiting-children')
|
|
2227
|
+
RETURNING ${jobs.id} AS id
|
|
2228
|
+
), purged AS (
|
|
2229
|
+
DELETE FROM ${flowChildren}
|
|
2230
|
+
WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
|
|
2231
|
+
)
|
|
2232
|
+
SELECT id FROM deleted
|
|
1488
2233
|
`).pipe(
|
|
1489
2234
|
Effect.mapError(storeError("remove failed")),
|
|
1490
2235
|
Effect.map((result) => rowsOf(result).length > 0)
|