effect-mq 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -17
- package/dist/Flow.d.ts +381 -0
- package/dist/Flow.d.ts.map +1 -0
- package/dist/Flow.js +340 -0
- package/dist/Flow.js.map +1 -0
- package/dist/Job.d.ts +31 -6
- package/dist/Job.d.ts.map +1 -1
- package/dist/Job.js +16 -2
- package/dist/Job.js.map +1 -1
- package/dist/JobStore.d.ts +312 -10
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +334 -7
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/Metrics.d.ts +31 -0
- package/dist/Metrics.d.ts.map +1 -1
- package/dist/Metrics.js +39 -0
- package/dist/Metrics.js.map +1 -1
- package/dist/Worker.d.ts +120 -11
- package/dist/Worker.d.ts.map +1 -1
- package/dist/Worker.js +452 -26
- package/dist/Worker.js.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.js +653 -77
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
- package/dist/drizzle-postgres/schema.d.ts +293 -3
- package/dist/drizzle-postgres/schema.d.ts.map +1 -1
- package/dist/drizzle-postgres/schema.js +66 -1
- package/dist/drizzle-postgres/schema.js.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -0
- package/dist/index.js.map +1 -1
- package/dist/redis/RedisJobStore.d.ts.map +1 -1
- package/dist/redis/RedisJobStore.js +219 -18
- package/dist/redis/RedisJobStore.js.map +1 -1
- package/dist/redis/scripts.d.ts +117 -10
- package/dist/redis/scripts.d.ts.map +1 -1
- package/dist/redis/scripts.js +492 -25
- package/dist/redis/scripts.js.map +1 -1
- package/dist/testing/conformance.d.ts +6 -0
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +728 -1
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +1 -1
- package/src/Flow.ts +778 -0
- package/src/Job.ts +35 -11
- package/src/JobStore.ts +339 -9
- package/src/MemoryJobStore.ts +370 -7
- package/src/Metrics.ts +43 -0
- package/src/Worker.ts +726 -37
- package/src/drizzle-postgres/DrizzleJobStore.ts +817 -78
- package/src/drizzle-postgres/schema.ts +92 -0
- package/src/index.ts +8 -0
- package/src/redis/RedisJobStore.ts +289 -8
- package/src/redis/scripts.ts +524 -24
- package/src/testing/conformance.ts +945 -1
|
@@ -18,11 +18,19 @@
|
|
|
18
18
|
*/
|
|
19
19
|
import * as JobStore from "../JobStore.ts"
|
|
20
20
|
import type { PgClient } from "@effect/sql-pg"
|
|
21
|
-
import { asc, eq, getTableColumns, sql } from "drizzle-orm"
|
|
21
|
+
import { asc, eq, getTableColumns, type SQL, sql } from "drizzle-orm"
|
|
22
22
|
import * as PgDrizzle from "drizzle-orm/effect-postgres"
|
|
23
23
|
import { getTableConfig } from "drizzle-orm/pg-core"
|
|
24
|
-
import { Clock, type Context, Deferred, Duration, Effect, Layer, Option, type Scope, Stream } from "effect"
|
|
25
|
-
import type {
|
|
24
|
+
import { Cause, Clock, type Context, Deferred, Duration, Effect, Layer, Option, Predicate, type Scope, Stream } from "effect"
|
|
25
|
+
import type {
|
|
26
|
+
MqDedupeTable,
|
|
27
|
+
MqFlowChildrenTable,
|
|
28
|
+
MqFlowOutboxTable,
|
|
29
|
+
MqJobAttemptsTable,
|
|
30
|
+
MqJobsTable,
|
|
31
|
+
MqQueueControlTable,
|
|
32
|
+
MqSchedulesTable
|
|
33
|
+
} from "./schema.ts"
|
|
26
34
|
|
|
27
35
|
const { JobId } = JobStore
|
|
28
36
|
|
|
@@ -40,6 +48,10 @@ export interface DrizzleJobStoreOptions<StoreId = JobStore.JobStore> {
|
|
|
40
48
|
readonly queues: MqQueueControlTable
|
|
41
49
|
/** The dedup-key registry table (from `mqDedupe`). */
|
|
42
50
|
readonly dedupe: MqDedupeTable
|
|
51
|
+
/** The flow dependency-rows table (from `mqFlowChildren`). */
|
|
52
|
+
readonly flowChildren: MqFlowChildrenTable
|
|
53
|
+
/** The child-report outbox table (from `mqFlowOutbox`). */
|
|
54
|
+
readonly flowOutbox: MqFlowOutboxTable
|
|
43
55
|
/**
|
|
44
56
|
* Values for columns added via `mqJobs({ extend })`, evaluated at enqueue
|
|
45
57
|
* (and on dedupe `replace`). Keys are the extended columns' TS names.
|
|
@@ -84,6 +96,50 @@ export type ExtraColumnValue = string | number | boolean | Date | null
|
|
|
84
96
|
const storeError = (message: string) => (cause: unknown) =>
|
|
85
97
|
new JobStore.JobStoreError({ message, cause })
|
|
86
98
|
|
|
99
|
+
// The exact spellings a bigserial ever produces: no leading zeros, no signs,
|
|
100
|
+
// no whitespace — what outbox ids look like and nothing Postgres would
|
|
101
|
+
// silently normalize into one.
|
|
102
|
+
const CANONICAL_BIGSERIAL = /^[1-9]\d*$/
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Whether an error surfaced by drizzle's Effect driver is a Postgres
|
|
106
|
+
* deadlock (40P01), i.e. safe to retry.
|
|
107
|
+
*
|
|
108
|
+
* The deadlock travels wrapped: drizzle's session fails with an
|
|
109
|
+
* `EffectDrizzleQueryError` whose `cause` field is `Cause.fail(SqlError)`
|
|
110
|
+
* (drizzle-orm `pg-core/effect/session.ts`), and `@effect/sql-pg` classifies
|
|
111
|
+
* pg code 40P01 into the `SqlError`'s `reason: DeadlockError` (PgClient.ts
|
|
112
|
+
* `classifyError`). Neither layer's `toString` renders the pg code, so this
|
|
113
|
+
* unwraps structurally instead of string-matching the rendered error.
|
|
114
|
+
*
|
|
115
|
+
* @internal
|
|
116
|
+
*/
|
|
117
|
+
// oxlint-disable-next-line anti-slop/no-unknown-parameters -- a retry-predicate classifier over whatever the driver threw IS the boundary parser
|
|
118
|
+
export const isDeadlockError = (error: unknown): boolean => {
|
|
119
|
+
// Unwrap drizzle's envelope: its `cause` is an Effect Cause holding the
|
|
120
|
+
// original failure. A bare SqlError (non-drizzle path) passes through.
|
|
121
|
+
const unwrapped = Predicate.hasProperty(error, "cause") && Cause.isCause(error.cause)
|
|
122
|
+
? Option.getOrUndefined(Cause.findErrorOption(error.cause))
|
|
123
|
+
: error
|
|
124
|
+
if (
|
|
125
|
+
!Predicate.hasProperty(unwrapped, "_tag") || unwrapped._tag !== "SqlError" ||
|
|
126
|
+
!Predicate.hasProperty(unwrapped, "reason")
|
|
127
|
+
) {
|
|
128
|
+
return false
|
|
129
|
+
}
|
|
130
|
+
const reason = unwrapped.reason
|
|
131
|
+
if (Predicate.hasProperty(reason, "_tag") && reason._tag === "DeadlockError") {
|
|
132
|
+
return true
|
|
133
|
+
}
|
|
134
|
+
// Fallback for classifiers that missed the code: the reason keeps the raw
|
|
135
|
+
// pg error, whose message for 40P01 is "deadlock detected".
|
|
136
|
+
if (Predicate.hasProperty(reason, "cause")) {
|
|
137
|
+
const raw = String(reason.cause)
|
|
138
|
+
return raw.includes("40P01") || raw.includes("deadlock detected")
|
|
139
|
+
}
|
|
140
|
+
return false
|
|
141
|
+
}
|
|
142
|
+
|
|
87
143
|
/**
|
|
88
144
|
* drizzle's Effect driver types raw `execute` results as row arrays, but at
|
|
89
145
|
* runtime the raw path yields the node-postgres `QueryResult` envelope
|
|
@@ -114,6 +170,12 @@ type JobRow = {
|
|
|
114
170
|
readonly cancelRequested: boolean
|
|
115
171
|
readonly dedupeKey: string | null
|
|
116
172
|
readonly trace: JobStore.TraceContext | null
|
|
173
|
+
readonly parent: JobStore.ParentEnvelope | null
|
|
174
|
+
readonly flowFailFast: boolean | null
|
|
175
|
+
readonly flowPending: number | null
|
|
176
|
+
readonly flowCompleted: number | null
|
|
177
|
+
readonly flowFailed: number | null
|
|
178
|
+
readonly flowCancelled: number | null
|
|
117
179
|
readonly runAt: Date
|
|
118
180
|
readonly enqueuedAt: Date
|
|
119
181
|
readonly processedAt: Date | null
|
|
@@ -175,6 +237,17 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
|
|
|
175
237
|
cancelRequested: row.cancelRequested,
|
|
176
238
|
dedupeKey: row.dedupeKey ?? undefined,
|
|
177
239
|
trace: row.trace ?? undefined,
|
|
240
|
+
parent: row.parent ?? undefined,
|
|
241
|
+
// The flow columns are NULL together; `flowPending` is the presence marker.
|
|
242
|
+
flow: row.flowPending === null || row.flowPending === undefined
|
|
243
|
+
? undefined
|
|
244
|
+
: {
|
|
245
|
+
failFast: row.flowFailFast === true,
|
|
246
|
+
pending: Number(row.flowPending),
|
|
247
|
+
completed: Number(row.flowCompleted ?? 0),
|
|
248
|
+
failed: Number(row.flowFailed ?? 0),
|
|
249
|
+
cancelled: Number(row.flowCancelled ?? 0)
|
|
250
|
+
},
|
|
178
251
|
runAt: row.runAt.getTime(),
|
|
179
252
|
enqueuedAt: row.enqueuedAt.getTime(),
|
|
180
253
|
processedAt: row.processedAt?.getTime(),
|
|
@@ -183,6 +256,31 @@ const toRecord = (row: JobRow): JobStore.JobRecord => ({
|
|
|
183
256
|
failedReason: row.failedReason ?? undefined
|
|
184
257
|
})
|
|
185
258
|
|
|
259
|
+
type FlowChildRow = {
|
|
260
|
+
readonly flowId: string
|
|
261
|
+
readonly childKey: string
|
|
262
|
+
readonly name: string
|
|
263
|
+
readonly storeKey: string
|
|
264
|
+
/** Projected from `spec->>'id'` (never NULL: the FanOut ack validated it). */
|
|
265
|
+
readonly childJobId: string
|
|
266
|
+
readonly status: JobStore.FlowChildRecord["status"]
|
|
267
|
+
readonly exit: unknown
|
|
268
|
+
readonly failedReason: string | null
|
|
269
|
+
readonly cascaded: boolean
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
const toFlowChildRecord = (row: FlowChildRow): JobStore.FlowChildRecord => ({
|
|
273
|
+
flowId: JobId(row.flowId),
|
|
274
|
+
childKey: row.childKey,
|
|
275
|
+
name: row.name,
|
|
276
|
+
storeKey: row.storeKey,
|
|
277
|
+
childJobId: JobId(row.childJobId),
|
|
278
|
+
status: row.status,
|
|
279
|
+
exit: row.exit ?? undefined,
|
|
280
|
+
failedReason: row.failedReason ?? undefined,
|
|
281
|
+
cascaded: row.cascaded
|
|
282
|
+
})
|
|
283
|
+
|
|
186
284
|
/**
|
|
187
285
|
* Build the store implementation. Requires `PgClient` and a `Scope` (for the
|
|
188
286
|
* LISTEN subscription).
|
|
@@ -200,6 +298,8 @@ export const make = (
|
|
|
200
298
|
const schedules = options.schedules
|
|
201
299
|
const queues = options.queues
|
|
202
300
|
const dedupe = options.dedupe
|
|
301
|
+
const flowChildren = options.flowChildren
|
|
302
|
+
const flowOutbox = options.flowOutbox
|
|
203
303
|
const jobsName = getTableConfig(jobs).name
|
|
204
304
|
const attemptsName = getTableConfig(attempts).name
|
|
205
305
|
const wakeChannel = `effect_mq_wake_${jobsName}`
|
|
@@ -225,6 +325,12 @@ export const make = (
|
|
|
225
325
|
"cancelRequested",
|
|
226
326
|
"dedupeKey",
|
|
227
327
|
"trace",
|
|
328
|
+
"parent",
|
|
329
|
+
"flowFailFast",
|
|
330
|
+
"flowPending",
|
|
331
|
+
"flowCompleted",
|
|
332
|
+
"flowFailed",
|
|
333
|
+
"flowCancelled",
|
|
228
334
|
"runAt",
|
|
229
335
|
"enqueuedAt",
|
|
230
336
|
"processedAt",
|
|
@@ -263,7 +369,9 @@ export const make = (
|
|
|
263
369
|
db.select({ jobId: attempts.jobId }).from(attempts).limit(0),
|
|
264
370
|
db.select({ key: schedules.key }).from(schedules).limit(0),
|
|
265
371
|
db.select({ queue: queues.queue }).from(queues).limit(0),
|
|
266
|
-
db.select({ key: dedupe.key }).from(dedupe).limit(0)
|
|
372
|
+
db.select({ key: dedupe.key }).from(dedupe).limit(0),
|
|
373
|
+
db.select({ flowId: flowChildren.flowId }).from(flowChildren).limit(0),
|
|
374
|
+
db.select({ id: flowOutbox.id }).from(flowOutbox).limit(0)
|
|
267
375
|
]).pipe(
|
|
268
376
|
Effect.mapError(storeError(
|
|
269
377
|
`effect-mq: tables "${jobsName}"/"${attemptsName}" are missing or mismatched — ` +
|
|
@@ -324,6 +432,28 @@ export const make = (
|
|
|
324
432
|
Effect.forever,
|
|
325
433
|
Effect.forkScoped
|
|
326
434
|
)
|
|
435
|
+
// Automatic retention (the store-level sweep and per-job `keep`) never
|
|
436
|
+
// prunes a flow parent that still owes cascade cancels: its dependency
|
|
437
|
+
// rows marked `cancelled` and not `cascaded` are the only record that
|
|
438
|
+
// real cancels are still due in the child stores. The explicit `remove`
|
|
439
|
+
// verb is not exempted.
|
|
440
|
+
const owesCascades = sql`EXISTS (
|
|
441
|
+
SELECT 1 FROM ${flowChildren}
|
|
442
|
+
WHERE ${flowChildren.flowId} = ${jobs.id}
|
|
443
|
+
AND ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
|
|
444
|
+
)`
|
|
445
|
+
// A pruned flow parent's dependency rows go with it in the same statement.
|
|
446
|
+
const purgeJobsWhere = (predicate: SQL) =>
|
|
447
|
+
sql`
|
|
448
|
+
WITH deleted AS (
|
|
449
|
+
DELETE FROM ${jobs}
|
|
450
|
+
WHERE ${predicate}
|
|
451
|
+
RETURNING ${jobs.id} AS id
|
|
452
|
+
)
|
|
453
|
+
DELETE FROM ${flowChildren}
|
|
454
|
+
WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
|
|
455
|
+
`
|
|
456
|
+
|
|
327
457
|
if (options.historyTtl !== undefined) {
|
|
328
458
|
const ttlByState = JobStore.normalizeHistoryTtl(options.historyTtl)
|
|
329
459
|
const sweepMs = Duration.toMillis(options.historySweepInterval ?? "1 minute")
|
|
@@ -334,9 +464,8 @@ export const make = (
|
|
|
334
464
|
// job name is pruned on the timer, not only when its group is acked.
|
|
335
465
|
for (const state of ["completed", "failed", "cancelled"] as const) {
|
|
336
466
|
const ttl = ttlByState[state]
|
|
337
|
-
yield* db.execute(sql`
|
|
338
|
-
|
|
339
|
-
WHERE ${jobs.state} = ${state} AND (
|
|
467
|
+
yield* db.execute(purgeJobsWhere(sql`
|
|
468
|
+
${jobs.state} = ${state} AND NOT ${owesCascades} AND (
|
|
340
469
|
${ttl !== undefined ? sql`${jobs.finishedAt} <= ${new Date(now.getTime() - ttl)}` : sql`FALSE`}
|
|
341
470
|
OR (
|
|
342
471
|
COALESCE(
|
|
@@ -351,7 +480,7 @@ export const make = (
|
|
|
351
480
|
ELSE ${jobs.keep}->>'ageMs' END
|
|
352
481
|
)::double precision) / 1000.0))
|
|
353
482
|
)
|
|
354
|
-
`)
|
|
483
|
+
`))
|
|
355
484
|
}
|
|
356
485
|
// Dead dedup rows: expired windows, or pointers at vanished jobs.
|
|
357
486
|
yield* db.execute(sql`
|
|
@@ -359,7 +488,7 @@ export const make = (
|
|
|
359
488
|
WHERE (${dedupe.windowExpiresAt} IS NOT NULL AND ${dedupe.windowExpiresAt} <= ${now})
|
|
360
489
|
OR (${dedupe.windowExpiresAt} IS NULL AND NOT EXISTS (
|
|
361
490
|
SELECT 1 FROM ${jobs} WHERE ${jobs.id} = ${dedupe.jobId}
|
|
362
|
-
AND ${jobs.state} IN ('waiting', 'delayed', 'active')
|
|
491
|
+
AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
|
|
363
492
|
))
|
|
364
493
|
`)
|
|
365
494
|
}).pipe(
|
|
@@ -433,23 +562,23 @@ export const make = (
|
|
|
433
562
|
const keep = keepPolicyFor(row.keep, row.state)
|
|
434
563
|
if (keep === undefined) return
|
|
435
564
|
if (keep.ageMs !== undefined) {
|
|
436
|
-
yield* tx.execute(sql`
|
|
437
|
-
|
|
438
|
-
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
565
|
+
yield* tx.execute(purgeJobsWhere(sql`
|
|
566
|
+
${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
439
567
|
AND ${jobs.finishedAt} <= ${new Date(now.getTime() - keep.ageMs)}
|
|
440
|
-
|
|
568
|
+
AND NOT ${owesCascades}
|
|
569
|
+
`))
|
|
441
570
|
}
|
|
442
571
|
if (keep.count !== undefined) {
|
|
443
|
-
yield* tx.execute(sql`
|
|
444
|
-
|
|
445
|
-
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
572
|
+
yield* tx.execute(purgeJobsWhere(sql`
|
|
573
|
+
${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
446
574
|
AND ${jobs.id} NOT IN (
|
|
447
575
|
SELECT ${jobs.id} FROM ${jobs}
|
|
448
576
|
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
449
577
|
ORDER BY ${jobs.finishedAt} DESC, ${jobs.seq} DESC
|
|
450
578
|
LIMIT ${keep.count}
|
|
451
579
|
)
|
|
452
|
-
|
|
580
|
+
AND NOT ${owesCascades}
|
|
581
|
+
`))
|
|
453
582
|
}
|
|
454
583
|
})
|
|
455
584
|
|
|
@@ -495,7 +624,7 @@ export const make = (
|
|
|
495
624
|
: sql`'j-' || ${seqExpr}::text`
|
|
496
625
|
const rows = rowsOf(yield* exec.execute<{ id: string }>(sql`
|
|
497
626
|
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
498
|
-
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
627
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
|
|
499
628
|
VALUES (${idExpr}, ${request.name}, ${request.queue}, ${state}, ${request.priority},
|
|
500
629
|
${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
|
|
501
630
|
${request.attemptsMax},
|
|
@@ -503,6 +632,7 @@ export const make = (
|
|
|
503
632
|
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
504
633
|
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
505
634
|
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
635
|
+
${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
|
|
506
636
|
${runAt}, ${now}${extraColumnValues(request)})
|
|
507
637
|
ON CONFLICT (id) DO NOTHING
|
|
508
638
|
RETURNING ${jobs.id} AS id
|
|
@@ -633,7 +763,7 @@ export const make = (
|
|
|
633
763
|
// killed and safe to retry.
|
|
634
764
|
Effect.retry({
|
|
635
765
|
times: 3,
|
|
636
|
-
while:
|
|
766
|
+
while: isDeadlockError
|
|
637
767
|
}),
|
|
638
768
|
Effect.mapError((error) =>
|
|
639
769
|
error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)
|
|
@@ -658,59 +788,139 @@ export const make = (
|
|
|
658
788
|
AND (${dedupe.windowExpiresAt} IS NULL OR ${dedupe.windowExpiresAt} <= ${now})
|
|
659
789
|
`).pipe(Effect.asVoid)
|
|
660
790
|
|
|
791
|
+
// The outbox invariant: every operation that moves a job carrying a
|
|
792
|
+
// `parent` envelope INTO a terminal state appends its report here in the
|
|
793
|
+
// same transaction (see `JobStore.OutboxEntry`). `exit`/`failedReason`
|
|
794
|
+
// mirror the job row AFTER the transition; JSON.stringify drops
|
|
795
|
+
// undefined fields so absent values read back as undefined.
|
|
796
|
+
const appendOutbox = (
|
|
797
|
+
exec: Pick<Db, "execute">,
|
|
798
|
+
parent: JobStore.ParentEnvelope | null | undefined,
|
|
799
|
+
outcome: JobStore.FlowChildReport["outcome"],
|
|
800
|
+
exit: JobStore.FlowChildReport["exit"],
|
|
801
|
+
failedReason: string | null | undefined
|
|
802
|
+
) =>
|
|
803
|
+
parent === null || parent === undefined
|
|
804
|
+
? Effect.void
|
|
805
|
+
: exec.execute(sql`
|
|
806
|
+
INSERT INTO ${flowOutbox} (flow_name, parent_store_key, report)
|
|
807
|
+
VALUES (${parent.flowName}, ${parent.parentStoreKey}, ${
|
|
808
|
+
JSON.stringify({
|
|
809
|
+
flowId: parent.flowId,
|
|
810
|
+
childKey: parent.childKey,
|
|
811
|
+
outcome,
|
|
812
|
+
exit,
|
|
813
|
+
failedReason: failedReason ?? undefined
|
|
814
|
+
})
|
|
815
|
+
}::jsonb)
|
|
816
|
+
`).pipe(Effect.asVoid)
|
|
817
|
+
|
|
818
|
+
// Flip a flow's still-pending dependency rows to `cancelled` and NOT
|
|
819
|
+
// `cascaded` (the sweeper still owes the child stores real cancels).
|
|
820
|
+
// Embedded as a CTE body so the caller can count the marked rows into
|
|
821
|
+
// the parent's `cancelled` counter in the same statement.
|
|
822
|
+
const cancelPendingChildren = (flowId: string) =>
|
|
823
|
+
sql`
|
|
824
|
+
UPDATE ${flowChildren} SET status = 'cancelled', cascaded = FALSE
|
|
825
|
+
WHERE ${flowChildren.flowId} = ${flowId} AND ${flowChildren.status} = 'pending'
|
|
826
|
+
RETURNING 1
|
|
827
|
+
`
|
|
828
|
+
|
|
661
829
|
// Shared by cancel and cancelByDedupe.
|
|
662
830
|
const cancelJob = (id: JobStore.JobId) =>
|
|
663
831
|
db.transaction((tx) =>
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
832
|
+
Effect.gen(function*() {
|
|
833
|
+
const now = yield* nowDate
|
|
834
|
+
// Contract lock order (dependency rows first, parent second): mark
|
|
835
|
+
// a waiting-children parent's remaining pending rows cancelled.
|
|
836
|
+
// Pending rows exist only while the parent is `waiting-children`,
|
|
837
|
+
// so this is a no-op for every other state; a non-cancellable
|
|
838
|
+
// parent rolls the transaction back anyway.
|
|
839
|
+
const firstPass = rowsOf(yield* tx.execute<{ marked: number }>(sql`
|
|
840
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
841
|
+
SELECT count(*)::int AS marked FROM marked
|
|
842
|
+
`))
|
|
843
|
+
const preMarked = firstPass[0]?.marked ?? 0
|
|
844
|
+
// One guarded statement: waiting/delayed/waiting-children become
|
|
845
|
+
// terminal, active gets the cancel-request flag; anything else is
|
|
846
|
+
// reported by state.
|
|
847
|
+
const rows = rowsOf(yield* tx.execute<
|
|
848
|
+
{
|
|
849
|
+
id: string
|
|
850
|
+
state: string
|
|
851
|
+
processedAt: Date | null
|
|
852
|
+
name: string
|
|
853
|
+
keep: JobStore.KeepPolicy | null
|
|
854
|
+
dedupeKey: string | null
|
|
855
|
+
hasFlow: boolean
|
|
856
|
+
parent: JobStore.ParentEnvelope | null
|
|
857
|
+
exit: unknown
|
|
858
|
+
failedReason: string | null
|
|
859
|
+
}
|
|
860
|
+
>(sql`
|
|
861
|
+
UPDATE ${jobs} SET
|
|
862
|
+
state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN 'cancelled' ELSE ${jobs.state} END,
|
|
863
|
+
finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
|
|
864
|
+
flow_pending = CASE WHEN ${jobs.state} = 'waiting-children' THEN 0 ELSE ${jobs.flowPending} END,
|
|
865
|
+
cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
|
|
866
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
|
|
867
|
+
RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
|
|
868
|
+
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
869
|
+
${jobs.dedupeKey} AS "dedupeKey", (${jobs.flowPending} IS NOT NULL) AS "hasFlow",
|
|
870
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
871
|
+
`))
|
|
872
|
+
const row = rows[0]
|
|
873
|
+
if (row === undefined) {
|
|
874
|
+
const existing = rowsOf(yield* tx.execute<{ state: JobStore.JobState }>(sql`
|
|
875
|
+
SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
|
|
686
876
|
`))
|
|
687
|
-
const
|
|
688
|
-
if (
|
|
689
|
-
|
|
690
|
-
SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
|
|
691
|
-
`))
|
|
692
|
-
const found = existing[0]
|
|
693
|
-
if (found === undefined) {
|
|
694
|
-
return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
695
|
-
}
|
|
696
|
-
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
|
|
877
|
+
const found = existing[0]
|
|
878
|
+
if (found === undefined) {
|
|
879
|
+
return yield* new JobStore.JobNotFoundError({ jobId: id })
|
|
697
880
|
}
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
881
|
+
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state })
|
|
882
|
+
}
|
|
883
|
+
if (row.state === "cancelled") {
|
|
884
|
+
if (row.hasFlow) {
|
|
885
|
+
// Re-run the marking now that the parent lock is held: the
|
|
886
|
+
// first UPDATE above races a concurrent FanOut — its
|
|
887
|
+
// uncommitted dependency-row INSERTs are invisible, while
|
|
888
|
+
// this UPDATE's own EPQ re-check can still see the parent as
|
|
889
|
+
// 'waiting-children' after the FanOut commits. Without this
|
|
890
|
+
// pass those rows would stay 'pending' forever, invisible to
|
|
891
|
+
// both sweep classes. Every row either pass marked lands in
|
|
892
|
+
// the `cancelled` counter.
|
|
893
|
+
yield* tx.execute(sql`
|
|
894
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
895
|
+
UPDATE ${jobs} SET flow_cancelled = ${jobs.flowCancelled} + ${preMarked}::int
|
|
896
|
+
+ (SELECT count(*)::int FROM marked)
|
|
897
|
+
WHERE ${jobs.id} = ${id}
|
|
898
|
+
`)
|
|
702
899
|
}
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
)
|
|
900
|
+
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
901
|
+
// A cancelled child reports upward through the outbox.
|
|
902
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
|
|
903
|
+
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
904
|
+
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
905
|
+
}
|
|
906
|
+
})
|
|
907
|
+
).pipe(
|
|
908
|
+
// Rare lock-order inversion (this row-then-parent flip vs a
|
|
909
|
+
// fail-fast settle's parent-then-rows marking) surfaces as a
|
|
910
|
+
// Postgres deadlock (40P01); one side is killed and safe to retry.
|
|
911
|
+
Effect.retry({
|
|
912
|
+
times: 3,
|
|
913
|
+
while: isDeadlockError
|
|
914
|
+
}),
|
|
915
|
+
Effect.mapError((error) =>
|
|
916
|
+
error instanceof JobStore.JobNotFoundError ||
|
|
917
|
+
error instanceof JobStore.JobNotCancellableError ||
|
|
918
|
+
error instanceof JobStore.JobStoreError
|
|
919
|
+
? error
|
|
920
|
+
: storeError("cancel failed")(error)
|
|
921
|
+
),
|
|
922
|
+
Effect.asVoid
|
|
923
|
+
)
|
|
714
924
|
|
|
715
925
|
const enqueueOne = (request: JobStore.EnqueueRequest) =>
|
|
716
926
|
Effect.gen(function*() {
|
|
@@ -800,11 +1010,12 @@ export const make = (
|
|
|
800
1010
|
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
801
1011
|
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
802
1012
|
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
1013
|
+
${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
|
|
803
1014
|
${new Date(now.getTime() + Math.max(0, request.delayMs))}, ${now}${extraColumnValues(request)})`
|
|
804
1015
|
)
|
|
805
1016
|
const rows = rowsOf(yield* db.execute<{ id: string }>(sql`
|
|
806
1017
|
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
807
|
-
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
1018
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
|
|
808
1019
|
VALUES ${sql.join(values, sql`, `)}
|
|
809
1020
|
ON CONFLICT (id) DO NOTHING
|
|
810
1021
|
RETURNING ${jobs.id} AS id
|
|
@@ -845,6 +1056,123 @@ export const make = (
|
|
|
845
1056
|
return resolved
|
|
846
1057
|
})
|
|
847
1058
|
|
|
1059
|
+
// The FanOut ack: land the manifest (flow columns + dependency rows) and
|
|
1060
|
+
// park the parent, all lock-token-guarded in one transaction. A fan-out
|
|
1061
|
+
// is a phase transition, not a completed run — `attempts_made` is not
|
|
1062
|
+
// incremented; the ledger records `fanned-out` with no exit.
|
|
1063
|
+
const ackFanOut = (
|
|
1064
|
+
id: JobStore.JobId,
|
|
1065
|
+
token: string,
|
|
1066
|
+
outcome: Extract<JobStore.AckOutcome, { _tag: "FanOut" }>
|
|
1067
|
+
): Effect.Effect<void, JobStore.JobStoreError | JobStore.JobNotFoundError | JobStore.LockLostError> =>
|
|
1068
|
+
Effect.gen(function*() {
|
|
1069
|
+
// Validate BEFORE any mutation, so a bad spec cannot leave the job
|
|
1070
|
+
// half-acked (lock cleared, ledger written, still active).
|
|
1071
|
+
if (outcome.children.some((child) => child.request.id === undefined)) {
|
|
1072
|
+
return yield* new JobStore.JobStoreError({
|
|
1073
|
+
message: "FanOut child specs require an explicit request.id"
|
|
1074
|
+
})
|
|
1075
|
+
}
|
|
1076
|
+
const wakeQueue = yield* db.transaction((tx) =>
|
|
1077
|
+
Effect.gen(function*() {
|
|
1078
|
+
const now = yield* nowDate
|
|
1079
|
+
const rows = rowsOf(yield* tx.execute<
|
|
1080
|
+
{
|
|
1081
|
+
processedAt: Date | null
|
|
1082
|
+
name: string
|
|
1083
|
+
keep: JobStore.KeepPolicy | null
|
|
1084
|
+
dedupeKey: string | null
|
|
1085
|
+
queue: string
|
|
1086
|
+
cancelRequested: boolean
|
|
1087
|
+
flowPending: number | null
|
|
1088
|
+
parent: JobStore.ParentEnvelope | null
|
|
1089
|
+
exit: unknown
|
|
1090
|
+
failedReason: string | null
|
|
1091
|
+
}
|
|
1092
|
+
>(sql`
|
|
1093
|
+
UPDATE ${jobs} SET lock_token = NULL, lock_expires_at = NULL
|
|
1094
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
1095
|
+
RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
|
|
1096
|
+
${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1097
|
+
${jobs.cancelRequested} AS "cancelRequested", ${jobs.flowPending} AS "flowPending",
|
|
1098
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
1099
|
+
`))
|
|
1100
|
+
const row = rows[0]
|
|
1101
|
+
if (row === undefined) {
|
|
1102
|
+
return yield* explainMiss(id)
|
|
1103
|
+
}
|
|
1104
|
+
yield* insertAttempt(tx, id, "fanned-out", row.processedAt, now, undefined)
|
|
1105
|
+
let pending: number
|
|
1106
|
+
if (row.flowPending === null || row.flowPending === undefined) {
|
|
1107
|
+
pending = outcome.children.length
|
|
1108
|
+
yield* tx.execute(sql`
|
|
1109
|
+
UPDATE ${jobs} SET flow_fail_fast = ${outcome.failFast}, flow_pending = ${pending},
|
|
1110
|
+
flow_completed = 0, flow_failed = 0, flow_cancelled = 0
|
|
1111
|
+
WHERE ${jobs.id} = ${id}
|
|
1112
|
+
`)
|
|
1113
|
+
// Chunked multi-row VALUES, like enqueueMany's insertBatch.
|
|
1114
|
+
for (let start = 0; start < outcome.children.length; start += 500) {
|
|
1115
|
+
const chunk = outcome.children.slice(start, start + 500)
|
|
1116
|
+
const values = chunk.map((child) =>
|
|
1117
|
+
sql`(${id}, ${child.childKey}, ${child.request.name}, ${child.storeKey},
|
|
1118
|
+
${JSON.stringify(child.request)}::jsonb, 'pending', NULL, NULL, FALSE, ${now})`
|
|
1119
|
+
)
|
|
1120
|
+
yield* tx.execute(sql`
|
|
1121
|
+
INSERT INTO ${flowChildren} (flow_id, child_key, name, store_key, spec,
|
|
1122
|
+
status, exit, failed_reason, cascaded, pending_since)
|
|
1123
|
+
VALUES ${sql.join(values, sql`, `)}
|
|
1124
|
+
`)
|
|
1125
|
+
}
|
|
1126
|
+
} else {
|
|
1127
|
+
// A manifest that was already present is kept untouched (double
|
|
1128
|
+
// fan-out converges on the persisted children); the state
|
|
1129
|
+
// transition follows the persisted pending count either way.
|
|
1130
|
+
pending = Number(row.flowPending)
|
|
1131
|
+
}
|
|
1132
|
+
if (row.cancelRequested) {
|
|
1133
|
+
// A cancel raced the fan-out: cancellation wins. The rows exist
|
|
1134
|
+
// and get marked (into the `cancelled` counter), so the sweeper
|
|
1135
|
+
// cascades (mostly no-op cancels for never-enqueued children).
|
|
1136
|
+
yield* tx.execute(sql`
|
|
1137
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
1138
|
+
UPDATE ${jobs} SET state = 'cancelled', finished_at = ${now},
|
|
1139
|
+
cancel_requested = FALSE, flow_pending = 0,
|
|
1140
|
+
flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
|
|
1141
|
+
WHERE ${jobs.id} = ${id}
|
|
1142
|
+
`)
|
|
1143
|
+
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
1144
|
+
// A cancelled NESTED parent reports upward through the outbox.
|
|
1145
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
|
|
1146
|
+
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
1147
|
+
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
1148
|
+
return undefined
|
|
1149
|
+
}
|
|
1150
|
+
if (pending > 0) {
|
|
1151
|
+
yield* tx.execute(sql`
|
|
1152
|
+
UPDATE ${jobs} SET state = 'waiting-children' WHERE ${jobs.id} = ${id}
|
|
1153
|
+
`)
|
|
1154
|
+
return undefined
|
|
1155
|
+
}
|
|
1156
|
+
// Empty (or fully recorded) manifest: straight to runnable collect.
|
|
1157
|
+
yield* tx.execute(sql`
|
|
1158
|
+
UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
|
|
1159
|
+
WHERE ${jobs.id} = ${id}
|
|
1160
|
+
`)
|
|
1161
|
+
return JobStore.QueueName(row.queue)
|
|
1162
|
+
})
|
|
1163
|
+
).pipe(
|
|
1164
|
+
Effect.mapError((error) =>
|
|
1165
|
+
error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
|
|
1166
|
+
error instanceof JobStore.JobStoreError
|
|
1167
|
+
? error
|
|
1168
|
+
: storeError("ack failed")(error)
|
|
1169
|
+
)
|
|
1170
|
+
)
|
|
1171
|
+
if (wakeQueue !== undefined) {
|
|
1172
|
+
yield* wakeUp(wakeQueue)
|
|
1173
|
+
}
|
|
1174
|
+
})
|
|
1175
|
+
|
|
848
1176
|
const store: JobStore.Service = {
|
|
849
1177
|
enqueue: enqueueOne,
|
|
850
1178
|
|
|
@@ -919,7 +1247,10 @@ export const make = (
|
|
|
919
1247
|
${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
|
|
920
1248
|
${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
|
|
921
1249
|
${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
|
|
922
|
-
${jobs.trace} AS "trace",
|
|
1250
|
+
${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
|
|
1251
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
|
|
1252
|
+
${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
|
|
1253
|
+
${jobs.flowCancelled} AS "flowCancelled",
|
|
923
1254
|
${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
|
|
924
1255
|
${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
|
|
925
1256
|
${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
@@ -946,8 +1277,11 @@ export const make = (
|
|
|
946
1277
|
error instanceof JobStore.JobStoreError ? error : storeError("claim failed")(error)
|
|
947
1278
|
)),
|
|
948
1279
|
|
|
949
|
-
ack: (id, token, outcome) =>
|
|
950
|
-
|
|
1280
|
+
ack: (id, token, outcome) => {
|
|
1281
|
+
if (outcome._tag === "FanOut") {
|
|
1282
|
+
return ackFanOut(id, token, outcome)
|
|
1283
|
+
}
|
|
1284
|
+
return db.transaction((tx) =>
|
|
951
1285
|
Effect.gen(function*() {
|
|
952
1286
|
const now = yield* nowDate
|
|
953
1287
|
const update = outcome._tag === "Complete"
|
|
@@ -973,6 +1307,9 @@ export const make = (
|
|
|
973
1307
|
keep: JobStore.KeepPolicy | null
|
|
974
1308
|
dedupeKey: string | null
|
|
975
1309
|
queue: string
|
|
1310
|
+
parent: JobStore.ParentEnvelope | null
|
|
1311
|
+
exit: unknown
|
|
1312
|
+
failedReason: string | null
|
|
976
1313
|
}
|
|
977
1314
|
>(sql`
|
|
978
1315
|
UPDATE ${jobs} SET ${update},
|
|
@@ -980,7 +1317,8 @@ export const make = (
|
|
|
980
1317
|
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
981
1318
|
RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
|
|
982
1319
|
${jobs.state} AS "state", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
|
|
983
|
-
${jobs.queue} AS "queue"
|
|
1320
|
+
${jobs.queue} AS "queue", ${jobs.parent} AS "parent", ${jobs.exit} AS "exit",
|
|
1321
|
+
${jobs.failedReason} AS "failedReason"
|
|
984
1322
|
`))
|
|
985
1323
|
const row = rows[0]
|
|
986
1324
|
if (row === undefined) {
|
|
@@ -1003,6 +1341,15 @@ export const make = (
|
|
|
1003
1341
|
outcome._tag === "Cancelled" || cancelledRetry ? undefined : outcome.exit
|
|
1004
1342
|
)
|
|
1005
1343
|
if (outcome._tag !== "Retry" || cancelledRetry) {
|
|
1344
|
+
// A terminal transition of an envelope-carrying child reports
|
|
1345
|
+
// upward through the outbox (exit/failedReason as persisted).
|
|
1346
|
+
yield* appendOutbox(
|
|
1347
|
+
tx,
|
|
1348
|
+
row.parent,
|
|
1349
|
+
ledgerOutcome === "retried" ? "cancelled" : ledgerOutcome,
|
|
1350
|
+
row.exit ?? undefined,
|
|
1351
|
+
row.failedReason
|
|
1352
|
+
)
|
|
1006
1353
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
1007
1354
|
yield* applyKeep(tx, row, now)
|
|
1008
1355
|
}
|
|
@@ -1019,7 +1366,8 @@ export const make = (
|
|
|
1019
1366
|
),
|
|
1020
1367
|
Effect.tap((queue) => queue !== undefined ? wakeUp(queue) : Effect.void),
|
|
1021
1368
|
Effect.asVoid
|
|
1022
|
-
)
|
|
1369
|
+
)
|
|
1370
|
+
},
|
|
1023
1371
|
|
|
1024
1372
|
release: (id, token) =>
|
|
1025
1373
|
Effect.gen(function*() {
|
|
@@ -1037,6 +1385,9 @@ export const make = (
|
|
|
1037
1385
|
keep: JobStore.KeepPolicy | null
|
|
1038
1386
|
dedupeKey: string | null
|
|
1039
1387
|
queue: string
|
|
1388
|
+
parent: JobStore.ParentEnvelope | null
|
|
1389
|
+
exit: unknown
|
|
1390
|
+
failedReason: string | null
|
|
1040
1391
|
}
|
|
1041
1392
|
>(sql`
|
|
1042
1393
|
UPDATE ${jobs} SET
|
|
@@ -1047,12 +1398,15 @@ export const make = (
|
|
|
1047
1398
|
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
1048
1399
|
RETURNING ${jobs.id} AS id, (${jobs.state} = 'cancelled') AS cancelled,
|
|
1049
1400
|
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
1050
|
-
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue"
|
|
1401
|
+
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1402
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
1051
1403
|
`))
|
|
1052
1404
|
const row = rows[0]
|
|
1053
1405
|
if (row === undefined) return undefined
|
|
1054
1406
|
if (row.cancelled) {
|
|
1055
1407
|
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined)
|
|
1408
|
+
// A cancel honoured at release is a terminal transition too.
|
|
1409
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason)
|
|
1056
1410
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now)
|
|
1057
1411
|
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now)
|
|
1058
1412
|
}
|
|
@@ -1119,6 +1473,9 @@ export const make = (
|
|
|
1119
1473
|
name: string
|
|
1120
1474
|
keep: JobStore.KeepPolicy | null
|
|
1121
1475
|
dedupeKey: string | null
|
|
1476
|
+
parent: JobStore.ParentEnvelope | null
|
|
1477
|
+
exit: unknown
|
|
1478
|
+
failedReason: string | null
|
|
1122
1479
|
}
|
|
1123
1480
|
>(sql`
|
|
1124
1481
|
UPDATE ${jobs} SET
|
|
@@ -1139,7 +1496,8 @@ export const make = (
|
|
|
1139
1496
|
cancel_requested = FALSE
|
|
1140
1497
|
WHERE ${jobs.state} = 'active' AND ${jobs.lockExpiresAt} <= ${now}::timestamptz
|
|
1141
1498
|
RETURNING ${jobs.id} AS "id", ${jobs.state} AS "state", ${jobs.processedAt} AS "processedAt",
|
|
1142
|
-
${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey"
|
|
1499
|
+
${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
|
|
1500
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
1143
1501
|
`))
|
|
1144
1502
|
const recovered: Array<{ id: JobStore.JobId; failed: boolean }> = []
|
|
1145
1503
|
for (const row of rows) {
|
|
@@ -1152,6 +1510,15 @@ export const make = (
|
|
|
1152
1510
|
undefined
|
|
1153
1511
|
)
|
|
1154
1512
|
if (row.state === "cancelled" || row.state === "failed") {
|
|
1513
|
+
// Honoured cancels and stall exhaustion are terminal
|
|
1514
|
+
// transitions: envelope-carrying children report upward.
|
|
1515
|
+
yield* appendOutbox(
|
|
1516
|
+
tx,
|
|
1517
|
+
row.parent,
|
|
1518
|
+
row.state === "cancelled" ? "cancelled" : "failed",
|
|
1519
|
+
row.exit ?? undefined,
|
|
1520
|
+
row.failedReason
|
|
1521
|
+
)
|
|
1155
1522
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, row.id, now)
|
|
1156
1523
|
}
|
|
1157
1524
|
if (row.state === "cancelled") {
|
|
@@ -1233,7 +1600,11 @@ export const make = (
|
|
|
1233
1600
|
${jobs.attemptsMax} AS "attemptsMax", ${jobs.attemptsMade} AS "attemptsMade",
|
|
1234
1601
|
${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
|
|
1235
1602
|
${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
|
|
1236
|
-
${jobs.cancelRequested} AS "cancelRequested",
|
|
1603
|
+
${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
|
|
1604
|
+
${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
|
|
1605
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
|
|
1606
|
+
${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
|
|
1607
|
+
${jobs.flowCancelled} AS "flowCancelled",
|
|
1237
1608
|
${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
|
|
1238
1609
|
${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
|
|
1239
1610
|
${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
@@ -1464,6 +1835,365 @@ export const make = (
|
|
|
1464
1835
|
return fired
|
|
1465
1836
|
}),
|
|
1466
1837
|
|
|
1838
|
+
recordChildResults: (reports) =>
|
|
1839
|
+
Effect.gen(function*() {
|
|
1840
|
+
if (reports.length === 0) {
|
|
1841
|
+
const none: Array<{ applied: boolean; parentSettled: boolean }> = []
|
|
1842
|
+
return none
|
|
1843
|
+
}
|
|
1844
|
+
const batch = yield* db.transaction((tx) =>
|
|
1845
|
+
Effect.gen(function*() {
|
|
1846
|
+
const now = yield* nowDate
|
|
1847
|
+
const results = reports.map(() => ({ applied: false, parentSettled: false }))
|
|
1848
|
+
|
|
1849
|
+
// Phase 1a — apply every row update (contract lock order:
|
|
1850
|
+
// dependency rows FIRST, parents second). Only a (flow, key)'s
|
|
1851
|
+
// first occurrence in the batch can apply — later duplicates
|
|
1852
|
+
// would find the row non-pending anyway, and UPDATE ... FROM
|
|
1853
|
+
// must never see two source rows for one target.
|
|
1854
|
+
const candidates: Array<{ readonly index: number; readonly report: JobStore.FlowChildReport }> = []
|
|
1855
|
+
const seen = new Set<string>()
|
|
1856
|
+
for (const [index, report] of reports.entries()) {
|
|
1857
|
+
const key = `${report.flowId}\u0000${report.childKey}`
|
|
1858
|
+
if (seen.has(key)) continue
|
|
1859
|
+
seen.add(key)
|
|
1860
|
+
candidates.push({ index, report })
|
|
1861
|
+
}
|
|
1862
|
+
for (let start = 0; start < candidates.length; start += 200) {
|
|
1863
|
+
const chunk = candidates.slice(start, start + 200)
|
|
1864
|
+
const values = chunk.map(({ index, report }) =>
|
|
1865
|
+
sql`(${index}::int, ${report.flowId}::text, ${report.childKey}::text,
|
|
1866
|
+
${report.outcome}::text,
|
|
1867
|
+
${report.exit === undefined ? null : JSON.stringify(report.exit)}::jsonb,
|
|
1868
|
+
${report.failedReason ?? null}::text)`
|
|
1869
|
+
)
|
|
1870
|
+
const appliedRows = rowsOf(yield* tx.execute<{ ord: number }>(sql`
|
|
1871
|
+
UPDATE ${flowChildren} SET status = v.outcome, exit = v.exit,
|
|
1872
|
+
failed_reason = v.failed_reason, cascaded = TRUE
|
|
1873
|
+
FROM (VALUES ${sql.join(values, sql`, `)})
|
|
1874
|
+
AS v(ord, flow_id, child_key, outcome, exit, failed_reason)
|
|
1875
|
+
WHERE ${flowChildren.flowId} = v.flow_id AND ${flowChildren.childKey} = v.child_key
|
|
1876
|
+
AND ${flowChildren.status} = 'pending'
|
|
1877
|
+
RETURNING v.ord AS "ord"
|
|
1878
|
+
`))
|
|
1879
|
+
for (const row of appliedRows) {
|
|
1880
|
+
const result = results[Number(row.ord)]
|
|
1881
|
+
if (result !== undefined) {
|
|
1882
|
+
result.applied = true
|
|
1883
|
+
}
|
|
1884
|
+
}
|
|
1885
|
+
}
|
|
1886
|
+
|
|
1887
|
+
// Tally the applied reports per touched flow, in batch order.
|
|
1888
|
+
interface Touch {
|
|
1889
|
+
appliedCount: number
|
|
1890
|
+
completed: number
|
|
1891
|
+
failed: number
|
|
1892
|
+
cancelled: number
|
|
1893
|
+
firstAppliedFailed: number | undefined
|
|
1894
|
+
lastApplied: number
|
|
1895
|
+
}
|
|
1896
|
+
const touched = new Map<string, Touch>()
|
|
1897
|
+
for (const [index, report] of reports.entries()) {
|
|
1898
|
+
if (results[index]?.applied !== true) continue
|
|
1899
|
+
const touch = touched.get(report.flowId) ?? {
|
|
1900
|
+
appliedCount: 0,
|
|
1901
|
+
completed: 0,
|
|
1902
|
+
failed: 0,
|
|
1903
|
+
cancelled: 0,
|
|
1904
|
+
firstAppliedFailed: undefined,
|
|
1905
|
+
lastApplied: index
|
|
1906
|
+
}
|
|
1907
|
+
touch.appliedCount += 1
|
|
1908
|
+
touch.lastApplied = index
|
|
1909
|
+
if (report.outcome === "completed") touch.completed += 1
|
|
1910
|
+
if (report.outcome === "cancelled") touch.cancelled += 1
|
|
1911
|
+
if (report.outcome === "failed") {
|
|
1912
|
+
touch.failed += 1
|
|
1913
|
+
touch.firstAppliedFailed ??= index
|
|
1914
|
+
}
|
|
1915
|
+
touched.set(report.flowId, touch)
|
|
1916
|
+
}
|
|
1917
|
+
|
|
1918
|
+
// Phase 1b + 2 — per touched flow (sorted, so concurrent
|
|
1919
|
+
// batches take parent locks in one order): move the applied
|
|
1920
|
+
// children from `pending` to their outcome counters, then make
|
|
1921
|
+
// at most one settle decision. Fail-fast wins the tie.
|
|
1922
|
+
const wakeQueues: Array<JobStore.QueueName> = []
|
|
1923
|
+
for (const flowId of [...touched.keys()].toSorted()) {
|
|
1924
|
+
const touch = touched.get(flowId)
|
|
1925
|
+
if (touch === undefined) continue
|
|
1926
|
+
const parents = rowsOf(yield* tx.execute<
|
|
1927
|
+
{
|
|
1928
|
+
state: JobStore.JobState
|
|
1929
|
+
flowPending: number
|
|
1930
|
+
flowFailFast: boolean | null
|
|
1931
|
+
processedAt: Date | null
|
|
1932
|
+
name: string
|
|
1933
|
+
keep: JobStore.KeepPolicy | null
|
|
1934
|
+
dedupeKey: string | null
|
|
1935
|
+
queue: string
|
|
1936
|
+
parent: JobStore.ParentEnvelope | null
|
|
1937
|
+
}
|
|
1938
|
+
>(sql`
|
|
1939
|
+
UPDATE ${jobs} SET
|
|
1940
|
+
flow_pending = GREATEST(${jobs.flowPending} - ${touch.appliedCount}::int, 0),
|
|
1941
|
+
flow_completed = ${jobs.flowCompleted} + ${touch.completed}::int,
|
|
1942
|
+
flow_failed = ${jobs.flowFailed} + ${touch.failed}::int,
|
|
1943
|
+
flow_cancelled = ${jobs.flowCancelled} + ${touch.cancelled}::int
|
|
1944
|
+
WHERE ${jobs.id} = ${flowId} AND ${jobs.flowPending} IS NOT NULL
|
|
1945
|
+
RETURNING ${jobs.state} AS "state", ${jobs.flowPending} AS "flowPending",
|
|
1946
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.processedAt} AS "processedAt",
|
|
1947
|
+
${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
1948
|
+
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1949
|
+
${jobs.parent} AS "parent"
|
|
1950
|
+
`))
|
|
1951
|
+
const parent = parents[0]
|
|
1952
|
+
if (parent === undefined || parent.state !== "waiting-children") continue
|
|
1953
|
+
const failedIndex = parent.flowFailFast === true ? touch.firstAppliedFailed : undefined
|
|
1954
|
+
const failedReport = failedIndex !== undefined ? reports[failedIndex] : undefined
|
|
1955
|
+
if (failedIndex !== undefined && failedReport !== undefined) {
|
|
1956
|
+
// The first applied failure settles the parent terminally
|
|
1957
|
+
// (store-side, like stall exhaustion: failedReason, no
|
|
1958
|
+
// exit) and marks the remaining rows in the same
|
|
1959
|
+
// transaction. A nested parent's own report goes to the
|
|
1960
|
+
// outbox here — this settle IS its terminal transition.
|
|
1961
|
+
const reason = `effect-mq: flow child "${failedReport.childKey}" failed`
|
|
1962
|
+
yield* tx.execute(sql`
|
|
1963
|
+
WITH marked AS (${cancelPendingChildren(flowId)})
|
|
1964
|
+
UPDATE ${jobs} SET state = 'failed', finished_at = ${now},
|
|
1965
|
+
failed_reason = ${reason}, cancel_requested = FALSE, flow_pending = 0,
|
|
1966
|
+
flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
|
|
1967
|
+
WHERE ${jobs.id} = ${flowId}
|
|
1968
|
+
`)
|
|
1969
|
+
yield* insertAttempt(tx, flowId, "failed", parent.processedAt, now, undefined)
|
|
1970
|
+
yield* appendOutbox(tx, parent.parent, "failed", undefined, reason)
|
|
1971
|
+
yield* releaseDedupe(tx, parent.name, parent.dedupeKey, flowId, now)
|
|
1972
|
+
yield* applyKeep(tx, { name: parent.name, state: "failed", keep: parent.keep }, now)
|
|
1973
|
+
const decided = results[failedIndex]
|
|
1974
|
+
if (decided !== undefined) {
|
|
1975
|
+
decided.parentSettled = true
|
|
1976
|
+
}
|
|
1977
|
+
continue
|
|
1978
|
+
}
|
|
1979
|
+
if (Number(parent.flowPending) === 0) {
|
|
1980
|
+
// All children settled: the parent resumes runnable, phase
|
|
1981
|
+
// collect, settled at the flow's last applied report.
|
|
1982
|
+
yield* tx.execute(sql`
|
|
1983
|
+
UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
|
|
1984
|
+
WHERE ${jobs.id} = ${flowId}
|
|
1985
|
+
`)
|
|
1986
|
+
wakeQueues.push(JobStore.QueueName(parent.queue))
|
|
1987
|
+
const decided = results[touch.lastApplied]
|
|
1988
|
+
if (decided !== undefined) {
|
|
1989
|
+
decided.parentSettled = true
|
|
1990
|
+
}
|
|
1991
|
+
}
|
|
1992
|
+
}
|
|
1993
|
+
return { results, wakeQueues }
|
|
1994
|
+
})
|
|
1995
|
+
).pipe(
|
|
1996
|
+
// Rare lock-order inversion (a concurrent cancel/settle marking
|
|
1997
|
+
// rows) surfaces as a Postgres deadlock (40P01); the killed side
|
|
1998
|
+
// is safe to retry — the row-state guard keeps it idempotent.
|
|
1999
|
+
Effect.retry({
|
|
2000
|
+
times: 3,
|
|
2001
|
+
while: isDeadlockError
|
|
2002
|
+
}),
|
|
2003
|
+
Effect.mapError((error) =>
|
|
2004
|
+
error instanceof JobStore.JobStoreError ? error : storeError("recordChildResults failed")(error)
|
|
2005
|
+
)
|
|
2006
|
+
)
|
|
2007
|
+
for (const queue of batch.wakeQueues) {
|
|
2008
|
+
yield* wakeUp(queue)
|
|
2009
|
+
}
|
|
2010
|
+
return batch.results
|
|
2011
|
+
}),
|
|
2012
|
+
|
|
2013
|
+
peekOutbox: (peekOptions) =>
|
|
2014
|
+
Effect.gen(function*() {
|
|
2015
|
+
const limit = Math.max(0, peekOptions.limit)
|
|
2016
|
+
if (limit === 0) {
|
|
2017
|
+
const none: Array<JobStore.OutboxEntry> = []
|
|
2018
|
+
return none
|
|
2019
|
+
}
|
|
2020
|
+
// `after` pages past a previously returned id (exclusive), whether
|
|
2021
|
+
// or not that entry still exists. Anything that is not a canonical
|
|
2022
|
+
// id this store issued is treated as unset.
|
|
2023
|
+
const after = peekOptions.after !== undefined && CANONICAL_BIGSERIAL.test(peekOptions.after)
|
|
2024
|
+
? peekOptions.after
|
|
2025
|
+
: undefined
|
|
2026
|
+
const rows = rowsOf(yield* db.execute<
|
|
2027
|
+
{
|
|
2028
|
+
id: string
|
|
2029
|
+
flowName: string
|
|
2030
|
+
parentStoreKey: string
|
|
2031
|
+
report: JobStore.FlowChildReport
|
|
2032
|
+
}
|
|
2033
|
+
>(sql`
|
|
2034
|
+
SELECT ${flowOutbox.id}::text AS "id", ${flowOutbox.flowName} AS "flowName",
|
|
2035
|
+
${flowOutbox.parentStoreKey} AS "parentStoreKey", ${flowOutbox.report} AS "report"
|
|
2036
|
+
FROM ${flowOutbox}
|
|
2037
|
+
${after === undefined ? sql`` : sql`WHERE ${flowOutbox.id} > ${after}::bigint`}
|
|
2038
|
+
ORDER BY ${flowOutbox.id} ASC
|
|
2039
|
+
LIMIT ${limit}
|
|
2040
|
+
`).pipe(Effect.mapError(storeError("peekOutbox failed"))))
|
|
2041
|
+
return rows.map((row): JobStore.OutboxEntry => ({
|
|
2042
|
+
id: row.id,
|
|
2043
|
+
flowName: row.flowName,
|
|
2044
|
+
parentStoreKey: row.parentStoreKey,
|
|
2045
|
+
report: row.report
|
|
2046
|
+
}))
|
|
2047
|
+
}),
|
|
2048
|
+
|
|
2049
|
+
deleteOutbox: (ids) =>
|
|
2050
|
+
Effect.suspend(() => {
|
|
2051
|
+
// Ids are opaque strings to callers; only CANONICAL ids this store
|
|
2052
|
+
// could have issued can match. The strictness matters twice: a
|
|
2053
|
+
// foreign/garbled id must not blow up the ::bigint cast, and a
|
|
2054
|
+
// non-canonical spelling ("007") must stay an unknown-id no-op
|
|
2055
|
+
// rather than cast to 7 and delete a live entry.
|
|
2056
|
+
const numeric = ids.filter((id) => CANONICAL_BIGSERIAL.test(id))
|
|
2057
|
+
if (numeric.length === 0) return Effect.void
|
|
2058
|
+
return db.execute(sql`
|
|
2059
|
+
DELETE FROM ${flowOutbox}
|
|
2060
|
+
WHERE ${flowOutbox.id} = ANY(${sql.param(numeric)}::bigint[])
|
|
2061
|
+
`).pipe(
|
|
2062
|
+
Effect.mapError(storeError("deleteOutbox failed")),
|
|
2063
|
+
Effect.asVoid
|
|
2064
|
+
)
|
|
2065
|
+
}),
|
|
2066
|
+
|
|
2067
|
+
listChildResults: (flowId, listOptions) =>
|
|
2068
|
+
Effect.gen(function*() {
|
|
2069
|
+
const limit = Math.max(1, listOptions?.limit ?? 1000)
|
|
2070
|
+
const cursor = listOptions?.cursor
|
|
2071
|
+
// `spec->>'id'` instead of the whole spec: at 10k children the full
|
|
2072
|
+
// payloads would transfer on every collect.
|
|
2073
|
+
const rows = rowsOf(yield* db.execute<FlowChildRow>(sql`
|
|
2074
|
+
SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
2075
|
+
${flowChildren.name} AS "name", ${flowChildren.storeKey} AS "storeKey",
|
|
2076
|
+
${flowChildren.spec}->>'id' AS "childJobId", ${flowChildren.status} AS "status",
|
|
2077
|
+
${flowChildren.exit} AS "exit", ${flowChildren.failedReason} AS "failedReason",
|
|
2078
|
+
${flowChildren.cascaded} AS "cascaded"
|
|
2079
|
+
FROM ${flowChildren}
|
|
2080
|
+
WHERE ${flowChildren.flowId} = ${flowId}
|
|
2081
|
+
${cursor === undefined ? sql`` : sql`AND ${flowChildren.childKey} > ${cursor}`}
|
|
2082
|
+
ORDER BY ${flowChildren.childKey} ASC
|
|
2083
|
+
LIMIT ${limit + 1}
|
|
2084
|
+
`).pipe(Effect.mapError(storeError("listChildResults failed"))))
|
|
2085
|
+
const page = rows.slice(0, limit)
|
|
2086
|
+
const items = page.map(toFlowChildRecord)
|
|
2087
|
+
const last = items[items.length - 1]
|
|
2088
|
+
return {
|
|
2089
|
+
items,
|
|
2090
|
+
cursor: rows.length > limit && last !== undefined ? last.childKey : undefined
|
|
2091
|
+
}
|
|
2092
|
+
}),
|
|
2093
|
+
|
|
2094
|
+
flowSweepWork: (sweepOptions) =>
|
|
2095
|
+
Effect.gen(function*() {
|
|
2096
|
+
const now = yield* nowDate
|
|
2097
|
+
const limit = Math.max(1, sweepOptions.limit ?? 1000)
|
|
2098
|
+
const threshold = new Date(now.getTime() - sweepOptions.pendingAgeMs)
|
|
2099
|
+
type SweepRow = {
|
|
2100
|
+
readonly flowId: string
|
|
2101
|
+
readonly childKey: string
|
|
2102
|
+
readonly storeKey: string
|
|
2103
|
+
readonly spec: JobStore.EnqueueRequest
|
|
2104
|
+
}
|
|
2105
|
+
// Reconcile: pending rows past the eligibility threshold whose
|
|
2106
|
+
// parent is still parked (a settled flow never re-drives work).
|
|
2107
|
+
// Returning a row re-arms `pending_since` in the same statement, so
|
|
2108
|
+
// a full page rotates across sweeps instead of pinning its head.
|
|
2109
|
+
// SKIP LOCKED: rows a concurrent report/settle holds are its
|
|
2110
|
+
// business, and never waiting means this statement cannot deadlock.
|
|
2111
|
+
// The raw column names are safe: only table names vary across
|
|
2112
|
+
// factory instances. RETURNING order is unspecified — sorted below.
|
|
2113
|
+
const reconcileRows = rowsOf(yield* db.execute<SweepRow>(sql`
|
|
2114
|
+
WITH due AS (
|
|
2115
|
+
SELECT c.flow_id, c.child_key
|
|
2116
|
+
FROM ${flowChildren} c
|
|
2117
|
+
JOIN ${jobs} j ON j.id = c.flow_id
|
|
2118
|
+
WHERE c.status = 'pending' AND c.pending_since <= ${threshold}
|
|
2119
|
+
AND j.state = 'waiting-children'
|
|
2120
|
+
ORDER BY c.flow_id, c.child_key
|
|
2121
|
+
LIMIT ${limit}
|
|
2122
|
+
FOR UPDATE OF c SKIP LOCKED
|
|
2123
|
+
)
|
|
2124
|
+
UPDATE ${flowChildren} SET pending_since = ${now}
|
|
2125
|
+
FROM due
|
|
2126
|
+
WHERE ${flowChildren.flowId} = due.flow_id AND ${flowChildren.childKey} = due.child_key
|
|
2127
|
+
RETURNING ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
2128
|
+
${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
|
|
2129
|
+
`).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
|
|
2130
|
+
.toSorted((a, b) =>
|
|
2131
|
+
a.flowId !== b.flowId
|
|
2132
|
+
? (a.flowId < b.flowId ? -1 : 1)
|
|
2133
|
+
: a.childKey < b.childKey
|
|
2134
|
+
? -1
|
|
2135
|
+
: a.childKey > b.childKey
|
|
2136
|
+
? 1
|
|
2137
|
+
: 0
|
|
2138
|
+
)
|
|
2139
|
+
// Cascade: cancelled rows whose cancel has not been delivered into
|
|
2140
|
+
// the child's store yet (any parent state).
|
|
2141
|
+
const cascadeRows = rowsOf(yield* db.execute<SweepRow>(sql`
|
|
2142
|
+
SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
2143
|
+
${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
|
|
2144
|
+
FROM ${flowChildren}
|
|
2145
|
+
WHERE ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
|
|
2146
|
+
ORDER BY ${flowChildren.flowId}, ${flowChildren.childKey}
|
|
2147
|
+
LIMIT ${limit}
|
|
2148
|
+
`).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
|
|
2149
|
+
const reconcile: Array<{
|
|
2150
|
+
flowId: JobStore.JobId
|
|
2151
|
+
children: Array<JobStore.FlowChildSpec>
|
|
2152
|
+
}> = []
|
|
2153
|
+
for (const row of reconcileRows) {
|
|
2154
|
+
const flowId = JobId(row.flowId)
|
|
2155
|
+
let group = reconcile[reconcile.length - 1]
|
|
2156
|
+
if (group === undefined || group.flowId !== flowId) {
|
|
2157
|
+
group = { flowId, children: [] }
|
|
2158
|
+
reconcile.push(group)
|
|
2159
|
+
}
|
|
2160
|
+
group.children.push({ childKey: row.childKey, storeKey: row.storeKey, request: row.spec })
|
|
2161
|
+
}
|
|
2162
|
+
const cascade: Array<{
|
|
2163
|
+
flowId: JobStore.JobId
|
|
2164
|
+
children: Array<{ childKey: string; storeKey: string; childJobId: JobStore.JobId }>
|
|
2165
|
+
}> = []
|
|
2166
|
+
for (const row of cascadeRows) {
|
|
2167
|
+
const flowId = JobId(row.flowId)
|
|
2168
|
+
let group = cascade[cascade.length - 1]
|
|
2169
|
+
if (group === undefined || group.flowId !== flowId) {
|
|
2170
|
+
group = { flowId, children: [] }
|
|
2171
|
+
cascade.push(group)
|
|
2172
|
+
}
|
|
2173
|
+
group.children.push({
|
|
2174
|
+
childKey: row.childKey,
|
|
2175
|
+
storeKey: row.storeKey,
|
|
2176
|
+
// SAFETY: the FanOut ack validated every spec id before
|
|
2177
|
+
// persisting it.
|
|
2178
|
+
childJobId: row.spec.id as JobStore.JobId
|
|
2179
|
+
})
|
|
2180
|
+
}
|
|
2181
|
+
const work: JobStore.FlowSweepWork = { reconcile, cascade }
|
|
2182
|
+
return work
|
|
2183
|
+
}),
|
|
2184
|
+
|
|
2185
|
+
markChildrenCascaded: (flowId, childKeys) =>
|
|
2186
|
+
childKeys.length === 0
|
|
2187
|
+
? Effect.void
|
|
2188
|
+
: db.execute(sql`
|
|
2189
|
+
UPDATE ${flowChildren} SET cascaded = TRUE
|
|
2190
|
+
WHERE ${flowChildren.flowId} = ${flowId}
|
|
2191
|
+
AND ${flowChildren.childKey} = ANY(${sql.param([...childKeys])})
|
|
2192
|
+
`).pipe(
|
|
2193
|
+
Effect.mapError(storeError("markChildrenCascaded failed")),
|
|
2194
|
+
Effect.asVoid
|
|
2195
|
+
),
|
|
2196
|
+
|
|
1467
2197
|
counts: (queue) =>
|
|
1468
2198
|
db.execute<{ state: JobStore.JobState; count: number }>(sql`
|
|
1469
2199
|
SELECT ${jobs.state} AS "state", count(*)::int AS "count" FROM ${jobs}
|
|
@@ -1477,6 +2207,7 @@ export const make = (
|
|
|
1477
2207
|
waiting: 0,
|
|
1478
2208
|
delayed: 0,
|
|
1479
2209
|
active: 0,
|
|
2210
|
+
"waiting-children": 0,
|
|
1480
2211
|
completed: 0,
|
|
1481
2212
|
failed: 0,
|
|
1482
2213
|
cancelled: 0
|
|
@@ -1487,10 +2218,18 @@ export const make = (
|
|
|
1487
2218
|
),
|
|
1488
2219
|
|
|
1489
2220
|
remove: (id) =>
|
|
2221
|
+
// The purge CTE takes a removed flow parent's dependency rows with it
|
|
2222
|
+
// in the same statement.
|
|
1490
2223
|
db.execute(sql`
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
2224
|
+
WITH deleted AS (
|
|
2225
|
+
DELETE FROM ${jobs}
|
|
2226
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} NOT IN ('active', 'waiting-children')
|
|
2227
|
+
RETURNING ${jobs.id} AS id
|
|
2228
|
+
), purged AS (
|
|
2229
|
+
DELETE FROM ${flowChildren}
|
|
2230
|
+
WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
|
|
2231
|
+
)
|
|
2232
|
+
SELECT id FROM deleted
|
|
1494
2233
|
`).pipe(
|
|
1495
2234
|
Effect.mapError(storeError("remove failed")),
|
|
1496
2235
|
Effect.map((result) => rowsOf(result).length > 0)
|