effect-mq 0.4.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +85 -17
- package/dist/Flow.d.ts +381 -0
- package/dist/Flow.d.ts.map +1 -0
- package/dist/Flow.js +340 -0
- package/dist/Flow.js.map +1 -0
- package/dist/Job.d.ts +37 -6
- package/dist/Job.d.ts.map +1 -1
- package/dist/Job.js +17 -2
- package/dist/Job.js.map +1 -1
- package/dist/JobSchedules.d.ts +112 -0
- package/dist/JobSchedules.d.ts.map +1 -0
- package/dist/JobSchedules.js +106 -0
- package/dist/JobSchedules.js.map +1 -0
- package/dist/JobStore.d.ts +320 -10
- package/dist/JobStore.d.ts.map +1 -1
- package/dist/JobStore.js.map +1 -1
- package/dist/MemoryJobStore.d.ts.map +1 -1
- package/dist/MemoryJobStore.js +336 -8
- package/dist/MemoryJobStore.js.map +1 -1
- package/dist/Metrics.d.ts +31 -0
- package/dist/Metrics.d.ts.map +1 -1
- package/dist/Metrics.js +39 -0
- package/dist/Metrics.js.map +1 -1
- package/dist/Worker.d.ts +120 -11
- package/dist/Worker.d.ts.map +1 -1
- package/dist/Worker.js +452 -26
- package/dist/Worker.js.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts +19 -1
- package/dist/drizzle-postgres/DrizzleJobStore.d.ts.map +1 -1
- package/dist/drizzle-postgres/DrizzleJobStore.js +662 -81
- package/dist/drizzle-postgres/DrizzleJobStore.js.map +1 -1
- package/dist/drizzle-postgres/schema.d.ts +310 -3
- package/dist/drizzle-postgres/schema.d.ts.map +1 -1
- package/dist/drizzle-postgres/schema.js +68 -1
- package/dist/drizzle-postgres/schema.js.map +1 -1
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +14 -0
- package/dist/index.js.map +1 -1
- package/dist/redis/RedisJobStore.d.ts.map +1 -1
- package/dist/redis/RedisJobStore.js +221 -19
- package/dist/redis/RedisJobStore.js.map +1 -1
- package/dist/redis/scripts.d.ts +118 -11
- package/dist/redis/scripts.d.ts.map +1 -1
- package/dist/redis/scripts.js +497 -28
- package/dist/redis/scripts.js.map +1 -1
- package/dist/testing/conformance.d.ts +6 -0
- package/dist/testing/conformance.d.ts.map +1 -1
- package/dist/testing/conformance.js +765 -1
- package/dist/testing/conformance.js.map +1 -1
- package/package.json +1 -1
- package/src/Flow.ts +778 -0
- package/src/Job.ts +42 -11
- package/src/JobSchedules.ts +223 -0
- package/src/JobStore.ts +347 -9
- package/src/MemoryJobStore.ts +372 -8
- package/src/Metrics.ts +43 -0
- package/src/Worker.ts +726 -37
- package/src/drizzle-postgres/DrizzleJobStore.ts +827 -82
- package/src/drizzle-postgres/schema.ts +94 -0
- package/src/index.ts +16 -0
- package/src/redis/RedisJobStore.ts +291 -8
- package/src/redis/scripts.ts +529 -26
- package/src/testing/conformance.ts +989 -1
|
@@ -20,9 +20,49 @@ import * as JobStore from "../JobStore.js";
|
|
|
20
20
|
import { asc, eq, getTableColumns, sql } from "drizzle-orm";
|
|
21
21
|
import * as PgDrizzle from "drizzle-orm/effect-postgres";
|
|
22
22
|
import { getTableConfig } from "drizzle-orm/pg-core";
|
|
23
|
-
import { Clock, Deferred, Duration, Effect, Layer, Option, Stream } from "effect";
|
|
23
|
+
import { Cause, Clock, Deferred, Duration, Effect, Layer, Option, Predicate, Stream } from "effect";
|
|
24
24
|
const { JobId } = JobStore;
|
|
25
25
|
const storeError = (message) => (cause) => new JobStore.JobStoreError({ message, cause });
|
|
26
|
+
// The exact spellings a bigserial ever produces: no leading zeros, no signs,
|
|
27
|
+
// no whitespace — what outbox ids look like and nothing Postgres would
|
|
28
|
+
// silently normalize into one.
|
|
29
|
+
const CANONICAL_BIGSERIAL = /^[1-9]\d*$/;
|
|
30
|
+
/**
|
|
31
|
+
* Whether an error surfaced by drizzle's Effect driver is a Postgres
|
|
32
|
+
* deadlock (40P01), i.e. safe to retry.
|
|
33
|
+
*
|
|
34
|
+
* The deadlock travels wrapped: drizzle's session fails with an
|
|
35
|
+
* `EffectDrizzleQueryError` whose `cause` field is `Cause.fail(SqlError)`
|
|
36
|
+
* (drizzle-orm `pg-core/effect/session.ts`), and `@effect/sql-pg` classifies
|
|
37
|
+
* pg code 40P01 into the `SqlError`'s `reason: DeadlockError` (PgClient.ts
|
|
38
|
+
* `classifyError`). Neither layer's `toString` renders the pg code, so this
|
|
39
|
+
* unwraps structurally instead of string-matching the rendered error.
|
|
40
|
+
*
|
|
41
|
+
* @internal
|
|
42
|
+
*/
|
|
43
|
+
// oxlint-disable-next-line anti-slop/no-unknown-parameters -- a retry-predicate classifier over whatever the driver threw IS the boundary parser
|
|
44
|
+
export const isDeadlockError = (error) => {
|
|
45
|
+
// Unwrap drizzle's envelope: its `cause` is an Effect Cause holding the
|
|
46
|
+
// original failure. A bare SqlError (non-drizzle path) passes through.
|
|
47
|
+
const unwrapped = Predicate.hasProperty(error, "cause") && Cause.isCause(error.cause)
|
|
48
|
+
? Option.getOrUndefined(Cause.findErrorOption(error.cause))
|
|
49
|
+
: error;
|
|
50
|
+
if (!Predicate.hasProperty(unwrapped, "_tag") || unwrapped._tag !== "SqlError" ||
|
|
51
|
+
!Predicate.hasProperty(unwrapped, "reason")) {
|
|
52
|
+
return false;
|
|
53
|
+
}
|
|
54
|
+
const reason = unwrapped.reason;
|
|
55
|
+
if (Predicate.hasProperty(reason, "_tag") && reason._tag === "DeadlockError") {
|
|
56
|
+
return true;
|
|
57
|
+
}
|
|
58
|
+
// Fallback for classifiers that missed the code: the reason keeps the raw
|
|
59
|
+
// pg error, whose message for 40P01 is "deadlock detected".
|
|
60
|
+
if (Predicate.hasProperty(reason, "cause")) {
|
|
61
|
+
const raw = String(reason.cause);
|
|
62
|
+
return raw.includes("40P01") || raw.includes("deadlock detected");
|
|
63
|
+
}
|
|
64
|
+
return false;
|
|
65
|
+
};
|
|
26
66
|
const rowsOf = (result) => "rows" in result ? result.rows : result;
|
|
27
67
|
const toSchedule = (row) => ({
|
|
28
68
|
key: JobStore.ScheduleKey(row.key),
|
|
@@ -38,6 +78,7 @@ const toSchedule = (row) => ({
|
|
|
38
78
|
backoff: row.backoff ?? undefined,
|
|
39
79
|
keep: row.keep ?? undefined,
|
|
40
80
|
timeoutMs: row.timeoutMs === null ? undefined : Number(row.timeoutMs),
|
|
81
|
+
group: row.group ?? undefined,
|
|
41
82
|
nextRunAt: row.nextRunAt.getTime()
|
|
42
83
|
});
|
|
43
84
|
const toRecord = (row) => ({
|
|
@@ -57,6 +98,17 @@ const toRecord = (row) => ({
|
|
|
57
98
|
cancelRequested: row.cancelRequested,
|
|
58
99
|
dedupeKey: row.dedupeKey ?? undefined,
|
|
59
100
|
trace: row.trace ?? undefined,
|
|
101
|
+
parent: row.parent ?? undefined,
|
|
102
|
+
// The flow columns are NULL together; `flowPending` is the presence marker.
|
|
103
|
+
flow: row.flowPending === null || row.flowPending === undefined
|
|
104
|
+
? undefined
|
|
105
|
+
: {
|
|
106
|
+
failFast: row.flowFailFast === true,
|
|
107
|
+
pending: Number(row.flowPending),
|
|
108
|
+
completed: Number(row.flowCompleted ?? 0),
|
|
109
|
+
failed: Number(row.flowFailed ?? 0),
|
|
110
|
+
cancelled: Number(row.flowCancelled ?? 0)
|
|
111
|
+
},
|
|
60
112
|
runAt: row.runAt.getTime(),
|
|
61
113
|
enqueuedAt: row.enqueuedAt.getTime(),
|
|
62
114
|
processedAt: row.processedAt?.getTime(),
|
|
@@ -64,6 +116,17 @@ const toRecord = (row) => ({
|
|
|
64
116
|
exit: row.exit ?? undefined,
|
|
65
117
|
failedReason: row.failedReason ?? undefined
|
|
66
118
|
});
|
|
119
|
+
const toFlowChildRecord = (row) => ({
|
|
120
|
+
flowId: JobId(row.flowId),
|
|
121
|
+
childKey: row.childKey,
|
|
122
|
+
name: row.name,
|
|
123
|
+
storeKey: row.storeKey,
|
|
124
|
+
childJobId: JobId(row.childJobId),
|
|
125
|
+
status: row.status,
|
|
126
|
+
exit: row.exit ?? undefined,
|
|
127
|
+
failedReason: row.failedReason ?? undefined,
|
|
128
|
+
cascaded: row.cascaded
|
|
129
|
+
});
|
|
67
130
|
/**
|
|
68
131
|
* Build the store implementation. Requires `PgClient` and a `Scope` (for the
|
|
69
132
|
* LISTEN subscription).
|
|
@@ -78,6 +141,8 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
78
141
|
const schedules = options.schedules;
|
|
79
142
|
const queues = options.queues;
|
|
80
143
|
const dedupe = options.dedupe;
|
|
144
|
+
const flowChildren = options.flowChildren;
|
|
145
|
+
const flowOutbox = options.flowOutbox;
|
|
81
146
|
const jobsName = getTableConfig(jobs).name;
|
|
82
147
|
const attemptsName = getTableConfig(attempts).name;
|
|
83
148
|
const wakeChannel = `effect_mq_wake_${jobsName}`;
|
|
@@ -102,6 +167,12 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
102
167
|
"cancelRequested",
|
|
103
168
|
"dedupeKey",
|
|
104
169
|
"trace",
|
|
170
|
+
"parent",
|
|
171
|
+
"flowFailFast",
|
|
172
|
+
"flowPending",
|
|
173
|
+
"flowCompleted",
|
|
174
|
+
"flowFailed",
|
|
175
|
+
"flowCancelled",
|
|
105
176
|
"runAt",
|
|
106
177
|
"enqueuedAt",
|
|
107
178
|
"processedAt",
|
|
@@ -135,7 +206,9 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
135
206
|
db.select({ jobId: attempts.jobId }).from(attempts).limit(0),
|
|
136
207
|
db.select({ key: schedules.key }).from(schedules).limit(0),
|
|
137
208
|
db.select({ queue: queues.queue }).from(queues).limit(0),
|
|
138
|
-
db.select({ key: dedupe.key }).from(dedupe).limit(0)
|
|
209
|
+
db.select({ key: dedupe.key }).from(dedupe).limit(0),
|
|
210
|
+
db.select({ flowId: flowChildren.flowId }).from(flowChildren).limit(0),
|
|
211
|
+
db.select({ id: flowOutbox.id }).from(flowOutbox).limit(0)
|
|
139
212
|
]).pipe(Effect.mapError(storeError(`effect-mq: tables "${jobsName}"/"${attemptsName}" are missing or mismatched — ` +
|
|
140
213
|
`re-export the effect-mq/drizzle schema factories from your drizzle schema and run your migrations (drizzle-kit generate)`)));
|
|
141
214
|
}
|
|
@@ -175,6 +248,26 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
175
248
|
// Resubscribe forever: if the LISTEN stream ends or fails, wake-ups
|
|
176
249
|
// degrade to the worker's pollInterval until the next attempt succeeds.
|
|
177
250
|
yield* client.listen(wakeChannel).pipe(Stream.runForEach((payload) => Effect.sync(() => signalWake(payload === "*" ? undefined : JobStore.QueueName(payload)))), Effect.catchCause((cause) => Effect.logWarning("effect-mq: LISTEN subscription failed; wake-ups degraded to polling until resubscribe", cause)), Effect.andThen(Effect.sleep("1 second")), Effect.forever, Effect.forkScoped);
|
|
251
|
+
// Automatic retention (the store-level sweep and per-job `keep`) never
|
|
252
|
+
// prunes a flow parent that still owes cascade cancels: its dependency
|
|
253
|
+
// rows marked `cancelled` and not `cascaded` are the only record that
|
|
254
|
+
// real cancels are still due in the child stores. The explicit `remove`
|
|
255
|
+
// verb is not exempted.
|
|
256
|
+
const owesCascades = sql `EXISTS (
|
|
257
|
+
SELECT 1 FROM ${flowChildren}
|
|
258
|
+
WHERE ${flowChildren.flowId} = ${jobs.id}
|
|
259
|
+
AND ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
|
|
260
|
+
)`;
|
|
261
|
+
// A pruned flow parent's dependency rows go with it in the same statement.
|
|
262
|
+
const purgeJobsWhere = (predicate) => sql `
|
|
263
|
+
WITH deleted AS (
|
|
264
|
+
DELETE FROM ${jobs}
|
|
265
|
+
WHERE ${predicate}
|
|
266
|
+
RETURNING ${jobs.id} AS id
|
|
267
|
+
)
|
|
268
|
+
DELETE FROM ${flowChildren}
|
|
269
|
+
WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
|
|
270
|
+
`;
|
|
178
271
|
if (options.historyTtl !== undefined) {
|
|
179
272
|
const ttlByState = JobStore.normalizeHistoryTtl(options.historyTtl);
|
|
180
273
|
const sweepMs = Duration.toMillis(options.historySweepInterval ?? "1 minute");
|
|
@@ -185,9 +278,8 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
185
278
|
// job name is pruned on the timer, not only when its group is acked.
|
|
186
279
|
for (const state of ["completed", "failed", "cancelled"]) {
|
|
187
280
|
const ttl = ttlByState[state];
|
|
188
|
-
yield* db.execute(sql `
|
|
189
|
-
|
|
190
|
-
WHERE ${jobs.state} = ${state} AND (
|
|
281
|
+
yield* db.execute(purgeJobsWhere(sql `
|
|
282
|
+
${jobs.state} = ${state} AND NOT ${owesCascades} AND (
|
|
191
283
|
${ttl !== undefined ? sql `${jobs.finishedAt} <= ${new Date(now.getTime() - ttl)}` : sql `FALSE`}
|
|
192
284
|
OR (
|
|
193
285
|
COALESCE(
|
|
@@ -202,7 +294,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
202
294
|
ELSE ${jobs.keep}->>'ageMs' END
|
|
203
295
|
)::double precision) / 1000.0))
|
|
204
296
|
)
|
|
205
|
-
`);
|
|
297
|
+
`));
|
|
206
298
|
}
|
|
207
299
|
// Dead dedup rows: expired windows, or pointers at vanished jobs.
|
|
208
300
|
yield* db.execute(sql `
|
|
@@ -210,7 +302,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
210
302
|
WHERE (${dedupe.windowExpiresAt} IS NOT NULL AND ${dedupe.windowExpiresAt} <= ${now})
|
|
211
303
|
OR (${dedupe.windowExpiresAt} IS NULL AND NOT EXISTS (
|
|
212
304
|
SELECT 1 FROM ${jobs} WHERE ${jobs.id} = ${dedupe.jobId}
|
|
213
|
-
AND ${jobs.state} IN ('waiting', 'delayed', 'active')
|
|
305
|
+
AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
|
|
214
306
|
))
|
|
215
307
|
`);
|
|
216
308
|
}).pipe(Effect.catchCause((cause) => Effect.logWarning("effect-mq: history sweep failed", cause)), Effect.forever, Effect.forkScoped);
|
|
@@ -258,23 +350,23 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
258
350
|
if (keep === undefined)
|
|
259
351
|
return;
|
|
260
352
|
if (keep.ageMs !== undefined) {
|
|
261
|
-
yield* tx.execute(sql `
|
|
262
|
-
|
|
263
|
-
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
353
|
+
yield* tx.execute(purgeJobsWhere(sql `
|
|
354
|
+
${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
264
355
|
AND ${jobs.finishedAt} <= ${new Date(now.getTime() - keep.ageMs)}
|
|
265
|
-
|
|
356
|
+
AND NOT ${owesCascades}
|
|
357
|
+
`));
|
|
266
358
|
}
|
|
267
359
|
if (keep.count !== undefined) {
|
|
268
|
-
yield* tx.execute(sql `
|
|
269
|
-
|
|
270
|
-
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
360
|
+
yield* tx.execute(purgeJobsWhere(sql `
|
|
361
|
+
${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
271
362
|
AND ${jobs.id} NOT IN (
|
|
272
363
|
SELECT ${jobs.id} FROM ${jobs}
|
|
273
364
|
WHERE ${jobs.name} = ${row.name} AND ${jobs.state} = ${row.state}
|
|
274
365
|
ORDER BY ${jobs.finishedAt} DESC, ${jobs.seq} DESC
|
|
275
366
|
LIMIT ${keep.count}
|
|
276
367
|
)
|
|
277
|
-
|
|
368
|
+
AND NOT ${owesCascades}
|
|
369
|
+
`));
|
|
278
370
|
}
|
|
279
371
|
});
|
|
280
372
|
// Distinguish JobNotFound vs LockLost after a guarded UPDATE hit 0 rows.
|
|
@@ -303,7 +395,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
303
395
|
: sql `'j-' || ${seqExpr}::text`;
|
|
304
396
|
const rows = rowsOf(yield* exec.execute(sql `
|
|
305
397
|
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
306
|
-
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
398
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
|
|
307
399
|
VALUES (${idExpr}, ${request.name}, ${request.queue}, ${state}, ${request.priority},
|
|
308
400
|
${JSON.stringify(request.payload ?? null)}::jsonb, ${JSON.stringify(request.metadata)}::jsonb,
|
|
309
401
|
${request.attemptsMax},
|
|
@@ -311,6 +403,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
311
403
|
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
312
404
|
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
313
405
|
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
406
|
+
${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
|
|
314
407
|
${runAt}, ${now}${extraColumnValues(request)})
|
|
315
408
|
ON CONFLICT (id) DO NOTHING
|
|
316
409
|
RETURNING ${jobs.id} AS id
|
|
@@ -437,7 +530,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
437
530
|
// killed and safe to retry.
|
|
438
531
|
Effect.retry({
|
|
439
532
|
times: 3,
|
|
440
|
-
while:
|
|
533
|
+
while: isDeadlockError
|
|
441
534
|
}), Effect.mapError((error) => error instanceof JobStore.JobStoreError ? error : storeError("enqueue failed")(error)));
|
|
442
535
|
// A job leaving the pending states frees its pending-mode dedup row; live
|
|
443
536
|
// throttle windows deliberately outlast the job.
|
|
@@ -449,26 +542,65 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
449
542
|
AND ${dedupe.jobId} = ${jobId}
|
|
450
543
|
AND (${dedupe.windowExpiresAt} IS NULL OR ${dedupe.windowExpiresAt} <= ${now})
|
|
451
544
|
`).pipe(Effect.asVoid);
|
|
545
|
+
// The outbox invariant: every operation that moves a job carrying a
|
|
546
|
+
// `parent` envelope INTO a terminal state appends its report here in the
|
|
547
|
+
// same transaction (see `JobStore.OutboxEntry`). `exit`/`failedReason`
|
|
548
|
+
// mirror the job row AFTER the transition; JSON.stringify drops
|
|
549
|
+
// undefined fields so absent values read back as undefined.
|
|
550
|
+
const appendOutbox = (exec, parent, outcome, exit, failedReason) => parent === null || parent === undefined
|
|
551
|
+
? Effect.void
|
|
552
|
+
: exec.execute(sql `
|
|
553
|
+
INSERT INTO ${flowOutbox} (flow_name, parent_store_key, report)
|
|
554
|
+
VALUES (${parent.flowName}, ${parent.parentStoreKey}, ${JSON.stringify({
|
|
555
|
+
flowId: parent.flowId,
|
|
556
|
+
childKey: parent.childKey,
|
|
557
|
+
outcome,
|
|
558
|
+
exit,
|
|
559
|
+
failedReason: failedReason ?? undefined
|
|
560
|
+
})}::jsonb)
|
|
561
|
+
`).pipe(Effect.asVoid);
|
|
562
|
+
// Flip a flow's still-pending dependency rows to `cancelled` and NOT
|
|
563
|
+
// `cascaded` (the sweeper still owes the child stores real cancels).
|
|
564
|
+
// Embedded as a CTE body so the caller can count the marked rows into
|
|
565
|
+
// the parent's `cancelled` counter in the same statement.
|
|
566
|
+
const cancelPendingChildren = (flowId) => sql `
|
|
567
|
+
UPDATE ${flowChildren} SET status = 'cancelled', cascaded = FALSE
|
|
568
|
+
WHERE ${flowChildren.flowId} = ${flowId} AND ${flowChildren.status} = 'pending'
|
|
569
|
+
RETURNING 1
|
|
570
|
+
`;
|
|
452
571
|
// Shared by cancel and cancelByDedupe.
|
|
453
572
|
const cancelJob = (id) => db.transaction((tx) => Effect.gen(function* () {
|
|
454
573
|
const now = yield* nowDate;
|
|
455
|
-
//
|
|
456
|
-
//
|
|
574
|
+
// Contract lock order (dependency rows first, parent second): mark
|
|
575
|
+
// a waiting-children parent's remaining pending rows cancelled.
|
|
576
|
+
// Pending rows exist only while the parent is `waiting-children`,
|
|
577
|
+
// so this is a no-op for every other state; a non-cancellable
|
|
578
|
+
// parent rolls the transaction back anyway.
|
|
579
|
+
const firstPass = rowsOf(yield* tx.execute(sql `
|
|
580
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
581
|
+
SELECT count(*)::int AS marked FROM marked
|
|
582
|
+
`));
|
|
583
|
+
const preMarked = firstPass[0]?.marked ?? 0;
|
|
584
|
+
// One guarded statement: waiting/delayed/waiting-children become
|
|
585
|
+
// terminal, active gets the cancel-request flag; anything else is
|
|
586
|
+
// reported by state.
|
|
457
587
|
const rows = rowsOf(yield* tx.execute(sql `
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
588
|
+
UPDATE ${jobs} SET
|
|
589
|
+
state = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN 'cancelled' ELSE ${jobs.state} END,
|
|
590
|
+
finished_at = CASE WHEN ${jobs.state} IN ('waiting', 'delayed', 'waiting-children') THEN ${now}::timestamptz ELSE ${jobs.finishedAt} END,
|
|
591
|
+
flow_pending = CASE WHEN ${jobs.state} = 'waiting-children' THEN 0 ELSE ${jobs.flowPending} END,
|
|
592
|
+
cancel_requested = CASE WHEN ${jobs.state} = 'active' THEN TRUE ELSE ${jobs.cancelRequested} END
|
|
593
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} IN ('waiting', 'delayed', 'active', 'waiting-children')
|
|
594
|
+
RETURNING ${jobs.id} AS id, ${jobs.state} AS state,
|
|
595
|
+
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
596
|
+
${jobs.dedupeKey} AS "dedupeKey", (${jobs.flowPending} IS NOT NULL) AS "hasFlow",
|
|
597
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
598
|
+
`));
|
|
467
599
|
const row = rows[0];
|
|
468
600
|
if (row === undefined) {
|
|
469
601
|
const existing = rowsOf(yield* tx.execute(sql `
|
|
470
|
-
|
|
471
|
-
|
|
602
|
+
SELECT ${jobs.state} AS state FROM ${jobs} WHERE ${jobs.id} = ${id}
|
|
603
|
+
`));
|
|
472
604
|
const found = existing[0];
|
|
473
605
|
if (found === undefined) {
|
|
474
606
|
return yield* new JobStore.JobNotFoundError({ jobId: id });
|
|
@@ -476,11 +608,36 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
476
608
|
return yield* new JobStore.JobNotCancellableError({ jobId: id, state: found.state });
|
|
477
609
|
}
|
|
478
610
|
if (row.state === "cancelled") {
|
|
611
|
+
if (row.hasFlow) {
|
|
612
|
+
// Re-run the marking now that the parent lock is held: the
|
|
613
|
+
// first UPDATE above races a concurrent FanOut — its
|
|
614
|
+
// uncommitted dependency-row INSERTs are invisible, while
|
|
615
|
+
// this UPDATE's own EPQ re-check can still see the parent as
|
|
616
|
+
// 'waiting-children' after the FanOut commits. Without this
|
|
617
|
+
// pass those rows would stay 'pending' forever, invisible to
|
|
618
|
+
// both sweep classes. Every row either pass marked lands in
|
|
619
|
+
// the `cancelled` counter.
|
|
620
|
+
yield* tx.execute(sql `
|
|
621
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
622
|
+
UPDATE ${jobs} SET flow_cancelled = ${jobs.flowCancelled} + ${preMarked}::int
|
|
623
|
+
+ (SELECT count(*)::int FROM marked)
|
|
624
|
+
WHERE ${jobs.id} = ${id}
|
|
625
|
+
`);
|
|
626
|
+
}
|
|
479
627
|
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined);
|
|
628
|
+
// A cancelled child reports upward through the outbox.
|
|
629
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason);
|
|
480
630
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
|
|
481
631
|
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now);
|
|
482
632
|
}
|
|
483
|
-
})).pipe(
|
|
633
|
+
})).pipe(
|
|
634
|
+
// Rare lock-order inversion (this row-then-parent flip vs a
|
|
635
|
+
// fail-fast settle's parent-then-rows marking) surfaces as a
|
|
636
|
+
// Postgres deadlock (40P01); one side is killed and safe to retry.
|
|
637
|
+
Effect.retry({
|
|
638
|
+
times: 3,
|
|
639
|
+
while: isDeadlockError
|
|
640
|
+
}), Effect.mapError((error) => error instanceof JobStore.JobNotFoundError ||
|
|
484
641
|
error instanceof JobStore.JobNotCancellableError ||
|
|
485
642
|
error instanceof JobStore.JobStoreError
|
|
486
643
|
? error
|
|
@@ -565,10 +722,11 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
565
722
|
${request.keep === undefined ? null : JSON.stringify(request.keep)}::jsonb,
|
|
566
723
|
${request.timeoutMs ?? null}, ${request.dedupe?.key ?? null},
|
|
567
724
|
${request.trace === undefined ? null : JSON.stringify(request.trace)}::jsonb,
|
|
725
|
+
${request.parent === undefined ? null : JSON.stringify(request.parent)}::jsonb,
|
|
568
726
|
${new Date(now.getTime() + Math.max(0, request.delayMs))}, ${now}${extraColumnValues(request)})`);
|
|
569
727
|
const rows = rowsOf(yield* db.execute(sql `
|
|
570
728
|
INSERT INTO ${jobs} (id, name, queue, state, priority, payload, metadata,
|
|
571
|
-
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, run_at, enqueued_at${extraColumnNames})
|
|
729
|
+
attempts_max, backoff, keep, timeout_ms, dedupe_key, trace, parent, run_at, enqueued_at${extraColumnNames})
|
|
572
730
|
VALUES ${sql.join(values, sql `, `)}
|
|
573
731
|
ON CONFLICT (id) DO NOTHING
|
|
574
732
|
RETURNING ${jobs.id} AS id
|
|
@@ -610,6 +768,97 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
610
768
|
}
|
|
611
769
|
return resolved;
|
|
612
770
|
});
|
|
771
|
+
// The FanOut ack: land the manifest (flow columns + dependency rows) and
|
|
772
|
+
// park the parent, all lock-token-guarded in one transaction. A fan-out
|
|
773
|
+
// is a phase transition, not a completed run — `attempts_made` is not
|
|
774
|
+
// incremented; the ledger records `fanned-out` with no exit.
|
|
775
|
+
const ackFanOut = (id, token, outcome) => Effect.gen(function* () {
|
|
776
|
+
// Validate BEFORE any mutation, so a bad spec cannot leave the job
|
|
777
|
+
// half-acked (lock cleared, ledger written, still active).
|
|
778
|
+
if (outcome.children.some((child) => child.request.id === undefined)) {
|
|
779
|
+
return yield* new JobStore.JobStoreError({
|
|
780
|
+
message: "FanOut child specs require an explicit request.id"
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
const wakeQueue = yield* db.transaction((tx) => Effect.gen(function* () {
|
|
784
|
+
const now = yield* nowDate;
|
|
785
|
+
const rows = rowsOf(yield* tx.execute(sql `
|
|
786
|
+
UPDATE ${jobs} SET lock_token = NULL, lock_expires_at = NULL
|
|
787
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
788
|
+
RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
|
|
789
|
+
${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
790
|
+
${jobs.cancelRequested} AS "cancelRequested", ${jobs.flowPending} AS "flowPending",
|
|
791
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
792
|
+
`));
|
|
793
|
+
const row = rows[0];
|
|
794
|
+
if (row === undefined) {
|
|
795
|
+
return yield* explainMiss(id);
|
|
796
|
+
}
|
|
797
|
+
yield* insertAttempt(tx, id, "fanned-out", row.processedAt, now, undefined);
|
|
798
|
+
let pending;
|
|
799
|
+
if (row.flowPending === null || row.flowPending === undefined) {
|
|
800
|
+
pending = outcome.children.length;
|
|
801
|
+
yield* tx.execute(sql `
|
|
802
|
+
UPDATE ${jobs} SET flow_fail_fast = ${outcome.failFast}, flow_pending = ${pending},
|
|
803
|
+
flow_completed = 0, flow_failed = 0, flow_cancelled = 0
|
|
804
|
+
WHERE ${jobs.id} = ${id}
|
|
805
|
+
`);
|
|
806
|
+
// Chunked multi-row VALUES, like enqueueMany's insertBatch.
|
|
807
|
+
for (let start = 0; start < outcome.children.length; start += 500) {
|
|
808
|
+
const chunk = outcome.children.slice(start, start + 500);
|
|
809
|
+
const values = chunk.map((child) => sql `(${id}, ${child.childKey}, ${child.request.name}, ${child.storeKey},
|
|
810
|
+
${JSON.stringify(child.request)}::jsonb, 'pending', NULL, NULL, FALSE, ${now})`);
|
|
811
|
+
yield* tx.execute(sql `
|
|
812
|
+
INSERT INTO ${flowChildren} (flow_id, child_key, name, store_key, spec,
|
|
813
|
+
status, exit, failed_reason, cascaded, pending_since)
|
|
814
|
+
VALUES ${sql.join(values, sql `, `)}
|
|
815
|
+
`);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
else {
|
|
819
|
+
// A manifest that was already present is kept untouched (double
|
|
820
|
+
// fan-out converges on the persisted children); the state
|
|
821
|
+
// transition follows the persisted pending count either way.
|
|
822
|
+
pending = Number(row.flowPending);
|
|
823
|
+
}
|
|
824
|
+
if (row.cancelRequested) {
|
|
825
|
+
// A cancel raced the fan-out: cancellation wins. The rows exist
|
|
826
|
+
// and get marked (into the `cancelled` counter), so the sweeper
|
|
827
|
+
// cascades (mostly no-op cancels for never-enqueued children).
|
|
828
|
+
yield* tx.execute(sql `
|
|
829
|
+
WITH marked AS (${cancelPendingChildren(id)})
|
|
830
|
+
UPDATE ${jobs} SET state = 'cancelled', finished_at = ${now},
|
|
831
|
+
cancel_requested = FALSE, flow_pending = 0,
|
|
832
|
+
flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
|
|
833
|
+
WHERE ${jobs.id} = ${id}
|
|
834
|
+
`);
|
|
835
|
+
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined);
|
|
836
|
+
// A cancelled NESTED parent reports upward through the outbox.
|
|
837
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason);
|
|
838
|
+
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
|
|
839
|
+
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now);
|
|
840
|
+
return undefined;
|
|
841
|
+
}
|
|
842
|
+
if (pending > 0) {
|
|
843
|
+
yield* tx.execute(sql `
|
|
844
|
+
UPDATE ${jobs} SET state = 'waiting-children' WHERE ${jobs.id} = ${id}
|
|
845
|
+
`);
|
|
846
|
+
return undefined;
|
|
847
|
+
}
|
|
848
|
+
// Empty (or fully recorded) manifest: straight to runnable collect.
|
|
849
|
+
yield* tx.execute(sql `
|
|
850
|
+
UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
|
|
851
|
+
WHERE ${jobs.id} = ${id}
|
|
852
|
+
`);
|
|
853
|
+
return JobStore.QueueName(row.queue);
|
|
854
|
+
})).pipe(Effect.mapError((error) => error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
|
|
855
|
+
error instanceof JobStore.JobStoreError
|
|
856
|
+
? error
|
|
857
|
+
: storeError("ack failed")(error)));
|
|
858
|
+
if (wakeQueue !== undefined) {
|
|
859
|
+
yield* wakeUp(wakeQueue);
|
|
860
|
+
}
|
|
861
|
+
});
|
|
613
862
|
const store = {
|
|
614
863
|
enqueue: enqueueOne,
|
|
615
864
|
enqueueMany: (requests) => Effect.gen(function* () {
|
|
@@ -680,7 +929,10 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
680
929
|
${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
|
|
681
930
|
${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
|
|
682
931
|
${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
|
|
683
|
-
${jobs.trace} AS "trace",
|
|
932
|
+
${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
|
|
933
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
|
|
934
|
+
${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
|
|
935
|
+
${jobs.flowCancelled} AS "flowCancelled",
|
|
684
936
|
${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
|
|
685
937
|
${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
|
|
686
938
|
${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
@@ -703,55 +955,64 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
703
955
|
return empty;
|
|
704
956
|
}));
|
|
705
957
|
}).pipe(Effect.mapError((error) => error instanceof JobStore.JobStoreError ? error : storeError("claim failed")(error))),
|
|
706
|
-
ack: (id, token, outcome) =>
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
:
|
|
958
|
+
ack: (id, token, outcome) => {
|
|
959
|
+
if (outcome._tag === "FanOut") {
|
|
960
|
+
return ackFanOut(id, token, outcome);
|
|
961
|
+
}
|
|
962
|
+
return db.transaction((tx) => Effect.gen(function* () {
|
|
963
|
+
const now = yield* nowDate;
|
|
964
|
+
const update = outcome._tag === "Complete"
|
|
965
|
+
? sql `state = 'completed', cancel_requested = FALSE, exit = ${JSON.stringify(outcome.exit ?? null)}::jsonb, finished_at = ${now}`
|
|
966
|
+
: outcome._tag === "Fail"
|
|
967
|
+
? sql `state = 'failed', cancel_requested = FALSE, exit = ${JSON.stringify(outcome.exit ?? null)}::jsonb, finished_at = ${now}`
|
|
968
|
+
: outcome._tag === "Cancelled"
|
|
969
|
+
? sql `state = 'cancelled', cancel_requested = FALSE, finished_at = ${now}`
|
|
970
|
+
// A cancel that raced this natural failure wins over revival
|
|
971
|
+
// (mirrors release/recoverStalled).
|
|
972
|
+
: sql `state = CASE WHEN ${jobs.cancelRequested} THEN 'cancelled'
|
|
717
973
|
ELSE ${outcome.delayMs > 0 ? "delayed" : "waiting"} END,
|
|
718
974
|
finished_at = CASE WHEN ${jobs.cancelRequested} THEN ${now}::timestamptz ELSE NULL END,
|
|
719
975
|
run_at = CASE WHEN ${jobs.cancelRequested} THEN ${jobs.runAt}
|
|
720
976
|
ELSE ${new Date(now.getTime() + Math.max(0, outcome.delayMs))}::timestamptz END,
|
|
721
977
|
seq = CASE WHEN ${jobs.cancelRequested} THEN ${jobs.seq} ELSE ${seqExpr} END,
|
|
722
978
|
cancel_requested = FALSE`;
|
|
723
|
-
|
|
979
|
+
const rows = rowsOf(yield* tx.execute(sql `
|
|
724
980
|
UPDATE ${jobs} SET ${update},
|
|
725
981
|
attempts_made = ${jobs.attemptsMade} + 1, lock_token = NULL, lock_expires_at = NULL
|
|
726
982
|
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
727
983
|
RETURNING ${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name",
|
|
728
984
|
${jobs.state} AS "state", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
|
|
729
|
-
${jobs.queue} AS "queue"
|
|
985
|
+
${jobs.queue} AS "queue", ${jobs.parent} AS "parent", ${jobs.exit} AS "exit",
|
|
986
|
+
${jobs.failedReason} AS "failedReason"
|
|
730
987
|
`));
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
988
|
+
const row = rows[0];
|
|
989
|
+
if (row === undefined) {
|
|
990
|
+
return yield* explainMiss(id);
|
|
991
|
+
}
|
|
992
|
+
const cancelledRetry = outcome._tag === "Retry" && row.state === "cancelled";
|
|
993
|
+
const ledgerOutcome = outcome._tag === "Complete"
|
|
994
|
+
? "completed"
|
|
995
|
+
: outcome._tag === "Fail"
|
|
996
|
+
? "failed"
|
|
997
|
+
: outcome._tag === "Cancelled" || cancelledRetry
|
|
998
|
+
? "cancelled"
|
|
999
|
+
: "retried";
|
|
1000
|
+
yield* insertAttempt(tx, id, ledgerOutcome, row.processedAt, now, outcome._tag === "Cancelled" || cancelledRetry ? undefined : outcome.exit);
|
|
1001
|
+
if (outcome._tag !== "Retry" || cancelledRetry) {
|
|
1002
|
+
// A terminal transition of an envelope-carrying child reports
|
|
1003
|
+
// upward through the outbox (exit/failedReason as persisted).
|
|
1004
|
+
yield* appendOutbox(tx, row.parent, ledgerOutcome === "retried" ? "cancelled" : ledgerOutcome, row.exit ?? undefined, row.failedReason);
|
|
1005
|
+
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
|
|
1006
|
+
yield* applyKeep(tx, row, now);
|
|
1007
|
+
}
|
|
1008
|
+
return outcome._tag === "Retry" && !cancelledRetry
|
|
1009
|
+
? JobStore.QueueName(row.queue)
|
|
1010
|
+
: undefined;
|
|
1011
|
+
})).pipe(Effect.mapError((error) => error instanceof JobStore.JobNotFoundError || error instanceof JobStore.LockLostError ||
|
|
1012
|
+
error instanceof JobStore.JobStoreError
|
|
1013
|
+
? error
|
|
1014
|
+
: storeError("ack failed")(error)), Effect.tap((queue) => queue !== undefined ? wakeUp(queue) : Effect.void), Effect.asVoid);
|
|
1015
|
+
},
|
|
755
1016
|
release: (id, token) => Effect.gen(function* () {
|
|
756
1017
|
const released = yield* db.transaction((tx) => Effect.gen(function* () {
|
|
757
1018
|
const now = yield* nowDate;
|
|
@@ -766,13 +1027,16 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
766
1027
|
WHERE ${jobs.id} = ${id} AND ${jobs.state} = 'active' AND ${jobs.lockToken} = ${token}
|
|
767
1028
|
RETURNING ${jobs.id} AS id, (${jobs.state} = 'cancelled') AS cancelled,
|
|
768
1029
|
${jobs.processedAt} AS "processedAt", ${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
769
|
-
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue"
|
|
1030
|
+
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1031
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
770
1032
|
`));
|
|
771
1033
|
const row = rows[0];
|
|
772
1034
|
if (row === undefined)
|
|
773
1035
|
return undefined;
|
|
774
1036
|
if (row.cancelled) {
|
|
775
1037
|
yield* insertAttempt(tx, id, "cancelled", row.processedAt, now, undefined);
|
|
1038
|
+
// A cancel honoured at release is a terminal transition too.
|
|
1039
|
+
yield* appendOutbox(tx, row.parent, "cancelled", row.exit ?? undefined, row.failedReason);
|
|
776
1040
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, id, now);
|
|
777
1041
|
yield* applyKeep(tx, { name: row.name, state: "cancelled", keep: row.keep }, now);
|
|
778
1042
|
}
|
|
@@ -842,12 +1106,16 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
842
1106
|
cancel_requested = FALSE
|
|
843
1107
|
WHERE ${jobs.state} = 'active' AND ${jobs.lockExpiresAt} <= ${now}::timestamptz
|
|
844
1108
|
RETURNING ${jobs.id} AS "id", ${jobs.state} AS "state", ${jobs.processedAt} AS "processedAt",
|
|
845
|
-
${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey"
|
|
1109
|
+
${jobs.name} AS "name", ${jobs.keep} AS "keep", ${jobs.dedupeKey} AS "dedupeKey",
|
|
1110
|
+
${jobs.parent} AS "parent", ${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
846
1111
|
`));
|
|
847
1112
|
const recovered = [];
|
|
848
1113
|
for (const row of rows) {
|
|
849
1114
|
yield* insertAttempt(tx, row.id, row.state === "cancelled" ? "cancelled" : "stalled", row.processedAt, now, undefined);
|
|
850
1115
|
if (row.state === "cancelled" || row.state === "failed") {
|
|
1116
|
+
// Honoured cancels and stall exhaustion are terminal
|
|
1117
|
+
// transitions: envelope-carrying children report upward.
|
|
1118
|
+
yield* appendOutbox(tx, row.parent, row.state === "cancelled" ? "cancelled" : "failed", row.exit ?? undefined, row.failedReason);
|
|
851
1119
|
yield* releaseDedupe(tx, row.name, row.dedupeKey, row.id, now);
|
|
852
1120
|
}
|
|
853
1121
|
if (row.state === "cancelled") {
|
|
@@ -905,7 +1173,11 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
905
1173
|
${jobs.attemptsMax} AS "attemptsMax", ${jobs.attemptsMade} AS "attemptsMade",
|
|
906
1174
|
${jobs.stalledCount} AS "stalledCount", ${jobs.backoff} AS "backoff",
|
|
907
1175
|
${jobs.keep} AS "keep", ${jobs.timeoutMs} AS "timeoutMs",
|
|
908
|
-
${jobs.cancelRequested} AS "cancelRequested",
|
|
1176
|
+
${jobs.cancelRequested} AS "cancelRequested", ${jobs.dedupeKey} AS "dedupeKey",
|
|
1177
|
+
${jobs.trace} AS "trace", ${jobs.parent} AS "parent",
|
|
1178
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.flowPending} AS "flowPending",
|
|
1179
|
+
${jobs.flowCompleted} AS "flowCompleted", ${jobs.flowFailed} AS "flowFailed",
|
|
1180
|
+
${jobs.flowCancelled} AS "flowCancelled",
|
|
909
1181
|
${jobs.runAt} AS "runAt", ${jobs.enqueuedAt} AS "enqueuedAt",
|
|
910
1182
|
${jobs.processedAt} AS "processedAt", ${jobs.finishedAt} AS "finishedAt",
|
|
911
1183
|
${jobs.exit} AS "exit", ${jobs.failedReason} AS "failedReason"
|
|
@@ -990,20 +1262,21 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
990
1262
|
`).pipe(Effect.mapError(storeError("pausedQueues failed")), Effect.map((result) => rowsOf(result).map((row) => JobStore.QueueName(row.queue)))),
|
|
991
1263
|
upsertSchedule: (schedule) => db.execute(sql `
|
|
992
1264
|
INSERT INTO ${schedules} (key, job_name, queue, cron, tz, every_ms, payload, metadata,
|
|
993
|
-
priority, attempts_max, backoff, keep, timeout_ms, next_run_at)
|
|
1265
|
+
priority, attempts_max, backoff, keep, timeout_ms, group_name, next_run_at)
|
|
994
1266
|
VALUES (${schedule.key}, ${schedule.jobName}, ${schedule.queue},
|
|
995
1267
|
${schedule.cron ?? null}, ${schedule.tz ?? null}, ${schedule.everyMs ?? null},
|
|
996
1268
|
${JSON.stringify(schedule.payload ?? null)}::jsonb, ${JSON.stringify(schedule.metadata)}::jsonb,
|
|
997
1269
|
${schedule.priority}, ${schedule.attemptsMax},
|
|
998
1270
|
${schedule.backoff === undefined ? null : JSON.stringify(schedule.backoff)}::jsonb,
|
|
999
1271
|
${schedule.keep === undefined ? null : JSON.stringify(schedule.keep)}::jsonb,
|
|
1000
|
-
${schedule.timeoutMs ?? null}, ${new Date(schedule.nextRunAt)})
|
|
1272
|
+
${schedule.timeoutMs ?? null}, ${schedule.group ?? null}, ${new Date(schedule.nextRunAt)})
|
|
1001
1273
|
ON CONFLICT (key) DO UPDATE SET
|
|
1002
1274
|
job_name = EXCLUDED.job_name, queue = EXCLUDED.queue, cron = EXCLUDED.cron,
|
|
1003
1275
|
tz = EXCLUDED.tz, every_ms = EXCLUDED.every_ms, payload = EXCLUDED.payload,
|
|
1004
1276
|
metadata = EXCLUDED.metadata, priority = EXCLUDED.priority,
|
|
1005
1277
|
attempts_max = EXCLUDED.attempts_max, backoff = EXCLUDED.backoff,
|
|
1006
1278
|
keep = EXCLUDED.keep, timeout_ms = EXCLUDED.timeout_ms,
|
|
1279
|
+
group_name = EXCLUDED.group_name,
|
|
1007
1280
|
next_run_at = CASE
|
|
1008
1281
|
WHEN ${schedules.cron} IS NOT DISTINCT FROM EXCLUDED.cron
|
|
1009
1282
|
AND ${schedules.tz} IS NOT DISTINCT FROM EXCLUDED.tz
|
|
@@ -1023,6 +1296,9 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
1023
1296
|
if (listOptions?.queue !== undefined) {
|
|
1024
1297
|
conditions.push(sql `${schedules.queue} = ${listOptions.queue}`);
|
|
1025
1298
|
}
|
|
1299
|
+
if (listOptions?.group !== undefined) {
|
|
1300
|
+
conditions.push(sql `${schedules.group} = ${listOptions.group}`);
|
|
1301
|
+
}
|
|
1026
1302
|
const rows = rowsOf(yield* db.execute(sql `
|
|
1027
1303
|
SELECT ${schedules.key} AS "key", ${schedules.jobName} AS "jobName",
|
|
1028
1304
|
${schedules.queue} AS "queue", ${schedules.cron} AS "cron", ${schedules.tz} AS "tz",
|
|
@@ -1030,7 +1306,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
1030
1306
|
${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
|
|
1031
1307
|
${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
|
|
1032
1308
|
${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
|
|
1033
|
-
${schedules.nextRunAt} AS "nextRunAt"
|
|
1309
|
+
${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
|
|
1034
1310
|
FROM ${schedules}
|
|
1035
1311
|
WHERE ${sql.join(conditions, sql ` AND `)}
|
|
1036
1312
|
ORDER BY ${schedules.key}
|
|
@@ -1046,7 +1322,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
1046
1322
|
${schedules.metadata} AS "metadata", ${schedules.priority} AS "priority",
|
|
1047
1323
|
${schedules.attemptsMax} AS "attemptsMax", ${schedules.backoff} AS "backoff",
|
|
1048
1324
|
${schedules.keep} AS "keep", ${schedules.timeoutMs} AS "timeoutMs",
|
|
1049
|
-
${schedules.nextRunAt} AS "nextRunAt"
|
|
1325
|
+
${schedules.group} AS "group", ${schedules.nextRunAt} AS "nextRunAt"
|
|
1050
1326
|
FROM ${schedules}
|
|
1051
1327
|
WHERE ${schedules.nextRunAt} <= ${now}
|
|
1052
1328
|
ORDER BY ${schedules.nextRunAt} ASC
|
|
@@ -1085,6 +1361,301 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
1085
1361
|
}
|
|
1086
1362
|
return fired;
|
|
1087
1363
|
}),
|
|
1364
|
+
recordChildResults: (reports) => Effect.gen(function* () {
|
|
1365
|
+
if (reports.length === 0) {
|
|
1366
|
+
const none = [];
|
|
1367
|
+
return none;
|
|
1368
|
+
}
|
|
1369
|
+
const batch = yield* db.transaction((tx) => Effect.gen(function* () {
|
|
1370
|
+
const now = yield* nowDate;
|
|
1371
|
+
const results = reports.map(() => ({ applied: false, parentSettled: false }));
|
|
1372
|
+
// Phase 1a — apply every row update (contract lock order:
|
|
1373
|
+
// dependency rows FIRST, parents second). Only a (flow, key)'s
|
|
1374
|
+
// first occurrence in the batch can apply — later duplicates
|
|
1375
|
+
// would find the row non-pending anyway, and UPDATE ... FROM
|
|
1376
|
+
// must never see two source rows for one target.
|
|
1377
|
+
const candidates = [];
|
|
1378
|
+
const seen = new Set();
|
|
1379
|
+
for (const [index, report] of reports.entries()) {
|
|
1380
|
+
const key = `${report.flowId}\u0000${report.childKey}`;
|
|
1381
|
+
if (seen.has(key))
|
|
1382
|
+
continue;
|
|
1383
|
+
seen.add(key);
|
|
1384
|
+
candidates.push({ index, report });
|
|
1385
|
+
}
|
|
1386
|
+
for (let start = 0; start < candidates.length; start += 200) {
|
|
1387
|
+
const chunk = candidates.slice(start, start + 200);
|
|
1388
|
+
const values = chunk.map(({ index, report }) => sql `(${index}::int, ${report.flowId}::text, ${report.childKey}::text,
|
|
1389
|
+
${report.outcome}::text,
|
|
1390
|
+
${report.exit === undefined ? null : JSON.stringify(report.exit)}::jsonb,
|
|
1391
|
+
${report.failedReason ?? null}::text)`);
|
|
1392
|
+
const appliedRows = rowsOf(yield* tx.execute(sql `
|
|
1393
|
+
UPDATE ${flowChildren} SET status = v.outcome, exit = v.exit,
|
|
1394
|
+
failed_reason = v.failed_reason, cascaded = TRUE
|
|
1395
|
+
FROM (VALUES ${sql.join(values, sql `, `)})
|
|
1396
|
+
AS v(ord, flow_id, child_key, outcome, exit, failed_reason)
|
|
1397
|
+
WHERE ${flowChildren.flowId} = v.flow_id AND ${flowChildren.childKey} = v.child_key
|
|
1398
|
+
AND ${flowChildren.status} = 'pending'
|
|
1399
|
+
RETURNING v.ord AS "ord"
|
|
1400
|
+
`));
|
|
1401
|
+
for (const row of appliedRows) {
|
|
1402
|
+
const result = results[Number(row.ord)];
|
|
1403
|
+
if (result !== undefined) {
|
|
1404
|
+
result.applied = true;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
}
|
|
1408
|
+
const touched = new Map();
|
|
1409
|
+
for (const [index, report] of reports.entries()) {
|
|
1410
|
+
if (results[index]?.applied !== true)
|
|
1411
|
+
continue;
|
|
1412
|
+
const touch = touched.get(report.flowId) ?? {
|
|
1413
|
+
appliedCount: 0,
|
|
1414
|
+
completed: 0,
|
|
1415
|
+
failed: 0,
|
|
1416
|
+
cancelled: 0,
|
|
1417
|
+
firstAppliedFailed: undefined,
|
|
1418
|
+
lastApplied: index
|
|
1419
|
+
};
|
|
1420
|
+
touch.appliedCount += 1;
|
|
1421
|
+
touch.lastApplied = index;
|
|
1422
|
+
if (report.outcome === "completed")
|
|
1423
|
+
touch.completed += 1;
|
|
1424
|
+
if (report.outcome === "cancelled")
|
|
1425
|
+
touch.cancelled += 1;
|
|
1426
|
+
if (report.outcome === "failed") {
|
|
1427
|
+
touch.failed += 1;
|
|
1428
|
+
touch.firstAppliedFailed ??= index;
|
|
1429
|
+
}
|
|
1430
|
+
touched.set(report.flowId, touch);
|
|
1431
|
+
}
|
|
1432
|
+
// Phase 1b + 2 — per touched flow (sorted, so concurrent
|
|
1433
|
+
// batches take parent locks in one order): move the applied
|
|
1434
|
+
// children from `pending` to their outcome counters, then make
|
|
1435
|
+
// at most one settle decision. Fail-fast wins the tie.
|
|
1436
|
+
const wakeQueues = [];
|
|
1437
|
+
for (const flowId of [...touched.keys()].toSorted()) {
|
|
1438
|
+
const touch = touched.get(flowId);
|
|
1439
|
+
if (touch === undefined)
|
|
1440
|
+
continue;
|
|
1441
|
+
const parents = rowsOf(yield* tx.execute(sql `
|
|
1442
|
+
UPDATE ${jobs} SET
|
|
1443
|
+
flow_pending = GREATEST(${jobs.flowPending} - ${touch.appliedCount}::int, 0),
|
|
1444
|
+
flow_completed = ${jobs.flowCompleted} + ${touch.completed}::int,
|
|
1445
|
+
flow_failed = ${jobs.flowFailed} + ${touch.failed}::int,
|
|
1446
|
+
flow_cancelled = ${jobs.flowCancelled} + ${touch.cancelled}::int
|
|
1447
|
+
WHERE ${jobs.id} = ${flowId} AND ${jobs.flowPending} IS NOT NULL
|
|
1448
|
+
RETURNING ${jobs.state} AS "state", ${jobs.flowPending} AS "flowPending",
|
|
1449
|
+
${jobs.flowFailFast} AS "flowFailFast", ${jobs.processedAt} AS "processedAt",
|
|
1450
|
+
${jobs.name} AS "name", ${jobs.keep} AS "keep",
|
|
1451
|
+
${jobs.dedupeKey} AS "dedupeKey", ${jobs.queue} AS "queue",
|
|
1452
|
+
${jobs.parent} AS "parent"
|
|
1453
|
+
`));
|
|
1454
|
+
const parent = parents[0];
|
|
1455
|
+
if (parent === undefined || parent.state !== "waiting-children")
|
|
1456
|
+
continue;
|
|
1457
|
+
const failedIndex = parent.flowFailFast === true ? touch.firstAppliedFailed : undefined;
|
|
1458
|
+
const failedReport = failedIndex !== undefined ? reports[failedIndex] : undefined;
|
|
1459
|
+
if (failedIndex !== undefined && failedReport !== undefined) {
|
|
1460
|
+
// The first applied failure settles the parent terminally
|
|
1461
|
+
// (store-side, like stall exhaustion: failedReason, no
|
|
1462
|
+
// exit) and marks the remaining rows in the same
|
|
1463
|
+
// transaction. A nested parent's own report goes to the
|
|
1464
|
+
// outbox here — this settle IS its terminal transition.
|
|
1465
|
+
const reason = `effect-mq: flow child "${failedReport.childKey}" failed`;
|
|
1466
|
+
yield* tx.execute(sql `
|
|
1467
|
+
WITH marked AS (${cancelPendingChildren(flowId)})
|
|
1468
|
+
UPDATE ${jobs} SET state = 'failed', finished_at = ${now},
|
|
1469
|
+
failed_reason = ${reason}, cancel_requested = FALSE, flow_pending = 0,
|
|
1470
|
+
flow_cancelled = ${jobs.flowCancelled} + (SELECT count(*)::int FROM marked)
|
|
1471
|
+
WHERE ${jobs.id} = ${flowId}
|
|
1472
|
+
`);
|
|
1473
|
+
yield* insertAttempt(tx, flowId, "failed", parent.processedAt, now, undefined);
|
|
1474
|
+
yield* appendOutbox(tx, parent.parent, "failed", undefined, reason);
|
|
1475
|
+
yield* releaseDedupe(tx, parent.name, parent.dedupeKey, flowId, now);
|
|
1476
|
+
yield* applyKeep(tx, { name: parent.name, state: "failed", keep: parent.keep }, now);
|
|
1477
|
+
const decided = results[failedIndex];
|
|
1478
|
+
if (decided !== undefined) {
|
|
1479
|
+
decided.parentSettled = true;
|
|
1480
|
+
}
|
|
1481
|
+
continue;
|
|
1482
|
+
}
|
|
1483
|
+
if (Number(parent.flowPending) === 0) {
|
|
1484
|
+
// All children settled: the parent resumes runnable, phase
|
|
1485
|
+
// collect, settled at the flow's last applied report.
|
|
1486
|
+
yield* tx.execute(sql `
|
|
1487
|
+
UPDATE ${jobs} SET state = 'waiting', run_at = ${now}, seq = ${seqExpr}
|
|
1488
|
+
WHERE ${jobs.id} = ${flowId}
|
|
1489
|
+
`);
|
|
1490
|
+
wakeQueues.push(JobStore.QueueName(parent.queue));
|
|
1491
|
+
const decided = results[touch.lastApplied];
|
|
1492
|
+
if (decided !== undefined) {
|
|
1493
|
+
decided.parentSettled = true;
|
|
1494
|
+
}
|
|
1495
|
+
}
|
|
1496
|
+
}
|
|
1497
|
+
return { results, wakeQueues };
|
|
1498
|
+
})).pipe(
|
|
1499
|
+
// Rare lock-order inversion (a concurrent cancel/settle marking
|
|
1500
|
+
// rows) surfaces as a Postgres deadlock (40P01); the killed side
|
|
1501
|
+
// is safe to retry — the row-state guard keeps it idempotent.
|
|
1502
|
+
Effect.retry({
|
|
1503
|
+
times: 3,
|
|
1504
|
+
while: isDeadlockError
|
|
1505
|
+
}), Effect.mapError((error) => error instanceof JobStore.JobStoreError ? error : storeError("recordChildResults failed")(error)));
|
|
1506
|
+
for (const queue of batch.wakeQueues) {
|
|
1507
|
+
yield* wakeUp(queue);
|
|
1508
|
+
}
|
|
1509
|
+
return batch.results;
|
|
1510
|
+
}),
|
|
1511
|
+
peekOutbox: (peekOptions) => Effect.gen(function* () {
|
|
1512
|
+
const limit = Math.max(0, peekOptions.limit);
|
|
1513
|
+
if (limit === 0) {
|
|
1514
|
+
const none = [];
|
|
1515
|
+
return none;
|
|
1516
|
+
}
|
|
1517
|
+
// `after` pages past a previously returned id (exclusive), whether
|
|
1518
|
+
// or not that entry still exists. Anything that is not a canonical
|
|
1519
|
+
// id this store issued is treated as unset.
|
|
1520
|
+
const after = peekOptions.after !== undefined && CANONICAL_BIGSERIAL.test(peekOptions.after)
|
|
1521
|
+
? peekOptions.after
|
|
1522
|
+
: undefined;
|
|
1523
|
+
const rows = rowsOf(yield* db.execute(sql `
|
|
1524
|
+
SELECT ${flowOutbox.id}::text AS "id", ${flowOutbox.flowName} AS "flowName",
|
|
1525
|
+
${flowOutbox.parentStoreKey} AS "parentStoreKey", ${flowOutbox.report} AS "report"
|
|
1526
|
+
FROM ${flowOutbox}
|
|
1527
|
+
${after === undefined ? sql `` : sql `WHERE ${flowOutbox.id} > ${after}::bigint`}
|
|
1528
|
+
ORDER BY ${flowOutbox.id} ASC
|
|
1529
|
+
LIMIT ${limit}
|
|
1530
|
+
`).pipe(Effect.mapError(storeError("peekOutbox failed"))));
|
|
1531
|
+
return rows.map((row) => ({
|
|
1532
|
+
id: row.id,
|
|
1533
|
+
flowName: row.flowName,
|
|
1534
|
+
parentStoreKey: row.parentStoreKey,
|
|
1535
|
+
report: row.report
|
|
1536
|
+
}));
|
|
1537
|
+
}),
|
|
1538
|
+
deleteOutbox: (ids) => Effect.suspend(() => {
|
|
1539
|
+
// Ids are opaque strings to callers; only CANONICAL ids this store
|
|
1540
|
+
// could have issued can match. The strictness matters twice: a
|
|
1541
|
+
// foreign/garbled id must not blow up the ::bigint cast, and a
|
|
1542
|
+
// non-canonical spelling ("007") must stay an unknown-id no-op
|
|
1543
|
+
// rather than cast to 7 and delete a live entry.
|
|
1544
|
+
const numeric = ids.filter((id) => CANONICAL_BIGSERIAL.test(id));
|
|
1545
|
+
if (numeric.length === 0)
|
|
1546
|
+
return Effect.void;
|
|
1547
|
+
return db.execute(sql `
|
|
1548
|
+
DELETE FROM ${flowOutbox}
|
|
1549
|
+
WHERE ${flowOutbox.id} = ANY(${sql.param(numeric)}::bigint[])
|
|
1550
|
+
`).pipe(Effect.mapError(storeError("deleteOutbox failed")), Effect.asVoid);
|
|
1551
|
+
}),
|
|
1552
|
+
listChildResults: (flowId, listOptions) => Effect.gen(function* () {
|
|
1553
|
+
const limit = Math.max(1, listOptions?.limit ?? 1000);
|
|
1554
|
+
const cursor = listOptions?.cursor;
|
|
1555
|
+
// `spec->>'id'` instead of the whole spec: at 10k children the full
|
|
1556
|
+
// payloads would transfer on every collect.
|
|
1557
|
+
const rows = rowsOf(yield* db.execute(sql `
|
|
1558
|
+
SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
1559
|
+
${flowChildren.name} AS "name", ${flowChildren.storeKey} AS "storeKey",
|
|
1560
|
+
${flowChildren.spec}->>'id' AS "childJobId", ${flowChildren.status} AS "status",
|
|
1561
|
+
${flowChildren.exit} AS "exit", ${flowChildren.failedReason} AS "failedReason",
|
|
1562
|
+
${flowChildren.cascaded} AS "cascaded"
|
|
1563
|
+
FROM ${flowChildren}
|
|
1564
|
+
WHERE ${flowChildren.flowId} = ${flowId}
|
|
1565
|
+
${cursor === undefined ? sql `` : sql `AND ${flowChildren.childKey} > ${cursor}`}
|
|
1566
|
+
ORDER BY ${flowChildren.childKey} ASC
|
|
1567
|
+
LIMIT ${limit + 1}
|
|
1568
|
+
`).pipe(Effect.mapError(storeError("listChildResults failed"))));
|
|
1569
|
+
const page = rows.slice(0, limit);
|
|
1570
|
+
const items = page.map(toFlowChildRecord);
|
|
1571
|
+
const last = items[items.length - 1];
|
|
1572
|
+
return {
|
|
1573
|
+
items,
|
|
1574
|
+
cursor: rows.length > limit && last !== undefined ? last.childKey : undefined
|
|
1575
|
+
};
|
|
1576
|
+
}),
|
|
1577
|
+
flowSweepWork: (sweepOptions) => Effect.gen(function* () {
|
|
1578
|
+
const now = yield* nowDate;
|
|
1579
|
+
const limit = Math.max(1, sweepOptions.limit ?? 1000);
|
|
1580
|
+
const threshold = new Date(now.getTime() - sweepOptions.pendingAgeMs);
|
|
1581
|
+
// Reconcile: pending rows past the eligibility threshold whose
|
|
1582
|
+
// parent is still parked (a settled flow never re-drives work).
|
|
1583
|
+
// Returning a row re-arms `pending_since` in the same statement, so
|
|
1584
|
+
// a full page rotates across sweeps instead of pinning its head.
|
|
1585
|
+
// SKIP LOCKED: rows a concurrent report/settle holds are its
|
|
1586
|
+
// business, and never waiting means this statement cannot deadlock.
|
|
1587
|
+
// The raw column names are safe: only table names vary across
|
|
1588
|
+
// factory instances. RETURNING order is unspecified — sorted below.
|
|
1589
|
+
const reconcileRows = rowsOf(yield* db.execute(sql `
|
|
1590
|
+
WITH due AS (
|
|
1591
|
+
SELECT c.flow_id, c.child_key
|
|
1592
|
+
FROM ${flowChildren} c
|
|
1593
|
+
JOIN ${jobs} j ON j.id = c.flow_id
|
|
1594
|
+
WHERE c.status = 'pending' AND c.pending_since <= ${threshold}
|
|
1595
|
+
AND j.state = 'waiting-children'
|
|
1596
|
+
ORDER BY c.flow_id, c.child_key
|
|
1597
|
+
LIMIT ${limit}
|
|
1598
|
+
FOR UPDATE OF c SKIP LOCKED
|
|
1599
|
+
)
|
|
1600
|
+
UPDATE ${flowChildren} SET pending_since = ${now}
|
|
1601
|
+
FROM due
|
|
1602
|
+
WHERE ${flowChildren.flowId} = due.flow_id AND ${flowChildren.childKey} = due.child_key
|
|
1603
|
+
RETURNING ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
1604
|
+
${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
|
|
1605
|
+
`).pipe(Effect.mapError(storeError("flowSweepWork failed"))))
|
|
1606
|
+
.toSorted((a, b) => a.flowId !== b.flowId
|
|
1607
|
+
? (a.flowId < b.flowId ? -1 : 1)
|
|
1608
|
+
: a.childKey < b.childKey
|
|
1609
|
+
? -1
|
|
1610
|
+
: a.childKey > b.childKey
|
|
1611
|
+
? 1
|
|
1612
|
+
: 0);
|
|
1613
|
+
// Cascade: cancelled rows whose cancel has not been delivered into
|
|
1614
|
+
// the child's store yet (any parent state).
|
|
1615
|
+
const cascadeRows = rowsOf(yield* db.execute(sql `
|
|
1616
|
+
SELECT ${flowChildren.flowId} AS "flowId", ${flowChildren.childKey} AS "childKey",
|
|
1617
|
+
${flowChildren.storeKey} AS "storeKey", ${flowChildren.spec} AS "spec"
|
|
1618
|
+
FROM ${flowChildren}
|
|
1619
|
+
WHERE ${flowChildren.status} = 'cancelled' AND ${flowChildren.cascaded} = FALSE
|
|
1620
|
+
ORDER BY ${flowChildren.flowId}, ${flowChildren.childKey}
|
|
1621
|
+
LIMIT ${limit}
|
|
1622
|
+
`).pipe(Effect.mapError(storeError("flowSweepWork failed"))));
|
|
1623
|
+
const reconcile = [];
|
|
1624
|
+
for (const row of reconcileRows) {
|
|
1625
|
+
const flowId = JobId(row.flowId);
|
|
1626
|
+
let group = reconcile[reconcile.length - 1];
|
|
1627
|
+
if (group === undefined || group.flowId !== flowId) {
|
|
1628
|
+
group = { flowId, children: [] };
|
|
1629
|
+
reconcile.push(group);
|
|
1630
|
+
}
|
|
1631
|
+
group.children.push({ childKey: row.childKey, storeKey: row.storeKey, request: row.spec });
|
|
1632
|
+
}
|
|
1633
|
+
const cascade = [];
|
|
1634
|
+
for (const row of cascadeRows) {
|
|
1635
|
+
const flowId = JobId(row.flowId);
|
|
1636
|
+
let group = cascade[cascade.length - 1];
|
|
1637
|
+
if (group === undefined || group.flowId !== flowId) {
|
|
1638
|
+
group = { flowId, children: [] };
|
|
1639
|
+
cascade.push(group);
|
|
1640
|
+
}
|
|
1641
|
+
group.children.push({
|
|
1642
|
+
childKey: row.childKey,
|
|
1643
|
+
storeKey: row.storeKey,
|
|
1644
|
+
// SAFETY: the FanOut ack validated every spec id before
|
|
1645
|
+
// persisting it.
|
|
1646
|
+
childJobId: row.spec.id
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
const work = { reconcile, cascade };
|
|
1650
|
+
return work;
|
|
1651
|
+
}),
|
|
1652
|
+
markChildrenCascaded: (flowId, childKeys) => childKeys.length === 0
|
|
1653
|
+
? Effect.void
|
|
1654
|
+
: db.execute(sql `
|
|
1655
|
+
UPDATE ${flowChildren} SET cascaded = TRUE
|
|
1656
|
+
WHERE ${flowChildren.flowId} = ${flowId}
|
|
1657
|
+
AND ${flowChildren.childKey} = ANY(${sql.param([...childKeys])})
|
|
1658
|
+
`).pipe(Effect.mapError(storeError("markChildrenCascaded failed")), Effect.asVoid),
|
|
1088
1659
|
counts: (queue) => db.execute(sql `
|
|
1089
1660
|
SELECT ${jobs.state} AS "state", count(*)::int AS "count" FROM ${jobs}
|
|
1090
1661
|
${queue === undefined ? sql `` : sql `WHERE ${jobs.queue} = ${queue}`}
|
|
@@ -1095,6 +1666,7 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
1095
1666
|
waiting: 0,
|
|
1096
1667
|
delayed: 0,
|
|
1097
1668
|
active: 0,
|
|
1669
|
+
"waiting-children": 0,
|
|
1098
1670
|
completed: 0,
|
|
1099
1671
|
failed: 0,
|
|
1100
1672
|
cancelled: 0
|
|
@@ -1103,10 +1675,19 @@ export const make = (options) => Effect.gen(function* () {
|
|
|
1103
1675
|
counts[row.state] = row.count;
|
|
1104
1676
|
return counts;
|
|
1105
1677
|
})),
|
|
1106
|
-
remove: (id) =>
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1678
|
+
remove: (id) =>
|
|
1679
|
+
// The purge CTE takes a removed flow parent's dependency rows with it
|
|
1680
|
+
// in the same statement.
|
|
1681
|
+
db.execute(sql `
|
|
1682
|
+
WITH deleted AS (
|
|
1683
|
+
DELETE FROM ${jobs}
|
|
1684
|
+
WHERE ${jobs.id} = ${id} AND ${jobs.state} NOT IN ('active', 'waiting-children')
|
|
1685
|
+
RETURNING ${jobs.id} AS id
|
|
1686
|
+
), purged AS (
|
|
1687
|
+
DELETE FROM ${flowChildren}
|
|
1688
|
+
WHERE ${flowChildren.flowId} IN (SELECT id FROM deleted)
|
|
1689
|
+
)
|
|
1690
|
+
SELECT id FROM deleted
|
|
1110
1691
|
`).pipe(Effect.mapError(storeError("remove failed")), Effect.map((result) => rowsOf(result).length > 0))
|
|
1111
1692
|
};
|
|
1112
1693
|
return store;
|