@makerkit/workers-outbox 0.0.0-stage → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +1120 -2
- package/dist/backlog-Bd62tjCT.mjs +686 -0
- package/dist/catalog-DzvOuOHQ.d.mts +6 -0
- package/dist/catalog-G6w4wb_w.mjs +46 -0
- package/dist/cloudflare.d.mts +117 -0
- package/dist/cloudflare.mjs +300 -0
- package/dist/create-outbox-BTMmdvLs.mjs +1758 -0
- package/dist/create-outbox-C4vdn6Dr.d.mts +959 -0
- package/dist/drizzle.d.mts +654 -0
- package/dist/drizzle.mjs +166 -0
- package/dist/index.d.mts +3 -0
- package/dist/index.mjs +1 -0
- package/dist/inspect.d.mts +2 -0
- package/dist/inspect.mjs +41 -0
- package/dist/migrations-CX7O2Lcq.mjs +98 -0
- package/dist/pg.d.mts +30 -0
- package/dist/pg.mjs +76 -0
- package/dist/sql.d.mts +7 -0
- package/dist/sql.mjs +1 -0
- package/dist/status-BZhIWoE0.mjs +30 -0
- package/dist/testing.d.mts +49 -0
- package/dist/testing.mjs +97 -0
- package/dist/types-ClomUJ_8.d.mts +33 -0
- package/package.json +94 -4
|
@@ -0,0 +1,686 @@
|
|
|
1
|
+
import { o as TERMINAL_STATUSES } from "./status-BZhIWoE0.mjs";
|
|
2
|
+
import { t as queryRows } from "./catalog-G6w4wb_w.mjs";
|
|
3
|
+
import { sql } from "drizzle-orm";
|
|
4
|
+
// src/core/uuid.ts
|
|
5
|
+
const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
6
|
+
/** Any-case hex in the 8-4-4-4-12 form the `uuid` column accepts. */
|
|
7
|
+
const isUuid = (value) => UUID_PATTERN.test(value);
|
|
8
|
+
// src/store/outbox-store.ts
|
|
9
|
+
const versionClause = (alias, gate) => gate === void 0 ? sql`` : sql` AND (${alias}.min_version_ms IS NULL OR ${alias}.min_version_ms <= ${gate.mineMs} OR ${alias}.created_at <= (SELECT t FROM clock) - ${ms(gate.gateMs)})`;
|
|
10
|
+
const DEFAULT_SUCCEEDED_RETENTION_MS = 6048e5;
|
|
11
|
+
const DEFAULT_TERMINAL_RETENTION_MS = 7776e6;
|
|
12
|
+
const MAX_MESSAGE_LENGTH = 2e3;
|
|
13
|
+
const MAX_NUDGE_ERROR_LENGTH = 500;
|
|
14
|
+
const isTerminal = (status) => TERMINAL_STATUSES.includes(status);
|
|
15
|
+
const dbNow = (now) => sql`COALESCE(${now?.toISOString() ?? null}::timestamptz, now())`;
|
|
16
|
+
const ms = (value) => sql`(${Math.round(value)}::bigint * interval '1 millisecond')`;
|
|
17
|
+
const jsonb = (value) => sql`${JSON.stringify(value)}::jsonb`;
|
|
18
|
+
const arrayLiteral = (values) => `{${values.map((value) => `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"")}"`).join(",")}}`;
|
|
19
|
+
const textArray = (values) => sql`${arrayLiteral(values)}::text[]`;
|
|
20
|
+
const uuidArray = (values) => sql`${arrayLiteral(values)}::uuid[]`;
|
|
21
|
+
const tenantClause = (column, tenantKey) => {
|
|
22
|
+
if (tenantKey === void 0) return sql``;
|
|
23
|
+
return tenantKey === null ? sql` AND ${column} IS NULL` : sql` AND ${column} = ${tenantKey}`;
|
|
24
|
+
};
|
|
25
|
+
/** float8 epoch ms: every driver returns it as a number. */
|
|
26
|
+
const epochMs = (column) => sql`(EXTRACT(EPOCH FROM ${column}) * 1000)::float8`;
|
|
27
|
+
const toDate = (value) => value === null ? null : new Date(value);
|
|
28
|
+
const kindsTable = (kinds) => sql`jsonb_to_recordset(${jsonb(kinds.map((k) => ({
|
|
29
|
+
kind: k.kind,
|
|
30
|
+
lease_ms: k.leaseMs,
|
|
31
|
+
fits: k.fits
|
|
32
|
+
})))}) AS k(kind text, lease_ms bigint, fits boolean)`;
|
|
33
|
+
const isDue = (alias) => sql`((${alias}.status = 'pending' AND ${alias}.available_at <= (SELECT t FROM clock)) OR (${alias}.status = 'retrying' AND ${alias}.next_attempt_at <= (SELECT t FROM clock)))`;
|
|
34
|
+
async function insertJob(tx, job, now) {
|
|
35
|
+
const created = (await queryRows(tx, sql`INSERT INTO "outbox"."job" (id, kind, dedupe_key, payload, tenant_key, correlation_id, metadata, min_version_ms, available_at, created_at, updated_at)
|
|
36
|
+
VALUES (${job.id}, ${job.kind}, ${job.dedupeKey}, ${jsonb(job.payload)}, ${job.tenantKey}, ${job.correlationId},
|
|
37
|
+
${jsonb(job.metadata)}, ${job.minVersionMs ?? null}::bigint, ${dbNow(now)} + ${ms(job.delayMs)}, ${dbNow(now)}, ${dbNow(now)})
|
|
38
|
+
ON CONFLICT ((COALESCE(tenant_key, '')), dedupe_key) DO NOTHING
|
|
39
|
+
RETURNING id, kind, tenant_key`))[0];
|
|
40
|
+
if (created) return {
|
|
41
|
+
id: created.id,
|
|
42
|
+
kind: created.kind,
|
|
43
|
+
tenantKey: created.tenant_key,
|
|
44
|
+
created: true
|
|
45
|
+
};
|
|
46
|
+
const row = (await queryRows(tx, sql`SELECT id, kind, tenant_key, now() AS read_at FROM "outbox"."job" WHERE dedupe_key = ${job.dedupeKey} AND tenant_key IS NOT DISTINCT FROM ${job.tenantKey}`))[0];
|
|
47
|
+
if (!row) throw new Error(`insertJob: dedupe key ${JSON.stringify(job.dedupeKey)} conflicted but no row holds it`);
|
|
48
|
+
return {
|
|
49
|
+
id: row.id,
|
|
50
|
+
kind: row.kind,
|
|
51
|
+
tenantKey: row.tenant_key,
|
|
52
|
+
created: false
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
async function claimDue(db, input) {
|
|
56
|
+
const claimable = input.kinds.filter((k) => k.fits);
|
|
57
|
+
if (claimable.length === 0 || input.limit < 1) return [];
|
|
58
|
+
return (await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
59
|
+
k AS (SELECT * FROM ${kindsTable(claimable)}),
|
|
60
|
+
picked AS (
|
|
61
|
+
SELECT j.id FROM "outbox"."job" j
|
|
62
|
+
WHERE j.kind = ANY(${textArray(claimable.map((k) => k.kind))}) AND ${isDue(sql`j`)}${tenantClause(sql`j.tenant_key`, input.tenantKey)}${versionClause(sql`j`, input.version)}
|
|
63
|
+
ORDER BY COALESCE(j.next_attempt_at, j.available_at), j.created_at, j.id
|
|
64
|
+
LIMIT ${input.limit}
|
|
65
|
+
FOR UPDATE SKIP LOCKED
|
|
66
|
+
),
|
|
67
|
+
${claimAndOpenAttempt(input.leaseToken, input.trigger)}
|
|
68
|
+
SELECT * FROM claimed`)).map(toClaimedJob);
|
|
69
|
+
}
|
|
70
|
+
/** Never claims a substitute; each id's diagnosis reads the pre-claim snapshot. */
|
|
71
|
+
async function claimByIds(db, input) {
|
|
72
|
+
const ids = [...new Set(input.ids)];
|
|
73
|
+
const valid = ids.filter((id) => isUuid(id));
|
|
74
|
+
const invalid = ids.filter((id) => !isUuid(id)).map((id) => ({
|
|
75
|
+
id,
|
|
76
|
+
result: "not_found"
|
|
77
|
+
}));
|
|
78
|
+
if (valid.length === 0 || input.kinds.length === 0) return [...invalid, ...valid.map((id) => ({
|
|
79
|
+
id,
|
|
80
|
+
result: "not_found"
|
|
81
|
+
}))];
|
|
82
|
+
const rows = await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
83
|
+
k AS (SELECT * FROM ${kindsTable(input.kinds)}),
|
|
84
|
+
req AS (SELECT DISTINCT unnest(${uuidArray(valid)}) AS id),
|
|
85
|
+
picked AS (
|
|
86
|
+
SELECT j.id FROM "outbox"."job" j
|
|
87
|
+
JOIN req ON req.id = j.id
|
|
88
|
+
WHERE j.kind = ANY(${textArray(input.kinds.filter((k) => k.fits).map((k) => k.kind))}) AND ${isDue(sql`j`)}${tenantClause(sql`j.tenant_key`, input.tenantKey)}${versionClause(sql`j`, input.version)}
|
|
89
|
+
FOR UPDATE OF j SKIP LOCKED
|
|
90
|
+
),
|
|
91
|
+
${claimAndOpenAttempt(input.leaseToken, "nudge")}
|
|
92
|
+
SELECT req.id::text AS req_id,
|
|
93
|
+
CASE
|
|
94
|
+
WHEN c.id IS NOT NULL THEN 'claimed'
|
|
95
|
+
WHEN j.id IS NULL THEN 'not_found'
|
|
96
|
+
WHEN NOT EXISTS (SELECT 1 FROM k WHERE k.kind = j.kind) THEN 'unregistered_kind'
|
|
97
|
+
WHEN j.status IN ('succeeded', 'dead', 'cancelled') THEN 'terminal'
|
|
98
|
+
WHEN j.status = 'processing' THEN 'leased'
|
|
99
|
+
WHEN NOT ${isDue(sql`j`)} THEN 'not_due'
|
|
100
|
+
WHEN ${input.version === void 0 ? sql`false` : sql`NOT (TRUE${versionClause(sql`j`, input.version)})`} THEN 'newer_version'
|
|
101
|
+
WHEN NOT (SELECT fits FROM k WHERE k.kind = j.kind) THEN 'over_budget'
|
|
102
|
+
ELSE 'locked'
|
|
103
|
+
END AS result,
|
|
104
|
+
c.*
|
|
105
|
+
FROM req
|
|
106
|
+
LEFT JOIN claimed c ON c.id = req.id
|
|
107
|
+
-- LATERAL forces a primary-key probe per id; a plain join may seq-scan the table.
|
|
108
|
+
LEFT JOIN LATERAL (SELECT * FROM "outbox"."job" WHERE id = req.id${tenantClause(sql`tenant_key`, input.tenantKey)}) j ON true`);
|
|
109
|
+
return [...invalid, ...rows.map((row) => row.result === "claimed" ? {
|
|
110
|
+
id: row.req_id,
|
|
111
|
+
result: "claimed",
|
|
112
|
+
job: toClaimedJob(row)
|
|
113
|
+
} : {
|
|
114
|
+
id: row.req_id,
|
|
115
|
+
result: row.result
|
|
116
|
+
})];
|
|
117
|
+
}
|
|
118
|
+
/** The lease token fences stale workers: they get `lease_lost` and write nothing. */
|
|
119
|
+
async function settleJob(db, input) {
|
|
120
|
+
const t = input.transition;
|
|
121
|
+
const failure = "code" in t ? t : null;
|
|
122
|
+
const code = failure?.code ?? null;
|
|
123
|
+
const message = failure ? truncate(failure.message) : null;
|
|
124
|
+
const codeStreak = failure?.codeStreak ?? 0;
|
|
125
|
+
const stickyCode = failure?.stickyCode ?? null;
|
|
126
|
+
const reason = t.to === "dead" || t.to === "cancelled" ? t.reason : null;
|
|
127
|
+
const delayMs = t.to === "retrying" ? t.delayMs : 0;
|
|
128
|
+
const finishes = t.to !== "retrying";
|
|
129
|
+
const expiredClause = input.requireExpiredLease ? sql` AND leased_until < (SELECT t FROM clock)` : sql``;
|
|
130
|
+
return firstRow(await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
131
|
+
settled AS (
|
|
132
|
+
UPDATE "outbox"."job" SET
|
|
133
|
+
status = ${t.to},
|
|
134
|
+
lease_token = NULL,
|
|
135
|
+
leased_until = NULL,
|
|
136
|
+
finished_attempts = finished_attempts + 1,
|
|
137
|
+
spent_attempts = spent_attempts + ${t.attemptOutcome === "lease_expired" ? 0 : 1},
|
|
138
|
+
next_attempt_at = ${t.to === "retrying" ? sql`(SELECT t FROM clock) + ${ms(delayMs)}` : sql`NULL`},
|
|
139
|
+
code_streak = ${codeStreak},
|
|
140
|
+
sticky_code = COALESCE(${stickyCode}, sticky_code),
|
|
141
|
+
last_error_code = COALESCE(${code}, last_error_code),
|
|
142
|
+
last_error_message = ${code === null ? sql`last_error_message` : sql`${message}`},
|
|
143
|
+
terminal_reason = ${reason},
|
|
144
|
+
finished_at = ${finishes ? sql`(SELECT t FROM clock)` : sql`NULL`},
|
|
145
|
+
updated_at = (SELECT t FROM clock)
|
|
146
|
+
WHERE id = ${input.jobId} AND lease_token = ${input.leaseToken} AND status = 'processing'${expiredClause}
|
|
147
|
+
RETURNING id, started_attempts
|
|
148
|
+
),
|
|
149
|
+
closed AS (
|
|
150
|
+
UPDATE "outbox"."attempt" a SET
|
|
151
|
+
finished_at = (SELECT t FROM clock),
|
|
152
|
+
outcome = ${t.attemptOutcome},
|
|
153
|
+
code = ${code},
|
|
154
|
+
message = ${message},
|
|
155
|
+
duration_ms = LEAST(2147483647, GREATEST(0, EXTRACT(EPOCH FROM ((SELECT t FROM clock) - a.started_at)) * 1000))::int
|
|
156
|
+
FROM settled
|
|
157
|
+
WHERE a.job_id = settled.id AND a.attempt_no = settled.started_attempts AND a.lease_token = ${input.leaseToken}
|
|
158
|
+
RETURNING 1
|
|
159
|
+
)
|
|
160
|
+
SELECT (SELECT count(*) FROM settled)::int AS applied`), "settleJob").applied === 1 ? "applied" : "lease_lost";
|
|
161
|
+
}
|
|
162
|
+
/** Lock-free pre-check before a settle hook; `settleJob` stays the authoritative fence. */
|
|
163
|
+
async function isLeaseHeld(db, input) {
|
|
164
|
+
if (!isUuid(input.jobId) || !isUuid(input.leaseToken)) return false;
|
|
165
|
+
const expiredClause = input.requireExpiredLease ? sql` AND leased_until < ${dbNow(input.now)}` : sql``;
|
|
166
|
+
return firstRow(await queryRows(db, sql`SELECT EXISTS (
|
|
167
|
+
SELECT 1 FROM "outbox"."job"
|
|
168
|
+
WHERE id = ${input.jobId} AND lease_token = ${input.leaseToken} AND status = 'processing'${expiredClause}
|
|
169
|
+
) AS held, now() AS read_at`), "isLeaseHeld").held;
|
|
170
|
+
}
|
|
171
|
+
/**
|
|
172
|
+
* Native bounds for the rest of the current transaction (PostgreSQL 17+), in one round trip:
|
|
173
|
+
* the whole transaction (FATAL 25P04: the session ends), each statement (57014), each lock wait (55P03).
|
|
174
|
+
*/
|
|
175
|
+
async function setSettlementTimeouts(tx, input) {
|
|
176
|
+
const toMs = (value) => `'${Math.max(1, Math.ceil(value))}ms'`;
|
|
177
|
+
await tx.execute(sql.raw(`SELECT set_config('transaction_timeout', ${toMs(input.transactionMs)}, true), set_config('statement_timeout', ${toMs(input.statementMs)}, true), set_config('lock_timeout', ${toMs(input.lockMs)}, true), now() AS read_at`));
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* How the attempt opened under this lease ended, if it did: resolves a settle whose
|
|
181
|
+
* connection dropped after the write may have committed.
|
|
182
|
+
*/
|
|
183
|
+
async function readAttemptOutcome(db, input) {
|
|
184
|
+
return (await queryRows(db, sql`SELECT outcome, now() AS read_at FROM "outbox"."attempt"
|
|
185
|
+
WHERE job_id = ${input.jobId} AND lease_token = ${input.leaseToken}`))[0]?.outcome ?? null;
|
|
186
|
+
}
|
|
187
|
+
async function listExpiredLeases(db, input) {
|
|
188
|
+
if (input.kinds?.length === 0 || input.limit < 1) return [];
|
|
189
|
+
const kindClause = input.kinds ? sql` AND kind = ANY(${textArray(input.kinds)})` : sql``;
|
|
190
|
+
return (await queryRows(db, sql`SELECT id, kind, lease_token, spent_attempts, code_streak, last_error_code, sticky_code,
|
|
191
|
+
tenant_key, correlation_id, metadata, started_attempts
|
|
192
|
+
FROM "outbox"."job"
|
|
193
|
+
WHERE status = 'processing' AND leased_until < ${dbNow(input.now)}${kindClause}
|
|
194
|
+
ORDER BY leased_until
|
|
195
|
+
LIMIT ${input.limit}`)).map((row) => ({
|
|
196
|
+
id: row.id,
|
|
197
|
+
kind: row.kind,
|
|
198
|
+
leaseToken: row.lease_token,
|
|
199
|
+
spentAttempts: row.spent_attempts,
|
|
200
|
+
codeStreak: row.code_streak,
|
|
201
|
+
lastErrorCode: row.last_error_code,
|
|
202
|
+
stickyCode: row.sticky_code,
|
|
203
|
+
tenantKey: row.tenant_key,
|
|
204
|
+
correlationId: row.correlation_id,
|
|
205
|
+
metadata: row.metadata,
|
|
206
|
+
attemptNo: row.started_attempts
|
|
207
|
+
}));
|
|
208
|
+
}
|
|
209
|
+
/** A job mid-attempt is `in_flight`: its worker owns it until it settles. */
|
|
210
|
+
async function cancelJob(db, input) {
|
|
211
|
+
if ("id" in input.ref && !isUuid(input.ref.id)) return "not_found";
|
|
212
|
+
const [row] = await updateMatching(db, {
|
|
213
|
+
match: "id" in input.ref ? sql`id = ${input.ref.id}` : sql`dedupe_key = ${input.ref.dedupeKey} AND tenant_key IS NOT DISTINCT FROM ${input.ref.tenantKey}`,
|
|
214
|
+
tenantKey: input.tenantKey,
|
|
215
|
+
now: input.now,
|
|
216
|
+
...cancellation(input.reason)
|
|
217
|
+
});
|
|
218
|
+
return row ? cancelResult(row) : "not_found";
|
|
219
|
+
}
|
|
220
|
+
/** Dead or cancelled → pending, with a fresh retry budget. */
|
|
221
|
+
async function replayJob(db, input) {
|
|
222
|
+
if (!isUuid(input.id)) return "not_found";
|
|
223
|
+
const [row] = await updateMatching(db, {
|
|
224
|
+
match: sql`id = ${input.id}`,
|
|
225
|
+
tenantKey: input.tenantKey,
|
|
226
|
+
now: input.now,
|
|
227
|
+
...REPLAY
|
|
228
|
+
});
|
|
229
|
+
return row ? replayResult(row) : "not_found";
|
|
230
|
+
}
|
|
231
|
+
async function deleteFinishedJobs(db, input) {
|
|
232
|
+
const expired = sql`((status = 'succeeded' AND finished_at < (SELECT t FROM clock) - ${ms(input.succeededRetentionMs)})
|
|
233
|
+
OR (status IN ('dead', 'cancelled') AND finished_at < (SELECT t FROM clock) - ${ms(input.terminalRetentionMs)}))`;
|
|
234
|
+
return firstRow(await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
235
|
+
doomed AS (
|
|
236
|
+
SELECT id FROM "outbox"."job"
|
|
237
|
+
WHERE ${expired}
|
|
238
|
+
LIMIT ${input.batch}
|
|
239
|
+
),
|
|
240
|
+
deleted AS (DELETE FROM "outbox"."job" WHERE id IN (SELECT id FROM doomed) AND ${expired} RETURNING 1)
|
|
241
|
+
SELECT count(*)::int AS deleted FROM deleted`), "deleteFinishedJobs").deleted;
|
|
242
|
+
}
|
|
243
|
+
async function hasDueJobs(db, input) {
|
|
244
|
+
const claimable = input.kinds.filter((k) => k.fits);
|
|
245
|
+
if (claimable.length === 0) return false;
|
|
246
|
+
return firstRow(await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t)
|
|
247
|
+
SELECT EXISTS (
|
|
248
|
+
SELECT 1 FROM "outbox"."job" j WHERE j.kind = ANY(${textArray(claimable.map((k) => k.kind))}) AND ${isDue(sql`j`)}${versionClause(sql`j`, input.version)}
|
|
249
|
+
) AS due`), "hasDueJobs").due;
|
|
250
|
+
}
|
|
251
|
+
async function listDueJobIds(db, input) {
|
|
252
|
+
if (input.kinds.length === 0 || input.limit < 1) return [];
|
|
253
|
+
return (await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t)
|
|
254
|
+
SELECT j.id FROM "outbox"."job" j
|
|
255
|
+
WHERE j.kind = ANY(${textArray(input.kinds)}) AND ${isDue(sql`j`)}
|
|
256
|
+
AND (j.nudge_sent_at IS NULL OR j.nudge_sent_at <= (SELECT t FROM clock) - ${ms(input.nudgeCooldownMs)})${tenantClause(sql`j.tenant_key`, input.tenantKey)}${versionClause(sql`j`, input.version)}
|
|
257
|
+
ORDER BY COALESCE(j.next_attempt_at, j.available_at), j.created_at, j.id
|
|
258
|
+
LIMIT ${input.limit}`)).map((row) => row.id);
|
|
259
|
+
}
|
|
260
|
+
async function countBacklog(db, input) {
|
|
261
|
+
if (input.kinds?.length === 0) return [];
|
|
262
|
+
const kindClause = input.kinds ? sql` AND j.kind = ANY(${textArray(input.kinds)})` : sql``;
|
|
263
|
+
return (await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
264
|
+
active AS (
|
|
265
|
+
SELECT j.kind, j.status, j.leased_until,
|
|
266
|
+
CASE j.status WHEN 'pending' THEN j.available_at WHEN 'retrying' THEN j.next_attempt_at END AS due_at
|
|
267
|
+
FROM "outbox"."job" j
|
|
268
|
+
WHERE j.status IN ('pending', 'retrying', 'processing')${kindClause}${tenantClause(sql`j.tenant_key`, input.tenantKey)}
|
|
269
|
+
)
|
|
270
|
+
SELECT kind,
|
|
271
|
+
count(*) FILTER (WHERE due_at <= (SELECT t FROM clock))::int AS due,
|
|
272
|
+
count(*) FILTER (WHERE due_at < (SELECT t FROM clock) - ${ms(input.agedMs)})::int AS aged,
|
|
273
|
+
count(*) FILTER (WHERE due_at > (SELECT t FROM clock))::int AS scheduled,
|
|
274
|
+
count(*) FILTER (WHERE status = 'processing')::int AS processing,
|
|
275
|
+
count(*) FILTER (WHERE status = 'processing' AND leased_until < (SELECT t FROM clock))::int AS stuck,
|
|
276
|
+
(max(EXTRACT(EPOCH FROM ((SELECT t FROM clock) - due_at))) FILTER (WHERE due_at <= (SELECT t FROM clock)) * 1000)::float8 AS oldest_due_age_ms
|
|
277
|
+
FROM active
|
|
278
|
+
GROUP BY kind
|
|
279
|
+
ORDER BY kind`)).map((row) => ({
|
|
280
|
+
kind: row.kind,
|
|
281
|
+
due: row.due,
|
|
282
|
+
aged: row.aged,
|
|
283
|
+
scheduled: row.scheduled,
|
|
284
|
+
processing: row.processing,
|
|
285
|
+
stuck: row.stuck,
|
|
286
|
+
oldestDueAgeMs: row.oldest_due_age_ms === null ? null : Math.round(row.oldest_due_age_ms)
|
|
287
|
+
}));
|
|
288
|
+
}
|
|
289
|
+
const optional = (value, clause) => value === void 0 ? sql`` : clause(value);
|
|
290
|
+
const jobFilter = (filter) => sql`j.status = ${filter.status}${optional(filter.kind, (v) => sql` AND j.kind = ${v}`)}${tenantClause(sql`j.tenant_key`, filter.tenantKey)}${optional(filter.dedupeKey, (v) => sql` AND j.dedupe_key = ${v}`)}${optional(filter.terminalReason, (v) => sql` AND j.terminal_reason = ${v}`)}${optional(filter.createdAfter, (v) => sql` AND j.created_at > ${v.toISOString()}::timestamptz`)}${optional(filter.createdBefore, (v) => sql` AND j.created_at < ${v.toISOString()}::timestamptz`)}${optional(filter.finishedAfter, (v) => sql` AND j.finished_at > ${v.toISOString()}::timestamptz`)}${optional(filter.finishedBefore, (v) => sql` AND j.finished_at < ${v.toISOString()}::timestamptz`)}`;
|
|
291
|
+
const jobSummaryColumns = sql`j.id, j.kind, j.status, j.dedupe_key, j.tenant_key, j.correlation_id, j.metadata,
|
|
292
|
+
j.started_attempts, j.finished_attempts, j.spent_attempts, j.code_streak, j.sticky_code,
|
|
293
|
+
j.last_error_code, j.last_error_message, j.terminal_reason, j.nudge_attempts, j.last_nudge_error,
|
|
294
|
+
${epochMs(sql`j.available_at`)} AS available_at, ${epochMs(sql`j.next_attempt_at`)} AS next_attempt_at,
|
|
295
|
+
${epochMs(sql`j.leased_until`)} AS leased_until, ${epochMs(sql`j.created_at`)} AS created_at,
|
|
296
|
+
${epochMs(sql`j.updated_at`)} AS updated_at, ${epochMs(sql`j.finished_at`)} AS finished_at,
|
|
297
|
+
${epochMs(sql`j.nudge_sent_at`)} AS nudge_sent_at, now() AS read_at`;
|
|
298
|
+
/** Keyset-paginated: pass `nextCursor` back as `cursor`. */
|
|
299
|
+
async function listJobs(db, input) {
|
|
300
|
+
const after = input.cursor ? parseCursor(input.cursor) : null;
|
|
301
|
+
const afterClause = after ? sql` AND (j.created_at, j.id) > (${after.createdAt}::timestamptz, ${after.id}::uuid)` : sql``;
|
|
302
|
+
const rows = await queryRows(db, sql`SELECT ${jobSummaryColumns}
|
|
303
|
+
FROM "outbox"."job" j
|
|
304
|
+
WHERE ${jobFilter(input)}${afterClause}
|
|
305
|
+
ORDER BY j.created_at, j.id
|
|
306
|
+
LIMIT ${Math.max(0, input.limit) + 1}`);
|
|
307
|
+
const page = rows.slice(0, Math.max(0, input.limit)).map(toJobSummary);
|
|
308
|
+
const last = page.at(-1);
|
|
309
|
+
return {
|
|
310
|
+
jobs: page,
|
|
311
|
+
nextCursor: rows.length > page.length && last ? `${last.createdAt.toISOString()}|${last.id}` : null
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
async function countJobs(db, input) {
|
|
315
|
+
return firstRow(await queryRows(db, sql`SELECT count(*)::int AS count, now() AS read_at FROM "outbox"."job" j WHERE ${jobFilter(input)}`), "countJobs").count;
|
|
316
|
+
}
|
|
317
|
+
async function aggregateAttemptOutcomes(db, input) {
|
|
318
|
+
const filters = sql.join([
|
|
319
|
+
input.kinds ? sql` AND j.kind = ANY(${textArray(input.kinds)})` : sql``,
|
|
320
|
+
tenantClause(sql`j.tenant_key`, input.tenantKey),
|
|
321
|
+
input.outcomes ? sql` AND a.outcome = ANY(${textArray(input.outcomes)})` : sql``,
|
|
322
|
+
input.code === void 0 ? sql`` : sql` AND a.code = ${input.code}`
|
|
323
|
+
], sql``);
|
|
324
|
+
const maxRows = Math.max(0, input.maxRows);
|
|
325
|
+
const row = firstRow(await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
326
|
+
w AS (SELECT (SELECT t FROM clock) - ${ms(input.windowMs)} AS cutoff),
|
|
327
|
+
g AS (
|
|
328
|
+
SELECT j.kind, j.tenant_key, a.outcome, a.code, count(*)::int AS count
|
|
329
|
+
FROM "outbox"."attempt" a
|
|
330
|
+
JOIN "outbox"."job" j ON j.id = a.job_id
|
|
331
|
+
WHERE a.finished_at >= (SELECT cutoff FROM w)${filters}
|
|
332
|
+
GROUP BY j.kind, j.tenant_key, a.outcome, a.code
|
|
333
|
+
ORDER BY count DESC, j.kind, j.tenant_key NULLS FIRST, a.outcome, a.code NULLS FIRST
|
|
334
|
+
LIMIT ${maxRows + 1}
|
|
335
|
+
)
|
|
336
|
+
SELECT ${epochMs(sql`(SELECT cutoff FROM w)`)} AS cutoff_ms,
|
|
337
|
+
COALESCE((SELECT json_agg(g ORDER BY g.count DESC, g.kind, g.tenant_key NULLS FIRST, g.outcome, g.code NULLS FIRST) FROM g), '[]'::json) AS groups`), "aggregateAttemptOutcomes");
|
|
338
|
+
return {
|
|
339
|
+
cutoff: new Date(row.cutoff_ms),
|
|
340
|
+
groups: row.groups.slice(0, maxRows).map((g) => ({
|
|
341
|
+
kind: g.kind,
|
|
342
|
+
tenantKey: g.tenant_key,
|
|
343
|
+
outcome: g.outcome,
|
|
344
|
+
code: g.code,
|
|
345
|
+
count: g.count
|
|
346
|
+
})),
|
|
347
|
+
truncated: row.groups.length > maxRows
|
|
348
|
+
};
|
|
349
|
+
}
|
|
350
|
+
/** Only waiting jobs are stamped: bookkeeping must never wait on a claimed row's settle lock. */
|
|
351
|
+
async function recordNudgeResult(db, input) {
|
|
352
|
+
const ids = [...new Set(input.ids)].filter((id) => isUuid(id));
|
|
353
|
+
if (ids.length === 0) return 0;
|
|
354
|
+
const stamp = input.error === null ? sql`nudge_sent_at = (SELECT t FROM clock), last_nudge_error = NULL` : sql`last_nudge_error = ${truncate(input.error, MAX_NUDGE_ERROR_LENGTH)}`;
|
|
355
|
+
return firstRow(await queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
356
|
+
stamped AS (
|
|
357
|
+
UPDATE "outbox"."job" SET nudge_attempts = nudge_attempts + 1, ${stamp}
|
|
358
|
+
WHERE id = ANY(${uuidArray(ids)}) AND status IN ('pending', 'retrying')
|
|
359
|
+
RETURNING 1
|
|
360
|
+
)
|
|
361
|
+
SELECT count(*)::int AS stamped FROM stamped`), "recordNudgeResult").stamped;
|
|
362
|
+
}
|
|
363
|
+
function toJobSummary(row) {
|
|
364
|
+
return {
|
|
365
|
+
id: row.id,
|
|
366
|
+
kind: row.kind,
|
|
367
|
+
status: row.status,
|
|
368
|
+
dedupeKey: row.dedupe_key,
|
|
369
|
+
tenantKey: row.tenant_key,
|
|
370
|
+
correlationId: row.correlation_id,
|
|
371
|
+
metadata: row.metadata,
|
|
372
|
+
startedAttempts: row.started_attempts,
|
|
373
|
+
finishedAttempts: row.finished_attempts,
|
|
374
|
+
spentAttempts: row.spent_attempts,
|
|
375
|
+
codeStreak: row.code_streak,
|
|
376
|
+
stickyCode: row.sticky_code,
|
|
377
|
+
lastErrorCode: row.last_error_code,
|
|
378
|
+
lastErrorMessage: row.last_error_message,
|
|
379
|
+
terminalReason: row.terminal_reason,
|
|
380
|
+
availableAt: new Date(row.available_at),
|
|
381
|
+
nextAttemptAt: toDate(row.next_attempt_at),
|
|
382
|
+
leasedUntil: toDate(row.leased_until),
|
|
383
|
+
createdAt: new Date(row.created_at),
|
|
384
|
+
updatedAt: new Date(row.updated_at),
|
|
385
|
+
finishedAt: toDate(row.finished_at),
|
|
386
|
+
nudgeSentAt: toDate(row.nudge_sent_at),
|
|
387
|
+
nudgeAttempts: row.nudge_attempts,
|
|
388
|
+
lastNudgeError: row.last_nudge_error
|
|
389
|
+
};
|
|
390
|
+
}
|
|
391
|
+
function parseCursor(cursor) {
|
|
392
|
+
const [createdAt = "", id = ""] = cursor.split("|");
|
|
393
|
+
if (!isUuid(id) || Number.isNaN(Date.parse(createdAt))) throw new Error(`listJobs: invalid cursor ${JSON.stringify(cursor)}`);
|
|
394
|
+
return {
|
|
395
|
+
createdAt,
|
|
396
|
+
id
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
/** One statement claims and opens the attempt: a claim never exists without its attempt. */
|
|
400
|
+
function claimAndOpenAttempt(leaseToken, trigger) {
|
|
401
|
+
return sql`claimed AS (
|
|
402
|
+
UPDATE "outbox"."job" j SET
|
|
403
|
+
status = 'processing',
|
|
404
|
+
started_attempts = j.started_attempts + 1,
|
|
405
|
+
next_attempt_at = NULL,
|
|
406
|
+
lease_token = ${leaseToken},
|
|
407
|
+
leased_until = (SELECT t FROM clock) + k.lease_ms * interval '1 millisecond',
|
|
408
|
+
updated_at = (SELECT t FROM clock)
|
|
409
|
+
FROM picked, k
|
|
410
|
+
WHERE j.id = picked.id AND k.kind = j.kind
|
|
411
|
+
RETURNING j.id, j.kind, j.payload, j.tenant_key, j.correlation_id, j.metadata,
|
|
412
|
+
j.lease_token, j.started_attempts, j.spent_attempts, j.code_streak, j.last_error_code,
|
|
413
|
+
j.sticky_code
|
|
414
|
+
),
|
|
415
|
+
opened AS (
|
|
416
|
+
INSERT INTO "outbox"."attempt" (job_id, attempt_no, kind, lease_token, trigger, started_at)
|
|
417
|
+
SELECT id, started_attempts, kind, lease_token, ${trigger}, (SELECT t FROM clock) FROM claimed
|
|
418
|
+
RETURNING 1
|
|
419
|
+
)`;
|
|
420
|
+
}
|
|
421
|
+
function toClaimedJob(row) {
|
|
422
|
+
return {
|
|
423
|
+
id: row.id,
|
|
424
|
+
kind: row.kind,
|
|
425
|
+
payload: row.payload,
|
|
426
|
+
tenantKey: row.tenant_key,
|
|
427
|
+
correlationId: row.correlation_id,
|
|
428
|
+
metadata: row.metadata,
|
|
429
|
+
leaseToken: row.lease_token,
|
|
430
|
+
attemptNo: row.started_attempts,
|
|
431
|
+
spentAttempts: row.spent_attempts,
|
|
432
|
+
codeStreak: row.code_streak,
|
|
433
|
+
lastErrorCode: row.last_error_code,
|
|
434
|
+
stickyCode: row.sticky_code
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
function truncate(message, max = MAX_MESSAGE_LENGTH) {
|
|
438
|
+
if (message === null || message.length <= max) return message;
|
|
439
|
+
return `${message.slice(0, max - 1)}…`;
|
|
440
|
+
}
|
|
441
|
+
function firstRow(rows, statement) {
|
|
442
|
+
const row = rows[0];
|
|
443
|
+
if (!row) throw new Error(`${statement}: statement returned no row`);
|
|
444
|
+
return row;
|
|
445
|
+
}
|
|
446
|
+
/** One job with its attempt ledger, for an operator view. Payload excluded. */
|
|
447
|
+
async function getJob(db, input) {
|
|
448
|
+
if (!isUuid(input.id)) return null;
|
|
449
|
+
const [row] = await queryRows(db, sql`SELECT ${jobSummaryColumns}
|
|
450
|
+
FROM "outbox"."job" j
|
|
451
|
+
WHERE j.id = ${input.id}${tenantClause(sql`j.tenant_key`, input.tenantKey)}`);
|
|
452
|
+
if (!row) return null;
|
|
453
|
+
const attempts = await queryRows(db, sql`SELECT attempt_no, trigger, outcome, code, message, duration_ms,
|
|
454
|
+
${epochMs(sql`started_at`)} AS started_at, ${epochMs(sql`finished_at`)} AS finished_at, now() AS read_at
|
|
455
|
+
FROM "outbox"."attempt" WHERE job_id = ${input.id}
|
|
456
|
+
ORDER BY attempt_no`);
|
|
457
|
+
return {
|
|
458
|
+
...toJobSummary(row),
|
|
459
|
+
attempts: attempts.map((a) => ({
|
|
460
|
+
attemptNo: a.attempt_no,
|
|
461
|
+
trigger: a.trigger,
|
|
462
|
+
outcome: a.outcome,
|
|
463
|
+
code: a.code,
|
|
464
|
+
message: a.message,
|
|
465
|
+
durationMs: a.duration_ms,
|
|
466
|
+
startedAt: new Date(a.started_at),
|
|
467
|
+
finishedAt: toDate(a.finished_at)
|
|
468
|
+
}))
|
|
469
|
+
};
|
|
470
|
+
}
|
|
471
|
+
/** Every job one request produced, newest first. */
|
|
472
|
+
async function findJobsByCorrelationId(db, input) {
|
|
473
|
+
return (await queryRows(db, sql`SELECT ${jobSummaryColumns}
|
|
474
|
+
FROM "outbox"."job" j
|
|
475
|
+
WHERE j.correlation_id = ${input.correlationId}${tenantClause(sql`j.tenant_key`, input.tenantKey)}
|
|
476
|
+
ORDER BY j.created_at DESC, j.id DESC
|
|
477
|
+
LIMIT ${Math.max(0, input.limit)}`)).map(toJobSummary);
|
|
478
|
+
}
|
|
479
|
+
/** Makes a pending or retrying job due now ("retry now"). */
|
|
480
|
+
async function rescheduleNow(db, input) {
|
|
481
|
+
if (!isUuid(input.id)) return {
|
|
482
|
+
result: "not_found",
|
|
483
|
+
kind: null
|
|
484
|
+
};
|
|
485
|
+
const [row] = await updateMatching(db, {
|
|
486
|
+
match: sql`id = ${input.id}`,
|
|
487
|
+
tenantKey: input.tenantKey,
|
|
488
|
+
now: input.now,
|
|
489
|
+
set: sql`available_at = CASE WHEN status = 'pending' THEN (SELECT t FROM clock) ELSE available_at END,
|
|
490
|
+
next_attempt_at = CASE WHEN status = 'retrying' THEN (SELECT t FROM clock) ELSE next_attempt_at END,
|
|
491
|
+
updated_at = (SELECT t FROM clock)`,
|
|
492
|
+
allowed: WAITING
|
|
493
|
+
});
|
|
494
|
+
if (!row) return {
|
|
495
|
+
result: "not_found",
|
|
496
|
+
kind: null
|
|
497
|
+
};
|
|
498
|
+
return {
|
|
499
|
+
result: row.done ? "rescheduled" : "not_waiting",
|
|
500
|
+
kind: row.kind
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
/**
|
|
504
|
+
* Erasure: deletes one tenant's jobs (attempts cascade), in flight or not. A worker
|
|
505
|
+
* mid-attempt then settles `lease_lost`. One bounded batch per call.
|
|
506
|
+
*/
|
|
507
|
+
async function deleteTenantJobs(db, input) {
|
|
508
|
+
return firstRow(await queryRows(db, sql`WITH doomed AS (
|
|
509
|
+
SELECT id FROM "outbox"."job" WHERE tenant_key = ${input.tenantKey} LIMIT ${input.batch}
|
|
510
|
+
),
|
|
511
|
+
deleted AS (DELETE FROM "outbox"."job" WHERE id IN (SELECT id FROM doomed) RETURNING 1)
|
|
512
|
+
SELECT count(*)::int AS deleted, now() AS read_at FROM deleted`), "deleteTenantJobs").deleted;
|
|
513
|
+
}
|
|
514
|
+
/** Cancels many waiting jobs in one statement; every requested id gets a result. */
|
|
515
|
+
async function cancelJobs(db, input) {
|
|
516
|
+
const rows = await updateByIds(db, input, cancellation(input.reason));
|
|
517
|
+
return mapBulk(input.ids, rows, cancelResult);
|
|
518
|
+
}
|
|
519
|
+
/** Replays many dead or cancelled jobs in one statement, each with a fresh budget. */
|
|
520
|
+
async function replayJobs(db, input) {
|
|
521
|
+
const rows = await updateByIds(db, input, REPLAY);
|
|
522
|
+
return mapBulk(input.ids, rows, replayResult);
|
|
523
|
+
}
|
|
524
|
+
const WAITING = sql`status IN ('pending', 'retrying')`;
|
|
525
|
+
const cancellation = (reason) => ({
|
|
526
|
+
set: sql`status = 'cancelled', terminal_reason = ${reason}, next_attempt_at = NULL,
|
|
527
|
+
finished_at = (SELECT t FROM clock), updated_at = (SELECT t FROM clock)`,
|
|
528
|
+
allowed: WAITING
|
|
529
|
+
});
|
|
530
|
+
const REPLAY = {
|
|
531
|
+
set: sql`status = 'pending', available_at = (SELECT t FROM clock), next_attempt_at = NULL,
|
|
532
|
+
terminal_reason = NULL, finished_at = NULL, spent_attempts = 0, code_streak = 0,
|
|
533
|
+
sticky_code = NULL, updated_at = (SELECT t FROM clock)`,
|
|
534
|
+
allowed: sql`status IN ('dead', 'cancelled')`
|
|
535
|
+
};
|
|
536
|
+
const cancelResult = (row) => row.done ? "cancelled" : isTerminal(row.status) ? "already_terminal" : "in_flight";
|
|
537
|
+
const replayResult = (row) => row.done ? "replayed" : "not_replayable";
|
|
538
|
+
/** One statement: every matching job (in the tenant scope) reports its status, and whether it changed. */
|
|
539
|
+
function updateMatching(db, input) {
|
|
540
|
+
return queryRows(db, sql`WITH clock AS (SELECT ${dbNow(input.now)} AS t),
|
|
541
|
+
target AS (
|
|
542
|
+
SELECT id, kind, status FROM "outbox"."job"
|
|
543
|
+
WHERE ${input.match}${tenantClause(sql`tenant_key`, input.tenantKey)}
|
|
544
|
+
),
|
|
545
|
+
updated AS (
|
|
546
|
+
UPDATE "outbox"."job" SET ${input.set}
|
|
547
|
+
WHERE id IN (SELECT id FROM target) AND ${input.allowed}
|
|
548
|
+
RETURNING id
|
|
549
|
+
)
|
|
550
|
+
SELECT target.id::text AS id, target.kind, target.status, (updated.id IS NOT NULL) AS done
|
|
551
|
+
FROM target LEFT JOIN updated USING (id)`);
|
|
552
|
+
}
|
|
553
|
+
async function updateByIds(db, input, change) {
|
|
554
|
+
const valid = [...new Set(input.ids)].filter((id) => isUuid(id));
|
|
555
|
+
if (valid.length === 0) return [];
|
|
556
|
+
return updateMatching(db, {
|
|
557
|
+
...change,
|
|
558
|
+
match: sql`id = ANY(${uuidArray(valid)})`,
|
|
559
|
+
tenantKey: input.tenantKey,
|
|
560
|
+
now: input.now
|
|
561
|
+
});
|
|
562
|
+
}
|
|
563
|
+
function mapBulk(ids, rows, decide) {
|
|
564
|
+
const byId = new Map(rows.map((row) => [row.id, row]));
|
|
565
|
+
const results = {};
|
|
566
|
+
for (const id of new Set(ids)) {
|
|
567
|
+
const row = byId.get(id.toLowerCase());
|
|
568
|
+
results[id] = row ? decide(row) : "not_found";
|
|
569
|
+
}
|
|
570
|
+
return results;
|
|
571
|
+
}
|
|
572
|
+
// src/core/backlog.ts
|
|
573
|
+
const DEFAULT_BACKLOG_THRESHOLDS = {
|
|
574
|
+
maxDue: 1e3,
|
|
575
|
+
maxAged: 100,
|
|
576
|
+
maxStuck: 0,
|
|
577
|
+
criticalFactor: 2,
|
|
578
|
+
maxSeverity: "critical"
|
|
579
|
+
};
|
|
580
|
+
/** Milliseconds a due job may wait before it counts as aged. */
|
|
581
|
+
const DEFAULT_BACKLOG_AGED_MS = 9e5;
|
|
582
|
+
const UNREGISTERED_THRESHOLDS = {
|
|
583
|
+
maxDue: 0,
|
|
584
|
+
maxAged: 0,
|
|
585
|
+
maxSeverity: "warning"
|
|
586
|
+
};
|
|
587
|
+
const SEVERITY_RANK = {
|
|
588
|
+
ok: 0,
|
|
589
|
+
warning: 1,
|
|
590
|
+
critical: 2
|
|
591
|
+
};
|
|
592
|
+
function evaluateBacklog(input) {
|
|
593
|
+
const byLane = /* @__PURE__ */ new Map();
|
|
594
|
+
for (const count of input.counts) {
|
|
595
|
+
const lane = input.laneOf(count.kind) ?? "(unregistered)";
|
|
596
|
+
byLane.set(lane, [...byLane.get(lane) ?? [], count]);
|
|
597
|
+
}
|
|
598
|
+
const lanes = [...byLane.entries()].map(([lane, kinds]) => gradeLane(lane, kinds, resolveThresholds(lane, input)));
|
|
599
|
+
const totals = sumTotals(lanes);
|
|
600
|
+
const severity = lanes.reduce((worst, l) => worse(worst, l.severity), "ok");
|
|
601
|
+
if (severity === "ok") return {
|
|
602
|
+
severity,
|
|
603
|
+
totals,
|
|
604
|
+
lanes,
|
|
605
|
+
alert: null
|
|
606
|
+
};
|
|
607
|
+
const breaching = lanes.filter((lane) => lane.severity !== "ok");
|
|
608
|
+
return {
|
|
609
|
+
severity,
|
|
610
|
+
totals,
|
|
611
|
+
lanes,
|
|
612
|
+
alert: {
|
|
613
|
+
severity,
|
|
614
|
+
message: alertMessage(severity, breaching),
|
|
615
|
+
lanes: breaching,
|
|
616
|
+
totals
|
|
617
|
+
}
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
function resolveThresholds(lane, input) {
|
|
621
|
+
return {
|
|
622
|
+
...DEFAULT_BACKLOG_THRESHOLDS,
|
|
623
|
+
...lane === "(unregistered)" ? UNREGISTERED_THRESHOLDS : input.thresholds,
|
|
624
|
+
...input.lanes?.[lane]
|
|
625
|
+
};
|
|
626
|
+
}
|
|
627
|
+
function gradeLane(lane, kinds, thresholds) {
|
|
628
|
+
const totals = sumTotals(kinds);
|
|
629
|
+
const breaches = [
|
|
630
|
+
[
|
|
631
|
+
"due",
|
|
632
|
+
totals.due,
|
|
633
|
+
thresholds.maxDue
|
|
634
|
+
],
|
|
635
|
+
[
|
|
636
|
+
"aged",
|
|
637
|
+
totals.aged,
|
|
638
|
+
thresholds.maxAged
|
|
639
|
+
],
|
|
640
|
+
[
|
|
641
|
+
"stuck",
|
|
642
|
+
totals.stuck,
|
|
643
|
+
thresholds.maxStuck
|
|
644
|
+
]
|
|
645
|
+
].flatMap(([metric, value, threshold]) => {
|
|
646
|
+
if (value <= threshold) return [];
|
|
647
|
+
return [{
|
|
648
|
+
metric,
|
|
649
|
+
value,
|
|
650
|
+
threshold,
|
|
651
|
+
severity: value >= threshold * thresholds.criticalFactor && thresholds.maxSeverity === "critical" ? "critical" : "warning"
|
|
652
|
+
}];
|
|
653
|
+
});
|
|
654
|
+
const ages = kinds.flatMap((k) => k.oldestDueAgeMs === null ? [] : [k.oldestDueAgeMs]);
|
|
655
|
+
return {
|
|
656
|
+
lane,
|
|
657
|
+
kinds,
|
|
658
|
+
...totals,
|
|
659
|
+
oldestDueAgeMs: ages.length === 0 ? null : Math.max(...ages),
|
|
660
|
+
severity: breaches.reduce((worst, b) => worse(worst, b.severity), "ok"),
|
|
661
|
+
breaches,
|
|
662
|
+
thresholds
|
|
663
|
+
};
|
|
664
|
+
}
|
|
665
|
+
function sumTotals(rows) {
|
|
666
|
+
return rows.reduce((sum, row) => ({
|
|
667
|
+
due: sum.due + row.due,
|
|
668
|
+
aged: sum.aged + row.aged,
|
|
669
|
+
scheduled: sum.scheduled + row.scheduled,
|
|
670
|
+
processing: sum.processing + row.processing,
|
|
671
|
+
stuck: sum.stuck + row.stuck
|
|
672
|
+
}), {
|
|
673
|
+
due: 0,
|
|
674
|
+
aged: 0,
|
|
675
|
+
scheduled: 0,
|
|
676
|
+
processing: 0,
|
|
677
|
+
stuck: 0
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
function worse(a, b) {
|
|
681
|
+
return SEVERITY_RANK[b] > SEVERITY_RANK[a] ? b : a;
|
|
682
|
+
}
|
|
683
|
+
function alertMessage(severity, lanes) {
|
|
684
|
+
return `outbox backlog ${severity}: ${lanes.flatMap((lane) => lane.breaches.map((b) => `${lane.lane} ${b.metric} ${b.value} > ${b.threshold}`)).join("; ")}`;
|
|
685
|
+
}
|
|
686
|
+
export { isUuid as A, readAttemptOutcome as C, rescheduleNow as D, replayJobs as E, setSettlementTimeouts as O, listJobs as S, replayJob as T, hasDueJobs as _, DEFAULT_TERMINAL_RETENTION_MS as a, listDueJobIds as b, cancelJobs as c, countBacklog as d, countJobs as f, getJob as g, findJobsByCorrelationId as h, DEFAULT_SUCCEEDED_RETENTION_MS as i, settleJob as k, claimByIds as l, deleteTenantJobs as m, DEFAULT_BACKLOG_THRESHOLDS as n, aggregateAttemptOutcomes as o, deleteFinishedJobs as p, evaluateBacklog as r, cancelJob as s, DEFAULT_BACKLOG_AGED_MS as t, claimDue as u, insertJob as v, recordNudgeResult as w, listExpiredLeases as x, isLeaseHeld as y };
|