@openlup/outbox 0.0.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +65 -0
- package/CHANGELOG.md +27 -0
- package/LICENSE +204 -0
- package/README.md +94 -2
- package/dist/config.d.ts +4 -0
- package/dist/config.js +73 -0
- package/dist/contracts.d.ts +88 -0
- package/dist/contracts.js +1 -0
- package/dist/enqueue.d.ts +15 -0
- package/dist/enqueue.js +9 -0
- package/dist/factory.d.ts +73 -0
- package/dist/factory.js +99 -0
- package/dist/index.d.ts +10 -0
- package/dist/index.js +8 -0
- package/dist/postgres.d.ts +10 -0
- package/dist/postgres.js +79 -0
- package/dist/registry.d.ts +4 -0
- package/dist/registry.js +48 -0
- package/dist/testing.d.ts +5 -0
- package/dist/testing.js +16 -0
- package/dist/trace.d.ts +8 -0
- package/dist/trace.js +44 -0
- package/dist/typedEvents.d.ts +4 -0
- package/dist/typedEvents.js +9 -0
- package/dist/worker.d.ts +11 -0
- package/dist/worker.js +232 -0
- package/package.json +59 -4
- package/release-gates.json +53 -0
- package/sql/0001_outbox.sql +329 -0
- package/sql/manifest.json +11 -0
- package/src/config.ts +79 -0
- package/src/contracts.ts +86 -0
- package/src/enqueue.ts +15 -0
- package/src/factory.ts +107 -0
- package/src/index.ts +10 -0
- package/src/postgres.ts +76 -0
- package/src/registry.ts +49 -0
- package/src/testing.ts +13 -0
- package/src/trace.ts +52 -0
- package/src/typedEvents.ts +10 -0
- package/src/worker.ts +281 -0
package/dist/worker.js
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
import { executeHandler } from "./trace.js";
|
|
2
|
+
// Tail reserved per row: worst-case handler.timeoutMs + this mark-RPC margin.
|
|
3
|
+
const MARK_MARGIN_MS = 2000;
|
|
4
|
+
// R-8: a hung queueStats RPC must never starve finishJobRun past Vercel's
|
|
5
|
+
// kill — best-effort stats race a hard deadline (try/catch only covers throws).
|
|
6
|
+
const QUEUE_STATS_DEADLINE_MS = 3000;
|
|
7
|
+
const DISCARDED_EVENT_IDS_CAP = 25;
|
|
8
|
+
const SAFE_MESSAGE_MAX_CHARS = 300;
|
|
9
|
+
export async function runOutboxDispatchWorker(deps) {
|
|
10
|
+
const { store, registry, config } = deps;
|
|
11
|
+
const suppliedLogger = deps.logger ?? console;
|
|
12
|
+
const log = (method, args) => {
|
|
13
|
+
try {
|
|
14
|
+
suppliedLogger[method](...args);
|
|
15
|
+
}
|
|
16
|
+
catch { /* Diagnostic failures cannot interrupt dispatch. */ }
|
|
17
|
+
};
|
|
18
|
+
const logger = { warn: (...args) => log("warn", args), error: (...args) => log("error", args) };
|
|
19
|
+
const observe = (event, count) => {
|
|
20
|
+
try {
|
|
21
|
+
deps.observe?.({ event, count });
|
|
22
|
+
}
|
|
23
|
+
catch { /* Diagnostics cannot rewrite an outcome. */ }
|
|
24
|
+
};
|
|
25
|
+
const clock = deps.clock ?? { now: () => Date.now() };
|
|
26
|
+
const startedAt = clock.now();
|
|
27
|
+
const counters = {
|
|
28
|
+
claimed: 0,
|
|
29
|
+
processed: 0,
|
|
30
|
+
retried: 0,
|
|
31
|
+
snoozed: 0,
|
|
32
|
+
discarded: 0,
|
|
33
|
+
released: 0,
|
|
34
|
+
duplicateInRun: 0,
|
|
35
|
+
leaseLost: 0,
|
|
36
|
+
batches: 0,
|
|
37
|
+
};
|
|
38
|
+
const byEventType = {};
|
|
39
|
+
const discardedEventIds = [];
|
|
40
|
+
const bump = (eventType, key) => {
|
|
41
|
+
const bucket = (byEventType[eventType] ??= {});
|
|
42
|
+
bucket[key] = (bucket[key] ?? 0) + 1;
|
|
43
|
+
};
|
|
44
|
+
const finish = async (input) => {
|
|
45
|
+
let queue;
|
|
46
|
+
if (!input.skipped && deps.diagnostics) {
|
|
47
|
+
let deadlineTimer;
|
|
48
|
+
try {
|
|
49
|
+
queue = await Promise.race([
|
|
50
|
+
deps.diagnostics.queueStats(),
|
|
51
|
+
new Promise((resolve) => {
|
|
52
|
+
deadlineTimer = setTimeout(() => resolve(undefined), QUEUE_STATS_DEADLINE_MS);
|
|
53
|
+
}),
|
|
54
|
+
]);
|
|
55
|
+
if (queue === undefined)
|
|
56
|
+
logger.warn("[outbox-dispatch] queue_stats_failed", "deadline_exceeded");
|
|
57
|
+
}
|
|
58
|
+
catch (error) {
|
|
59
|
+
logger.warn("[outbox-dispatch] queue_stats_failed", safeMessage(error));
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
if (deadlineTimer !== undefined)
|
|
63
|
+
clearTimeout(deadlineTimer);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
observe("finished", counters.processed);
|
|
67
|
+
return {
|
|
68
|
+
ok: true,
|
|
69
|
+
checked: counters.claimed,
|
|
70
|
+
updated: counters.processed,
|
|
71
|
+
failures: counters.retried + counters.discarded,
|
|
72
|
+
skipped: input.skipped,
|
|
73
|
+
...(input.reason !== undefined ? { reason: input.reason } : {}),
|
|
74
|
+
...counters,
|
|
75
|
+
runMs: clock.now() - startedAt,
|
|
76
|
+
byEventType,
|
|
77
|
+
discardedEventIds,
|
|
78
|
+
...(queue !== undefined ? { queue } : {}),
|
|
79
|
+
};
|
|
80
|
+
};
|
|
81
|
+
const applyMarkFailed = async (row, claimToken, outcome, reason, benign = false) => {
|
|
82
|
+
const { status } = await store.markFailed({
|
|
83
|
+
eventId: row.id,
|
|
84
|
+
claimToken,
|
|
85
|
+
error: reason,
|
|
86
|
+
outcome,
|
|
87
|
+
baseDelaySeconds: config.backoffBaseSeconds,
|
|
88
|
+
maxDelaySeconds: config.backoffCapSeconds,
|
|
89
|
+
maxAttempts: config.maxAttempts,
|
|
90
|
+
snoozeSeconds: config.snoozeSeconds,
|
|
91
|
+
});
|
|
92
|
+
if (status === "failed") {
|
|
93
|
+
counters.retried += 1;
|
|
94
|
+
bump(row.event_type, "retried");
|
|
95
|
+
}
|
|
96
|
+
else if (status === "discarded") {
|
|
97
|
+
counters.discarded += 1;
|
|
98
|
+
bump(row.event_type, "discarded");
|
|
99
|
+
if (discardedEventIds.length < DISCARDED_EVENT_IDS_CAP)
|
|
100
|
+
discardedEventIds.push(row.id);
|
|
101
|
+
// Benign discard = expected terminal drop (deleted aggregate): same tag, logged at warn not error so it does not read as a failure.
|
|
102
|
+
const payload = { eventId: row.id, eventType: row.event_type, aggregateId: row.aggregate_id, attempts: row.attempts, reason, ...(benign ? { benign: true } : {}) };
|
|
103
|
+
(benign ? logger.warn : logger.error)("[outbox-dispatch] event_discarded", JSON.stringify(payload));
|
|
104
|
+
}
|
|
105
|
+
else if (status === "missed") {
|
|
106
|
+
counters.leaseLost += 1;
|
|
107
|
+
bump(row.event_type, "leaseLost");
|
|
108
|
+
logger.warn("[outbox-dispatch] outbox_lease_lost", row.id);
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
counters.snoozed += 1;
|
|
112
|
+
bump(row.event_type, "snoozed");
|
|
113
|
+
}
|
|
114
|
+
return status;
|
|
115
|
+
};
|
|
116
|
+
const allowlist = [...registry.keys()];
|
|
117
|
+
if (allowlist.length === 0) {
|
|
118
|
+
return finish({ skipped: true, reason: "outbox_registry_empty" });
|
|
119
|
+
}
|
|
120
|
+
const seen = new Set();
|
|
121
|
+
let circuitBroken = false;
|
|
122
|
+
drain: while (clock.now() - startedAt < config.softBudgetMs) {
|
|
123
|
+
const claimed = await store.claimBatch({
|
|
124
|
+
eventTypes: allowlist,
|
|
125
|
+
knownEventTypes: deps.knownEventTypes,
|
|
126
|
+
batchSize: config.batchSize,
|
|
127
|
+
visibilitySeconds: config.visibilitySeconds,
|
|
128
|
+
maxAttempts: config.maxAttempts,
|
|
129
|
+
});
|
|
130
|
+
if (claimed.length === 0)
|
|
131
|
+
break;
|
|
132
|
+
counters.batches += 1;
|
|
133
|
+
counters.claimed += claimed.length;
|
|
134
|
+
observe("claimed", claimed.length);
|
|
135
|
+
// RETURNING order is unspecified — re-establish (created_at, id) order.
|
|
136
|
+
const batch = [...claimed].sort(compareByCreatedAtThenId);
|
|
137
|
+
const unstarted = [];
|
|
138
|
+
for (let index = 0; index < batch.length; index += 1) {
|
|
139
|
+
const row = batch[index];
|
|
140
|
+
const claimToken = row.metadata?.claimToken;
|
|
141
|
+
if (typeof claimToken !== "string" || claimToken.length === 0) {
|
|
142
|
+
counters.leaseLost += 1;
|
|
143
|
+
bump(row.event_type, "leaseLost");
|
|
144
|
+
logger.error("[outbox-dispatch] missing_claim_token", row.id);
|
|
145
|
+
continue;
|
|
146
|
+
}
|
|
147
|
+
if (seen.has(row.id)) {
|
|
148
|
+
// An in-process aborted orphan may still race — never re-execute in-run.
|
|
149
|
+
counters.released += await store.releaseUnprocessed([{ eventId: row.id, claimToken }], config.backoffBaseSeconds);
|
|
150
|
+
counters.duplicateInRun += 1;
|
|
151
|
+
bump(row.event_type, "duplicateInRun");
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
seen.add(row.id);
|
|
155
|
+
const handler = registry.get(row.event_type);
|
|
156
|
+
if (!handler) {
|
|
157
|
+
await applyMarkFailed(row, claimToken, "retry", "outbox_handler_missing");
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
const elapsed = clock.now() - startedAt;
|
|
161
|
+
if (elapsed + handler.timeoutMs + MARK_MARGIN_MS > config.softBudgetMs) {
|
|
162
|
+
unstarted.push({ eventId: row.id, claimToken });
|
|
163
|
+
continue;
|
|
164
|
+
}
|
|
165
|
+
const outcome = await executeHandler(handler, row);
|
|
166
|
+
if (outcome.kind === "processed") {
|
|
167
|
+
const { applied } = await store.markProcessed({
|
|
168
|
+
eventId: row.id,
|
|
169
|
+
claimToken,
|
|
170
|
+
metadata: outcome.detail,
|
|
171
|
+
});
|
|
172
|
+
if (applied) {
|
|
173
|
+
counters.processed += 1;
|
|
174
|
+
bump(row.event_type, "processed");
|
|
175
|
+
}
|
|
176
|
+
else {
|
|
177
|
+
counters.leaseLost += 1;
|
|
178
|
+
bump(row.event_type, "leaseLost");
|
|
179
|
+
logger.warn("[outbox-dispatch] outbox_lease_lost", row.id);
|
|
180
|
+
}
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
if (outcome.kind === "snooze") {
|
|
184
|
+
const snoozeCount = Number(row.metadata?.snoozeCount) || 0;
|
|
185
|
+
if (snoozeCount >= config.maxSnoozes) {
|
|
186
|
+
// Liveness bound: a misclassified permanent error must not snooze
|
|
187
|
+
// forever — consume an attempt instead. No circuit-break.
|
|
188
|
+
await applyMarkFailed(row, claimToken, "retry", `snooze_budget_exhausted:${outcome.reason}`);
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
await applyMarkFailed(row, claimToken, "snooze", outcome.reason);
|
|
192
|
+
// Circuit-break: the provider is down — refund everything still
|
|
193
|
+
// claimed (tail-gated rows included) and stop hammering it.
|
|
194
|
+
const remainder = [...unstarted];
|
|
195
|
+
for (let rest = index + 1; rest < batch.length; rest += 1) {
|
|
196
|
+
const restToken = batch[rest].metadata?.claimToken;
|
|
197
|
+
if (typeof restToken === "string" && restToken.length > 0) {
|
|
198
|
+
remainder.push({ eventId: batch[rest].id, claimToken: restToken });
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
if (remainder.length > 0) {
|
|
202
|
+
counters.released += await store.releaseUnprocessed(remainder, config.snoozeSeconds);
|
|
203
|
+
}
|
|
204
|
+
circuitBroken = true;
|
|
205
|
+
break drain;
|
|
206
|
+
}
|
|
207
|
+
await applyMarkFailed(row, claimToken, outcome.kind, outcome.reason, outcome.kind === "discard" && outcome.benign === true);
|
|
208
|
+
}
|
|
209
|
+
if (unstarted.length > 0) {
|
|
210
|
+
counters.released += await store.releaseUnprocessed(unstarted, 0);
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
if (claimed.length < config.batchSize)
|
|
214
|
+
break;
|
|
215
|
+
}
|
|
216
|
+
return finish({
|
|
217
|
+
skipped: false,
|
|
218
|
+
...(circuitBroken ? { reason: "provider_outage" } : {}),
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
function compareByCreatedAtThenId(a, b) {
|
|
222
|
+
if (a.created_at < b.created_at)
|
|
223
|
+
return -1;
|
|
224
|
+
if (a.created_at > b.created_at)
|
|
225
|
+
return 1;
|
|
226
|
+
if (a.id < b.id)
|
|
227
|
+
return -1;
|
|
228
|
+
if (a.id > b.id)
|
|
229
|
+
return 1;
|
|
230
|
+
return 0;
|
|
231
|
+
}
|
|
232
|
+
function safeMessage(error) { return (error instanceof Error ? error.message : String(error)).slice(0, SAFE_MESSAGE_MAX_CHARS); }
|
package/package.json
CHANGED
|
@@ -1,9 +1,64 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@openlup/outbox",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
3
|
+
"version": "0.14.0",
|
|
4
|
+
"private": false,
|
|
5
|
+
"description": "Transactional PostgreSQL outbox rail; development preview.",
|
|
5
6
|
"license": "Apache-2.0",
|
|
6
7
|
"type": "module",
|
|
7
|
-
"
|
|
8
|
-
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/openlup/openlup.git",
|
|
11
|
+
"directory": "packages/outbox"
|
|
12
|
+
},
|
|
13
|
+
"engines": {
|
|
14
|
+
"node": "24.x"
|
|
15
|
+
},
|
|
16
|
+
"packageManager": "npm@11.19.0",
|
|
17
|
+
"sideEffects": false,
|
|
18
|
+
"files": [
|
|
19
|
+
"dist/**",
|
|
20
|
+
"src/**",
|
|
21
|
+
"sql/**",
|
|
22
|
+
"AGENTS.md",
|
|
23
|
+
"README.md",
|
|
24
|
+
"CHANGELOG.md",
|
|
25
|
+
"LICENSE",
|
|
26
|
+
"release-gates.json"
|
|
27
|
+
],
|
|
28
|
+
"scripts": {
|
|
29
|
+
"build": "tsc -p tsconfig.build.json",
|
|
30
|
+
"prepack": "npm run build",
|
|
31
|
+
"test": "vitest run test",
|
|
32
|
+
"prepublishOnly": "node -e \"throw new Error('OUTBOX_DIRECTORY_PUBLISH_REFUSED')\"",
|
|
33
|
+
"api:check": "node --experimental-strip-types ./scripts/api-contract.ts",
|
|
34
|
+
"api:update": "npm run build && node --experimental-strip-types ./scripts/api-contract.ts --update",
|
|
35
|
+
"ci": "npm run build && npm run test && npm run api:check",
|
|
36
|
+
"ci:required": "npm run test:consumer",
|
|
37
|
+
"test:consumer": "node --experimental-strip-types ./scripts/consumer-proof.ts"
|
|
38
|
+
},
|
|
39
|
+
"peerDependencies": {
|
|
40
|
+
"@openlup/core": "0.14.0"
|
|
41
|
+
},
|
|
42
|
+
"exports": {
|
|
43
|
+
".": {
|
|
44
|
+
"types": "./dist/index.d.ts",
|
|
45
|
+
"import": "./dist/index.js",
|
|
46
|
+
"default": "./dist/index.js"
|
|
47
|
+
},
|
|
48
|
+
"./postgres": {
|
|
49
|
+
"types": "./dist/postgres.d.ts",
|
|
50
|
+
"import": "./dist/postgres.js",
|
|
51
|
+
"default": "./dist/postgres.js"
|
|
52
|
+
},
|
|
53
|
+
"./testing": {
|
|
54
|
+
"types": "./dist/testing.d.ts",
|
|
55
|
+
"import": "./dist/testing.js",
|
|
56
|
+
"default": "./dist/testing.js"
|
|
57
|
+
}
|
|
58
|
+
},
|
|
59
|
+
"publishConfig": {
|
|
60
|
+
"access": "public",
|
|
61
|
+
"provenance": true,
|
|
62
|
+
"tag": "preview"
|
|
63
|
+
}
|
|
9
64
|
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schemaVersion": 5,
|
|
3
|
+
"kind": "rail",
|
|
4
|
+
"compatibilityPolicies": {
|
|
5
|
+
"internal-candidate": "Development-preview API; no stable compatibility claim.",
|
|
6
|
+
"testing-internal": "Synthetic protocol test support."
|
|
7
|
+
},
|
|
8
|
+
"packageSurface": {
|
|
9
|
+
".": {
|
|
10
|
+
"role": "rail",
|
|
11
|
+
"maturity": "experimental",
|
|
12
|
+
"compatibilityPolicy": "internal-candidate",
|
|
13
|
+
"packageSmokeEvidence": [
|
|
14
|
+
"test/packageContract.test.ts"
|
|
15
|
+
],
|
|
16
|
+
"entrypoint": "dist/index.d.ts",
|
|
17
|
+
"snapshot": "api/index.api.md"
|
|
18
|
+
},
|
|
19
|
+
"./postgres": {
|
|
20
|
+
"role": "rail",
|
|
21
|
+
"maturity": "experimental",
|
|
22
|
+
"compatibilityPolicy": "internal-candidate",
|
|
23
|
+
"packageSmokeEvidence": [
|
|
24
|
+
"test/packageContract.test.ts"
|
|
25
|
+
],
|
|
26
|
+
"entrypoint": "dist/postgres.d.ts",
|
|
27
|
+
"snapshot": "api/postgres.api.md"
|
|
28
|
+
},
|
|
29
|
+
"./testing": {
|
|
30
|
+
"role": "testing",
|
|
31
|
+
"maturity": "testing",
|
|
32
|
+
"compatibilityPolicy": "testing-internal",
|
|
33
|
+
"packageSmokeEvidence": [
|
|
34
|
+
"test/packageContract.test.ts"
|
|
35
|
+
],
|
|
36
|
+
"entrypoint": "dist/testing.d.ts",
|
|
37
|
+
"snapshot": "api/testing.api.md"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"pack": {
|
|
41
|
+
"code": {
|
|
42
|
+
"maxPackedBytes": 32768,
|
|
43
|
+
"maxFiles": 48
|
|
44
|
+
},
|
|
45
|
+
"docs": {
|
|
46
|
+
"maxPackedBytes": 16384,
|
|
47
|
+
"maxFiles": 12
|
|
48
|
+
},
|
|
49
|
+
"maxPackedBytes": 65536,
|
|
50
|
+
"maxUnpackedBytes": 262144,
|
|
51
|
+
"maxFiles": 64
|
|
52
|
+
}
|
|
53
|
+
}
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
-- Public platform communications outbox rail: the queue table, the dormant-event-type
|
|
2
|
+
-- allowlist, and the claim/ack protocol that drains them.
|
|
3
|
+
--
|
|
4
|
+
-- WHAT THIS FORWARD SHIPS, AND WHY IT IS THE FIRST ONE CARRYING FUNCTIONS.
|
|
5
|
+
-- The baseline and the read-parity forward both hold `grep -c 'CREATE FUNCTION'` = 0 on
|
|
6
|
+
-- purpose. This migration breaks that property deliberately, because the dispatch rail
|
|
7
|
+
-- has no non-RPC surface: claim is an atomic `UPDATE ... FOR UPDATE SKIP LOCKED` that
|
|
8
|
+
-- hands back a claim token and evaluates a per-aggregate ordering barrier inside the same
|
|
9
|
+
-- statement. Expressing it through the gateway's query builder would mean reimplementing
|
|
10
|
+
-- pessimistic locking in the application, i.e. a second and divergent rail. Four functions
|
|
11
|
+
-- is the whole crossing: no privilege model, no `private` schema, no queue statistics, no
|
|
12
|
+
-- pruning, no preview-matrix claim scope, and above all no enqueue -- rows are written by
|
|
13
|
+
-- capabilities that own their own business logic, and none of them is authored here.
|
|
14
|
+
--
|
|
15
|
+
-- AUTHORED, NOT COPIED. The protocol below was written for this migration against the
|
|
16
|
+
-- observable contract the platform outbox store port issues (five named RPC arguments sets,
|
|
17
|
+
-- a boolean/text/integer/SETOF return shape). It is not a cut of any historical migration:
|
|
18
|
+
-- the provider-specific idempotency window and the hardcoded product event-type vocabulary
|
|
19
|
+
-- that the managed implementation calls into are absent, the preview-matrix claim scope is
|
|
20
|
+
-- absent, row-level security and role grants are absent because this kernel creates no
|
|
21
|
+
-- roles to grant to, `SECURITY DEFINER` is absent for the same reason, every constraint is
|
|
22
|
+
-- named rather than left to a generated identifier, and `CREATE` is unconditional because a
|
|
23
|
+
-- manifest-ordered forward runs exactly once against a known prefix -- `IF NOT EXISTS` there
|
|
24
|
+
-- would hide the drift the migration ledger exists to catch. `SET search_path` is kept: it
|
|
25
|
+
-- is a real injection-hardening property that does not depend on a role model.
|
|
26
|
+
|
|
27
|
+
CREATE TABLE public.outbox_events (
|
|
28
|
+
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
|
29
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
30
|
+
available_at timestamptz NOT NULL DEFAULT now(),
|
|
31
|
+
processed_at timestamptz,
|
|
32
|
+
aggregate_type text NOT NULL,
|
|
33
|
+
aggregate_id uuid NOT NULL,
|
|
34
|
+
event_type text NOT NULL,
|
|
35
|
+
idempotency_key text NOT NULL,
|
|
36
|
+
status text NOT NULL DEFAULT 'pending',
|
|
37
|
+
attempts integer NOT NULL DEFAULT 0,
|
|
38
|
+
payload jsonb NOT NULL,
|
|
39
|
+
error text,
|
|
40
|
+
metadata jsonb NOT NULL DEFAULT '{}'::jsonb,
|
|
41
|
+
CONSTRAINT outbox_events_aggregate_type_nonempty_check CHECK (btrim(aggregate_type) <> ''),
|
|
42
|
+
CONSTRAINT outbox_events_event_type_nonempty_check CHECK (btrim(event_type) <> ''),
|
|
43
|
+
CONSTRAINT outbox_events_idempotency_key_nonempty_check CHECK (btrim(idempotency_key) <> ''),
|
|
44
|
+
CONSTRAINT outbox_events_attempts_check CHECK (attempts >= 0),
|
|
45
|
+
CONSTRAINT outbox_events_status_check
|
|
46
|
+
CHECK (status IN ('pending', 'processing', 'processed', 'failed', 'discarded')),
|
|
47
|
+
CONSTRAINT outbox_events_event_type_idempotency_key_key UNIQUE (event_type, idempotency_key)
|
|
48
|
+
);
|
|
49
|
+
|
|
50
|
+
CREATE INDEX idx_outbox_events_status_available
|
|
51
|
+
ON public.outbox_events (status, available_at);
|
|
52
|
+
|
|
53
|
+
CREATE INDEX idx_outbox_events_aggregate
|
|
54
|
+
ON public.outbox_events (aggregate_type, aggregate_id);
|
|
55
|
+
|
|
56
|
+
-- The claim barrier below refuses to skip an older unprocessed row for the same aggregate,
|
|
57
|
+
-- INCLUDING one whose event type the caller does not know about. Without a way to declare a
|
|
58
|
+
-- type intentionally unconsumed, a single retired producer would stall its aggregate
|
|
59
|
+
-- forever. This table is that declaration, and it ships empty: which types are dormant is
|
|
60
|
+
-- operator data, not platform content.
|
|
61
|
+
CREATE TABLE public.outbox_dormant_event_types (
|
|
62
|
+
event_type text PRIMARY KEY,
|
|
63
|
+
owner text NOT NULL,
|
|
64
|
+
reason text NOT NULL,
|
|
65
|
+
created_at timestamptz NOT NULL DEFAULT now(),
|
|
66
|
+
updated_at timestamptz NOT NULL DEFAULT now(),
|
|
67
|
+
CONSTRAINT outbox_dormant_event_types_event_type_nonempty_check CHECK (btrim(event_type) <> ''),
|
|
68
|
+
CONSTRAINT outbox_dormant_event_types_owner_nonempty_check CHECK (btrim(owner) <> ''),
|
|
69
|
+
CONSTRAINT outbox_dormant_event_types_reason_nonempty_check CHECK (btrim(reason) <> '')
|
|
70
|
+
);
|
|
71
|
+
|
|
72
|
+
-- Claim a batch for one allowlist of event types.
|
|
73
|
+
--
|
|
74
|
+
-- Three properties make this more than a SELECT, and each is load-bearing:
|
|
75
|
+
-- 1. an empty allowlist is refused, never widened -- a caller that forgot its filter must
|
|
76
|
+
-- not drain every event type on the instance;
|
|
77
|
+
-- 2. rows whose attempts already reached the ceiling are discarded before the scan, so an
|
|
78
|
+
-- exhausted row cannot be re-selected on every tick;
|
|
79
|
+
-- 3. per aggregate, an older unprocessed row blocks the newer one. A prior of an event
|
|
80
|
+
-- type the caller neither claims nor knows also blocks, unless it has been declared
|
|
81
|
+
-- dormant -- ordering within an aggregate is the guarantee producers rely on.
|
|
82
|
+
-- The claim token written into metadata is the authorization for every later acknowledgement:
|
|
83
|
+
-- knowing the row id is not enough, which is what makes a stale ack detectable rather than
|
|
84
|
+
-- destructive.
|
|
85
|
+
CREATE FUNCTION public.outbox_claim_batch(
|
|
86
|
+
p_event_types text[],
|
|
87
|
+
p_known_event_types text[] DEFAULT ARRAY[]::text[],
|
|
88
|
+
p_batch_size integer DEFAULT 25,
|
|
89
|
+
p_visibility_seconds integer DEFAULT 300,
|
|
90
|
+
p_max_attempts integer DEFAULT 8
|
|
91
|
+
)
|
|
92
|
+
RETURNS SETOF public.outbox_events
|
|
93
|
+
LANGUAGE plpgsql
|
|
94
|
+
SET search_path = public, pg_catalog
|
|
95
|
+
AS $$
|
|
96
|
+
DECLARE
|
|
97
|
+
v_batch integer := LEAST(GREATEST(COALESCE(p_batch_size, 25), 1), 100);
|
|
98
|
+
v_visibility integer := LEAST(GREATEST(COALESCE(p_visibility_seconds, 300), 300), 3600);
|
|
99
|
+
v_max integer := LEAST(GREATEST(COALESCE(p_max_attempts, 8), 1), 20);
|
|
100
|
+
v_known text[];
|
|
101
|
+
BEGIN
|
|
102
|
+
IF p_event_types IS NULL OR cardinality(p_event_types) = 0 THEN
|
|
103
|
+
RAISE EXCEPTION 'outbox_claim_allowlist_required' USING ERRCODE = '22023';
|
|
104
|
+
END IF;
|
|
105
|
+
|
|
106
|
+
-- Everything the caller can recognise: what it claims now, plus what it declares it knows
|
|
107
|
+
-- about. A prior outside this set is treated as foreign, and only the dormant table can
|
|
108
|
+
-- release it.
|
|
109
|
+
SELECT array_agg(DISTINCT event_type)
|
|
110
|
+
INTO v_known
|
|
111
|
+
FROM unnest(COALESCE(p_known_event_types, ARRAY[]::text[]) || p_event_types) AS known(event_type)
|
|
112
|
+
WHERE event_type IS NOT NULL AND btrim(event_type) <> '';
|
|
113
|
+
|
|
114
|
+
WITH exhausted AS (
|
|
115
|
+
SELECT event.id
|
|
116
|
+
FROM public.outbox_events event
|
|
117
|
+
WHERE event.event_type = ANY (p_event_types)
|
|
118
|
+
AND event.status IN ('pending', 'failed', 'processing')
|
|
119
|
+
AND event.available_at <= now()
|
|
120
|
+
AND event.attempts >= v_max
|
|
121
|
+
ORDER BY event.created_at, event.id
|
|
122
|
+
LIMIT 100
|
|
123
|
+
FOR UPDATE SKIP LOCKED
|
|
124
|
+
)
|
|
125
|
+
UPDATE public.outbox_events event
|
|
126
|
+
SET status = 'discarded',
|
|
127
|
+
error = left(COALESCE(event.error, '') || ' | max_attempts_exhausted_at_claim', 2000),
|
|
128
|
+
metadata = event.metadata || jsonb_build_object(
|
|
129
|
+
'discardReason', 'max_attempts',
|
|
130
|
+
'discardedAt', now(),
|
|
131
|
+
'finalAttempts', event.attempts)
|
|
132
|
+
FROM exhausted
|
|
133
|
+
WHERE event.id = exhausted.id;
|
|
134
|
+
|
|
135
|
+
RETURN QUERY
|
|
136
|
+
WITH candidates AS (
|
|
137
|
+
SELECT event.id
|
|
138
|
+
FROM public.outbox_events event
|
|
139
|
+
WHERE event.event_type = ANY (p_event_types)
|
|
140
|
+
AND event.available_at <= now()
|
|
141
|
+
AND event.status IN ('pending', 'failed', 'processing')
|
|
142
|
+
AND event.attempts < v_max
|
|
143
|
+
AND NOT EXISTS (
|
|
144
|
+
SELECT 1
|
|
145
|
+
FROM public.outbox_events prior
|
|
146
|
+
WHERE prior.aggregate_type = event.aggregate_type
|
|
147
|
+
AND prior.aggregate_id = event.aggregate_id
|
|
148
|
+
AND prior.status IN ('pending', 'failed', 'processing')
|
|
149
|
+
AND (prior.created_at, prior.id) < (event.created_at, event.id)
|
|
150
|
+
AND (
|
|
151
|
+
prior.event_type = ANY (p_event_types)
|
|
152
|
+
OR (
|
|
153
|
+
prior.event_type <> ALL (COALESCE(v_known, ARRAY[]::text[]))
|
|
154
|
+
AND NOT EXISTS (
|
|
155
|
+
SELECT 1
|
|
156
|
+
FROM public.outbox_dormant_event_types dormant
|
|
157
|
+
WHERE dormant.event_type = prior.event_type
|
|
158
|
+
)
|
|
159
|
+
)
|
|
160
|
+
))
|
|
161
|
+
ORDER BY event.created_at, event.id
|
|
162
|
+
LIMIT v_batch
|
|
163
|
+
FOR UPDATE SKIP LOCKED
|
|
164
|
+
)
|
|
165
|
+
UPDATE public.outbox_events event
|
|
166
|
+
SET status = 'processing',
|
|
167
|
+
attempts = event.attempts + 1,
|
|
168
|
+
available_at = now() + make_interval(secs => v_visibility),
|
|
169
|
+
metadata = event.metadata || jsonb_build_object(
|
|
170
|
+
'lastClaimedAt', now(),
|
|
171
|
+
'claimToken', gen_random_uuid()::text)
|
|
172
|
+
FROM candidates
|
|
173
|
+
WHERE event.id = candidates.id
|
|
174
|
+
RETURNING event.*;
|
|
175
|
+
END;
|
|
176
|
+
$$;
|
|
177
|
+
|
|
178
|
+
-- Acknowledge success. Returns false rather than raising when the row is no longer the
|
|
179
|
+
-- caller's: a lease that expired and was re-claimed elsewhere is an ordinary race, and the
|
|
180
|
+
-- caller logs it instead of aborting a batch.
|
|
181
|
+
CREATE FUNCTION public.outbox_mark_processed(
|
|
182
|
+
p_event_id uuid,
|
|
183
|
+
p_claim_token text,
|
|
184
|
+
p_metadata jsonb DEFAULT '{}'::jsonb
|
|
185
|
+
)
|
|
186
|
+
RETURNS boolean
|
|
187
|
+
LANGUAGE sql
|
|
188
|
+
SET search_path = public, pg_catalog
|
|
189
|
+
AS $$
|
|
190
|
+
WITH updated AS (
|
|
191
|
+
UPDATE public.outbox_events
|
|
192
|
+
SET status = 'processed',
|
|
193
|
+
processed_at = now(),
|
|
194
|
+
error = NULL,
|
|
195
|
+
metadata = metadata || COALESCE(p_metadata, '{}'::jsonb)
|
|
196
|
+
WHERE id = p_event_id
|
|
197
|
+
AND status = 'processing'
|
|
198
|
+
AND metadata->>'claimToken' = p_claim_token
|
|
199
|
+
RETURNING id
|
|
200
|
+
)
|
|
201
|
+
SELECT EXISTS (SELECT 1 FROM updated);
|
|
202
|
+
$$;
|
|
203
|
+
|
|
204
|
+
-- Acknowledge failure. The outcome the handler asks for is a request, not a verdict: a
|
|
205
|
+
-- 'retry' on a row that has already spent its attempts becomes a discard, so the ceiling is
|
|
206
|
+
-- enforced in one place instead of in every caller.
|
|
207
|
+
--
|
|
208
|
+
-- 'snooze' means the downstream is unavailable rather than the row being bad, so the attempt
|
|
209
|
+
-- taken at claim is refunded -- an outage must not consume the DLQ budget of the events it
|
|
210
|
+
-- happened to touch. Retry delay is exponential with equal jitter in [d/2, d]: without the
|
|
211
|
+
-- jitter every row that failed in the same tick returns in the same later tick, and the
|
|
212
|
+
-- convoy reforms on each retry instead of dispersing.
|
|
213
|
+
CREATE FUNCTION public.outbox_mark_failed(
|
|
214
|
+
p_event_id uuid,
|
|
215
|
+
p_claim_token text,
|
|
216
|
+
p_error text,
|
|
217
|
+
p_outcome text DEFAULT 'retry',
|
|
218
|
+
p_base_delay_seconds integer DEFAULT 60,
|
|
219
|
+
p_max_delay_seconds integer DEFAULT 3600,
|
|
220
|
+
p_max_attempts integer DEFAULT 8,
|
|
221
|
+
p_snooze_seconds integer DEFAULT 300
|
|
222
|
+
)
|
|
223
|
+
RETURNS text
|
|
224
|
+
LANGUAGE plpgsql
|
|
225
|
+
SET search_path = public, pg_catalog
|
|
226
|
+
AS $$
|
|
227
|
+
DECLARE
|
|
228
|
+
v_row public.outbox_events%ROWTYPE;
|
|
229
|
+
v_base integer := LEAST(GREATEST(COALESCE(p_base_delay_seconds, 60), 5), 3600);
|
|
230
|
+
v_cap integer;
|
|
231
|
+
v_max integer := LEAST(GREATEST(COALESCE(p_max_attempts, 8), 1), 20);
|
|
232
|
+
v_snooze integer := LEAST(GREATEST(COALESCE(p_snooze_seconds, 300), 60), 3600);
|
|
233
|
+
v_delay double precision;
|
|
234
|
+
BEGIN
|
|
235
|
+
IF p_outcome IS NULL OR p_outcome NOT IN ('retry', 'discard', 'snooze') THEN
|
|
236
|
+
RAISE EXCEPTION 'outbox_mark_failed_invalid_outcome' USING ERRCODE = '22023';
|
|
237
|
+
END IF;
|
|
238
|
+
|
|
239
|
+
v_cap := LEAST(GREATEST(v_base, COALESCE(p_max_delay_seconds, 3600)), 86400);
|
|
240
|
+
|
|
241
|
+
SELECT *
|
|
242
|
+
INTO v_row
|
|
243
|
+
FROM public.outbox_events
|
|
244
|
+
WHERE id = p_event_id
|
|
245
|
+
AND status = 'processing'
|
|
246
|
+
AND metadata->>'claimToken' = p_claim_token
|
|
247
|
+
FOR UPDATE;
|
|
248
|
+
|
|
249
|
+
IF NOT FOUND THEN
|
|
250
|
+
RETURN 'missed';
|
|
251
|
+
END IF;
|
|
252
|
+
|
|
253
|
+
IF p_outcome = 'discard' OR (p_outcome = 'retry' AND v_row.attempts >= v_max) THEN
|
|
254
|
+
UPDATE public.outbox_events
|
|
255
|
+
SET status = 'discarded',
|
|
256
|
+
error = left(p_error, 2000),
|
|
257
|
+
metadata = metadata || jsonb_build_object(
|
|
258
|
+
'discardReason',
|
|
259
|
+
CASE WHEN p_outcome = 'discard' THEN 'permanent_error' ELSE 'max_attempts' END,
|
|
260
|
+
'discardedAt', now(),
|
|
261
|
+
'finalAttempts', v_row.attempts)
|
|
262
|
+
WHERE id = v_row.id;
|
|
263
|
+
RETURN 'discarded';
|
|
264
|
+
END IF;
|
|
265
|
+
|
|
266
|
+
IF p_outcome = 'snooze' THEN
|
|
267
|
+
UPDATE public.outbox_events
|
|
268
|
+
SET status = 'failed',
|
|
269
|
+
attempts = GREATEST(v_row.attempts - 1, 0),
|
|
270
|
+
available_at = now() + make_interval(secs => v_snooze),
|
|
271
|
+
error = left(p_error, 2000),
|
|
272
|
+
metadata = metadata || jsonb_build_object(
|
|
273
|
+
'snoozeCount', COALESCE((metadata->>'snoozeCount')::integer, 0) + 1,
|
|
274
|
+
'lastSnoozedAt', now())
|
|
275
|
+
WHERE id = v_row.id;
|
|
276
|
+
RETURN 'snoozed';
|
|
277
|
+
END IF;
|
|
278
|
+
|
|
279
|
+
v_delay := LEAST(
|
|
280
|
+
v_cap::double precision,
|
|
281
|
+
v_base::double precision * power(2, LEAST(GREATEST(v_row.attempts - 1, 0), 16))
|
|
282
|
+
) * (0.5 + random() * 0.5);
|
|
283
|
+
|
|
284
|
+
UPDATE public.outbox_events
|
|
285
|
+
SET status = 'failed',
|
|
286
|
+
available_at = now() + make_interval(secs => v_delay),
|
|
287
|
+
error = left(p_error, 2000)
|
|
288
|
+
WHERE id = v_row.id;
|
|
289
|
+
RETURN 'failed';
|
|
290
|
+
END;
|
|
291
|
+
$$;
|
|
292
|
+
|
|
293
|
+
-- Hand back rows the caller claimed but never ran -- a drain loop that aborts mid-batch, or
|
|
294
|
+
-- one handler reporting the downstream is out. The attempt taken at claim is refunded for
|
|
295
|
+
-- exactly the same reason as in 'snooze': nothing was tried, so nothing was spent. Ids and
|
|
296
|
+
-- tokens are supplied as parallel arrays and must agree in length, because a mismatch means
|
|
297
|
+
-- the caller lost track of which token belongs to which row and releasing the wrong pairing
|
|
298
|
+
-- would silently release nothing.
|
|
299
|
+
CREATE FUNCTION public.outbox_release_unprocessed(
|
|
300
|
+
p_event_ids uuid[],
|
|
301
|
+
p_claim_tokens text[],
|
|
302
|
+
p_delay_seconds integer DEFAULT 0
|
|
303
|
+
)
|
|
304
|
+
RETURNS integer
|
|
305
|
+
LANGUAGE plpgsql
|
|
306
|
+
SET search_path = public, pg_catalog
|
|
307
|
+
AS $$
|
|
308
|
+
DECLARE
|
|
309
|
+
v_delay integer := LEAST(GREATEST(COALESCE(p_delay_seconds, 0), 0), 3600);
|
|
310
|
+
v_count integer;
|
|
311
|
+
BEGIN
|
|
312
|
+
IF p_event_ids IS NULL OR p_claim_tokens IS NULL
|
|
313
|
+
OR cardinality(p_event_ids) <> cardinality(p_claim_tokens) THEN
|
|
314
|
+
RAISE EXCEPTION 'outbox_release_invalid_input' USING ERRCODE = '22023';
|
|
315
|
+
END IF;
|
|
316
|
+
|
|
317
|
+
UPDATE public.outbox_events event
|
|
318
|
+
SET status = 'pending',
|
|
319
|
+
available_at = now() + make_interval(secs => v_delay),
|
|
320
|
+
attempts = GREATEST(event.attempts - 1, 0)
|
|
321
|
+
FROM unnest(p_event_ids, p_claim_tokens) AS released(id, token)
|
|
322
|
+
WHERE event.id = released.id
|
|
323
|
+
AND event.status = 'processing'
|
|
324
|
+
AND event.metadata->>'claimToken' = released.token;
|
|
325
|
+
|
|
326
|
+
GET DIAGNOSTICS v_count = ROW_COUNT;
|
|
327
|
+
RETURN v_count;
|
|
328
|
+
END;
|
|
329
|
+
$$;
|