@nestarc/inbox 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +72 -0
- package/LICENSE +21 -0
- package/README.md +309 -2
- package/dist/contracts/errors.d.ts +34 -0
- package/dist/contracts/errors.js +73 -0
- package/dist/contracts/errors.js.map +1 -0
- package/dist/contracts/types.d.ts +174 -0
- package/dist/contracts/types.js +3 -0
- package/dist/contracts/types.js.map +1 -0
- package/dist/inbox-admin.service.d.ts +44 -0
- package/dist/inbox-admin.service.js +102 -0
- package/dist/inbox-admin.service.js.map +1 -0
- package/dist/inbox.module.d.ts +13 -0
- package/dist/inbox.module.js +59 -0
- package/dist/inbox.module.js.map +1 -0
- package/dist/inbox.service.d.ts +22 -0
- package/dist/inbox.service.js +74 -0
- package/dist/inbox.service.js.map +1 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +28 -0
- package/dist/index.js.map +1 -0
- package/dist/integrations/tenancy.d.ts +25 -0
- package/dist/integrations/tenancy.js +56 -0
- package/dist/integrations/tenancy.js.map +1 -0
- package/dist/internal/handler-context.d.ts +3 -0
- package/dist/internal/handler-context.js +39 -0
- package/dist/internal/handler-context.js.map +1 -0
- package/dist/internal/options.d.ts +5 -0
- package/dist/internal/options.js +86 -0
- package/dist/internal/options.js.map +1 -0
- package/dist/internal/reliability-core/abort.d.ts +2 -0
- package/dist/internal/reliability-core/abort.js +21 -0
- package/dist/internal/reliability-core/abort.js.map +1 -0
- package/dist/internal/reliability-core/index.d.ts +3 -0
- package/dist/internal/reliability-core/index.js +14 -0
- package/dist/internal/reliability-core/index.js.map +1 -0
- package/dist/internal/reliability-core/json.d.ts +13 -0
- package/dist/internal/reliability-core/json.js +125 -0
- package/dist/internal/reliability-core/json.js.map +1 -0
- package/dist/internal/reliability-core/retry.d.ts +8 -0
- package/dist/internal/reliability-core/retry.js +26 -0
- package/dist/internal/reliability-core/retry.js.map +1 -0
- package/dist/internal/validation.d.ts +6 -0
- package/dist/internal/validation.js +123 -0
- package/dist/internal/validation.js.map +1 -0
- package/dist/storage/prisma/prisma-inbox.storage.d.ts +30 -0
- package/dist/storage/prisma/prisma-inbox.storage.js +369 -0
- package/dist/storage/prisma/prisma-inbox.storage.js.map +1 -0
- package/dist/testing/index.d.ts +40 -0
- package/dist/testing/index.js +120 -0
- package/dist/testing/index.js.map +1 -0
- package/dist/worker/inbox-worker.d.ts +31 -0
- package/dist/worker/inbox-worker.js +343 -0
- package/dist/worker/inbox-worker.js.map +1 -0
- package/docs/adr/0001-durable-inbox.md +40 -0
- package/docs/benchmarks/2026-10-27-local.md +31 -0
- package/docs/benchmarks/2026-10-28-preview1-local.md +87 -0
- package/docs/benchmarks/2026-10-28-preview1-run1.json +120 -0
- package/docs/benchmarks/2026-10-28-preview1-run2.json +120 -0
- package/docs/benchmarks/2026-10-28-preview1-run3.json +120 -0
- package/docs/benchmarks/2026-10-28-sustained-local.md +104 -0
- package/docs/benchmarks/2026-10-28-sustained-run1.json +2553 -0
- package/docs/benchmarks/2026-10-28-sustained-run2.json +2553 -0
- package/docs/broker-recovery-results.json +52 -0
- package/docs/broker-recovery.md +30 -0
- package/docs/db-operations.md +41 -0
- package/docs/implementation-roadmap.md +129 -0
- package/docs/operations/2026-10-28-database.json +34 -0
- package/docs/package-plan.md +250 -0
- package/docs/releasing.md +91 -0
- package/docs/testing.md +86 -0
- package/docs/usage.md +266 -0
- package/docs/validation.md +174 -0
- package/examples/README.md +124 -0
- package/examples/compose.yaml +26 -0
- package/examples/package.json +42 -0
- package/examples/prisma/schema.prisma +29 -0
- package/examples/prisma.config.ts +10 -0
- package/examples/scripts/prepare-db.cjs +45 -0
- package/examples/scripts/smoke.cjs +37 -0
- package/examples/scripts/with-brokers.cjs +249 -0
- package/examples/scripts/with-rabbitmq.cjs +131 -0
- package/examples/sql/application.sql +14 -0
- package/examples/sql/tenancy.sql +26 -0
- package/examples/src/broker-common.ts +79 -0
- package/examples/src/bullmq-ingress/main.ts +165 -0
- package/examples/src/common.ts +112 -0
- package/examples/src/idempotency-ingress/main.ts +211 -0
- package/examples/src/kafka-ingress/main.ts +209 -0
- package/examples/src/outbox-to-inbox/main.ts +198 -0
- package/examples/src/rabbitmq-ingress/main.ts +203 -0
- package/examples/src/sqs-ingress/main.ts +273 -0
- package/examples/src/tenancy/main.ts +210 -0
- package/examples/src/webhook-ingress/main.ts +172 -0
- package/examples/tsconfig.json +16 -0
- package/package.json +99 -4
- package/src/sql/create-inbox-table.sql +100 -0
- package/src/sql/migrate-v1-to-v2.sql +53 -0
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.InboxWorker = void 0;
|
|
4
|
+
const node_crypto_1 = require("node:crypto");
|
|
5
|
+
const reliability_core_1 = require("../internal/reliability-core");
|
|
6
|
+
const handler_context_1 = require("../internal/handler-context");
|
|
7
|
+
const errors_1 = require("../contracts/errors");
|
|
8
|
+
/** Internal polling worker. Business effects and completion are committed by storage.run(). */
|
|
9
|
+
class InboxWorker {
|
|
10
|
+
storage;
|
|
11
|
+
options;
|
|
12
|
+
observer;
|
|
13
|
+
handlerContext;
|
|
14
|
+
workerId = (0, node_crypto_1.randomUUID)();
|
|
15
|
+
handlers = new Map();
|
|
16
|
+
active = new Set();
|
|
17
|
+
controllers = new Set();
|
|
18
|
+
claimPromise = null;
|
|
19
|
+
timer;
|
|
20
|
+
running = false;
|
|
21
|
+
stopped = false;
|
|
22
|
+
stopPromise;
|
|
23
|
+
constructor(storage, handlers, options, observer, handlerContext) {
|
|
24
|
+
this.storage = storage;
|
|
25
|
+
this.options = options;
|
|
26
|
+
this.observer = observer;
|
|
27
|
+
this.handlerContext = handlerContext;
|
|
28
|
+
for (const handler of handlers) {
|
|
29
|
+
if (this.handlers.has(handler.consumer)) {
|
|
30
|
+
throw new errors_1.InboxConfigurationError("Each consumer must have exactly one registered handler.");
|
|
31
|
+
}
|
|
32
|
+
this.handlers.set(handler.consumer, handler);
|
|
33
|
+
}
|
|
34
|
+
for (const consumer of options.consumers) {
|
|
35
|
+
if (!this.handlers.has(consumer)) {
|
|
36
|
+
throw new errors_1.InboxConfigurationError("Every worker-owned consumer must have a registered handler.");
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/** Start background polling. Schema validation must have succeeded first. */
|
|
41
|
+
start() {
|
|
42
|
+
if (this.running || this.stopped || !this.options.enabled)
|
|
43
|
+
return;
|
|
44
|
+
this.running = true;
|
|
45
|
+
this.schedule(0);
|
|
46
|
+
}
|
|
47
|
+
/** Claim only free slots and wait for this batch. Concurrent calls cannot oversubscribe. */
|
|
48
|
+
async tick() {
|
|
49
|
+
if (this.stopped ||
|
|
50
|
+
this.claimPromise ||
|
|
51
|
+
this.options.consumers.length === 0)
|
|
52
|
+
return 0;
|
|
53
|
+
const capacity = this.options.concurrency - this.active.size;
|
|
54
|
+
if (capacity <= 0)
|
|
55
|
+
return 0;
|
|
56
|
+
const claiming = Promise.resolve().then(() => this.stopped
|
|
57
|
+
? []
|
|
58
|
+
: this.storage.claim(this.options.consumers, capacity, this.workerId, this.options.leaseMs, this.options.maxAttempts));
|
|
59
|
+
this.claimPromise = claiming;
|
|
60
|
+
let records;
|
|
61
|
+
try {
|
|
62
|
+
records = await claiming;
|
|
63
|
+
// Shutdown can race the DB claim. Do not execute or renew newly returned claims;
|
|
64
|
+
// their durable leases will expire and another worker can recover them.
|
|
65
|
+
if (this.stopped)
|
|
66
|
+
return records.length;
|
|
67
|
+
const batch = records.map((record) => {
|
|
68
|
+
// Defer execution until the promise is tracked and capacity is reserved.
|
|
69
|
+
const task = Promise.resolve().then(() => this.process(record));
|
|
70
|
+
this.active.add(task);
|
|
71
|
+
void task.then(() => this.active.delete(task), () => this.active.delete(task));
|
|
72
|
+
return task;
|
|
73
|
+
});
|
|
74
|
+
this.claimPromise = null;
|
|
75
|
+
await Promise.all(batch);
|
|
76
|
+
return records.length;
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
if (this.claimPromise === claiming)
|
|
80
|
+
this.claimPromise = null;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
/** Stop is terminal and idempotent. Expired in-flight work is recovered via its lease. */
|
|
84
|
+
stop() {
|
|
85
|
+
if (this.stopPromise)
|
|
86
|
+
return this.stopPromise;
|
|
87
|
+
this.stopped = true;
|
|
88
|
+
this.running = false;
|
|
89
|
+
if (this.timer)
|
|
90
|
+
clearTimeout(this.timer);
|
|
91
|
+
this.stopPromise = this.drain();
|
|
92
|
+
return this.stopPromise;
|
|
93
|
+
}
|
|
94
|
+
schedule(delayMs) {
|
|
95
|
+
if (!this.running || this.stopped)
|
|
96
|
+
return;
|
|
97
|
+
this.timer = setTimeout(() => {
|
|
98
|
+
this.timer = undefined;
|
|
99
|
+
if (!this.running || this.stopped)
|
|
100
|
+
return;
|
|
101
|
+
void this.tick().catch(() => this.observe({ type: "error", code: "CLAIM_FAILED" }));
|
|
102
|
+
this.schedule(this.options.pollIntervalMs);
|
|
103
|
+
}, delayMs);
|
|
104
|
+
this.timer.unref?.();
|
|
105
|
+
}
|
|
106
|
+
async drain() {
|
|
107
|
+
let deadline;
|
|
108
|
+
const expired = new Promise((resolve) => {
|
|
109
|
+
deadline = setTimeout(() => {
|
|
110
|
+
for (const controller of this.controllers) {
|
|
111
|
+
controller.abort(new errors_1.InboxLeaseLostError());
|
|
112
|
+
}
|
|
113
|
+
resolve();
|
|
114
|
+
}, this.options.shutdownTimeoutMs);
|
|
115
|
+
});
|
|
116
|
+
const settled = (async () => {
|
|
117
|
+
await this.claimPromise?.catch(() => undefined);
|
|
118
|
+
await Promise.allSettled([...this.active]);
|
|
119
|
+
})();
|
|
120
|
+
try {
|
|
121
|
+
await Promise.race([settled, expired]);
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
if (deadline)
|
|
125
|
+
clearTimeout(deadline);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
async process(record) {
|
|
129
|
+
const controller = new AbortController();
|
|
130
|
+
this.controllers.add(controller);
|
|
131
|
+
let heartbeatTimer;
|
|
132
|
+
let finished = false;
|
|
133
|
+
// Include connection-pool waiting. Lease renewals cannot keep a hung handler alive forever.
|
|
134
|
+
const deadline = setTimeout(() => controller.abort(new errors_1.InboxTimeoutError()), this.options.transactionMaxWaitMs + this.options.transactionTimeoutMs);
|
|
135
|
+
deadline.unref?.();
|
|
136
|
+
const scheduleHeartbeat = () => {
|
|
137
|
+
if (finished || controller.signal.aborted)
|
|
138
|
+
return;
|
|
139
|
+
heartbeatTimer = setTimeout(() => {
|
|
140
|
+
heartbeatTimer = undefined;
|
|
141
|
+
if (finished || controller.signal.aborted)
|
|
142
|
+
return;
|
|
143
|
+
void (async () => {
|
|
144
|
+
try {
|
|
145
|
+
const owned = await this.storage.heartbeat(record.id, record.claimToken, this.options.leaseMs);
|
|
146
|
+
if (!owned)
|
|
147
|
+
controller.abort(new errors_1.InboxLeaseLostError());
|
|
148
|
+
}
|
|
149
|
+
catch {
|
|
150
|
+
// A failed renewal cannot prove continued ownership. Stop business work;
|
|
151
|
+
// never extend the same token after its lease has expired.
|
|
152
|
+
controller.abort(new errors_1.InboxLeaseLostError());
|
|
153
|
+
}
|
|
154
|
+
finally {
|
|
155
|
+
scheduleHeartbeat();
|
|
156
|
+
}
|
|
157
|
+
})();
|
|
158
|
+
}, this.options.heartbeatIntervalMs);
|
|
159
|
+
heartbeatTimer.unref?.();
|
|
160
|
+
};
|
|
161
|
+
scheduleHeartbeat();
|
|
162
|
+
let classified;
|
|
163
|
+
try {
|
|
164
|
+
if (this.stopped)
|
|
165
|
+
throw new errors_1.InboxLeaseLostError();
|
|
166
|
+
const handler = this.handlers.get(record.consumer);
|
|
167
|
+
if (!handler ||
|
|
168
|
+
handler.eventType !== record.eventType ||
|
|
169
|
+
handler.schemaVersion !== record.schemaVersion) {
|
|
170
|
+
classified = {
|
|
171
|
+
code: "UNSUPPORTED_MESSAGE",
|
|
172
|
+
message: "The owned consumer does not support this event type or schema version.",
|
|
173
|
+
permanent: true,
|
|
174
|
+
};
|
|
175
|
+
throw new errors_1.InboxPermanentError();
|
|
176
|
+
}
|
|
177
|
+
try {
|
|
178
|
+
const valid = handler.validate?.(record.payload);
|
|
179
|
+
// Runtime callers can bypass TypeScript. Async validators are not supported.
|
|
180
|
+
if (valid !== undefined && valid !== true) {
|
|
181
|
+
if (valid &&
|
|
182
|
+
typeof valid.then === "function") {
|
|
183
|
+
void Promise.resolve(valid).catch(() => undefined);
|
|
184
|
+
}
|
|
185
|
+
throw new errors_1.InboxPermanentError();
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
catch {
|
|
189
|
+
classified = {
|
|
190
|
+
code: "INVALID_PAYLOAD",
|
|
191
|
+
message: "The payload failed consumer validation.",
|
|
192
|
+
permanent: true,
|
|
193
|
+
};
|
|
194
|
+
throw new errors_1.InboxPermanentError();
|
|
195
|
+
}
|
|
196
|
+
await this.storage.run(record, async (tx) => {
|
|
197
|
+
const context = {
|
|
198
|
+
tx,
|
|
199
|
+
scope: record.scope,
|
|
200
|
+
messageId: record.messageId,
|
|
201
|
+
source: record.source,
|
|
202
|
+
consumer: record.consumer,
|
|
203
|
+
inboxId: record.id,
|
|
204
|
+
attempt: record.attemptCount,
|
|
205
|
+
signal: controller.signal,
|
|
206
|
+
replay: record.replay,
|
|
207
|
+
correlationId: record.correlationId,
|
|
208
|
+
causationId: record.causationId,
|
|
209
|
+
};
|
|
210
|
+
await (0, reliability_core_1.withAbort)(controller.signal, () => (0, handler_context_1.runInHandlerContext)(context, () => handler.handle(record.payload, context), this.handlerContext));
|
|
211
|
+
// An ignored signal must still cause rollback before the completion CAS.
|
|
212
|
+
if (controller.signal.aborted)
|
|
213
|
+
throw controller.signal.reason;
|
|
214
|
+
}, {
|
|
215
|
+
timeoutMs: this.options.transactionTimeoutMs,
|
|
216
|
+
maxWaitMs: this.options.transactionMaxWaitMs,
|
|
217
|
+
lockTimeoutMs: this.options.lockTimeoutMs,
|
|
218
|
+
});
|
|
219
|
+
this.observe({ type: "completed", consumer: record.consumer });
|
|
220
|
+
}
|
|
221
|
+
catch (error) {
|
|
222
|
+
// Stop renewing as soon as a transaction fails, including while reconciling an
|
|
223
|
+
// unknown commit outcome on a new connection.
|
|
224
|
+
finished = true;
|
|
225
|
+
if (heartbeatTimer)
|
|
226
|
+
clearTimeout(heartbeatTimer);
|
|
227
|
+
if (error instanceof errors_1.InboxLeaseLostError ||
|
|
228
|
+
controller.signal.reason instanceof errors_1.InboxLeaseLostError) {
|
|
229
|
+
this.observe({
|
|
230
|
+
type: "lease_lost",
|
|
231
|
+
consumer: record.consumer,
|
|
232
|
+
code: "LEASE_LOST",
|
|
233
|
+
});
|
|
234
|
+
return;
|
|
235
|
+
}
|
|
236
|
+
const failure = classified ?? classifyFailure(error);
|
|
237
|
+
await this.recordFailure(record, failure);
|
|
238
|
+
}
|
|
239
|
+
finally {
|
|
240
|
+
finished = true;
|
|
241
|
+
clearTimeout(deadline);
|
|
242
|
+
if (heartbeatTimer)
|
|
243
|
+
clearTimeout(heartbeatTimer);
|
|
244
|
+
this.controllers.delete(controller);
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
async recordFailure(record, failure) {
|
|
248
|
+
const controller = new AbortController();
|
|
249
|
+
this.controllers.add(controller);
|
|
250
|
+
// Driver/network promises can remain pending beyond a transaction failure.
|
|
251
|
+
// Reconciliation must not retain a concurrency slot or start a late failure write.
|
|
252
|
+
const deadline = setTimeout(() => controller.abort(new errors_1.InboxTimeoutError()), Math.min(this.options.transactionMaxWaitMs + this.options.transactionTimeoutMs, 5000));
|
|
253
|
+
deadline.unref?.();
|
|
254
|
+
try {
|
|
255
|
+
// A commit can succeed even when the caller receives a transport error.
|
|
256
|
+
const current = await (0, reliability_core_1.withAbort)(controller.signal, () => this.storage.get(record.id));
|
|
257
|
+
if (current?.status === "COMPLETED") {
|
|
258
|
+
this.observe({
|
|
259
|
+
type: current.claimToken === record.claimToken
|
|
260
|
+
? "completed"
|
|
261
|
+
: "lease_lost",
|
|
262
|
+
consumer: record.consumer,
|
|
263
|
+
});
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
266
|
+
if (!current ||
|
|
267
|
+
current.status !== "PROCESSING" ||
|
|
268
|
+
current.claimToken !== record.claimToken) {
|
|
269
|
+
this.observe({
|
|
270
|
+
type: "lease_lost",
|
|
271
|
+
consumer: record.consumer,
|
|
272
|
+
code: "LEASE_LOST",
|
|
273
|
+
});
|
|
274
|
+
return;
|
|
275
|
+
}
|
|
276
|
+
const delayMs = this.retryDelay(record.cycleAttemptCount);
|
|
277
|
+
const changed = await (0, reliability_core_1.withAbort)(controller.signal, () => this.storage.fail(record.id, record.claimToken, { code: failure.code, message: failure.message }, failure.permanent, delayMs, this.options.maxAttempts));
|
|
278
|
+
this.observe({
|
|
279
|
+
type: changed
|
|
280
|
+
? failure.permanent ||
|
|
281
|
+
record.cycleAttemptCount >= this.options.maxAttempts
|
|
282
|
+
? "dead"
|
|
283
|
+
: "retry"
|
|
284
|
+
: "lease_lost",
|
|
285
|
+
consumer: record.consumer,
|
|
286
|
+
code: changed ? failure.code : "LEASE_LOST",
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
catch {
|
|
290
|
+
// Unavailable state is not evidence of failure. Leave the durable claim alone
|
|
291
|
+
// so another worker can reconcile it after its lease expires.
|
|
292
|
+
this.observe({
|
|
293
|
+
type: "error",
|
|
294
|
+
consumer: record.consumer,
|
|
295
|
+
code: "RECONCILIATION_FAILED",
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
finally {
|
|
299
|
+
clearTimeout(deadline);
|
|
300
|
+
this.controllers.delete(controller);
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
retryDelay(attempt) {
|
|
304
|
+
return (0, reliability_core_1.retryDelayMs)(attempt, {
|
|
305
|
+
baseMs: this.options.retryBaseMs,
|
|
306
|
+
maxMs: this.options.retryMaxMs,
|
|
307
|
+
jitter: "full",
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
observe(event) {
|
|
311
|
+
if (!this.observer)
|
|
312
|
+
return;
|
|
313
|
+
try {
|
|
314
|
+
void Promise.resolve(this.observer(event)).catch(() => undefined);
|
|
315
|
+
}
|
|
316
|
+
catch {
|
|
317
|
+
// Observation is best effort and cannot affect persisted processing state.
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
exports.InboxWorker = InboxWorker;
|
|
322
|
+
function classifyFailure(error) {
|
|
323
|
+
if (error instanceof errors_1.InboxPermanentError) {
|
|
324
|
+
return {
|
|
325
|
+
code: "PERMANENT_ERROR",
|
|
326
|
+
message: "The handler rejected the message permanently.",
|
|
327
|
+
permanent: true,
|
|
328
|
+
};
|
|
329
|
+
}
|
|
330
|
+
if (error instanceof errors_1.InboxTimeoutError) {
|
|
331
|
+
return {
|
|
332
|
+
code: "PROCESSING_TIMEOUT",
|
|
333
|
+
message: "The handler exceeded its transaction deadline.",
|
|
334
|
+
permanent: false,
|
|
335
|
+
};
|
|
336
|
+
}
|
|
337
|
+
return {
|
|
338
|
+
code: "HANDLER_ERROR",
|
|
339
|
+
message: "Message processing failed.",
|
|
340
|
+
permanent: false,
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
//# sourceMappingURL=inbox-worker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"inbox-worker.js","sourceRoot":"","sources":["../../src/worker/inbox-worker.ts"],"names":[],"mappings":";;;AAAA,6CAAyC;AACzC,mEAAuE;AACvE,iEAAkE;AAClE,gDAK6B;AAY7B,+FAA+F;AAC/F,MAAa,WAAW;IAYH;IAEA;IACA;IACA;IAfF,QAAQ,GAAG,IAAA,wBAAU,GAAE,CAAC;IACxB,QAAQ,GAAG,IAAI,GAAG,EAAkC,CAAC;IACrD,MAAM,GAAG,IAAI,GAAG,EAAiB,CAAC;IAClC,WAAW,GAAG,IAAI,GAAG,EAAmB,CAAC;IAClD,YAAY,GAAkC,IAAI,CAAC;IACnD,KAAK,CAA4C;IACjD,OAAO,GAAG,KAAK,CAAC;IAChB,OAAO,GAAG,KAAK,CAAC;IAChB,WAAW,CAA4B;IAE/C,YACmB,OAA2B,EAC5C,QAAkC,EACjB,OAA8B,EAC9B,QAAmC,EACnC,cAA+C;QAJ/C,YAAO,GAAP,OAAO,CAAoB;QAE3B,YAAO,GAAP,OAAO,CAAuB;QAC9B,aAAQ,GAAR,QAAQ,CAA2B;QACnC,mBAAc,GAAd,cAAc,CAAiC;QAEhE,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACxC,MAAM,IAAI,gCAAuB,CAC/B,yDAAyD,CAC1D,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACzC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,gCAAuB,CAC/B,6DAA6D,CAC9D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,6EAA6E;IAC7E,KAAK;QACH,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO;YAAE,OAAO;QAClE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC;IAED,4FAA4F;IAC5F,KAAK,CAAC,IAAI;QACR,IACE,IAAI,CAAC,OAAO;YACZ,IAAI,CAAC,YAAY;YACjB,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC;YAEnC,OAAO,CAAC,CAAC;QACX,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAC7D,IAAI,QAAQ,IAAI,CAAC;YAAE,OAAO,CAAC,CAAC;QAE5B,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAC3C,IAAI,CAAC,OAAO;YACV,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAChB,IAAI,CAAC,OAAO,CAAC,SAAS,EACtB,QAAQ,EACR,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,CAAC,OAAO,EACpB,IAAI,CAAC,OAAO,CAAC,WAAW,CACzB,CACN,CAAC;QACF,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC;QAC7B,IAAI,OAAsB,CAAC;QAC3B,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,QAAQ,CAAC;YACzB,iFAAiF;YACjF,wEAAwE;YACxE,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO,OAAO,CAAC,MAAM,CAAC;YACxC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;gBACnC,yEAAyE;gBACzE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;gBAChE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACtB,KAAK,IAAI,CAAC,IAAI,CACZ,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,EAC9B,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAC/B,CAAC;gBACF,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YACzB,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzB,OAAO,OAAO,CAAC,MAAM,CAAC;QACxB,CAAC;gBAAS,CAAC;YACT,IAAI,IAAI,CAAC,YAAY,KAAK,QAAQ;gBAAE,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,0FAA0F;IAC1F,IAAI;QACF,IAAI,IAAI,CAAC,WAAW;YAAE,OAAO,IAAI,CAAC,WAAW,CAAC;QAC9C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACrB,IAAI,IAAI,CAAC,KAAK;YAAE,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QAChC,OAAO,IAAI,CAAC,WAAW,CAAC;IAC1B,CAAC;IAEO,QAAQ,CAAC,OAAe;QAC9B,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1C,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC3B,IAAI,CAAC,KAAK,GAAG,SAAS,CAAC;YACvB,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO;gBAAE,OAAO;YAC1C,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAC1B,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CACtD,CAAC;YACF,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QAC7C,CAAC,EAAE,OAAO,CAAC,CAAC;QACZ,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IACvB,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,QAAmD,CAAC;QACxD,MAAM,OAAO,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAC5C,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE;gBACzB,KAAK,MAAM,UAAU,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBAC1C,UAAU,CAAC,KAAK,CAAC,IAAI,4BAAmB,EAAE,CAAC,CAAC;gBAC9C,CAAC;gBACD,OAAO,EAAE,CAAC;YACZ,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,iBAAiB,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QACH,MAAM,OAAO,GAAG,CAAC,KAAK,IAAI,EAAE;YAC1B,MAAM,IAAI,CAAC,YAAY,EAAE,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAChD,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7C,CAAC,CAAC,EAAE,CAAC;QACL,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACzC,CAAC;gBAAS,CAAC;YACT,IAAI,QAAQ;gBAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,MAAmB;QACvC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACjC,IAAI,cAAyD,CAAC;QAC9D,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,4FAA4F;QAC5F,MAAM,QAAQ,GAAG,UAAU,CACzB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,0BAAiB,EAAE,CAAC,EAC/C,IAAI,CAAC,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CACtE,CAAC;QACF,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;QAEnB,MAAM,iBAAiB,GAAG,GAAG,EAAE;YAC7B,IAAI,QAAQ,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;gBAAE,OAAO;YAClD,cAAc,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,cAAc,GAAG,SAAS,CAAC;gBAC3B,IAAI,QAAQ,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;oBAAE,OAAO;gBAClD,KAAK,CAAC,KAAK,IAAI,EAAE;oBACf,IAAI,CAAC;wBACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CACxC,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,UAAU,EACjB,IAAI,CAAC,OAAO,CAAC,OAAO,CACrB,CAAC;wBACF,IAAI,CAAC,KAAK;4BAAE,UAAU,CAAC,KAAK,CAAC,IAAI,4BAAmB,EAAE,CAAC,CAAC;oBAC1D,CAAC;oBAAC,MAAM,CAAC;wBACP,yEAAyE;wBACzE,2DAA2D;wBAC3D,UAAU,CAAC,KAAK,CAAC,IAAI,4BAAmB,EAAE,CAAC,CAAC;oBAC9C,CAAC;4BAAS,CAAC;wBACT,iBAAiB,EAAE,CAAC;oBACtB,CAAC;gBACH,CAAC,CAAC,EAAE,CAAC;YACP,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;YACrC,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC;QAC3B,CAAC,CAAC;QACF,iBAAiB,EAAE,CAAC;QAEpB,IAAI,UAA+B,CAAC;QACpC,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,4BAAmB,EAAE,CAAC;YAClD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACnD,IACE,CAAC,OAAO;gBACR,OAAO,CAAC,SAAS,KAAK,MAAM,CAAC,SAAS;gBACtC,OAAO,CAAC,aAAa,KAAK,MAAM,CAAC,aAAa,EAC9C,CAAC;gBACD,UAAU,GAAG;oBACX,IAAI,EAAE,qBAAqB;oBAC3B,OAAO,EACL,wEAAwE;oBAC1E,SAAS,EAAE,IAAI;iBAChB,CAAC;gBACF,MAAM,IAAI,4BAAmB,EAAE,CAAC;YAClC,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,KAAK,GAAY,OAAO,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC1D,6EAA6E;gBAC7E,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;oBAC1C,IACE,KAAK;wBACL,OAAQ,KAAuC,CAAC,IAAI,KAAK,UAAU,EACnE,CAAC;wBACD,KAAK,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;oBACrD,CAAC;oBACD,MAAM,IAAI,4BAAmB,EAAE,CAAC;gBAClC,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU,GAAG;oBACX,IAAI,EAAE,iBAAiB;oBACvB,OAAO,EAAE,yCAAyC;oBAClD,SAAS,EAAE,IAAI;iBAChB,CAAC;gBACF,MAAM,IAAI,4BAAmB,EAAE,CAAC;YAClC,CAAC;YAED,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CACpB,MAAM,EACN,KAAK,EAAE,EAAE,EAAE,EAAE;gBACX,MAAM,OAAO,GAAG;oBACd,EAAE;oBACF,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,SAAS,EAAE,MAAM,CAAC,SAAS;oBAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,OAAO,EAAE,MAAM,CAAC,EAAE;oBAClB,OAAO,EAAE,MAAM,CAAC,YAAY;oBAC5B,MAAM,EAAE,UAAU,CAAC,MAAM;oBACzB,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,aAAa,EAAE,MAAM,CAAC,aAAa;oBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;iBAChC,CAAC;gBACF,MAAM,IAAA,4BAAS,EAAC,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE,CACtC,IAAA,qCAAmB,EACjB,OAAO,EACP,GAAG,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,EAC7C,IAAI,CAAC,cAAc,CACpB,CACF,CAAC;gBACF,yEAAyE;gBACzE,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO;oBAAE,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC;YAChE,CAAC,EACD;gBACE,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;gBAC5C,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,oBAAoB;gBAC5C,aAAa,EAAE,IAAI,CAAC,OAAO,CAAC,aAAa;aAC1C,CACF,CAAC;YACF,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;QACjE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,+EAA+E;YAC/E,8CAA8C;YAC9C,QAAQ,GAAG,IAAI,CAAC;YAChB,IAAI,cAAc;gBAAE,YAAY,CAAC,cAAc,CAAC,CAAC;YACjD,IACE,KAAK,YAAY,4BAAmB;gBACpC,UAAU,CAAC,MAAM,CAAC,MAAM,YAAY,4BAAmB,EACvD,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC;oBACX,IAAI,EAAE,YAAY;oBAClB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,IAAI,EAAE,YAAY;iBACnB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,UAAU,IAAI,eAAe,CAAC,KAAK,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC;gBAAS,CAAC;YACT,QAAQ,GAAG,IAAI,CAAC;YAChB,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvB,IAAI,cAAc;gBAAE,YAAY,CAAC,cAAc,CAAC,CAAC;YACjD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,MAAmB,EACnB,OAAgB;QAEhB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACjC,2EAA2E;QAC3E,mFAAmF;QACnF,MAAM,QAAQ,GAAG,UAAU,CACzB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,0BAAiB,EAAE,CAAC,EAC/C,IAAI,CAAC,GAAG,CACN,IAAI,CAAC,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,EACrE,IAAI,CACL,CACF,CAAC;QACF,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC;QACnB,IAAI,CAAC;YACH,wEAAwE;YACxE,MAAM,OAAO,GAAG,MAAM,IAAA,4BAAS,EAAC,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE,CACtD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAC5B,CAAC;YACF,IAAI,OAAO,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;gBACpC,IAAI,CAAC,OAAO,CAAC;oBACX,IAAI,EACF,OAAO,CAAC,UAAU,KAAK,MAAM,CAAC,UAAU;wBACtC,CAAC,CAAC,WAAW;wBACb,CAAC,CAAC,YAAY;oBAClB,QAAQ,EAAE,MAAM,CAAC,QAAQ;iBAC1B,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,IACE,CAAC,OAAO;gBACR,OAAO,CAAC,MAAM,KAAK,YAAY;gBAC/B,OAAO,CAAC,UAAU,KAAK,MAAM,CAAC,UAAU,EACxC,CAAC;gBACD,IAAI,CAAC,OAAO,CAAC;oBACX,IAAI,EAAE,YAAY;oBAClB,QAAQ,EAAE,MAAM,CAAC,QAAQ;oBACzB,IAAI,EAAE,YAAY;iBACnB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;YAC1D,MAAM,OAAO,GAAG,MAAM,IAAA,4BAAS,EAAC,UAAU,CAAC,MAAM,EAAE,GAAG,EAAE,CACtD,IAAI,CAAC,OAAO,CAAC,IAAI,CACf,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,UAAU,EACjB,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,EAChD,OAAO,CAAC,SAAS,EACjB,OAAO,EACP,IAAI,CAAC,OAAO,CAAC,WAAW,CACzB,CACF,CAAC;YACF,IAAI,CAAC,OAAO,CAAC;gBACX,IAAI,EAAE,OAAO;oBACX,CAAC,CAAC,OAAO,CAAC,SAAS;wBACjB,MAAM,CAAC,iBAAiB,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW;wBACpD,CAAC,CAAC,MAAM;wBACR,CAAC,CAAC,OAAO;oBACX,CAAC,CAAC,YAAY;gBAChB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY;aAC5C,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,8EAA8E;YAC9E,8DAA8D;YAC9D,IAAI,CAAC,OAAO,CAAC;gBACX,IAAI,EAAE,OAAO;gBACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,IAAI,EAAE,uBAAuB;aAC9B,CAAC,CAAC;QACL,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,QAAQ,CAAC,CAAC;YACvB,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,OAAe;QAChC,OAAO,IAAA,+BAAY,EAAC,OAAO,EAAE;YAC3B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW;YAChC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;YAC9B,MAAM,EAAE,MAAM;SACf,CAAC,CAAC;IACL,CAAC;IAEO,OAAO,CAAC,KAAuB;QACrC,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC3B,IAAI,CAAC;YACH,KAAK,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpE,CAAC;QAAC,MAAM,CAAC;YACP,2EAA2E;QAC7E,CAAC;IACH,CAAC;CACF;AA1WD,kCA0WC;AAED,SAAS,eAAe,CAAC,KAAc;IACrC,IAAI,KAAK,YAAY,4BAAmB,EAAE,CAAC;QACzC,OAAO;YACL,IAAI,EAAE,iBAAiB;YACvB,OAAO,EAAE,+CAA+C;YACxD,SAAS,EAAE,IAAI;SAChB,CAAC;IACJ,CAAC;IACD,IAAI,KAAK,YAAY,0BAAiB,EAAE,CAAC;QACvC,OAAO;YACL,IAAI,EAAE,oBAAoB;YAC1B,OAAO,EAAE,gDAAgD;YACzD,SAAS,EAAE,KAAK;SACjB,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,EAAE,eAAe;QACrB,OAAO,EAAE,4BAA4B;QACrC,SAAS,EAAE,KAAK;KACjB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# Durable inbox acceptance and transaction ownership
|
|
2
|
+
|
|
3
|
+
Status: accepted for the 0.1 Preview implementation.
|
|
4
|
+
|
|
5
|
+
Preview.1 extension: COMPLETED replay is a new receipt with a scoped idempotent request UUID and durable lineage. It never resets the original receipt. Schema v2 adds this lineage, with an explicit v1 upgrade. Optional tenancy context wraps the existing handler transaction; it cannot replace that transaction or skip handler execution.
|
|
6
|
+
|
|
7
|
+
## Decision
|
|
8
|
+
|
|
9
|
+
The package exposes durable acceptance followed by asynchronous processing. `accept()` owns its database transaction and resolves only after that transaction commits. The application may then acknowledge the broker delivery or return HTTP success. It must not interpret acceptance as completed business processing.
|
|
10
|
+
|
|
11
|
+
After acceptance, the Inbox worker owns handler retries. A redelivery from the broker checks the existing identity and returns `duplicate`; it does not create a second retry schedule. Storage failures and unknown acceptance commit results remain the ingress transport's responsibility.
|
|
12
|
+
|
|
13
|
+
Identity is the tuple `(scopeKind, scopeId, source, consumer, messageId)`. Global scope has an explicit non-null representation. Consumer names identify logical effects, not worker instances or deployment builds. A fingerprint covers event type, schema version, and canonical JSON payload; trace metadata is excluded. Conflicting content never overwrites the original record.
|
|
14
|
+
|
|
15
|
+
## Transaction boundaries
|
|
16
|
+
|
|
17
|
+
1. Acceptance inserts or observes a matching durable record and commits.
|
|
18
|
+
2. Claiming commits a new token, an expiring lease, and an incremented attempt count in a short transaction.
|
|
19
|
+
3. The handler executes in a new Prisma transaction supplied by Inbox. Its business writes, optional outbox writes, and fenced `COMPLETED` update commit together.
|
|
20
|
+
|
|
21
|
+
The completion update requires `PROCESSING`, the current token, and a valid lease. Updating zero rows throws inside the transaction, rolling back all handler writes. A successful completion update holds its row lock through commit; lease expiry between that update and commit does not permit a second completion.
|
|
22
|
+
|
|
23
|
+
Heartbeats and failure writes require the current processing token and unexpired lease. Recovery requires an expired processing lease. DEAD requeue requires the expected record version and preserves lifetime attempts and token history. Workers claim and recover only their configured consumers.
|
|
24
|
+
|
|
25
|
+
## Consequences
|
|
26
|
+
|
|
27
|
+
- Handler invocation is at least once; successfully committed effects in the same database transaction are deduplicated while the receipt is retained.
|
|
28
|
+
- External HTTP requests, email, other database clients, and other databases cannot join this transaction. Handlers use an outbox for subsequent delivery.
|
|
29
|
+
- Claim-time attempt accounting makes repeated crashes consume the retry budget. Exhaustion and permanent errors retain a DEAD record for operator action.
|
|
30
|
+
- Completed identity and fingerprint retention are separate from payload retention. Removing a receipt ends its deduplication guarantee.
|
|
31
|
+
- Business uniqueness across different message IDs and event ordering remain application concerns.
|
|
32
|
+
- The library does not perform schema migrations, create the Prisma connection pool, implement broker clients, or offer a second synchronous processing mode in Preview.
|
|
33
|
+
|
|
34
|
+
## Rejected alternative
|
|
35
|
+
|
|
36
|
+
A caller-owned transactional deduplication helper would be smaller but leave retry scheduling and dead-letter handling to the broker. It may be considered as a separate future API. Combining it with durable acceptance under one ambiguous `process()` method would obscure ACK timing and retry ownership.
|
|
37
|
+
|
|
38
|
+
## Verification
|
|
39
|
+
|
|
40
|
+
The [implementation roadmap](../implementation-roadmap.md) defines acceptance criteria for concurrent receipt creation, stale claims, ambiguous commits, crash recovery, tenant scope, retention, and bounded shutdown. These guarantees require real PostgreSQL tests; in-memory simulations are insufficient evidence.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# 로컬 MVP 성능 기준값
|
|
2
|
+
|
|
3
|
+
실행일: 2026년 10월 27일 22시 30분 KST. 이 결과는 재현 가능한 소규모 로컬 기준값이며 서비스 처리 용량이나 SLA가 아니다.
|
|
4
|
+
|
|
5
|
+
## 환경
|
|
6
|
+
|
|
7
|
+
- Node 24.11.1, Prisma 7.10.1, PostgreSQL 16.15
|
|
8
|
+
- macOS arm64, Apple M1 Pro, 논리 CPU 10개
|
|
9
|
+
- Docker `postgres:16-alpine`, 임시 tmpfs 데이터, loopback 연결
|
|
10
|
+
- payload 265바이트, Prisma pool 최대 12개와 관리 연결 2개
|
|
11
|
+
- 수신 동시성 8, worker 동시성 4, 시나리오당 수신 요청 500회
|
|
12
|
+
|
|
13
|
+
## 결과
|
|
14
|
+
|
|
15
|
+
| 시나리오 | 수신 요청/초 | 수신 p50 | 수신 p95 | 수신 p99 | worker 처리 소요 | 새 기록 / 중복 |
|
|
16
|
+
| --------------------------- | -----------: | -------: | -------: | -------: | ---------------: | -------------: |
|
|
17
|
+
| 새 테이블 | 2,197.939 | 3.338 ms | 5.390 ms | 8.086 ms | 813.740 ms | 500 / 0 |
|
|
18
|
+
| 완료 기록 1만 개 보존 | 2,735.363 | 2.742 ms | 4.440 ms | 6.553 ms | 774.389 ms | 500 / 0 |
|
|
19
|
+
| 완료 기록 1만 개와 중복 90% | 2,242.860 | 3.420 ms | 4.930 ms | 5.971 ms | 76.884 ms | 50 / 450 |
|
|
20
|
+
|
|
21
|
+
모든 시나리오에서 수신과 worker 오류는 0건이었다. 같은 transaction에서 기록한 업무 효과 수는 각각 500개, 500개, 50개로 확인했다. 표의 요청/초는 수신 단계의 처리량이며 handler의 지속 처리량과 다르다.
|
|
22
|
+
|
|
23
|
+
## 재현과 해석
|
|
24
|
+
|
|
25
|
+
```sh
|
|
26
|
+
npm run benchmark
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
[benchmark.mjs](../../scripts/benchmark.mjs)는 테스트 실행기가 만든 임시 DB만 허용하며, 환경·지연 백분위·처리량·업무 효과 수·오류를 JSON으로 출력한다. 기존 애플리케이션 DB에는 실행하지 않는다.
|
|
30
|
+
|
|
31
|
+
세 시나리오는 고정 순서로 한 번 실행했다. 뒤 시나리오의 JIT·cache 예열이 더 높은 수신 처리량의 원인일 수 있으므로 완료 기록이 많을수록 빨라진다는 결론을 내리지 않는다. tmpfs, 작은 payload, 짧은 handler, 동일 호스트 연결은 운영 환경과 다르다. 큰 payload, 디스크 I/O, 장시간 backlog, DB pool 포화, tenant별 공정성, 반복 표본과 분산 측정은 별도 부하 시험이 필요하다.
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# Preview 1 local performance measurements
|
|
2
|
+
|
|
3
|
+
Measured on **28 October 2026, 20:49 KST**, using `@nestarc/inbox` **0.1.0-preview.1** and inbox **SQL schema version 2**. Three complete benchmark invocations ran serially, each against a newly created disposable PostgreSQL container. Every result is reported, including the slower fresh-table result in run 3.
|
|
4
|
+
|
|
5
|
+
The measurements show the cost of durable receipt acceptance and a separate worker drain with one small database write per message. They do not establish sustained production capacity or a latency SLA.
|
|
6
|
+
|
|
7
|
+
## Observed ranges across all three runs
|
|
8
|
+
|
|
9
|
+
These are minimum–maximum values from the three runs, not confidence intervals. The p95 range describes three separately calculated p95 values; it is not a pooled percentile of all requests.
|
|
10
|
+
|
|
11
|
+
| Scenario | Receive requests/s | Receive p95 | Worker completions/s | Worker drain time |
|
|
12
|
+
| --------------------------------------------- | ------------------: | --------------: | -------------------: | -----------------: |
|
|
13
|
+
| Fresh table, 500 unique messages | 1,400.499–2,123.920 | 5.286–11.612 ms | 531.734–542.775 | 921.193–940.319 ms |
|
|
14
|
+
| 10,000 retained receipts, 500 unique messages | 2,331.351–2,467.967 | 4.602–5.275 ms | 543.328–561.850 | 889.917–920.254 ms |
|
|
15
|
+
| 10,000 retained receipts, 90% duplicates | 1,812.640–2,061.929 | 5.290–6.622 ms | 523.475–561.857 | 88.991–95.516 ms |
|
|
16
|
+
|
|
17
|
+
**All nine scenario results passed their correctness checks.** Across 4,500 receive calls, the benchmark observed 3,150 newly accepted messages, 1,350 duplicate receipts, and exactly 3,150 committed business effects. No acceptance or worker errors were observed. Each scenario finished with the expected completed count and no pending, processing, retryable, or dead records.
|
|
18
|
+
|
|
19
|
+
Receive requests/s includes both newly accepted and duplicate calls. It measures `accept()` returning after its receipt transaction, **before business processing**. After the receive phase finishes, the benchmark calls `runOnce()` repeatedly to drain the backlog. Worker completions/s measures unique committed effects divided by that drain duration, including claims, handler transactions, fenced completion, and the final empty poll. The duplicate scenario drains only 50 unique messages rather than 500, so its shorter drain is not evidence of faster processing per message.
|
|
20
|
+
|
|
21
|
+
## Every measured run
|
|
22
|
+
|
|
23
|
+
| Run | Scenario | Receive requests/s | Receive p95 | Worker completions/s | Worker drain time | Accepted / duplicate |
|
|
24
|
+
| --- | ------------------------------- | -----------------: | ----------: | -------------------: | ----------------: | -------------------: |
|
|
25
|
+
| 1 | Fresh unique | 2,123.920 | 5.286 ms | 531.734 | 940.319 ms | 500 / 0 |
|
|
26
|
+
| 1 | Retained 10,000, unique | 2,331.351 | 5.275 ms | 543.328 | 920.254 ms | 500 / 0 |
|
|
27
|
+
| 1 | Retained 10,000, 90% duplicates | 1,812.640 | 5.901 ms | 523.475 | 95.516 ms | 50 / 450 |
|
|
28
|
+
| 2 | Fresh unique | 2,117.532 | 5.406 ms | 538.703 | 928.155 ms | 500 / 0 |
|
|
29
|
+
| 2 | Retained 10,000, unique | 2,467.967 | 5.244 ms | 561.850 | 889.917 ms | 500 / 0 |
|
|
30
|
+
| 2 | Retained 10,000, 90% duplicates | 1,864.466 | 6.622 ms | 547.005 | 91.407 ms | 50 / 450 |
|
|
31
|
+
| 3 | Fresh unique | 1,400.499 | 11.612 ms | 542.775 | 921.193 ms | 500 / 0 |
|
|
32
|
+
| 3 | Retained 10,000, unique | 2,358.995 | 4.602 ms | 558.711 | 894.918 ms | 500 / 0 |
|
|
33
|
+
| 3 | Retained 10,000, 90% duplicates | 2,061.929 | 5.290 ms | 561.857 | 88.991 ms | 50 / 450 |
|
|
34
|
+
|
|
35
|
+
The original JSON reports include p50, p99, elapsed times, exact counters, error maps, and environment metadata:
|
|
36
|
+
|
|
37
|
+
- [Run 1](2026-10-28-preview1-run1.json), completed at `2026-10-28T11:49:05.582Z` / 20:49:05.582 KST.
|
|
38
|
+
- [Run 2](2026-10-28-preview1-run2.json), completed at `2026-10-28T11:49:36.479Z` / 20:49:36.479 KST.
|
|
39
|
+
- [Run 3](2026-10-28-preview1-run3.json), completed at `2026-10-28T11:49:53.903Z` / 20:49:53.903 KST.
|
|
40
|
+
|
|
41
|
+
Each JSON is preserved from the benchmark's output, with the outer runner's setup log removed. The JSON's single-sample limitation applies to that individual invocation; this document combines three such invocations without dropping any sample.
|
|
42
|
+
|
|
43
|
+
## Environment and workload
|
|
44
|
+
|
|
45
|
+
| Setting | Value |
|
|
46
|
+
| ----------------------------------------- | ---------------------------------------------------------------------------------- |
|
|
47
|
+
| Package / schema | `0.1.0-preview.1` / SQL version 2 |
|
|
48
|
+
| Node.js | `24.22.1` |
|
|
49
|
+
| NestJS / Prisma | `11.2.8` / `7.10.1` |
|
|
50
|
+
| PostgreSQL | `16.15`, aarch64 Linux musl |
|
|
51
|
+
| Host | macOS arm64, Apple M1 Pro, 10 logical CPUs |
|
|
52
|
+
| Docker engine | `29.8.2`, 10 CPUs and 8,319,504,384 bytes of memory reported by Docker |
|
|
53
|
+
| PostgreSQL image | `postgres:16-alpine` |
|
|
54
|
+
| Image digest | `postgres@sha256:f4610795587492cd4fb887c2dc3de301b813278bb6f616992061141b5f15ac36` |
|
|
55
|
+
| Storage and transport | Docker tmpfs database; loopback forwarded TCP |
|
|
56
|
+
| Prisma pool / fixture administration pool | Maximum 12 / 2 connections |
|
|
57
|
+
| Receive / worker concurrency | 8 / 4 |
|
|
58
|
+
| Receive calls per scenario | 500 |
|
|
59
|
+
| JSON payload | 265 UTF-8 bytes |
|
|
60
|
+
| Retained history | 10,000 completed receipts with payloads purged |
|
|
61
|
+
| Worker effect | One `INSERT` in the same Prisma transaction as inbox completion |
|
|
62
|
+
| Automatic polling | Disabled; explicit bounded `runOnce()` drain |
|
|
63
|
+
|
|
64
|
+
Each scenario truncates the fixture tables, optionally seeds retained receipts, runs `ANALYZE`, and issues concurrent `SELECT 1` queries to establish connections. This preparation is excluded from the timing. There are no warmup inbox writes. Within each invocation, scenarios always run in this order: fresh unique, retained unique, retained with 90% duplicates.
|
|
65
|
+
|
|
66
|
+
## Reproduce
|
|
67
|
+
|
|
68
|
+
With Docker running and the repository dependencies installed:
|
|
69
|
+
|
|
70
|
+
```sh
|
|
71
|
+
npm run benchmark
|
|
72
|
+
npm run benchmark
|
|
73
|
+
npm run benchmark
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
Use Node.js `24.22.1` to match this measurement. The runs here used that runtime with the npm cache set to `/private/tmp/nestarc-inbox-npm-cache`; the cache location does not affect the timed database operations.
|
|
77
|
+
|
|
78
|
+
[The benchmark](../../scripts/benchmark.mjs) builds the package and uses [the fixture runner](../../scripts/with-test-db.mjs) to create a database on a random loopback port. The runner ignores an existing application `DATABASE_URL`, generates the test Prisma client, and removes its container afterward. All three invocations exited successfully and no labeled inbox fixture containers remained after the measurements.
|
|
79
|
+
|
|
80
|
+
## Interpretation limits
|
|
81
|
+
|
|
82
|
+
- Three short samples reveal some local variation, not a reliable estimate of long-running throughput. Run 3's fresh-table receive rate and p95 illustrate that variation; its cause was not measured.
|
|
83
|
+
- Fixed scenario order allows later cases to benefit from process/JIT/cache warmup. These results do not imply that retaining more receipts makes the package faster.
|
|
84
|
+
- A tmpfs database, loopback networking, 265-byte payloads, and one simple business write differ from disk-backed production databases, remote services, larger payloads, and application handlers.
|
|
85
|
+
- Receive and worker phases run separately. This does not measure concurrent ingress and processing under a sustained backlog, pool saturation, or backpressure.
|
|
86
|
+
- Replay, tenancy/RLS hooks, outbox emission, broker traffic, network faults, cold caches, and tenant fairness are not exercised by this benchmark. Their correctness tests are separate evidence.
|
|
87
|
+
- The older Preview 0 report used a different Node version and code/schema. This is a fresh baseline, not a controlled before/after comparison or a comparison with other products.
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
{
|
|
2
|
+
"kind": "nestarc-inbox-local-baseline",
|
|
3
|
+
"measuredAt": "2026-10-28T11:49:05.582Z",
|
|
4
|
+
"packageVersion": "0.1.0-preview.1",
|
|
5
|
+
"metadata": {
|
|
6
|
+
"node": "v24.22.1",
|
|
7
|
+
"platform": "darwin",
|
|
8
|
+
"arch": "arm64",
|
|
9
|
+
"cpuModel": "Apple M1 Pro",
|
|
10
|
+
"hostLogicalCpus": 10,
|
|
11
|
+
"postgres": "PostgreSQL 16.15 on aarch64-unknown-linux-musl, compiled by gcc (Alpine 15.2.0) 15.2.0, 64-bit",
|
|
12
|
+
"image": "postgres:16-alpine",
|
|
13
|
+
"storage": "Docker tmpfs; loopback forwarded TCP",
|
|
14
|
+
"prisma": "7.10.1",
|
|
15
|
+
"connectionMax": 12,
|
|
16
|
+
"fixtureAdminConnectionMax": 2,
|
|
17
|
+
"acceptConcurrency": 8,
|
|
18
|
+
"workerConcurrency": 4,
|
|
19
|
+
"iterationsPerScenario": 500,
|
|
20
|
+
"payloadBytes": 265,
|
|
21
|
+
"retainedPayloads": "purged tombstones",
|
|
22
|
+
"warmup": "SELECT 1 at accept concurrency before each scenario; no warmup inbox writes",
|
|
23
|
+
"workerEffect": "One INSERT through the same Prisma transaction as inbox completion"
|
|
24
|
+
},
|
|
25
|
+
"scenarios": [
|
|
26
|
+
{
|
|
27
|
+
"name": "fresh_unique",
|
|
28
|
+
"retainedCompletedReceipts": 0,
|
|
29
|
+
"iterations": 500,
|
|
30
|
+
"expectedUniqueMessages": 500,
|
|
31
|
+
"duplicateRatio": 0,
|
|
32
|
+
"accepted": 500,
|
|
33
|
+
"duplicates": 0,
|
|
34
|
+
"accept": {
|
|
35
|
+
"elapsedMs": 235.414,
|
|
36
|
+
"requestsPerSecond": 2123.92,
|
|
37
|
+
"latencyMs": {
|
|
38
|
+
"p50": 3.501,
|
|
39
|
+
"p95": 5.286,
|
|
40
|
+
"p99": 9.834
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
"worker": {
|
|
44
|
+
"drainMs": 940.319,
|
|
45
|
+
"completedPerSecond": 531.734,
|
|
46
|
+
"batches": 125,
|
|
47
|
+
"committedEffects": 500
|
|
48
|
+
},
|
|
49
|
+
"errors": {
|
|
50
|
+
"accept": {},
|
|
51
|
+
"worker": {}
|
|
52
|
+
},
|
|
53
|
+
"verified": true
|
|
54
|
+
},
|
|
55
|
+
{
|
|
56
|
+
"name": "retained_10000_unique",
|
|
57
|
+
"retainedCompletedReceipts": 10000,
|
|
58
|
+
"iterations": 500,
|
|
59
|
+
"expectedUniqueMessages": 500,
|
|
60
|
+
"duplicateRatio": 0,
|
|
61
|
+
"accepted": 500,
|
|
62
|
+
"duplicates": 0,
|
|
63
|
+
"accept": {
|
|
64
|
+
"elapsedMs": 214.468,
|
|
65
|
+
"requestsPerSecond": 2331.351,
|
|
66
|
+
"latencyMs": {
|
|
67
|
+
"p50": 3.131,
|
|
68
|
+
"p95": 5.275,
|
|
69
|
+
"p99": 7.647
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
"worker": {
|
|
73
|
+
"drainMs": 920.254,
|
|
74
|
+
"completedPerSecond": 543.328,
|
|
75
|
+
"batches": 125,
|
|
76
|
+
"committedEffects": 500
|
|
77
|
+
},
|
|
78
|
+
"errors": {
|
|
79
|
+
"accept": {},
|
|
80
|
+
"worker": {}
|
|
81
|
+
},
|
|
82
|
+
"verified": true
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"name": "retained_10000_duplicates_90pct",
|
|
86
|
+
"retainedCompletedReceipts": 10000,
|
|
87
|
+
"iterations": 500,
|
|
88
|
+
"expectedUniqueMessages": 50,
|
|
89
|
+
"duplicateRatio": 0.9,
|
|
90
|
+
"accepted": 50,
|
|
91
|
+
"duplicates": 450,
|
|
92
|
+
"accept": {
|
|
93
|
+
"elapsedMs": 275.841,
|
|
94
|
+
"requestsPerSecond": 1812.64,
|
|
95
|
+
"latencyMs": {
|
|
96
|
+
"p50": 4.26,
|
|
97
|
+
"p95": 5.901,
|
|
98
|
+
"p99": 6.941
|
|
99
|
+
}
|
|
100
|
+
},
|
|
101
|
+
"worker": {
|
|
102
|
+
"drainMs": 95.516,
|
|
103
|
+
"completedPerSecond": 523.475,
|
|
104
|
+
"batches": 13,
|
|
105
|
+
"committedEffects": 50
|
|
106
|
+
},
|
|
107
|
+
"errors": {
|
|
108
|
+
"accept": {},
|
|
109
|
+
"worker": {}
|
|
110
|
+
},
|
|
111
|
+
"verified": true
|
|
112
|
+
}
|
|
113
|
+
],
|
|
114
|
+
"limitations": [
|
|
115
|
+
"One local sample on a disposable tmpfs database; no SLA or production capacity claim.",
|
|
116
|
+
"Scenarios run in a fixed order; later scenarios may benefit from JIT/cache warmup.",
|
|
117
|
+
"No sustained load, network faults, broker traffic, cold-cache runs, or fairness measurement.",
|
|
118
|
+
"Receipt seed and schema preparation are excluded from measurements."
|
|
119
|
+
]
|
|
120
|
+
}
|