@siglo/kafka 1.0.3 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +303 -47
- package/dist/index.d.ts +66 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +279 -53
- package/dist/index.js.map +1 -1
- package/docs/outbox.md +134 -0
- package/package.json +22 -6
package/README.md
CHANGED
|
@@ -1,27 +1,152 @@
|
|
|
1
1
|
# @siglo/kafka Adapter Guide
|
|
2
2
|
|
|
3
|
-
`@siglo/kafka`
|
|
3
|
+
`@siglo/kafka` runs Siglo workflows for messages received by a Kafka consumer. Start with a single-step workflow, then add options such as idempotency, timeouts, and tenant routing as needed.
|
|
4
4
|
|
|
5
5
|
---
|
|
6
6
|
|
|
7
|
-
## 1.
|
|
7
|
+
## 1. Getting Started
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
This example publishes a JSON order to Kafka and processes it with one workflow step. You need Node.js (`^22.13.0 || >=24.0.0`) and a running Kafka broker reachable at `localhost:9092`. If your broker uses a different address or requires authentication, update the `Kafka` configuration in both files below.
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
### Install
|
|
12
|
+
|
|
13
|
+
In your Node.js project, install the runtime packages and a TypeScript runner:
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
npm install @siglo/core @siglo/kafka kafkajs
|
|
17
|
+
npm install --save-dev typescript tsx @types/node
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
The KafkaJS peer dependency is `^2.0.0`.
|
|
21
|
+
|
|
22
|
+
### Create a consumer
|
|
23
|
+
|
|
24
|
+
Save this as `consumer.ts`:
|
|
25
|
+
|
|
26
|
+
```typescript
|
|
27
|
+
import { Kafka } from 'kafkajs';
|
|
28
|
+
import { Step, flow } from '@siglo/core';
|
|
29
|
+
import { createKafkaMessageHandler } from '@siglo/kafka';
|
|
30
|
+
|
|
31
|
+
const processOrder = Step.create('processOrder', async (order: { orderId: string }) => {
|
|
32
|
+
console.log(`Processing order ${order.orderId}`);
|
|
33
|
+
return { processed: true };
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const handleMessage = createKafkaMessageHandler(flow(processOrder));
|
|
37
|
+
const kafka = new Kafka({ clientId: 'orders-consumer', brokers: ['localhost:9092'] });
|
|
38
|
+
const consumer = kafka.consumer({ groupId: 'orders-getting-started' });
|
|
39
|
+
|
|
40
|
+
async function main() {
|
|
41
|
+
await consumer.connect();
|
|
42
|
+
await consumer.subscribe({ topic: 'orders', fromBeginning: true });
|
|
43
|
+
await consumer.run({
|
|
44
|
+
eachMessage: handleMessage,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
process.on('SIGINT', async () => {
|
|
49
|
+
await consumer.disconnect();
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
main().catch((error) => {
|
|
53
|
+
console.error(error);
|
|
54
|
+
process.exit(1);
|
|
55
|
+
});
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
The handler accepts KafkaJS payloads directly. It adds `topic` and `partition` to the message passed to adapter callbacks.
|
|
59
|
+
|
|
60
|
+
### Publish a test order and run
|
|
61
|
+
|
|
62
|
+
Save this as `publish.ts`. It creates the demo topic if needed, sends one order, and disconnects:
|
|
63
|
+
|
|
64
|
+
```typescript
|
|
65
|
+
import { Kafka } from 'kafkajs';
|
|
66
|
+
|
|
67
|
+
const kafka = new Kafka({ clientId: 'orders-producer', brokers: ['localhost:9092'] });
|
|
68
|
+
|
|
69
|
+
async function main() {
|
|
70
|
+
const admin = kafka.admin();
|
|
71
|
+
await admin.connect();
|
|
72
|
+
try {
|
|
73
|
+
await admin.createTopics({
|
|
74
|
+
topics: [{ topic: 'orders', numPartitions: 1, replicationFactor: 1 }],
|
|
75
|
+
});
|
|
76
|
+
} finally {
|
|
77
|
+
await admin.disconnect();
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const producer = kafka.producer();
|
|
81
|
+
await producer.connect();
|
|
82
|
+
try {
|
|
83
|
+
await producer.send({
|
|
84
|
+
topic: 'orders',
|
|
85
|
+
messages: [{ key: 'order-1', value: JSON.stringify({ orderId: 'order-1' }) }],
|
|
86
|
+
});
|
|
87
|
+
} finally {
|
|
88
|
+
await producer.disconnect();
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
main().catch((error) => {
|
|
93
|
+
console.error(error);
|
|
94
|
+
process.exit(1);
|
|
95
|
+
});
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Run the publisher first so the topic and message exist, then start the consumer:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
npx tsx publish.ts
|
|
102
|
+
npx tsx consumer.ts
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
Alongside KafkaJS logs, the consumer prints:
|
|
106
|
+
|
|
107
|
+
```text
|
|
108
|
+
Processing order order-1
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
Leave the consumer running and run `npx tsx publish.ts` in another terminal to send another order. Press **Ctrl+C** to disconnect the consumer. `fromBeginning: true` reads existing records when the consumer group has no committed offset; subsequent runs continue from that group's committed offset.
|
|
112
|
+
|
|
113
|
+
This example requires no database or idempotency store. Each delivered message runs the workflow from the beginning, and redelivery can repeat processing. For the next steps, see [configuration options](#7-configuration-options-siglokafkaoptions) and the [production consumer example](#9-complete-production-consumer-example).
|
|
114
|
+
|
|
115
|
+
---
|
|
116
|
+
|
|
117
|
+
## 2. How It Works
|
|
118
|
+
|
|
119
|
+
The example follows this path for each order:
|
|
12
120
|
|
|
13
121
|
```mermaid
|
|
14
|
-
flowchart
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
WF -->|On Failure| Failure["Fail Claim (idempotencyStore)<br/>Re-throw / DLQ"]
|
|
122
|
+
flowchart LR
|
|
123
|
+
Producer["publish.ts"] -->|JSON order| Topic["Kafka: orders"]
|
|
124
|
+
Topic --> Consumer["KafkaJS eachMessage"]
|
|
125
|
+
Consumer --> Adapter["handleMessage"]
|
|
126
|
+
Adapter --> Workflow["Siglo: processOrder"]
|
|
20
127
|
```
|
|
21
128
|
|
|
129
|
+
1. **The producer writes a record.** `publish.ts` sends a record to the `orders` topic with key `order-1` and a JSON value containing `{ "orderId": "order-1" }`. Kafka stores the record in a partition and assigns it an offset, which identifies its position in that partition.
|
|
130
|
+
2. **KafkaJS delivers the record.** The consumer subscribes to `orders` as part of the `orders-getting-started` consumer group. KafkaJS calls `eachMessage` with the record, topic, and partition.
|
|
131
|
+
3. **The adapter prepares the workflow input.** `createKafkaMessageHandler(flow(processOrder))` creates a reusable handler. Each call decodes the record's value and parses its JSON, so `processOrder` receives `{ orderId: 'order-1' }`. The Kafka record's key and headers are separate from this input. A custom `input` option can include that metadata when needed.
|
|
132
|
+
4. **Siglo runs the workflow.** `flow(processOrder)` contains one step, which logs the order ID and returns `{ processed: true }`. Adding more steps to `flow(...)` runs them in sequence within the same consumer process, with earlier results available to later steps.
|
|
133
|
+
5. **The handler finishes.** The adapter returns the workflow result to the caller. In the example, `eachMessage` simply awaits it; returning a result does not publish another Kafka message. With the default auto-commit settings, KafkaJS manages the consumer group's committed offsets after successful processing.
|
|
134
|
+
|
|
135
|
+
### What happens on failure?
|
|
136
|
+
|
|
137
|
+
If the workflow throws, the handler rejects and the error propagates to KafkaJS. The failed record is not marked successfully processed by `eachMessage`; retry and restart behavior depends on the KafkaJS consumer configuration. Configure `deadLetter` to quarantine invalid messages or exhausted workflow failures; without it, those errors propagate to KafkaJS. Workflow retries are bounded by `retry.maxAttempts` per delivery. When KafkaJS supplies a `heartbeat` callback, the adapter calls it before and periodically during record processing (every 5 seconds by default). A heartbeat failure requests workflow cancellation through its abort signal and rejects processing. Heartbeats require a responsive event loop; synchronous CPU work can still block them.
|
|
138
|
+
|
|
139
|
+
A crash before an offset is committed can cause a record to be delivered again. Each delivery starts the workflow from the beginning, so external actions such as charging a payment should use an idempotency key to avoid repeating the action.
|
|
140
|
+
|
|
141
|
+
### What do the optional settings add?
|
|
142
|
+
|
|
143
|
+
The basic handler works without options. An `idempotencyStore` with `groupId` tracks claims and completed results within the consumer group and tenant, `validate` checks input, and `timeout` sets a processing deadline. Malformed JSON is rejected by default, while null-valued tombstones are skipped. See [production policies](#11-production-policies) for retry, dead-letter, and shutdown examples. The adapter can also read the namespace from the record's `x-namespace` header. See the [configuration options](#7-configuration-options-siglokafkaoptions) for details.
|
|
144
|
+
|
|
145
|
+
> **Note on guarantees:** An idempotency claim does not make external effects atomic with workflow completion. If a payment succeeds and the process crashes before the claim is completed, a later retry may repeat the payment after the claim expires. Use provider-level deduplication keys for those effects.
|
|
146
|
+
|
|
22
147
|
---
|
|
23
148
|
|
|
24
|
-
##
|
|
149
|
+
## 3. Execution Model: In-Process vs. Distributed Step Execution
|
|
25
150
|
|
|
26
151
|
A common question in distributed event-driven systems is:
|
|
27
152
|
|
|
@@ -78,7 +203,7 @@ Each consumer server processes independent messages in parallel at full hardware
|
|
|
78
203
|
|
|
79
204
|
---
|
|
80
205
|
|
|
81
|
-
##
|
|
206
|
+
## 4. How to Distribute Steps Across Different Servers (When Needed)
|
|
82
207
|
|
|
83
208
|
Distributing individual steps across physically separate machines is only necessary in specific architectural scenarios:
|
|
84
209
|
1. **Heterogeneous Compute**: Steps 1–3 run on standard Node.js CPU instances, but Step 4 requires dedicated GPU worker nodes (e.g. AI model inference).
|
|
@@ -141,28 +266,14 @@ const stage2Handler = createKafkaMessageHandler(stage2Flow);
|
|
|
141
266
|
|
|
142
267
|
---
|
|
143
268
|
|
|
144
|
-
##
|
|
269
|
+
## 5. Crash Recovery, Rebalances & Failure Semantics
|
|
145
270
|
|
|
146
271
|
What happens if Server 1 is executing **Step 6** of 10 and the process crashes?
|
|
147
272
|
|
|
148
|
-
1. **Uncommitted Offset**: The
|
|
273
|
+
1. **Uncommitted Offset**: The handler rejects when a workflow fails. With the default KafkaJS processing settings, that record remains unresolved; earlier successful records may already have committed offsets.
|
|
149
274
|
2. **Kafka Partition Rebalance**: Kafka detects consumer heartbeat loss and reassigns that topic partition to another consumer (Server 2).
|
|
150
|
-
3. **Idempotency Protection**: Server 2 re-receives
|
|
151
|
-
4. **
|
|
152
|
-
|
|
153
|
-
---
|
|
154
|
-
|
|
155
|
-
## 5. Installation & Requirements
|
|
156
|
-
|
|
157
|
-
Install `@siglo/kafka` and `kafkajs`:
|
|
158
|
-
|
|
159
|
-
```bash
|
|
160
|
-
npm install @siglo/core @siglo/kafka kafkajs
|
|
161
|
-
```
|
|
162
|
-
|
|
163
|
-
Peer dependency matrix:
|
|
164
|
-
- `Siglo`: `*` (v1.0.0+)
|
|
165
|
-
- `kafkajs`: `^2.0.0`
|
|
275
|
+
3. **Idempotency Protection**: Server 2 re-receives records from the last committed offset. A configured `IdempotencyStore` skips completed records and rejects records with active claims. An expired claim permits replay, so external effects still need their own deduplication keys.
|
|
276
|
+
4. **Workflow Restart**: This adapter starts each workflow from the beginning. It does not automatically restore durable checkpoints.
|
|
166
277
|
|
|
167
278
|
---
|
|
168
279
|
|
|
@@ -170,13 +281,22 @@ Peer dependency matrix:
|
|
|
170
281
|
|
|
171
282
|
### `createKafkaMessageHandler(workflow, options)`
|
|
172
283
|
|
|
173
|
-
Creates
|
|
284
|
+
Creates a handler for `consumer.run({ eachMessage: handler })`:
|
|
174
285
|
|
|
175
286
|
```typescript
|
|
176
287
|
function createKafkaMessageHandler<TInput = any, TOutput = any>(
|
|
177
288
|
workflow: Flow<TInput, TOutput>,
|
|
178
289
|
options?: SigloKafkaOptions<TInput, TOutput>
|
|
179
|
-
): (payload:
|
|
290
|
+
): (payload: SigloKafkaPayload) => Promise<any>;
|
|
291
|
+
|
|
292
|
+
interface SigloKafkaPayload {
|
|
293
|
+
topic: string;
|
|
294
|
+
partition: number;
|
|
295
|
+
message: SigloKafkaRecord; // Kafka record; nested topic/partition are optional
|
|
296
|
+
heartbeat?: () => Promise<void>;
|
|
297
|
+
isRunning?: () => boolean;
|
|
298
|
+
isStale?: () => boolean;
|
|
299
|
+
}
|
|
180
300
|
```
|
|
181
301
|
|
|
182
302
|
### `SigloKafkaMessage` Interface
|
|
@@ -188,7 +308,8 @@ export interface SigloKafkaMessage {
|
|
|
188
308
|
partition: number;
|
|
189
309
|
topic: string;
|
|
190
310
|
offset: string;
|
|
191
|
-
|
|
311
|
+
timestamp?: string;
|
|
312
|
+
headers?: Record<string, Buffer | string | (Buffer | string)[] | undefined>;
|
|
192
313
|
}
|
|
193
314
|
```
|
|
194
315
|
|
|
@@ -198,28 +319,56 @@ export interface SigloKafkaMessage {
|
|
|
198
319
|
|
|
199
320
|
| Option | Type | Default | Description |
|
|
200
321
|
| :--- | :--- | :--- | :--- |
|
|
201
|
-
| `input` | `(message
|
|
202
|
-
| `
|
|
203
|
-
| `
|
|
204
|
-
| `
|
|
205
|
-
| `
|
|
206
|
-
| `
|
|
322
|
+
| `input` | `(message) => TInput \| Promise<TInput>` | JSON parser | Custom deserializer with record metadata. |
|
|
323
|
+
| `validate` | `(input, message) => void \| Promise<void>` | `undefined` | Throw to reject invalid workflow input before execution. |
|
|
324
|
+
| `invalidJson` | `'error' \| 'raw'` | `'error'` | Reject malformed JSON, or pass `{ raw: string }` for legacy behavior. |
|
|
325
|
+
| `tombstone` | `'skip' \| 'error' \| 'process'` | `'skip'` | Skip null values, reject them, or run the input mapper/workflow. |
|
|
326
|
+
| `output` | `(result, message) => TOutput \| Promise<TOutput>` | Workflow result | Formats successful workflow results, including cached ones. Not called for skips or dead letters. |
|
|
327
|
+
| `groupId` | `string` | `undefined` | Required with an idempotency store. Set to the KafkaJS consumer group ID. |
|
|
328
|
+
| `idempotencyStore` | `IdempotencyStore` | `undefined` | Tracks claims and results. Use a shared store across replicas of one consumer group. |
|
|
329
|
+
| `idempotencyKey` | `(message) => string` | `getKafkaRecordId(message)` | Business identity; automatically scoped by group and tenant. |
|
|
330
|
+
| `idempotencyTtlMs` | `number` | `86400000` | TTL passed explicitly to the store. Processing stops at 80% of this budget; claims are not renewed. |
|
|
331
|
+
| `heartbeatIntervalMs` | `number` | `5000` | Kafka heartbeat cadence; set below the session timeout. `0` disables heartbeats, while cancellation checks remain active. |
|
|
332
|
+
| `timeout` | `number` | `undefined` | Record processing budget in milliseconds, including retries. With a store, must be less than 80% of its TTL. |
|
|
333
|
+
| `namespace` | `string \| (message) => string \| undefined` | `x-namespace` header | Tenant scope, also included in idempotency keys. |
|
|
334
|
+
| `signal` | `AbortSignal` | `undefined` | Application shutdown/cancellation signal, forwarded to the workflow. |
|
|
335
|
+
| `retry` | `{ maxAttempts, delayMs?, shouldRetry? }` | One attempt | Retries workflow errors only; `maxAttempts` includes the initial run. |
|
|
336
|
+
| `deadLetter` | `(failure: KafkaDeadLetter) => Promise<void>` | `undefined` | Awaited quarantine callback after invalid input or exhausted workflow attempts. |
|
|
207
337
|
|
|
208
338
|
---
|
|
209
339
|
|
|
210
340
|
## 8. Idempotency Key Semantics
|
|
211
341
|
|
|
212
|
-
|
|
342
|
+
`getKafkaRecordId(message)` returns `kafka:<topic>:<partition>:<offset>`. This distinguishes events that share an entity key and stays stable across record redelivery. It is also useful for deduplication at external providers when a business event ID is unavailable.
|
|
343
|
+
|
|
344
|
+
Stored claim keys use `kafka:` followed by a JSON array of `[groupId, namespaceOrNull, identity]`. The default identity is the record ID. JSON encoding keeps group and tenant boundaries unambiguous. Independent consumer groups can safely share a store when each supplies its own `groupId`.
|
|
345
|
+
|
|
346
|
+
For business deduplication across records, provide a stable event ID through a header:
|
|
347
|
+
|
|
348
|
+
```typescript
|
|
349
|
+
const handler = createKafkaMessageHandler(orderWorkflow, {
|
|
350
|
+
groupId: 'orders-service',
|
|
351
|
+
idempotencyStore,
|
|
352
|
+
idempotencyKey: message => {
|
|
353
|
+
const eventId = message.headers?.['event-id'];
|
|
354
|
+
return eventId === undefined ? getKafkaRecordId(message) : String(eventId);
|
|
355
|
+
},
|
|
356
|
+
});
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
Import `getKafkaRecordId` from `@siglo/kafka`. A Kafka key commonly identifies an entity shared by multiple events; only use it as a business deduplication key when it uniquely identifies an event.
|
|
360
|
+
|
|
361
|
+
### Claim states
|
|
362
|
+
|
|
363
|
+
1. `'completed'`: Reads the cached outcome through `store.get()` without rerunning the workflow. Successful results still pass through `output`; cached skips and dead letters do not. Stores without cached-result access return `{ deduplicated: true }`.
|
|
364
|
+
2. `'in-progress'`: Rejects processing so an active claim is not treated as successfully handled.
|
|
365
|
+
3. `'claimed'`: Executes the workflow and saves its result or handled disposition. An ordinary processing failure releases the claim so a retry is possible.
|
|
213
366
|
|
|
214
|
-
|
|
367
|
+
Cancellation, heartbeat loss, timeouts, and uncertain completion writes retain claims until expiry. Output-mapping errors retain completed claims. This prevents immediate replay while external work may still be running. The default claim TTL is 24 hours; choose a shorter explicit TTL when faster recovery is needed, while keeping it comfortably above the total processing budget.
|
|
215
368
|
|
|
216
|
-
|
|
217
|
-
- If `message.key` is null, the key falls back to partition and offset: `kafka:orders:2:104921`.
|
|
369
|
+
**Migration:** Deduplication now requires `groupId`, and keys include the group and tenant. Previous stored keys do not match this format, even when `idempotencyKey` uses the earlier identity algorithm. Plan the deployment with idempotent external effects and avoid overlapping old and new consumers with different claim formats. Cached outcomes now use a reserved `$sigloKafka` envelope internally; adapter callers receive the result or disposition, not the envelope.
|
|
218
370
|
|
|
219
|
-
|
|
220
|
-
1. `'completed'`: The message was already processed in a previous attempt. The adapter immediately returns the cached result without executing the workflow.
|
|
221
|
-
2. `'in-progress'`: Throws `Error('Kafka message ... is already being processed concurrently')`, preventing duplicate parallel writes.
|
|
222
|
-
3. `'claimed'`: Executes workflow. On completion, saves result; on error, calls `store.fail()` so subsequent retries are permitted.
|
|
371
|
+
Malformed JSON now rejects by default and tombstones are skipped. Set `invalidJson: 'raw'` and `tombstone: 'process'` to retain the earlier parsing behavior.
|
|
223
372
|
|
|
224
373
|
---
|
|
225
374
|
|
|
@@ -266,6 +415,7 @@ const kafka = new Kafka({
|
|
|
266
415
|
const consumer = kafka.consumer({ groupId: 'payment-engine-group' });
|
|
267
416
|
|
|
268
417
|
const messageHandler = createKafkaMessageHandler(paymentPipeline, {
|
|
418
|
+
groupId: 'payment-engine-group',
|
|
269
419
|
idempotencyStore,
|
|
270
420
|
idempotencyTtlMs: 3600000, // 1 hour claim lock
|
|
271
421
|
timeout: 5000, // 5s execution SLA
|
|
@@ -296,3 +446,109 @@ process.on('SIGTERM', async () => {
|
|
|
296
446
|
process.exit(0);
|
|
297
447
|
});
|
|
298
448
|
```
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
---
|
|
452
|
+
|
|
453
|
+
## 10. Batch Processing
|
|
454
|
+
|
|
455
|
+
Use `createKafkaBatchHandler` when you want explicit control over progress within each partition batch. After connecting and subscribing a KafkaJS consumer:
|
|
456
|
+
|
|
457
|
+
```typescript
|
|
458
|
+
import { createKafkaBatchHandler } from '@siglo/kafka';
|
|
459
|
+
|
|
460
|
+
await consumer.run({
|
|
461
|
+
eachBatchAutoResolve: false,
|
|
462
|
+
eachBatch: createKafkaBatchHandler(orderWorkflow, {
|
|
463
|
+
groupId: 'orders-service',
|
|
464
|
+
idempotencyStore,
|
|
465
|
+
heartbeatIntervalMs: 3000,
|
|
466
|
+
}),
|
|
467
|
+
});
|
|
468
|
+
```
|
|
469
|
+
|
|
470
|
+
Records run sequentially within each batch. After each successful handler call, the adapter checks that the consumer is running and the batch is still current, heartbeats, resolves that record's offset, and calls KafkaJS's `commitOffsetsIfNecessary()`. The callback checks KafkaJS's configured commit interval and threshold. It can commit when those conditions are met even if consumer auto-commit is disabled. For fully manual commits, set both KafkaJS's `autoCommit: false` and the adapter's batch option `commitOffsets: false`; records will still be resolved locally.
|
|
471
|
+
|
|
472
|
+
An error stops the batch and leaves the failed and subsequent records unresolved. Stopping the consumer or marking a batch stale also stops further processing and offset resolution; the adapter requests cancellation of an active workflow. External operations must honor the workflow abort signal to stop their work. Keep `eachBatchAutoResolve: false` so KafkaJS does not automatically resolve the rest of a partially processed batch.
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
---
|
|
476
|
+
|
|
477
|
+
## 11. Production Policies
|
|
478
|
+
|
|
479
|
+
After defining `orderWorkflow`, an idempotency store, and a connected KafkaJS producer, configure validation, retries, and quarantine:
|
|
480
|
+
|
|
481
|
+
```typescript
|
|
482
|
+
const shutdown = new AbortController();
|
|
483
|
+
const handler = createKafkaMessageHandler(orderWorkflow, {
|
|
484
|
+
groupId: 'orders-service', // Match the KafkaJS consumer group.
|
|
485
|
+
idempotencyStore,
|
|
486
|
+
idempotencyTtlMs: 60000,
|
|
487
|
+
timeout: 30000,
|
|
488
|
+
signal: shutdown.signal,
|
|
489
|
+
validate: order => {
|
|
490
|
+
if (!order || typeof order.orderId !== 'string') {
|
|
491
|
+
throw new Error('orderId is required');
|
|
492
|
+
}
|
|
493
|
+
},
|
|
494
|
+
retry: {
|
|
495
|
+
maxAttempts: 3,
|
|
496
|
+
delayMs: 500,
|
|
497
|
+
shouldRetry: error => !(error instanceof PermanentOrderError),
|
|
498
|
+
},
|
|
499
|
+
deadLetter: async ({ message, recordId, error, attempts }) => {
|
|
500
|
+
const value = message.value == null ? null : Buffer.isBuffer(message.value)
|
|
501
|
+
? message.value.toString('base64')
|
|
502
|
+
: Buffer.from(message.value).toString('base64');
|
|
503
|
+
await producer.send({
|
|
504
|
+
topic: 'orders.dead-letter',
|
|
505
|
+
messages: [{
|
|
506
|
+
key: recordId,
|
|
507
|
+
value: JSON.stringify({
|
|
508
|
+
recordId, sourceTopic: message.topic, partition: message.partition,
|
|
509
|
+
offset: message.offset, valueBase64: value,
|
|
510
|
+
error: String(error), attempts,
|
|
511
|
+
}),
|
|
512
|
+
}],
|
|
513
|
+
});
|
|
514
|
+
},
|
|
515
|
+
});
|
|
516
|
+
```
|
|
517
|
+
|
|
518
|
+
`PermanentOrderError` represents an application-defined error class. Create the dead-letter topic before consuming and make the callback reject if publication fails. Invalid JSON, rejected tombstones, and validation errors go directly to the callback with zero workflow attempts. Transient workflow errors retry up to the configured limit, restarting the whole workflow each time. The limit applies per delivery; without successful quarantine, later redeliveries may retry again.
|
|
519
|
+
|
|
520
|
+
The adapter waits for dead-letter publication before acknowledging a handled failure. When a store is configured, it caches the disposition to avoid publishing again on ordinary redelivery. A crash between publication and recording that disposition can still publish a duplicate; the destination should deduplicate with `recordId`. Cancellation, heartbeat failures, store failures, and output-mapping errors do not go to the dead-letter callback.
|
|
521
|
+
|
|
522
|
+
For graceful shutdown, abort processing before disconnecting. For example, after creating `consumer` and `producer`:
|
|
523
|
+
|
|
524
|
+
```typescript
|
|
525
|
+
process.once('SIGTERM', async () => {
|
|
526
|
+
shutdown.abort();
|
|
527
|
+
await consumer.disconnect();
|
|
528
|
+
await producer.disconnect();
|
|
529
|
+
});
|
|
530
|
+
```
|
|
531
|
+
|
|
532
|
+
The signal is forwarded to Siglo steps. Pass the step's abort signal to external clients that support cancellation. Batch processing also checks for stopped consumers and stale batches during execution, even when automatic heartbeats are disabled. A cancelled record remains unresolved.
|
|
533
|
+
|
|
534
|
+
### Long workflows and external effects
|
|
535
|
+
|
|
536
|
+
With a store configured, the adapter stops processing at 80% of `idempotencyTtlMs` or at the earlier `timeout`. This leaves a margin before claim expiry; it does not renew claims or provide ownership fencing at the storage layer. Timers require a responsive event loop. Checks after callbacks also reject late results instead of completing the claim after the deadline.
|
|
537
|
+
|
|
538
|
+
Cancellation prevents further adapter work, but cannot undo a payment or forcibly stop an external operation that ignores its signal. Use a stable business event ID as the provider's idempotency key. An expired claim, a long process pause, or a delayed store write can still race with another consumer; the generic store API cannot eliminate those races. Split work that exceeds the claim budget into separately persisted stages rather than relying on an arbitrarily long claim.
|
|
539
|
+
|
|
540
|
+
For a database update followed by a Kafka event, use the [transactional inbox and outbox example](docs/outbox.md). It commits the database effect, processed-event marker, and pending outgoing event together, then relays the event with a stable ID.
|
|
541
|
+
|
|
542
|
+
---
|
|
543
|
+
|
|
544
|
+
## 12. Live Kafka Recovery Tests
|
|
545
|
+
|
|
546
|
+
From the repository root, with Docker available:
|
|
547
|
+
|
|
548
|
+
```bash
|
|
549
|
+
docker compose -p siglo-kafka-recovery-tests -f packages/kafka/integration/compose.yaml up -d --wait
|
|
550
|
+
npm run test:integration --workspace=@siglo/kafka
|
|
551
|
+
docker compose -p siglo-kafka-recovery-tests -f packages/kafka/integration/compose.yaml down -v
|
|
552
|
+
```
|
|
553
|
+
|
|
554
|
+
The isolated broker listens on `localhost:19092`, and PostgreSQL for the outbox recipe listens on `localhost:15434`. Set `KAFKA_BROKER` and `OUTBOX_DATABASE_URL` to use other test services. Tests create uniquely named topics and delete them afterward. They exercise shared keys across independent groups, poison-message quarantine, tombstones, offset recovery after a process is killed, provider-level effect deduplication, and rebalances during long workflows. The PostgreSQL test executes the published outbox recipe, checking concurrent duplicate events, transaction rollback, and stable outgoing event IDs after an uncertain send. Unit tests cover cancellation, lease deadlines, validation, retry exhaustion, and failed dead-letter publication.
|
package/dist/index.d.ts
CHANGED
|
@@ -1,28 +1,85 @@
|
|
|
1
1
|
import { Flow, IdempotencyStore } from '@siglo/core';
|
|
2
|
+
/** A Kafka record enriched with its topic and partition for adapter callbacks. */
|
|
2
3
|
export interface SigloKafkaMessage {
|
|
3
4
|
key?: Buffer | string | null;
|
|
4
5
|
value?: Buffer | string | null;
|
|
5
6
|
partition: number;
|
|
6
7
|
topic: string;
|
|
7
8
|
offset: string;
|
|
8
|
-
|
|
9
|
+
timestamp?: string;
|
|
10
|
+
headers?: Record<string, Buffer | string | (Buffer | string)[] | undefined>;
|
|
11
|
+
}
|
|
12
|
+
export type SigloKafkaRecord = Omit<SigloKafkaMessage, 'topic' | 'partition'> & Partial<Pick<SigloKafkaMessage, 'topic' | 'partition'>>;
|
|
13
|
+
export interface SigloKafkaPayload {
|
|
14
|
+
topic: string;
|
|
15
|
+
partition: number;
|
|
16
|
+
message: SigloKafkaRecord;
|
|
17
|
+
heartbeat?: () => Promise<void>;
|
|
18
|
+
isRunning?: () => boolean;
|
|
19
|
+
isStale?: () => boolean;
|
|
20
|
+
}
|
|
21
|
+
export interface SigloKafkaBatchPayload {
|
|
22
|
+
batch: {
|
|
23
|
+
topic: string;
|
|
24
|
+
partition: number;
|
|
25
|
+
messages: SigloKafkaRecord[];
|
|
26
|
+
};
|
|
27
|
+
resolveOffset: (offset: string) => void;
|
|
28
|
+
heartbeat: () => Promise<void>;
|
|
29
|
+
commitOffsetsIfNecessary: () => Promise<void>;
|
|
30
|
+
isRunning: () => boolean;
|
|
31
|
+
isStale: () => boolean;
|
|
32
|
+
}
|
|
33
|
+
export interface KafkaDeadLetter {
|
|
34
|
+
message: SigloKafkaMessage;
|
|
35
|
+
/** Stable across deliveries; use as a deduplication key at the destination. */
|
|
36
|
+
recordId: string;
|
|
37
|
+
error: unknown;
|
|
38
|
+
attempts: number;
|
|
9
39
|
}
|
|
10
40
|
export interface SigloKafkaOptions<TInput = any, TOutput = any> {
|
|
11
41
|
input?: (message: SigloKafkaMessage) => Promise<TInput> | TInput;
|
|
42
|
+
validate?: (input: TInput, message: SigloKafkaMessage) => Promise<void> | void;
|
|
12
43
|
output?: (result: any, message: SigloKafkaMessage) => Promise<TOutput> | TOutput;
|
|
44
|
+
/** Required when using an idempotency store; use the KafkaJS consumer group ID. */
|
|
45
|
+
groupId?: string;
|
|
13
46
|
idempotencyStore?: IdempotencyStore;
|
|
47
|
+
/** Default: 24 hours. Processing is cancelled at 80% of this budget. */
|
|
14
48
|
idempotencyTtlMs?: number;
|
|
49
|
+
/** Business identity within the group and tenant scope. */
|
|
50
|
+
idempotencyKey?: (message: SigloKafkaMessage) => string;
|
|
15
51
|
timeout?: number;
|
|
16
52
|
namespace?: string | ((message: SigloKafkaMessage) => string | undefined);
|
|
53
|
+
heartbeatIntervalMs?: number;
|
|
54
|
+
signal?: AbortSignal;
|
|
55
|
+
/** Default: error. Use raw for the previous { raw: string } behavior. */
|
|
56
|
+
invalidJson?: 'error' | 'raw';
|
|
57
|
+
/** Default: skip. process passes tombstones through input (or {} by default). */
|
|
58
|
+
tombstone?: 'skip' | 'error' | 'process';
|
|
59
|
+
retry?: {
|
|
60
|
+
/** Includes the initial attempt. Default: 1. Applies to workflow errors only. */
|
|
61
|
+
maxAttempts: number;
|
|
62
|
+
delayMs?: number;
|
|
63
|
+
shouldRetry?: (error: unknown, message: SigloKafkaMessage) => boolean;
|
|
64
|
+
};
|
|
65
|
+
/** Awaited before a failed record is considered handled. Failure rejects processing. */
|
|
66
|
+
deadLetter?: (failure: KafkaDeadLetter) => Promise<void>;
|
|
17
67
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
export declare
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
68
|
+
export interface SigloKafkaBatchOptions<TInput = any, TOutput = any> extends SigloKafkaOptions<TInput, TOutput> {
|
|
69
|
+
commitOffsets?: boolean;
|
|
70
|
+
}
|
|
71
|
+
export declare class KafkaInvalidMessageError extends Error {
|
|
72
|
+
constructor(message: string, options?: ErrorOptions);
|
|
73
|
+
}
|
|
74
|
+
export declare class KafkaProcessingCancelledError extends Error {
|
|
75
|
+
constructor(message: string, options?: ErrorOptions);
|
|
76
|
+
}
|
|
77
|
+
/** Stable record identity for downstream deduplication, independent of consumer group. */
|
|
78
|
+
export declare function getKafkaRecordId(message: Pick<SigloKafkaMessage, 'topic' | 'partition' | 'offset'>): string;
|
|
79
|
+
/** Creates a handler that can be passed directly to KafkaJS eachMessage. */
|
|
80
|
+
export declare function createKafkaMessageHandler<TInput = any, TOutput = any>(workflow: Flow<TInput, TOutput>, options?: SigloKafkaOptions<TInput, TOutput>): (payload: SigloKafkaPayload) => Promise<any>;
|
|
81
|
+
/** Processes partition batches sequentially; use eachBatchAutoResolve: false. */
|
|
82
|
+
export declare function createKafkaBatchHandler<TInput = any, TOutput = any>(workflow: Flow<TInput, TOutput>, options?: SigloKafkaBatchOptions<TInput, TOutput>): (payload: SigloKafkaBatchPayload) => Promise<void>;
|
|
26
83
|
export type RivuletKafkaMessage = SigloKafkaMessage;
|
|
27
84
|
export type RivuletKafkaOptions<TInput = any, TOutput = any> = SigloKafkaOptions<TInput, TOutput>;
|
|
28
85
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAoB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAoB,gBAAgB,EAA8C,MAAM,aAAa,CAAC;AAEnH,kFAAkF;AAClF,MAAM,WAAW,iBAAiB;IAC9B,GAAG,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC;CAC/E;AAED,MAAM,MAAM,gBAAgB,GAAG,IAAI,CAAC,iBAAiB,EAAE,OAAO,GAAG,WAAW,CAAC,GACzE,OAAO,CAAC,IAAI,CAAC,iBAAiB,EAAE,OAAO,GAAG,WAAW,CAAC,CAAC,CAAC;AAE5D,MAAM,WAAW,iBAAiB;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,gBAAgB,CAAC;IAC1B,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,MAAM,OAAO,CAAC;IAC1B,OAAO,CAAC,EAAE,MAAM,OAAO,CAAC;CAC3B;AAED,MAAM,WAAW,sBAAsB;IACnC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,gBAAgB,EAAE,CAAA;KAAE,CAAC;IAC1E,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B,wBAAwB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,SAAS,EAAE,MAAM,OAAO,CAAC;IACzB,OAAO,EAAE,MAAM,OAAO,CAAC;CAC1B;AAED,MAAM,WAAW,eAAe;IAC5B,OAAO,EAAE,iBAAiB,CAAC;IAC3B,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,GAAG;IAC1D,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC;IACjE,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IAC/E,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAC;IACjF,mFAAmF;IACnF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,wEAAwE;IACxE,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,2DAA2D;IAC3D,cAAc,CAAC,EAAE,CAAC,OAAO,EAAE,iBAAiB,KAAK,MAAM,CAAC;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,OAAO,EAAE,iBAAiB,KAAK,MAAM,GAAG,SAAS,CAAC,CAAC;IAC1E,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,WAAW,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;IAC9B,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACzC,KAAK,CAAC,EAAE;QACJ,iFAAiF;QACjF,WAAW,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,iBAAiB,KAAK,OAAO,CAAC;KACzE,CAAC;IACF,wFAAwF;IACxF,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC5D;AAED,MAAM,WAAW,sBAAsB,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,GAAG,CAAE,SAAQ,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC;IAC3G,aAAa,CAAC,EAAE,OAAO,CAAC;CAC3B;AAED,qBAAa,wBAAyB,SAAQ,KAAK;gBACnC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY;CAItD;AAED,qBAAa,6BAA8B,SAAQ,KAAK;gBACxC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY;CAItD;AAED,0FAA0F;AAC1F,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,iBAAiB,EAAE,OAAO,GAAG,WAAW,GAAG,QAAQ,CAAC,GAAG,MAAM,CAE3G;AAqGD,4EAA4E;AAC5E,wBAAgB,yBAAyB,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,GAAG,EACjE,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,OAAO,GAAE,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAM,aAkBzB,iBAAiB,KAAG,OAAO,CAAC,GAAG,CAAC,CA8G1D;AAED,iFAAiF;AACjF,wBAAgB,uBAAuB,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,GAAG,EAC/D,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,OAAO,GAAE,sBAAsB,CAAC,MAAM,EAAE,OAAO,CAAM,aAG9B,sBAAsB,KAAG,OAAO,CAAC,IAAI,CAAC,CAoBhE;AAED,MAAM,MAAM,mBAAmB,GAAG,iBAAiB,CAAC;AACpD,MAAM,MAAM,mBAAmB,CAAC,MAAM,GAAG,GAAG,EAAE,OAAO,GAAG,GAAG,IAAI,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,82 +1,308 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.KafkaProcessingCancelledError = exports.KafkaInvalidMessageError = void 0;
|
|
4
|
+
exports.getKafkaRecordId = getKafkaRecordId;
|
|
3
5
|
exports.createKafkaMessageHandler = createKafkaMessageHandler;
|
|
6
|
+
exports.createKafkaBatchHandler = createKafkaBatchHandler;
|
|
7
|
+
const core_1 = require("@siglo/core");
|
|
8
|
+
class KafkaInvalidMessageError extends Error {
|
|
9
|
+
constructor(message, options) {
|
|
10
|
+
super(message, options);
|
|
11
|
+
this.name = 'KafkaInvalidMessageError';
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
exports.KafkaInvalidMessageError = KafkaInvalidMessageError;
|
|
15
|
+
class KafkaProcessingCancelledError extends Error {
|
|
16
|
+
constructor(message, options) {
|
|
17
|
+
super(message, options);
|
|
18
|
+
this.name = 'KafkaProcessingCancelledError';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
exports.KafkaProcessingCancelledError = KafkaProcessingCancelledError;
|
|
22
|
+
/** Stable record identity for downstream deduplication, independent of consumer group. */
|
|
23
|
+
function getKafkaRecordId(message) {
|
|
24
|
+
return `kafka:${message.topic}:${message.partition}:${message.offset}`;
|
|
25
|
+
}
|
|
4
26
|
function unwrapContext(obj) {
|
|
5
|
-
if (!obj || typeof obj !== 'object')
|
|
27
|
+
if (!obj || typeof obj !== 'object')
|
|
6
28
|
return obj;
|
|
7
|
-
}
|
|
8
29
|
const clean = {};
|
|
9
30
|
for (const key of Object.keys(obj)) {
|
|
10
|
-
if (key.startsWith('$'))
|
|
31
|
+
if (key.startsWith('$'))
|
|
11
32
|
continue;
|
|
12
|
-
}
|
|
13
33
|
const val = typeof obj[key] === 'function' ? (obj[key].val !== undefined ? obj[key].val : (obj[key].length === 0 ? obj[key]() : obj[key])) : obj[key];
|
|
34
|
+
if (key === 'signal' && val instanceof AbortSignal)
|
|
35
|
+
continue;
|
|
14
36
|
clean[key] = val;
|
|
15
37
|
}
|
|
16
38
|
return clean;
|
|
17
39
|
}
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
40
|
+
function duration(name, value, allowZero = false) {
|
|
41
|
+
if (!Number.isFinite(value) || value < (allowZero ? 0 : 1) || value > 2147483647) {
|
|
42
|
+
throw new RangeError(`${name} must be between ${allowZero ? 0 : 1} and 2147483647`);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
function isTimeout(error) {
|
|
46
|
+
return error instanceof core_1.FlowTimeoutException || error instanceof core_1.StepTimeoutException;
|
|
47
|
+
}
|
|
48
|
+
function delay(ms, signal) {
|
|
49
|
+
signal.throwIfAborted();
|
|
50
|
+
return new Promise((resolve, reject) => {
|
|
51
|
+
const onAbort = () => { clearTimeout(timer); reject(signal.reason); };
|
|
52
|
+
const timer = setTimeout(() => {
|
|
53
|
+
signal.removeEventListener('abort', onAbort);
|
|
54
|
+
resolve();
|
|
55
|
+
}, ms);
|
|
56
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
/** Cancellation also fences adapter work after callbacks that ignore the signal settle. */
|
|
60
|
+
async function guarded(run, payload, options, controller, deadline) {
|
|
61
|
+
const abort = (message, cause) => {
|
|
62
|
+
if (!controller.signal.aborted)
|
|
63
|
+
controller.abort(new KafkaProcessingCancelledError(message, { cause }));
|
|
64
|
+
};
|
|
65
|
+
const checkState = () => {
|
|
66
|
+
try {
|
|
67
|
+
if (options.signal?.aborted)
|
|
68
|
+
abort('Kafka processing cancelled', options.signal.reason);
|
|
69
|
+
if (payload.isRunning?.() === false || payload.isStale?.() === true)
|
|
70
|
+
abort('Kafka consumer stopped or batch became stale');
|
|
71
|
+
if (deadline !== undefined && Date.now() >= deadline)
|
|
72
|
+
abort('Kafka processing deadline exceeded');
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
abort('Kafka consumer state check failed', error);
|
|
76
|
+
}
|
|
77
|
+
};
|
|
78
|
+
const onAbort = () => abort('Kafka processing cancelled', options.signal?.reason);
|
|
79
|
+
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
80
|
+
let rejectCancellation = () => { };
|
|
81
|
+
const cancelled = new Promise((_resolve, reject) => { rejectCancellation = reject; });
|
|
82
|
+
const onCancelled = () => rejectCancellation(controller.signal.reason);
|
|
83
|
+
controller.signal.addEventListener('abort', onCancelled, { once: true });
|
|
84
|
+
let pending;
|
|
85
|
+
let heartbeatTimer;
|
|
86
|
+
const stateTimer = setInterval(checkState, 50);
|
|
87
|
+
const deadlineTimer = deadline === undefined ? undefined : setTimeout(checkState, Math.max(1, Math.ceil(deadline - Date.now())));
|
|
88
|
+
const heartbeat = () => {
|
|
89
|
+
if (pending || controller.signal.aborted || !payload.heartbeat)
|
|
90
|
+
return;
|
|
91
|
+
pending = Promise.resolve().then(() => payload.heartbeat()).catch(error => {
|
|
92
|
+
abort(`Kafka heartbeat failed: ${String(error)}`, error);
|
|
93
|
+
}).finally(() => { pending = undefined; });
|
|
94
|
+
};
|
|
95
|
+
try {
|
|
96
|
+
checkState();
|
|
97
|
+
const operation = async () => {
|
|
98
|
+
controller.signal.throwIfAborted();
|
|
99
|
+
if (payload.heartbeat && (options.heartbeatIntervalMs ?? 5000) > 0) {
|
|
100
|
+
heartbeat();
|
|
101
|
+
await pending;
|
|
102
|
+
controller.signal.throwIfAborted();
|
|
103
|
+
heartbeatTimer = setInterval(heartbeat, options.heartbeatIntervalMs ?? 5000);
|
|
104
|
+
}
|
|
105
|
+
const result = await run(controller.signal);
|
|
106
|
+
if (heartbeatTimer)
|
|
107
|
+
clearInterval(heartbeatTimer);
|
|
108
|
+
await pending;
|
|
109
|
+
checkState();
|
|
110
|
+
controller.signal.throwIfAborted();
|
|
111
|
+
return result;
|
|
112
|
+
};
|
|
113
|
+
return await Promise.race([operation(), cancelled]);
|
|
114
|
+
}
|
|
115
|
+
finally {
|
|
116
|
+
clearInterval(stateTimer);
|
|
117
|
+
if (deadlineTimer)
|
|
118
|
+
clearTimeout(deadlineTimer);
|
|
119
|
+
if (heartbeatTimer)
|
|
120
|
+
clearInterval(heartbeatTimer);
|
|
121
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
122
|
+
controller.signal.removeEventListener('abort', onCancelled);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
/** Creates a handler that can be passed directly to KafkaJS eachMessage. */
|
|
21
126
|
function createKafkaMessageHandler(workflow, options = {}) {
|
|
127
|
+
duration('heartbeatIntervalMs', options.heartbeatIntervalMs ?? 5000, true);
|
|
128
|
+
if (options.timeout !== undefined)
|
|
129
|
+
duration('timeout', options.timeout);
|
|
130
|
+
const ttl = options.idempotencyTtlMs ?? 86400000;
|
|
131
|
+
duration('idempotencyTtlMs', ttl);
|
|
132
|
+
if (options.idempotencyStore && !options.groupId?.trim()) {
|
|
133
|
+
throw new Error('groupId is required when using an idempotencyStore');
|
|
134
|
+
}
|
|
135
|
+
if (options.idempotencyStore && options.timeout !== undefined && options.timeout >= ttl * 0.8) {
|
|
136
|
+
throw new RangeError('timeout must be less than 80% of idempotencyTtlMs');
|
|
137
|
+
}
|
|
138
|
+
if (!['error', 'raw'].includes(options.invalidJson ?? 'error'))
|
|
139
|
+
throw new Error('Invalid invalidJson policy');
|
|
140
|
+
if (!['skip', 'error', 'process'].includes(options.tombstone ?? 'skip'))
|
|
141
|
+
throw new Error('Invalid tombstone policy');
|
|
142
|
+
const maxAttempts = options.retry?.maxAttempts ?? 1;
|
|
143
|
+
if (!Number.isSafeInteger(maxAttempts) || maxAttempts < 1)
|
|
144
|
+
throw new RangeError('retry.maxAttempts must be a positive integer');
|
|
145
|
+
duration('retry.delayMs', options.retry?.delayMs ?? 0, true);
|
|
22
146
|
return async (payload) => {
|
|
23
|
-
const { message, topic, partition }
|
|
24
|
-
const
|
|
25
|
-
const
|
|
26
|
-
|
|
27
|
-
|
|
147
|
+
const message = { ...payload.message, topic: payload.topic, partition: payload.partition };
|
|
148
|
+
const recordId = getKafkaRecordId(message);
|
|
149
|
+
const store = options.idempotencyStore;
|
|
150
|
+
let namespace;
|
|
151
|
+
if (typeof options.namespace === 'function')
|
|
152
|
+
namespace = options.namespace(message);
|
|
153
|
+
else if (options.namespace !== undefined)
|
|
154
|
+
namespace = options.namespace;
|
|
155
|
+
else if (message.headers?.['x-namespace'])
|
|
156
|
+
namespace = String(message.headers['x-namespace']);
|
|
157
|
+
// JSON encoding preserves scope boundaries even when IDs contain delimiters.
|
|
158
|
+
const idempKey = `kafka:${JSON.stringify([options.groupId, namespace ?? null, options.idempotencyKey?.(message) ?? recordId])}`;
|
|
159
|
+
const controller = new AbortController();
|
|
160
|
+
let completionAttempted = false;
|
|
161
|
+
const mapOutcome = (outcome) => outcome.kind === 'processed' && options.output
|
|
162
|
+
? options.output(outcome.value, message) : outcome.value;
|
|
163
|
+
if (options.signal?.aborted || payload.isRunning?.() === false || payload.isStale?.() === true) {
|
|
164
|
+
throw new KafkaProcessingCancelledError('Kafka processing cancelled before claim');
|
|
165
|
+
}
|
|
166
|
+
const startedAt = Date.now();
|
|
167
|
+
if (store) {
|
|
168
|
+
const status = await store.claim(idempKey, ttl);
|
|
28
169
|
if (status === 'completed') {
|
|
29
|
-
return
|
|
170
|
+
return guarded(async (signal) => {
|
|
171
|
+
const result = store.get ? await store.get(idempKey) : undefined;
|
|
172
|
+
signal.throwIfAborted();
|
|
173
|
+
return result?.$sigloKafka === 1 ? mapOutcome(result)
|
|
174
|
+
: mapOutcome({ $sigloKafka: 1, kind: 'processed', value: result === undefined ? { deduplicated: true } : result });
|
|
175
|
+
}, payload, options, controller, options.timeout === undefined ? undefined : startedAt + options.timeout);
|
|
30
176
|
}
|
|
31
|
-
if (status === 'in-progress')
|
|
177
|
+
if (status === 'in-progress')
|
|
32
178
|
throw new Error(`Kafka message ${idempKey} is already being processed concurrently`);
|
|
33
|
-
}
|
|
34
179
|
}
|
|
180
|
+
const budget = Math.min(options.timeout ?? Infinity, store ? ttl * 0.8 : Infinity);
|
|
181
|
+
const deadline = Number.isFinite(budget) ? startedAt + budget : undefined;
|
|
182
|
+
let attempts = 0;
|
|
35
183
|
try {
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
184
|
+
return await guarded(async (signal) => {
|
|
185
|
+
const checkActive = () => {
|
|
186
|
+
if (deadline !== undefined && Date.now() >= deadline && !signal.aborted) {
|
|
187
|
+
controller.abort(new KafkaProcessingCancelledError('Kafka processing deadline exceeded'));
|
|
188
|
+
}
|
|
189
|
+
signal.throwIfAborted();
|
|
190
|
+
};
|
|
191
|
+
const finish = async (kind, value) => {
|
|
192
|
+
checkActive();
|
|
193
|
+
const outcome = { $sigloKafka: 1, kind, value };
|
|
194
|
+
if (store) {
|
|
195
|
+
completionAttempted = true;
|
|
196
|
+
await store.complete(idempKey, outcome);
|
|
197
|
+
}
|
|
198
|
+
checkActive();
|
|
199
|
+
return mapOutcome(outcome);
|
|
200
|
+
};
|
|
201
|
+
let outcome;
|
|
44
202
|
try {
|
|
45
|
-
|
|
203
|
+
if (message.value == null && options.tombstone !== 'process') {
|
|
204
|
+
if (options.tombstone === 'error')
|
|
205
|
+
throw new KafkaInvalidMessageError('Kafka tombstone is not accepted');
|
|
206
|
+
return finish('skipped', { skipped: true, recordId });
|
|
207
|
+
}
|
|
208
|
+
let input = {};
|
|
209
|
+
if (options.input)
|
|
210
|
+
input = await options.input(message);
|
|
211
|
+
else if (message.value != null) {
|
|
212
|
+
const text = String(message.value);
|
|
213
|
+
try {
|
|
214
|
+
input = JSON.parse(text);
|
|
215
|
+
}
|
|
216
|
+
catch (cause) {
|
|
217
|
+
if (options.invalidJson === 'raw')
|
|
218
|
+
input = { raw: text };
|
|
219
|
+
else
|
|
220
|
+
throw new KafkaInvalidMessageError('Kafka message contains invalid JSON', { cause });
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
checkActive();
|
|
224
|
+
if (options.validate) {
|
|
225
|
+
try {
|
|
226
|
+
await options.validate(input, message);
|
|
227
|
+
}
|
|
228
|
+
catch (cause) {
|
|
229
|
+
throw new KafkaInvalidMessageError('Kafka workflow input validation failed', { cause });
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
for (;;) {
|
|
233
|
+
checkActive();
|
|
234
|
+
attempts++;
|
|
235
|
+
try {
|
|
236
|
+
const execOptions = { namespace, signal, timeout: options.timeout };
|
|
237
|
+
const result = await workflow.run(input, execOptions);
|
|
238
|
+
checkActive();
|
|
239
|
+
outcome = { $sigloKafka: 1, kind: 'processed', value: unwrapContext(result) };
|
|
240
|
+
break;
|
|
241
|
+
}
|
|
242
|
+
catch (error) {
|
|
243
|
+
checkActive();
|
|
244
|
+
if (isTimeout(error))
|
|
245
|
+
throw new KafkaProcessingCancelledError('Kafka workflow timed out', { cause: error });
|
|
246
|
+
if (error instanceof KafkaInvalidMessageError || attempts >= maxAttempts ||
|
|
247
|
+
options.retry?.shouldRetry?.(error, message) === false)
|
|
248
|
+
throw error;
|
|
249
|
+
await delay(options.retry?.delayMs ?? 0, signal);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
46
252
|
}
|
|
47
|
-
catch {
|
|
48
|
-
|
|
253
|
+
catch (error) {
|
|
254
|
+
checkActive();
|
|
255
|
+
if (error instanceof KafkaProcessingCancelledError || !options.deadLetter)
|
|
256
|
+
throw error;
|
|
257
|
+
await options.deadLetter({ message, recordId, error, attempts });
|
|
258
|
+
checkActive();
|
|
259
|
+
outcome = { $sigloKafka: 1, kind: 'dead-lettered', value: { deadLettered: true, recordId, attempts } };
|
|
260
|
+
}
|
|
261
|
+
return finish(outcome.kind, outcome.value);
|
|
262
|
+
}, payload, options, controller, deadline);
|
|
263
|
+
}
|
|
264
|
+
catch (error) {
|
|
265
|
+
// Cancellation/timeouts may leave uncancelled external work in flight. Retain
|
|
266
|
+
// the claim until expiry instead of making an immediate duplicate possible.
|
|
267
|
+
if (store && !completionAttempted && !controller.signal.aborted && !(error instanceof KafkaProcessingCancelledError)) {
|
|
268
|
+
try {
|
|
269
|
+
await store.fail(idempKey, error);
|
|
270
|
+
}
|
|
271
|
+
catch (cleanupError) {
|
|
272
|
+
throw new AggregateError([error, cleanupError], 'Kafka processing and claim cleanup failed');
|
|
49
273
|
}
|
|
50
274
|
}
|
|
51
|
-
|
|
52
|
-
if (typeof options.namespace === 'function') {
|
|
53
|
-
namespace = options.namespace(message);
|
|
54
|
-
}
|
|
55
|
-
else if (options.namespace) {
|
|
56
|
-
namespace = options.namespace;
|
|
57
|
-
}
|
|
58
|
-
else if (message.headers?.['x-namespace']) {
|
|
59
|
-
namespace = String(message.headers['x-namespace']);
|
|
60
|
-
}
|
|
61
|
-
const execOptions = {
|
|
62
|
-
timeout: options.timeout,
|
|
63
|
-
namespace,
|
|
64
|
-
};
|
|
65
|
-
const rawResult = await workflow.run(parsedValue, execOptions);
|
|
66
|
-
const cleanResult = unwrapContext(rawResult);
|
|
67
|
-
if (options.idempotencyStore) {
|
|
68
|
-
await options.idempotencyStore.complete(idempKey, cleanResult);
|
|
69
|
-
}
|
|
70
|
-
if (options.output) {
|
|
71
|
-
return await options.output(cleanResult, message);
|
|
72
|
-
}
|
|
73
|
-
return cleanResult;
|
|
275
|
+
throw error;
|
|
74
276
|
}
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
/** Processes partition batches sequentially; use eachBatchAutoResolve: false. */
|
|
280
|
+
function createKafkaBatchHandler(workflow, options = {}) {
|
|
281
|
+
const handleMessage = createKafkaMessageHandler(workflow, options);
|
|
282
|
+
return async (payload) => {
|
|
283
|
+
for (const message of payload.batch.messages) {
|
|
284
|
+
if (!payload.isRunning() || payload.isStale())
|
|
285
|
+
return;
|
|
286
|
+
try {
|
|
287
|
+
await handleMessage({
|
|
288
|
+
topic: payload.batch.topic, partition: payload.batch.partition, message,
|
|
289
|
+
heartbeat: () => payload.heartbeat(),
|
|
290
|
+
isRunning: () => payload.isRunning(), isStale: () => payload.isStale(),
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
catch (error) {
|
|
294
|
+
if (error instanceof KafkaProcessingCancelledError && (!payload.isRunning() || payload.isStale()))
|
|
295
|
+
return;
|
|
296
|
+
throw error;
|
|
78
297
|
}
|
|
79
|
-
|
|
298
|
+
if (!payload.isRunning() || payload.isStale())
|
|
299
|
+
return;
|
|
300
|
+
await payload.heartbeat();
|
|
301
|
+
if (!payload.isRunning() || payload.isStale())
|
|
302
|
+
return;
|
|
303
|
+
payload.resolveOffset(message.offset);
|
|
304
|
+
if (options.commitOffsets !== false)
|
|
305
|
+
await payload.commitOffsetsIfNecessary();
|
|
80
306
|
}
|
|
81
307
|
};
|
|
82
308
|
}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAsCA,8DAmEC;AArFD,SAAS,aAAa,CAAC,GAAQ;IAC3B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAClC,OAAO,GAAG,CAAC;IACf,CAAC;IACD,MAAM,KAAK,GAAwB,EAAE,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,SAAS;QACb,CAAC;QACD,MAAM,GAAG,GAAG,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtJ,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACrB,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED;;GAEG;AACH,SAAgB,yBAAyB,CACrC,QAA+B,EAC/B,UAA8C,EAAE;IAEhD,OAAO,KAAK,EAAE,OAAyE,EAAgB,EAAE;QACrG,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,OAAO,CAAC;QAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QAC7F,MAAM,QAAQ,GAAG,SAAS,KAAK,IAAI,MAAM,EAAE,CAAC;QAE5C,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;YACxF,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzB,OAAQ,OAAO,CAAC,gBAAwB,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC;YAC7F,CAAC;YACD,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,0CAA0C,CAAC,CAAC;YACzF,CAAC;QACL,CAAC;QAED,IAAI,CAAC;YACD,IAAI,WAAW,GAAQ,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBAChB,WAAW,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/C,CAAC;iBAAM,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACvB,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,CAAC;oBACtC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC;oBACjC,CAAC,CAAC,CAAC,OAAO,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;gBAClF,IAAI,CAAC;oBACD,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAClC,CAAC;gBAAC,MAAM,CAAC;oBACL,WAAW,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;gBAC/B,CAAC;YACL,CAAC;YAED,IAAI,SAA6B,CAAC;YAClC,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;gBAC1C,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YAC3C,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;gBAC3B,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;YAClC,CAAC;iBAAM,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,EAAE,CAAC;gBAC1C,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;YACvD,CAAC;YAED,MAAM,WAAW,GAAqB;gBAClC,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,SAAS;aACZ,CAAC;YAEF,MAAM,SAAS,GAAQ,MAAM,QAAQ,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YACpE,MAAM,WAAW,GAAG,aAAa,CAAC,SAAS,CAAC,CAAC;YAE7C,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;gBAC3B,MAAM,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;YACnE,CAAC;YAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACjB,OAAO,MAAM,OAAO,CAAC,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACtD,CAAC;YAED,OAAO,WAAW,CAAC;QACvB,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAChB,IAAI,OAAO,CAAC,gBAAgB,EAAE,CAAC;gBAC3B,MAAM,OAAO,CAAC,gBAAgB,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YACvD,CAAC;YACD,MAAM,GAAG,CAAC;QACd,CAAC;IACL,CAAC,CAAC;AACN,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AA0FA,4CAEC;AAsGD,8DAkIC;AAGD,0DAyBC;AAhWD,sCAAmH;AA2EnH,MAAa,wBAAyB,SAAQ,KAAK;IAC/C,YAAY,OAAe,EAAE,OAAsB;QAC/C,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;IAC3C,CAAC;CACJ;AALD,4DAKC;AAED,MAAa,6BAA8B,SAAQ,KAAK;IACpD,YAAY,OAAe,EAAE,OAAsB;QAC/C,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACxB,IAAI,CAAC,IAAI,GAAG,+BAA+B,CAAC;IAChD,CAAC;CACJ;AALD,sEAKC;AAED,0FAA0F;AAC1F,SAAgB,gBAAgB,CAAC,OAAkE;IAC/F,OAAO,SAAS,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC;AAID,SAAS,aAAa,CAAC,GAAQ;IAC3B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IAChD,MAAM,KAAK,GAAwB,EAAE,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAClC,MAAM,GAAG,GAAG,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACtJ,IAAI,GAAG,KAAK,QAAQ,IAAI,GAAG,YAAY,WAAW;YAAE,SAAS;QAC7D,KAAK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACrB,CAAC;IACD,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,KAAa,EAAE,SAAS,GAAG,KAAK;IAC5D,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,GAAG,UAAU,EAAE,CAAC;QAC/E,MAAM,IAAI,UAAU,CAAC,GAAG,IAAI,oBAAoB,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IACxF,CAAC;AACL,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC7B,OAAO,KAAK,YAAY,2BAAoB,IAAI,KAAK,YAAY,2BAAoB,CAAC;AAC1F,CAAC;AAED,SAAS,KAAK,CAAC,EAAU,EAAE,MAAmB;IAC1C,MAAM,CAAC,cAAc,EAAE,CAAC;IACxB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,MAAM,OAAO,GAAG,GAAG,EAAE,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC1B,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,EAAE,CAAC;QACd,CAAC,EAAE,EAAE,CAAC,CAAC;QACP,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACP,CAAC;AAED,2FAA2F;AAC3F,KAAK,UAAU,OAAO,CAClB,GAAwC,EACxC,OAA0B,EAC1B,OAA0B,EAC1B,UAA2B,EAC3B,QAA4B;IAE5B,MAAM,KAAK,GAAG,CAAC,OAAe,EAAE,KAAe,EAAE,EAAE;QAC/C,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO;YAAE,UAAU,CAAC,KAAK,CAAC,IAAI,6BAA6B,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC5G,CAAC,CAAC;IACF,MAAM,UAAU,GAAG,GAAG,EAAE;QACpB,IAAI,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO;gBAAE,KAAK,CAAC,4BAA4B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACxF,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,KAAK,IAAI;gBAAE,KAAK,CAAC,8CAA8C,CAAC,CAAC;YAC3H,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ;gBAAE,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACtG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,KAAK,CAAC,mCAAmC,EAAE,KAAK,CAAC,CAAC;QACtD,CAAC;IACL,CAAC,CAAC;IACF,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,KAAK,CAAC,4BAA4B,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClF,OAAO,CAAC,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,IAAI,kBAAkB,GAA6B,GAAG,EAAE,GAAE,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,GAAG,kBAAkB,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7F,MAAM,WAAW,GAAG,GAAG,EAAE,CAAC,kBAAkB,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACvE,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;IACzE,IAAI,OAAkC,CAAC;IACvC,IAAI,cAA0D,CAAC;IAC/D,MAAM,UAAU,GAAG,WAAW,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAG,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACjI,MAAM,SAAS,GAAG,GAAG,EAAE;QACnB,IAAI,OAAO,IAAI,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS;YAAE,OAAO;QACvE,OAAO,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,SAAU,EAAE,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;YACvE,KAAK,CAAC,2BAA2B,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAC7D,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC,CAAC;IACF,IAAI,CAAC;QACD,UAAU,EAAE,CAAC;QACb,MAAM,SAAS,GAAG,KAAK,IAAI,EAAE;YACzB,UAAU,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACnC,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,SAAS,EAAE,CAAC;gBACZ,MAAM,OAAO,CAAC;gBACd,UAAU,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;gBACnC,cAAc,GAAG,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,mBAAmB,IAAI,IAAI,CAAC,CAAC;YACjF,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC5C,IAAI,cAAc;gBAAE,aAAa,CAAC,cAAc,CAAC,CAAC;YAClD,MAAM,OAAO,CAAC;YACd,UAAU,EAAE,CAAC;YACb,UAAU,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;YACnC,OAAO,MAAM,CAAC;QAClB,CAAC,CAAC;QACF,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC;IACxD,CAAC;YAAS,CAAC;QACP,aAAa,CAAC,UAAU,CAAC,CAAC;QAC1B,IAAI,aAAa;YAAE,YAAY,CAAC,aAAa,CAAC,CAAC;QAC/C,IAAI,cAAc;YAAE,aAAa,CAAC,cAAc,CAAC,CAAC;QAClD,OAAO,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACtD,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAChE,CAAC;AACL,CAAC;AAED,4EAA4E;AAC5E,SAAgB,yBAAyB,CACrC,QAA+B,EAC/B,UAA8C,EAAE;IAEhD,QAAQ,CAAC,qBAAqB,EAAE,OAAO,CAAC,mBAAmB,IAAI,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS;QAAE,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;IACxE,MAAM,GAAG,GAAG,OAAO,CAAC,gBAAgB,IAAI,QAAQ,CAAC;IACjD,QAAQ,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC;IAClC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,OAAO,CAAC,gBAAgB,IAAI,OAAO,CAAC,OAAO,KAAK,SAAS,IAAI,OAAO,CAAC,OAAO,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;QAC5F,MAAM,IAAI,UAAU,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC9G,IAAI,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACrH,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,EAAE,WAAW,IAAI,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,8CAA8C,CAAC,CAAC;IAChI,QAAQ,CAAC,eAAe,EAAE,OAAO,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAE7D,OAAO,KAAK,EAAE,OAA0B,EAAgB,EAAE;QACtD,MAAM,OAAO,GAAsB,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;QAC9G,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACvC,IAAI,SAA6B,CAAC;QAClC,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,UAAU;YAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;aAC/E,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS;YAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;aACnE,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC;YAAE,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;QAC9F,6EAA6E;QAC7E,MAAM,QAAQ,GAAG,SAAS,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,SAAS,IAAI,IAAI,EAAE,OAAO,CAAC,cAAc,EAAE,CAAC,OAAO,CAAC,IAAI,QAAQ,CAAC,CAAC,EAAE,CAAC;QAChI,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,IAAI,mBAAmB,GAAG,KAAK,CAAC;QAChC,MAAM,UAAU,GAAG,CAAC,OAAgB,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,KAAK,WAAW,IAAI,OAAO,CAAC,MAAM;YACnF,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;QAC7D,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,IAAI,OAAO,CAAC,SAAS,EAAE,EAAE,KAAK,KAAK,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7F,MAAM,IAAI,6BAA6B,CAAC,yCAAyC,CAAC,CAAC;QACvF,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,KAAK,EAAE,CAAC;YACR,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAChD,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;gBACzB,OAAO,OAAO,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;oBAC1B,MAAM,MAAM,GAAQ,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;oBACtE,MAAM,CAAC,cAAc,EAAE,CAAC;oBACxB,OAAO,MAAM,EAAE,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;wBACjD,CAAC,CAAC,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC3H,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;YAC9G,CAAC;YACD,IAAI,MAAM,KAAK,aAAa;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,0CAA0C,CAAC,CAAC;QACvH,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACnF,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC1E,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,CAAC;YACD,OAAO,MAAM,OAAO,CAAC,KAAK,EAAC,MAAM,EAAC,EAAE;gBAChC,MAAM,WAAW,GAAG,GAAG,EAAE;oBACrB,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,IAAI,QAAQ,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;wBACtE,UAAU,CAAC,KAAK,CAAC,IAAI,6BAA6B,CAAC,oCAAoC,CAAC,CAAC,CAAC;oBAC9F,CAAC;oBACD,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC5B,CAAC,CAAC;gBACF,MAAM,MAAM,GAAG,KAAK,EAAE,IAAqB,EAAE,KAAU,EAAE,EAAE;oBACvD,WAAW,EAAE,CAAC;oBACd,MAAM,OAAO,GAAY,EAAE,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;oBACzD,IAAI,KAAK,EAAE,CAAC;wBACR,mBAAmB,GAAG,IAAI,CAAC;wBAC3B,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;oBAC5C,CAAC;oBACD,WAAW,EAAE,CAAC;oBACd,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC;gBAC/B,CAAC,CAAC;gBACF,IAAI,OAAgB,CAAC;gBACrB,IAAI,CAAC;oBACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;wBAC3D,IAAI,OAAO,CAAC,SAAS,KAAK,OAAO;4BAAE,MAAM,IAAI,wBAAwB,CAAC,iCAAiC,CAAC,CAAC;wBACzG,OAAO,MAAM,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;oBAC1D,CAAC;oBACD,IAAI,KAAK,GAAQ,EAAE,CAAC;oBACpB,IAAI,OAAO,CAAC,KAAK;wBAAE,KAAK,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;yBACnD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;wBAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBACnC,IAAI,CAAC;4BAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;wBAAC,CAAC;wBACjC,OAAO,KAAK,EAAE,CAAC;4BACX,IAAI,OAAO,CAAC,WAAW,KAAK,KAAK;gCAAE,KAAK,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;;gCACpD,MAAM,IAAI,wBAAwB,CAAC,qCAAqC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;wBAC9F,CAAC;oBACL,CAAC;oBACD,WAAW,EAAE,CAAC;oBACd,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;wBACnB,IAAI,CAAC;4BAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;wBAAC,CAAC;wBAC/C,OAAO,KAAK,EAAE,CAAC;4BAAC,MAAM,IAAI,wBAAwB,CAAC,wCAAwC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;wBAAC,CAAC;oBAC9G,CAAC;oBACD,SAAS,CAAC;wBACN,WAAW,EAAE,CAAC;wBACd,QAAQ,EAAE,CAAC;wBACX,IAAI,CAAC;4BACD,MAAM,WAAW,GAAqB,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;4BACtF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;4BACtD,WAAW,EAAE,CAAC;4BACd,OAAO,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,CAAC;4BAC9E,MAAM;wBACV,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACb,WAAW,EAAE,CAAC;4BACd,IAAI,SAAS,CAAC,KAAK,CAAC;gCAAE,MAAM,IAAI,6BAA6B,CAAC,0BAA0B,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;4BAC5G,IAAI,KAAK,YAAY,wBAAwB,IAAI,QAAQ,IAAI,WAAW;gCACpE,OAAO,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,KAAK;gCAAE,MAAM,KAAK,CAAC;4BACxE,MAAM,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,IAAI,CAAC,EAAE,MAAM,CAAC,CAAC;wBACrD,CAAC;oBACL,CAAC;gBACL,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACb,WAAW,EAAE,CAAC;oBACd,IAAI,KAAK,YAAY,6BAA6B,IAAI,CAAC,OAAO,CAAC,UAAU;wBAAE,MAAM,KAAK,CAAC;oBACvF,MAAM,OAAO,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;oBACjE,WAAW,EAAE,CAAC;oBACd,OAAO,GAAG,EAAE,WAAW,EAAE,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,KAAK,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC;gBAC3G,CAAC;gBACD,OAAO,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;YAC/C,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACb,8EAA8E;YAC9E,4EAA4E;YAC5E,IAAI,KAAK,IAAI,CAAC,mBAAmB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,YAAY,6BAA6B,CAAC,EAAE,CAAC;gBACnH,IAAI,CAAC;oBAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;gBAAC,CAAC;gBAC1C,OAAO,YAAY,EAAE,CAAC;oBAClB,MAAM,IAAI,cAAc,CAAC,CAAC,KAAK,EAAE,YAAY,CAAC,EAAE,2CAA2C,CAAC,CAAC;gBACjG,CAAC;YACL,CAAC;YACD,MAAM,KAAK,CAAC;QAChB,CAAC;IACL,CAAC,CAAC;AACN,CAAC;AAED,iFAAiF;AACjF,SAAgB,uBAAuB,CACnC,QAA+B,EAC/B,UAAmD,EAAE;IAErD,MAAM,aAAa,GAAG,yBAAyB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnE,OAAO,KAAK,EAAE,OAA+B,EAAiB,EAAE;QAC5D,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC3C,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,OAAO,CAAC,OAAO,EAAE;gBAAE,OAAO;YACtD,IAAI,CAAC;gBACD,MAAM,aAAa,CAAC;oBAChB,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,OAAO;oBACvE,SAAS,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE;oBACpC,SAAS,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE;iBACzE,CAAC,CAAC;YACP,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,KAAK,YAAY,6BAA6B,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBAAE,OAAO;gBAC1G,MAAM,KAAK,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,OAAO,CAAC,OAAO,EAAE;gBAAE,OAAO;YACtD,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,OAAO,CAAC,OAAO,EAAE;gBAAE,OAAO;YACtD,OAAO,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YACtC,IAAI,OAAO,CAAC,aAAa,KAAK,KAAK;gBAAE,MAAM,OAAO,CAAC,wBAAwB,EAAE,CAAC;QAClF,CAAC;IACL,CAAC,CAAC;AACN,CAAC"}
|
package/docs/outbox.md
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# Database changes and follow-up Kafka events
|
|
2
|
+
|
|
3
|
+
A workflow cannot atomically commit a PostgreSQL update and a Kafka send. Put the database change, a processed-event marker, and an outbox row in **one PostgreSQL transaction**. A separate relay publishes committed outbox rows. This makes database effects safe to replay and prevents losing a follow-up event between the update and the send.
|
|
4
|
+
|
|
5
|
+
The adapter's `idempotencyStore` is optional for this pattern: the database transaction owns deduplication for these database effects. Other effects, such as a payment-provider request, still need the provider's own idempotency key.
|
|
6
|
+
|
|
7
|
+
## Schema
|
|
8
|
+
|
|
9
|
+
Run this migration in your application's database:
|
|
10
|
+
|
|
11
|
+
```sql
|
|
12
|
+
CREATE TABLE processed_order_events (
|
|
13
|
+
consumer_group text NOT NULL,
|
|
14
|
+
tenant text NOT NULL,
|
|
15
|
+
event_id text NOT NULL,
|
|
16
|
+
PRIMARY KEY (consumer_group, tenant, event_id)
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
CREATE TABLE accepted_orders (
|
|
20
|
+
tenant text NOT NULL,
|
|
21
|
+
order_id text NOT NULL,
|
|
22
|
+
PRIMARY KEY (tenant, order_id)
|
|
23
|
+
);
|
|
24
|
+
|
|
25
|
+
CREATE TABLE order_outbox (
|
|
26
|
+
event_id text PRIMARY KEY,
|
|
27
|
+
payload jsonb NOT NULL,
|
|
28
|
+
published_at timestamptz
|
|
29
|
+
);
|
|
30
|
+
CREATE INDEX order_outbox_pending ON order_outbox (event_id)
|
|
31
|
+
WHERE published_at IS NULL;
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Workflow step
|
|
35
|
+
|
|
36
|
+
Install `pg` and its TypeScript types alongside the getting-started dependencies. The producer of an order must supply an `eventId` that stays the same on business-event retries; the Kafka record key can still identify the order.
|
|
37
|
+
|
|
38
|
+
```typescript
|
|
39
|
+
import { Pool } from 'pg';
|
|
40
|
+
import { Step, flow } from '@siglo/core';
|
|
41
|
+
import { createKafkaMessageHandler } from '@siglo/kafka';
|
|
42
|
+
|
|
43
|
+
const pool = new Pool({ connectionString: process.env.DATABASE_URL });
|
|
44
|
+
const groupId = 'order-acceptance';
|
|
45
|
+
interface OrderEvent { eventId: string; tenant: string; orderId: string }
|
|
46
|
+
|
|
47
|
+
const acceptOrder = Step.create('acceptOrder', async (event: OrderEvent) => {
|
|
48
|
+
const client = await pool.connect();
|
|
49
|
+
try {
|
|
50
|
+
await client.query('BEGIN');
|
|
51
|
+
const inserted = await client.query(
|
|
52
|
+
`INSERT INTO processed_order_events VALUES ($1, $2, $3)
|
|
53
|
+
ON CONFLICT DO NOTHING RETURNING event_id`,
|
|
54
|
+
[groupId, event.tenant, event.eventId],
|
|
55
|
+
);
|
|
56
|
+
if (inserted.rowCount === 0) {
|
|
57
|
+
await client.query('COMMIT');
|
|
58
|
+
return { duplicate: true };
|
|
59
|
+
}
|
|
60
|
+
await client.query(
|
|
61
|
+
'INSERT INTO accepted_orders VALUES ($1, $2) ON CONFLICT DO NOTHING',
|
|
62
|
+
[event.tenant, event.orderId],
|
|
63
|
+
);
|
|
64
|
+
const outgoingId = JSON.stringify([groupId, event.tenant, event.eventId, 'order.accepted']);
|
|
65
|
+
await client.query(
|
|
66
|
+
'INSERT INTO order_outbox(event_id, payload) VALUES ($1, $2::jsonb)',
|
|
67
|
+
[outgoingId, JSON.stringify({
|
|
68
|
+
eventId: outgoingId, tenant: event.tenant, orderId: event.orderId,
|
|
69
|
+
})],
|
|
70
|
+
);
|
|
71
|
+
await client.query('COMMIT');
|
|
72
|
+
return { accepted: true };
|
|
73
|
+
} catch (error) {
|
|
74
|
+
await client.query('ROLLBACK');
|
|
75
|
+
throw error;
|
|
76
|
+
} finally {
|
|
77
|
+
client.release();
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
const handler = createKafkaMessageHandler<OrderEvent>(flow(acceptOrder), {
|
|
82
|
+
validate: event => {
|
|
83
|
+
if (!event || ![event.eventId, event.tenant, event.orderId]
|
|
84
|
+
.every(value => typeof value === 'string' && value.length > 0)) {
|
|
85
|
+
throw new Error('eventId, tenant, and orderId are required');
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
});
|
|
89
|
+
// Pass handler to a connected KafkaJS consumer's eachMessage option.
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
A crash before the transaction commits rolls everything back. A crash after commit but before Kafka offset completion causes redelivery; the primary key prevents repeating the database change or creating another outbox event. Keep processed-event markers for at least as long as you permit source-event replay.
|
|
93
|
+
|
|
94
|
+
## Relay
|
|
95
|
+
|
|
96
|
+
With a connected KafkaJS `producer`, call this function repeatedly from a worker, backing off when there is no pending row or an error occurs:
|
|
97
|
+
|
|
98
|
+
```typescript
|
|
99
|
+
async function publishNextOutbox(): Promise<boolean> {
|
|
100
|
+
const client = await pool.connect();
|
|
101
|
+
try {
|
|
102
|
+
await client.query('BEGIN');
|
|
103
|
+
const { rows } = await client.query(
|
|
104
|
+
`SELECT event_id, payload FROM order_outbox
|
|
105
|
+
WHERE published_at IS NULL ORDER BY event_id
|
|
106
|
+
LIMIT 1 FOR UPDATE SKIP LOCKED`,
|
|
107
|
+
);
|
|
108
|
+
if (rows.length === 0) {
|
|
109
|
+
await client.query('COMMIT');
|
|
110
|
+
return false;
|
|
111
|
+
}
|
|
112
|
+
const row = rows[0];
|
|
113
|
+
await producer.send({
|
|
114
|
+
topic: 'orders.accepted',
|
|
115
|
+
messages: [{ key: row.event_id, value: JSON.stringify(row.payload) }],
|
|
116
|
+
});
|
|
117
|
+
await client.query(
|
|
118
|
+
'UPDATE order_outbox SET published_at = NOW() WHERE event_id = $1',
|
|
119
|
+
[row.event_id],
|
|
120
|
+
);
|
|
121
|
+
await client.query('COMMIT');
|
|
122
|
+
return true;
|
|
123
|
+
} catch (error) {
|
|
124
|
+
await client.query('ROLLBACK');
|
|
125
|
+
throw error;
|
|
126
|
+
} finally {
|
|
127
|
+
client.release();
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
Multiple relay workers can claim different rows using `SKIP LOCKED`. Configure bounded database and producer timeouts because this simple relay holds a database transaction while sending. It does not guarantee per-order publication ordering across workers; use partitioned relay ownership or sequence enforcement when ordering is required.
|
|
133
|
+
|
|
134
|
+
A crash after sending but before marking the row published can publish the event again. Downstream consumers must deduplicate by `payload.eventId`, scoped to their group and tenant, using a transactional processed-event marker or a suitable idempotency store. A Kafka message key alone does not deduplicate records.
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@siglo/kafka",
|
|
3
|
-
"version": "1.0
|
|
4
|
-
"description": "Kafka consumer
|
|
3
|
+
"version": "1.1.0",
|
|
4
|
+
"description": "Apache Kafka consumer and event streaming adapter for running Siglo TypeScript workflows.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
7
7
|
"exports": {
|
|
@@ -16,7 +16,8 @@
|
|
|
16
16
|
"files": [
|
|
17
17
|
"dist",
|
|
18
18
|
"README.md",
|
|
19
|
-
"LICENSE"
|
|
19
|
+
"LICENSE",
|
|
20
|
+
"docs"
|
|
20
21
|
],
|
|
21
22
|
"publishConfig": {
|
|
22
23
|
"access": "public",
|
|
@@ -25,7 +26,8 @@
|
|
|
25
26
|
"scripts": {
|
|
26
27
|
"build": "tsc -p tsconfig.json",
|
|
27
28
|
"prepublishOnly": "npm run build",
|
|
28
|
-
"test": "npm run build && mocha -r tsx 'src/**/*.test.ts'"
|
|
29
|
+
"test": "npm run build && mocha -r tsx 'src/**/*.test.ts'",
|
|
30
|
+
"test:integration": "npm run build && mocha -r tsx 'integration/**/*.test.ts'"
|
|
29
31
|
},
|
|
30
32
|
"repository": {
|
|
31
33
|
"type": "git",
|
|
@@ -37,7 +39,7 @@
|
|
|
37
39
|
"bugs": {
|
|
38
40
|
"url": "https://github.com/jsoftgem/siglo/issues"
|
|
39
41
|
},
|
|
40
|
-
"homepage": "https://github.com/jsoftgem/siglo#readme",
|
|
42
|
+
"homepage": "https://github.com/jsoftgem/siglo/tree/main/packages/kafka#readme",
|
|
41
43
|
"dependencies": {
|
|
42
44
|
"@siglo/core": "^1.0.0"
|
|
43
45
|
},
|
|
@@ -53,9 +55,23 @@
|
|
|
53
55
|
"@types/chai": "^5.2.3",
|
|
54
56
|
"@types/mocha": "^10.0.10",
|
|
55
57
|
"@types/node": "^26.6.5",
|
|
58
|
+
"@types/pg": "^8.23.1",
|
|
56
59
|
"chai": "^3.5.0",
|
|
60
|
+
"kafkajs": "^2.2.4",
|
|
57
61
|
"mocha": "^12.0.3",
|
|
62
|
+
"pg": "^8.23.1",
|
|
58
63
|
"tsx": "^4.23.15",
|
|
59
64
|
"typescript": "~5.7.0"
|
|
60
|
-
}
|
|
65
|
+
},
|
|
66
|
+
"keywords": [
|
|
67
|
+
"siglo",
|
|
68
|
+
"workflow",
|
|
69
|
+
"typescript",
|
|
70
|
+
"kafka",
|
|
71
|
+
"apache-kafka",
|
|
72
|
+
"kafkajs",
|
|
73
|
+
"event-streaming",
|
|
74
|
+
"consumer",
|
|
75
|
+
"event-driven"
|
|
76
|
+
]
|
|
61
77
|
}
|