@eventferry/kafka 3.0.0 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +249 -0
- package/dist/index.cjs +292 -20
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +331 -13
- package/dist/index.d.ts +331 -13
- package/dist/index.js +287 -19
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -35,6 +35,255 @@ const publisher = new KafkaPublisher({
|
|
|
35
35
|
|
|
36
36
|
You can also pass a `customDriver` implementing the `KafkaDriver` interface.
|
|
37
37
|
|
|
38
|
+
## Authentication & TLS
|
|
39
|
+
|
|
40
|
+
### One-way TLS
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
new KafkaPublisher({
|
|
44
|
+
brokers: ["broker:9093"],
|
|
45
|
+
ssl: true, // uses the driver's default trust store
|
|
46
|
+
});
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### mTLS (mutual TLS)
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
import { readFileSync } from "node:fs";
|
|
53
|
+
|
|
54
|
+
new KafkaPublisher({
|
|
55
|
+
brokers: ["broker:9093"],
|
|
56
|
+
ssl: {
|
|
57
|
+
ca: readFileSync("/etc/ssl/kafka-ca.pem"),
|
|
58
|
+
cert: readFileSync("/etc/ssl/client.pem"),
|
|
59
|
+
key: readFileSync("/etc/ssl/client-key.pem"),
|
|
60
|
+
passphrase: "optional",
|
|
61
|
+
// servername: "broker.example.com", // SNI override if cert SAN differs
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
> `rejectUnauthorized` is intentionally NOT a knob. TLS verification is
|
|
67
|
+
> non-negotiable. For dev clusters with self-signed certs, pass the cluster
|
|
68
|
+
> CA via `ca` so verification succeeds.
|
|
69
|
+
|
|
70
|
+
### SASL — username + password (PLAIN / SCRAM)
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
new KafkaPublisher({
|
|
74
|
+
brokers: ["broker:9093"],
|
|
75
|
+
ssl: true,
|
|
76
|
+
sasl: {
|
|
77
|
+
mechanism: "scram-sha-512", // or "plain" | "scram-sha-256"
|
|
78
|
+
username: process.env.KAFKA_USER!,
|
|
79
|
+
password: process.env.KAFKA_PASSWORD!,
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### SASL/OAUTHBEARER (Azure Event Hubs, OIDC, MSK IAM)
|
|
85
|
+
|
|
86
|
+
```ts
|
|
87
|
+
new KafkaPublisher({
|
|
88
|
+
brokers: ["broker:9093"],
|
|
89
|
+
ssl: true,
|
|
90
|
+
sasl: {
|
|
91
|
+
mechanism: "oauthbearer",
|
|
92
|
+
oauthBearerProvider: async () => {
|
|
93
|
+
const token = await myTokenIssuer();
|
|
94
|
+
return {
|
|
95
|
+
value: token.value, // required for both drivers
|
|
96
|
+
principal: token.principal, // required for confluent driver
|
|
97
|
+
lifetime: token.expiresInMs, // required for confluent driver
|
|
98
|
+
extensions: token.extensions, // optional
|
|
99
|
+
};
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
> **Driver asymmetry:** `kafkajs` reads only `value`; `@confluentinc/kafka-javascript` requires `value` + `principal` + `lifetime` (in milliseconds) and accepts an optional `extensions` map. Cross-driver portable providers should populate all four fields.
|
|
106
|
+
|
|
107
|
+
## Producer tuning
|
|
108
|
+
|
|
109
|
+
The high-throughput recipe (confluent driver):
|
|
110
|
+
|
|
111
|
+
```ts
|
|
112
|
+
new KafkaPublisher({
|
|
113
|
+
driver: "confluent",
|
|
114
|
+
brokers: ["broker:9092"],
|
|
115
|
+
idempotent: true,
|
|
116
|
+
compression: "zstd",
|
|
117
|
+
lingerMs: 25, // batch up to 25ms for higher throughput
|
|
118
|
+
batchSize: 131_072, // 128 KB per partition batch
|
|
119
|
+
maxInFlightRequests: 5,
|
|
120
|
+
maxRequestSize: 2_000_000,
|
|
121
|
+
});
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Driver support matrix:
|
|
125
|
+
|
|
126
|
+
| Knob | `kafkajs` | `confluent` |
|
|
127
|
+
|---|:--:|:--:|
|
|
128
|
+
| `transactionTimeoutMs` | ✅ | ✅ |
|
|
129
|
+
| `requestTimeoutMs` | ✅ | ✅ |
|
|
130
|
+
| `maxInFlightRequests` | ✅ | ✅ |
|
|
131
|
+
| `lingerMs` | ⚠️ warn + ignore | ✅ |
|
|
132
|
+
| `batchSize` | ⚠️ warn + ignore | ✅ |
|
|
133
|
+
| `deliveryTimeoutMs` | ⚠️ warn + ignore | ✅ |
|
|
134
|
+
| `maxRequestSize` | ⚠️ warn + ignore | ✅ |
|
|
135
|
+
|
|
136
|
+
`kafkajs` has no equivalent producer-level config for the last four — its batching is sticky-partitioner + hardcoded internals. The typed API accepts them for portability; on the kafkajs driver they log a one-time warning and are otherwise ignored. Use the confluent driver when you need fine-grained tuning.
|
|
137
|
+
|
|
138
|
+
## Partitioning
|
|
139
|
+
|
|
140
|
+
### Default (key-based, java-compatible)
|
|
141
|
+
|
|
142
|
+
By default a record's `key` is hashed (murmur2, matching the Java client) and the partition derived from it. Same key → same partition → ordered stream per aggregate. No config needed.
|
|
143
|
+
|
|
144
|
+
### Explicit partition override
|
|
145
|
+
|
|
146
|
+
Pin a record to a specific partition by setting `partition` on the
|
|
147
|
+
`PublishableMessage` — for compacted topics with application-managed sharding, tenant-affinity routing, or geo-pinning:
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
const msg: PublishableMessage = {
|
|
151
|
+
topic: "orders.created",
|
|
152
|
+
key: "tenant-a:order-42",
|
|
153
|
+
value: encoded,
|
|
154
|
+
headers: {},
|
|
155
|
+
recordId: row.id,
|
|
156
|
+
messageId: row.message_id,
|
|
157
|
+
partition: 3, // ← pins this record to partition 3
|
|
158
|
+
};
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### kafkajs partitioner choice
|
|
162
|
+
|
|
163
|
+
The kafkajs driver exposes the v2 partitioner selection (and silences the
|
|
164
|
+
`KafkaJSPartitionerNotSpecified` warning):
|
|
165
|
+
|
|
166
|
+
```ts
|
|
167
|
+
new KafkaPublisher({
|
|
168
|
+
driver: "kafkajs",
|
|
169
|
+
brokers: ["broker:9092"],
|
|
170
|
+
partitioner: "java-compatible", // (default) | "legacy" | "default"
|
|
171
|
+
});
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
- `"java-compatible"` — kafkajs's `JavaCompatiblePartitioner`; greenfield recommendation, matches the Java client's murmur2.
|
|
175
|
+
- `"legacy"` — pre-v2 hashing. Use when migrating an existing topic to keep hash continuity.
|
|
176
|
+
- `"default"` — kafkajs's current default. May change in future major versions.
|
|
177
|
+
|
|
178
|
+
## Transactions (EOS)
|
|
179
|
+
|
|
180
|
+
### Callable `transactionalId`
|
|
181
|
+
|
|
182
|
+
`transactionalId` accepts a sync or async resolver — useful when the id depends on runtime context that isn't known at construction time (pod name, AZ + replica index, k8s ordinal):
|
|
183
|
+
|
|
184
|
+
```ts
|
|
185
|
+
new KafkaPublisher({
|
|
186
|
+
brokers,
|
|
187
|
+
transactional: true,
|
|
188
|
+
transactionalId: () =>
|
|
189
|
+
`${process.env.POD_NAME}-${process.env.REPLICA_INDEX}`,
|
|
190
|
+
});
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
For multi-instance EOS, the resolved id MUST be stable across a single instance's restarts but UNIQUE across instances. The plain-string form remains supported and unchanged.
|
|
194
|
+
|
|
195
|
+
### Abort-aware `onTransactionAbort`
|
|
196
|
+
|
|
197
|
+
When a transactional `sendBatch` triggers the abort path (mid-batch error, broker rejection), the publisher fires `hooks.onTransactionAbort(err)` so dashboards and metrics catch EOS failure rates:
|
|
198
|
+
|
|
199
|
+
```ts
|
|
200
|
+
new KafkaPublisher({
|
|
201
|
+
brokers,
|
|
202
|
+
transactional: true,
|
|
203
|
+
transactionalId: "orders-tx",
|
|
204
|
+
hooks: {
|
|
205
|
+
onTransactionAbort: (err) => metrics.txAborts.inc({ reason: err.name }),
|
|
206
|
+
},
|
|
207
|
+
});
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Best-effort: the hook is safe-wrapped (a throwing hook never breaks the abort path).
|
|
211
|
+
|
|
212
|
+
## Observability
|
|
213
|
+
|
|
214
|
+
### Hooks
|
|
215
|
+
|
|
216
|
+
Wire lifecycle hooks into your metrics / logging stack without subclassing or wrapping the publisher:
|
|
217
|
+
|
|
218
|
+
```ts
|
|
219
|
+
new KafkaPublisher({
|
|
220
|
+
brokers,
|
|
221
|
+
hooks: {
|
|
222
|
+
onConnect: () => readinessProbe.up(),
|
|
223
|
+
onDisconnect: () => readinessProbe.down(),
|
|
224
|
+
onPublish: (r, msg) => metrics.publishCounter.inc({ ok: String(r.ok) }),
|
|
225
|
+
onError: (e, msg) => sentry.captureException(e, { msg }),
|
|
226
|
+
onTransactionAbort: (e) => metrics.txAborts.inc(),
|
|
227
|
+
},
|
|
228
|
+
});
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
Hooks are **safe by construction**: a throwing hook never breaks publishing; the publisher swallows the error and logs it via the configured `logger`.
|
|
232
|
+
|
|
233
|
+
### OpenTelemetry tracing
|
|
234
|
+
|
|
235
|
+
The publisher wraps each `publish()` in a span that follows the current stable [OpenTelemetry messaging semantic conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/messaging/kafka.md). No dependency on `@opentelemetry/api` — wire your tracer through a thin adapter:
|
|
236
|
+
|
|
237
|
+
```ts
|
|
238
|
+
import { trace, SpanKind, SpanStatusCode } from "@opentelemetry/api";
|
|
239
|
+
import type { KafkaTracer, SpanLike } from "@eventferry/kafka";
|
|
240
|
+
|
|
241
|
+
const otel = trace.getTracer("@eventferry/kafka");
|
|
242
|
+
|
|
243
|
+
const tracer: KafkaTracer = {
|
|
244
|
+
startPublishSpan(name, attributes) {
|
|
245
|
+
const span = otel.startSpan(name, { kind: SpanKind.PRODUCER, attributes });
|
|
246
|
+
return {
|
|
247
|
+
setAttribute: (k, v) => span.setAttribute(k, v),
|
|
248
|
+
setAttributes: (a) => span.setAttributes(a),
|
|
249
|
+
setStatus: (s) =>
|
|
250
|
+
span.setStatus({
|
|
251
|
+
code: s.code === "ok" ? SpanStatusCode.OK : SpanStatusCode.ERROR,
|
|
252
|
+
message: s.message,
|
|
253
|
+
}),
|
|
254
|
+
recordException: (e) => span.recordException(e),
|
|
255
|
+
end: () => span.end(),
|
|
256
|
+
} satisfies SpanLike;
|
|
257
|
+
},
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
new KafkaPublisher({ brokers, tracer });
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
Per the spec, eventferry emits **one span per `publish()` call**, named `"{topic} publish"`, with attributes:
|
|
264
|
+
|
|
265
|
+
| Attribute | Always | Notes |
|
|
266
|
+
|---|:--:|---|
|
|
267
|
+
| `messaging.system` | ✅ | `"kafka"` |
|
|
268
|
+
| `messaging.operation.type` | ✅ | `"publish"` |
|
|
269
|
+
| `messaging.destination.name` | ✅ | First topic in the batch |
|
|
270
|
+
| `messaging.batch.message_count` | ✅ | Including single-message batches |
|
|
271
|
+
|
|
272
|
+
The user-supplied tracer SHOULD set `SpanKind.PRODUCER` on the span; the adapter above does this explicitly.
|
|
273
|
+
|
|
274
|
+
### Logger
|
|
275
|
+
|
|
276
|
+
Pass a `Logger` (the same interface used by `@eventferry/core`) to route the publisher's own diagnostics — driver warnings, hook failures — through your logging stack:
|
|
277
|
+
|
|
278
|
+
```ts
|
|
279
|
+
new KafkaPublisher({
|
|
280
|
+
brokers,
|
|
281
|
+
logger: pinoLoggerAdapter, // anything implementing { debug, info, warn, error }
|
|
282
|
+
});
|
|
283
|
+
```
|
|
284
|
+
|
|
285
|
+
When omitted, the publisher is silent and the driver falls back to `console.warn` for its diagnostics (preserves prior behavior).
|
|
286
|
+
|
|
38
287
|
📖 **Full documentation:** [github.com/SametGoktepe/eventferry](https://github.com/SametGoktepe/eventferry#readme)
|
|
39
288
|
|
|
40
289
|
## License
|
package/dist/index.cjs
CHANGED
|
@@ -33,8 +33,12 @@ __export(index_exports, {
|
|
|
33
33
|
ConfluentDriver: () => ConfluentDriver,
|
|
34
34
|
KafkaJsDriver: () => KafkaJsDriver,
|
|
35
35
|
KafkaPublisher: () => KafkaPublisher,
|
|
36
|
+
NoopKafkaTracer: () => NoopKafkaTracer,
|
|
37
|
+
_resetKafkajsWarnDedup: () => _resetKafkajsWarnDedup,
|
|
38
|
+
buildConfluentClientConfig: () => buildConfluentClientConfig,
|
|
36
39
|
classifyConfluentError: () => classifyConfluentError,
|
|
37
|
-
classifyKafkajsError: () => classifyKafkajsError
|
|
40
|
+
classifyKafkajsError: () => classifyKafkajsError,
|
|
41
|
+
safeHook: () => safeHook
|
|
38
42
|
});
|
|
39
43
|
module.exports = __toCommonJS(index_exports);
|
|
40
44
|
|
|
@@ -125,7 +129,27 @@ var CODE_TO_KIND = /* @__PURE__ */ new Map([
|
|
|
125
129
|
// INVALID_RECORD
|
|
126
130
|
]);
|
|
127
131
|
|
|
132
|
+
// src/transactional-id.ts
|
|
133
|
+
async function resolveTransactionalId(input) {
|
|
134
|
+
if (input === void 0) {
|
|
135
|
+
throw new Error("transactionalId is required when transactional=true");
|
|
136
|
+
}
|
|
137
|
+
const raw = typeof input === "function" ? await input() : input;
|
|
138
|
+
if (typeof raw !== "string" || raw.length === 0) {
|
|
139
|
+
throw new Error(
|
|
140
|
+
"transactionalId resolver must return a non-empty string"
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
return raw;
|
|
144
|
+
}
|
|
145
|
+
|
|
128
146
|
// src/kafkajs-driver.ts
|
|
147
|
+
var UNSUPPORTED_BY_KAFKAJS = [
|
|
148
|
+
"lingerMs",
|
|
149
|
+
"batchSize",
|
|
150
|
+
"deliveryTimeoutMs",
|
|
151
|
+
"maxRequestSize"
|
|
152
|
+
];
|
|
129
153
|
var KafkaJsDriver = class {
|
|
130
154
|
transactional;
|
|
131
155
|
producer = null;
|
|
@@ -138,6 +162,7 @@ var KafkaJsDriver = class {
|
|
|
138
162
|
"KafkaJsDriver: transactionalId is required when transactional=true"
|
|
139
163
|
);
|
|
140
164
|
}
|
|
165
|
+
warnUnsupportedKafkajsOptions(opts);
|
|
141
166
|
}
|
|
142
167
|
async connect() {
|
|
143
168
|
this.producer = await this.createProducer();
|
|
@@ -152,13 +177,37 @@ var KafkaJsDriver = class {
|
|
|
152
177
|
const kafka = new mod.Kafka({
|
|
153
178
|
clientId: this.opts.clientId ?? "eventferry",
|
|
154
179
|
brokers: this.opts.brokers,
|
|
180
|
+
// kafkajs accepts `ssl: tls.ConnectionOptions` directly — Buffer + PEM
|
|
181
|
+
// string both supported. Our TlsConfig is a structural subset of that
|
|
182
|
+
// (`rejectUnauthorized` intentionally omitted; the cluster CA goes via
|
|
183
|
+
// `ca`). No translation needed.
|
|
155
184
|
ssl: this.opts.ssl,
|
|
185
|
+
// SASL: PLAIN / SCRAM-SHA-256 / SCRAM-SHA-512 / OAUTHBEARER. kafkajs's
|
|
186
|
+
// shape matches ours; for OAUTHBEARER kafkajs reads only `value` from
|
|
187
|
+
// the provider's returned token (other fields are ignored).
|
|
156
188
|
sasl: this.opts.sasl
|
|
157
189
|
});
|
|
190
|
+
const createPartitioner = resolveCreatePartitioner(
|
|
191
|
+
mod.Partitioners,
|
|
192
|
+
this.opts.partitioner,
|
|
193
|
+
this.transactional
|
|
194
|
+
);
|
|
195
|
+
const resolvedTxId = this.transactional ? await resolveTransactionalId(this.opts.transactionalId) : void 0;
|
|
158
196
|
return kafka.producer({
|
|
159
197
|
idempotent: this.opts.idempotent ?? true,
|
|
160
|
-
|
|
161
|
-
|
|
198
|
+
// Idempotent / transactional producers cap maxInFlight at 5. When the
|
|
199
|
+
// user picks transactional we force 1 to keep strict ordering across
|
|
200
|
+
// retries on classic (non-idempotent) clusters that haven't migrated
|
|
201
|
+
// to the broker-side fence.
|
|
202
|
+
maxInFlightRequests: this.transactional ? 1 : this.opts.maxInFlightRequests,
|
|
203
|
+
transactionalId: resolvedTxId,
|
|
204
|
+
// kafkajs accepts these directly when set; undefined falls through to
|
|
205
|
+
// the kafkajs default.
|
|
206
|
+
requestTimeout: this.opts.requestTimeoutMs,
|
|
207
|
+
transactionTimeout: this.opts.transactionTimeoutMs,
|
|
208
|
+
// Setting any partitioner choice silences kafkajs's
|
|
209
|
+
// KafkaJSPartitionerNotSpecified warning.
|
|
210
|
+
createPartitioner
|
|
162
211
|
});
|
|
163
212
|
}
|
|
164
213
|
async disconnect() {
|
|
@@ -176,6 +225,11 @@ var KafkaJsDriver = class {
|
|
|
176
225
|
return messages.map((m) => ({ recordId: m.recordId, ok: true }));
|
|
177
226
|
} catch (err) {
|
|
178
227
|
await txn.abort().catch(() => void 0);
|
|
228
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
229
|
+
try {
|
|
230
|
+
this.opts.onTransactionAbort?.(error);
|
|
231
|
+
} catch {
|
|
232
|
+
}
|
|
179
233
|
return failedResults(messages, err);
|
|
180
234
|
}
|
|
181
235
|
}
|
|
@@ -204,7 +258,12 @@ function groupByTopic(messages, compression) {
|
|
|
204
258
|
arr.push({
|
|
205
259
|
key: m.key,
|
|
206
260
|
value: m.value,
|
|
207
|
-
headers: m.headers
|
|
261
|
+
headers: m.headers,
|
|
262
|
+
// Per-message partition override. When set, kafkajs routes the record
|
|
263
|
+
// to this exact partition; when undefined, the configured partitioner
|
|
264
|
+
// chooses. We keep the key here too because compacted topics need it
|
|
265
|
+
// even when partition is pinned.
|
|
266
|
+
...m.partition !== void 0 ? { partition: m.partition } : {}
|
|
208
267
|
});
|
|
209
268
|
byTopic.set(m.topic, arr);
|
|
210
269
|
}
|
|
@@ -214,6 +273,35 @@ function groupByTopic(messages, compression) {
|
|
|
214
273
|
...compression && compression !== "none" ? { compression } : {}
|
|
215
274
|
}));
|
|
216
275
|
}
|
|
276
|
+
function resolveCreatePartitioner(partitioners, choice, transactional) {
|
|
277
|
+
if (!partitioners) return void 0;
|
|
278
|
+
const effective = choice ?? (transactional ? "default" : "java-compatible");
|
|
279
|
+
switch (effective) {
|
|
280
|
+
case "java-compatible":
|
|
281
|
+
return partitioners.JavaCompatiblePartitioner;
|
|
282
|
+
case "legacy":
|
|
283
|
+
return partitioners.LegacyPartitioner;
|
|
284
|
+
case "default":
|
|
285
|
+
return partitioners.DefaultPartitioner;
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
var warnedKafkajsKeys = /* @__PURE__ */ new Set();
|
|
289
|
+
function warnUnsupportedKafkajsOptions(opts) {
|
|
290
|
+
for (const key of UNSUPPORTED_BY_KAFKAJS) {
|
|
291
|
+
if (opts[key] === void 0) continue;
|
|
292
|
+
if (warnedKafkajsKeys.has(key)) continue;
|
|
293
|
+
warnedKafkajsKeys.add(key);
|
|
294
|
+
const message = `'${key}' is not configurable on the kafkajs driver and was ignored. Switch to the confluent driver (driver: "confluent") for fine-grained tuning, or remove the option to silence this warning.`;
|
|
295
|
+
if (opts.logger) {
|
|
296
|
+
opts.logger.warn(`[@eventferry/kafka] ${message}`, { option: key });
|
|
297
|
+
} else {
|
|
298
|
+
console.warn(`[@eventferry/kafka] ${message}`);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
function _resetKafkajsWarnDedup() {
|
|
303
|
+
warnedKafkajsKeys.clear();
|
|
304
|
+
}
|
|
217
305
|
async function importKafkaJs() {
|
|
218
306
|
try {
|
|
219
307
|
return await import("kafkajs");
|
|
@@ -322,6 +410,71 @@ var NAME_TO_KIND = /* @__PURE__ */ new Map([
|
|
|
322
410
|
["ERR_THROTTLING_QUOTA_EXCEEDED", "quota"]
|
|
323
411
|
]);
|
|
324
412
|
|
|
413
|
+
// src/confluent-config.ts
|
|
414
|
+
function buildConfluentClientConfig(opts) {
|
|
415
|
+
const kafkaJS = {
|
|
416
|
+
clientId: opts.clientId ?? "eventferry",
|
|
417
|
+
brokers: opts.brokers
|
|
418
|
+
};
|
|
419
|
+
const librdkafka = {};
|
|
420
|
+
if (opts.lingerMs !== void 0) librdkafka["linger.ms"] = opts.lingerMs;
|
|
421
|
+
if (opts.batchSize !== void 0) librdkafka["batch.size"] = opts.batchSize;
|
|
422
|
+
if (opts.maxInFlightRequests !== void 0) {
|
|
423
|
+
librdkafka["max.in.flight.requests.per.connection"] = opts.maxInFlightRequests;
|
|
424
|
+
}
|
|
425
|
+
if (opts.requestTimeoutMs !== void 0) {
|
|
426
|
+
librdkafka["request.timeout.ms"] = opts.requestTimeoutMs;
|
|
427
|
+
}
|
|
428
|
+
if (opts.deliveryTimeoutMs !== void 0) {
|
|
429
|
+
librdkafka["delivery.timeout.ms"] = opts.deliveryTimeoutMs;
|
|
430
|
+
}
|
|
431
|
+
if (opts.maxRequestSize !== void 0) {
|
|
432
|
+
librdkafka["message.max.bytes"] = opts.maxRequestSize;
|
|
433
|
+
}
|
|
434
|
+
if (opts.transactionTimeoutMs !== void 0) {
|
|
435
|
+
librdkafka["transaction.timeout.ms"] = opts.transactionTimeoutMs;
|
|
436
|
+
}
|
|
437
|
+
const tlsRequested = opts.ssl === true || isTlsConfig(opts.ssl);
|
|
438
|
+
const saslRequested = !!opts.sasl;
|
|
439
|
+
if (saslRequested && tlsRequested) {
|
|
440
|
+
librdkafka["security.protocol"] = "sasl_ssl";
|
|
441
|
+
} else if (tlsRequested) {
|
|
442
|
+
librdkafka["security.protocol"] = "ssl";
|
|
443
|
+
} else if (saslRequested) {
|
|
444
|
+
librdkafka["security.protocol"] = "sasl_plaintext";
|
|
445
|
+
}
|
|
446
|
+
if (isTlsConfig(opts.ssl)) {
|
|
447
|
+
const tls = opts.ssl;
|
|
448
|
+
if (tls.ca !== void 0) {
|
|
449
|
+
librdkafka["ssl.ca.pem"] = stringifyPem(tls.ca);
|
|
450
|
+
}
|
|
451
|
+
if (tls.cert !== void 0) {
|
|
452
|
+
librdkafka["ssl.certificate.pem"] = stringifyPem(tls.cert);
|
|
453
|
+
}
|
|
454
|
+
if (tls.key !== void 0) {
|
|
455
|
+
librdkafka["ssl.key.pem"] = stringifyPem(tls.key);
|
|
456
|
+
}
|
|
457
|
+
if (tls.passphrase !== void 0) {
|
|
458
|
+
librdkafka["ssl.key.password"] = tls.passphrase;
|
|
459
|
+
}
|
|
460
|
+
} else if (opts.ssl === true) {
|
|
461
|
+
kafkaJS["ssl"] = true;
|
|
462
|
+
}
|
|
463
|
+
if (opts.sasl) {
|
|
464
|
+
kafkaJS["sasl"] = opts.sasl;
|
|
465
|
+
}
|
|
466
|
+
return { kafkaJS, librdkafka };
|
|
467
|
+
}
|
|
468
|
+
function isTlsConfig(v) {
|
|
469
|
+
return typeof v === "object" && v !== null;
|
|
470
|
+
}
|
|
471
|
+
function stringifyPem(input) {
|
|
472
|
+
if (Array.isArray(input)) {
|
|
473
|
+
return input.map((x) => typeof x === "string" ? x : x.toString("utf8")).join("\n");
|
|
474
|
+
}
|
|
475
|
+
return typeof input === "string" ? input : input.toString("utf8");
|
|
476
|
+
}
|
|
477
|
+
|
|
325
478
|
// src/confluent-driver.ts
|
|
326
479
|
var ConfluentDriver = class {
|
|
327
480
|
transactional;
|
|
@@ -346,18 +499,16 @@ var ConfluentDriver = class {
|
|
|
346
499
|
*/
|
|
347
500
|
async createProducer() {
|
|
348
501
|
const mod = await importConfluent();
|
|
502
|
+
const { kafkaJS, librdkafka } = buildConfluentClientConfig(this.opts);
|
|
349
503
|
const kafka = new mod.KafkaJS.Kafka({
|
|
350
|
-
kafkaJS
|
|
351
|
-
|
|
352
|
-
brokers: this.opts.brokers,
|
|
353
|
-
ssl: this.opts.ssl,
|
|
354
|
-
sasl: this.opts.sasl
|
|
355
|
-
}
|
|
504
|
+
kafkaJS,
|
|
505
|
+
...librdkafka
|
|
356
506
|
});
|
|
507
|
+
const resolvedTxId = this.transactional ? await resolveTransactionalId(this.opts.transactionalId) : void 0;
|
|
357
508
|
return kafka.producer({
|
|
358
509
|
kafkaJS: {
|
|
359
510
|
idempotent: this.opts.idempotent ?? true,
|
|
360
|
-
...
|
|
511
|
+
...resolvedTxId ? { transactionalId: resolvedTxId } : {}
|
|
361
512
|
}
|
|
362
513
|
});
|
|
363
514
|
}
|
|
@@ -388,6 +539,11 @@ var ConfluentDriver = class {
|
|
|
388
539
|
return messages.map((m) => ({ recordId: m.recordId, ok: true }));
|
|
389
540
|
} catch (err) {
|
|
390
541
|
await txn.abort().catch(() => void 0);
|
|
542
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
543
|
+
try {
|
|
544
|
+
this.opts.onTransactionAbort?.(error);
|
|
545
|
+
} catch {
|
|
546
|
+
}
|
|
391
547
|
return failedResults2(messages, err);
|
|
392
548
|
}
|
|
393
549
|
}
|
|
@@ -413,7 +569,14 @@ function groupByTopic2(messages) {
|
|
|
413
569
|
const byTopic = /* @__PURE__ */ new Map();
|
|
414
570
|
for (const m of messages) {
|
|
415
571
|
const arr = byTopic.get(m.topic) ?? [];
|
|
416
|
-
arr.push({
|
|
572
|
+
arr.push({
|
|
573
|
+
key: m.key,
|
|
574
|
+
value: m.value,
|
|
575
|
+
headers: m.headers,
|
|
576
|
+
// Per-message partition override. librdkafka honors an explicit
|
|
577
|
+
// partition value; undefined leaves the default partitioner in charge.
|
|
578
|
+
...m.partition !== void 0 ? { partition: m.partition } : {}
|
|
579
|
+
});
|
|
417
580
|
byTopic.set(m.topic, arr);
|
|
418
581
|
}
|
|
419
582
|
return [...byTopic.entries()].map(([topic, msgs]) => ({
|
|
@@ -431,20 +594,108 @@ async function importConfluent() {
|
|
|
431
594
|
}
|
|
432
595
|
}
|
|
433
596
|
|
|
597
|
+
// src/hooks.ts
|
|
598
|
+
async function safeHook(logger, hookName, invoke) {
|
|
599
|
+
try {
|
|
600
|
+
const r = invoke();
|
|
601
|
+
if (r && typeof r.then === "function") {
|
|
602
|
+
await r;
|
|
603
|
+
}
|
|
604
|
+
} catch (err) {
|
|
605
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
606
|
+
logger?.warn(`[@eventferry/kafka] hook ${hookName} threw; ignored`, {
|
|
607
|
+
error: error.message
|
|
608
|
+
});
|
|
609
|
+
}
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
// src/tracing.ts
|
|
613
|
+
var NoopKafkaTracer = class {
|
|
614
|
+
startPublishSpan() {
|
|
615
|
+
return NOOP_SPAN;
|
|
616
|
+
}
|
|
617
|
+
};
|
|
618
|
+
var NOOP_SPAN = {
|
|
619
|
+
setAttribute() {
|
|
620
|
+
},
|
|
621
|
+
setAttributes() {
|
|
622
|
+
},
|
|
623
|
+
setStatus() {
|
|
624
|
+
},
|
|
625
|
+
recordException() {
|
|
626
|
+
},
|
|
627
|
+
end() {
|
|
628
|
+
}
|
|
629
|
+
};
|
|
630
|
+
|
|
434
631
|
// src/publisher.ts
|
|
435
632
|
var KafkaPublisher = class {
|
|
436
633
|
driver;
|
|
634
|
+
logger;
|
|
635
|
+
hooks;
|
|
636
|
+
tracer;
|
|
437
637
|
constructor(opts) {
|
|
438
|
-
this.
|
|
638
|
+
this.logger = opts.logger;
|
|
639
|
+
this.hooks = opts.hooks ?? {};
|
|
640
|
+
this.tracer = opts.tracer ?? new NoopKafkaTracer();
|
|
641
|
+
const onTransactionAbort = this.hooks.onTransactionAbort ? (error) => {
|
|
642
|
+
void safeHook(
|
|
643
|
+
this.logger,
|
|
644
|
+
"onTransactionAbort",
|
|
645
|
+
() => this.hooks.onTransactionAbort?.(error)
|
|
646
|
+
);
|
|
647
|
+
} : void 0;
|
|
648
|
+
this.driver = opts.customDriver ?? selectDriver({ ...opts, onTransactionAbort });
|
|
439
649
|
}
|
|
440
|
-
connect() {
|
|
441
|
-
|
|
650
|
+
async connect() {
|
|
651
|
+
await this.driver.connect();
|
|
652
|
+
await safeHook(this.logger, "onConnect", () => this.hooks.onConnect?.());
|
|
442
653
|
}
|
|
443
|
-
disconnect() {
|
|
444
|
-
|
|
654
|
+
async disconnect() {
|
|
655
|
+
await this.driver.disconnect();
|
|
656
|
+
await safeHook(
|
|
657
|
+
this.logger,
|
|
658
|
+
"onDisconnect",
|
|
659
|
+
() => this.hooks.onDisconnect?.()
|
|
660
|
+
);
|
|
445
661
|
}
|
|
446
|
-
publish(messages) {
|
|
447
|
-
|
|
662
|
+
async publish(messages) {
|
|
663
|
+
if (messages.length === 0) return [];
|
|
664
|
+
const span = this.startBatchSpan(messages);
|
|
665
|
+
let results;
|
|
666
|
+
try {
|
|
667
|
+
results = await this.driver.sendBatch(messages);
|
|
668
|
+
} catch (err) {
|
|
669
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
670
|
+
span.setStatus({ code: "error", message: error.message });
|
|
671
|
+
span.recordException(error);
|
|
672
|
+
span.end();
|
|
673
|
+
await safeHook(this.logger, "onError", () => this.hooks.onError?.(error));
|
|
674
|
+
throw err;
|
|
675
|
+
}
|
|
676
|
+
const byId = new Map(messages.map((m) => [m.recordId, m]));
|
|
677
|
+
let allOk = true;
|
|
678
|
+
for (const r of results) {
|
|
679
|
+
const msg = byId.get(r.recordId);
|
|
680
|
+
if (!msg) continue;
|
|
681
|
+
await safeHook(
|
|
682
|
+
this.logger,
|
|
683
|
+
"onPublish",
|
|
684
|
+
() => this.hooks.onPublish?.(r, msg)
|
|
685
|
+
);
|
|
686
|
+
if (!r.ok) {
|
|
687
|
+
allOk = false;
|
|
688
|
+
const err = r.error ?? new Error("publish failed");
|
|
689
|
+
await safeHook(
|
|
690
|
+
this.logger,
|
|
691
|
+
"onError",
|
|
692
|
+
() => this.hooks.onError?.(err, msg)
|
|
693
|
+
);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
span.setStatus(allOk ? { code: "ok" } : { code: "error" });
|
|
697
|
+
span.end();
|
|
698
|
+
return results;
|
|
448
699
|
}
|
|
449
700
|
/**
|
|
450
701
|
* Send a single dead-lettered message. The message already carries the
|
|
@@ -469,6 +720,23 @@ var KafkaPublisher = class {
|
|
|
469
720
|
get transactional() {
|
|
470
721
|
return this.driver.transactional;
|
|
471
722
|
}
|
|
723
|
+
/**
|
|
724
|
+
* Start a span for the batch following the OTel messaging conventions.
|
|
725
|
+
*
|
|
726
|
+
* Multi-topic batches: per the OTel spec, the span name uses the
|
|
727
|
+
* destination — we pick the FIRST topic in the batch and document the
|
|
728
|
+
* limitation. Callers that publish heterogeneous batches and care about
|
|
729
|
+
* per-topic spans should split their batches upstream.
|
|
730
|
+
*/
|
|
731
|
+
startBatchSpan(messages) {
|
|
732
|
+
const topic = messages[0]?.topic ?? "unknown";
|
|
733
|
+
return this.tracer.startPublishSpan(`${topic} publish`, {
|
|
734
|
+
"messaging.system": "kafka",
|
|
735
|
+
"messaging.operation.type": "publish",
|
|
736
|
+
"messaging.destination.name": topic,
|
|
737
|
+
"messaging.batch.message_count": messages.length
|
|
738
|
+
});
|
|
739
|
+
}
|
|
472
740
|
};
|
|
473
741
|
function selectDriver(opts) {
|
|
474
742
|
const kind = opts.driver ?? "kafkajs";
|
|
@@ -486,7 +754,11 @@ function selectDriver(opts) {
|
|
|
486
754
|
ConfluentDriver,
|
|
487
755
|
KafkaJsDriver,
|
|
488
756
|
KafkaPublisher,
|
|
757
|
+
NoopKafkaTracer,
|
|
758
|
+
_resetKafkajsWarnDedup,
|
|
759
|
+
buildConfluentClientConfig,
|
|
489
760
|
classifyConfluentError,
|
|
490
|
-
classifyKafkajsError
|
|
761
|
+
classifyKafkajsError,
|
|
762
|
+
safeHook
|
|
491
763
|
});
|
|
492
764
|
//# sourceMappingURL=index.cjs.map
|