@eventferry/kafka 2.0.0 → 3.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +140 -0
- package/dist/index.cjs +359 -20
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +195 -8
- package/dist/index.d.ts +195 -8
- package/dist/index.js +354 -19
- package/dist/index.js.map +1 -1
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -35,6 +35,146 @@ const publisher = new KafkaPublisher({
|
|
|
35
35
|
|
|
36
36
|
You can also pass a `customDriver` implementing the `KafkaDriver` interface.
|
|
37
37
|
|
|
38
|
+
## Authentication & TLS
|
|
39
|
+
|
|
40
|
+
### One-way TLS
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
new KafkaPublisher({
|
|
44
|
+
brokers: ["broker:9093"],
|
|
45
|
+
ssl: true, // uses the driver's default trust store
|
|
46
|
+
});
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### mTLS (mutual TLS)
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
import { readFileSync } from "node:fs";
|
|
53
|
+
|
|
54
|
+
new KafkaPublisher({
|
|
55
|
+
brokers: ["broker:9093"],
|
|
56
|
+
ssl: {
|
|
57
|
+
ca: readFileSync("/etc/ssl/kafka-ca.pem"),
|
|
58
|
+
cert: readFileSync("/etc/ssl/client.pem"),
|
|
59
|
+
key: readFileSync("/etc/ssl/client-key.pem"),
|
|
60
|
+
passphrase: "optional",
|
|
61
|
+
// servername: "broker.example.com", // SNI override if cert SAN differs
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
> `rejectUnauthorized` is intentionally NOT a knob. TLS verification is
|
|
67
|
+
> non-negotiable. For dev clusters with self-signed certs, pass the cluster
|
|
68
|
+
> CA via `ca` so verification succeeds.
|
|
69
|
+
|
|
70
|
+
### SASL — username + password (PLAIN / SCRAM)
|
|
71
|
+
|
|
72
|
+
```ts
|
|
73
|
+
new KafkaPublisher({
|
|
74
|
+
brokers: ["broker:9093"],
|
|
75
|
+
ssl: true,
|
|
76
|
+
sasl: {
|
|
77
|
+
mechanism: "scram-sha-512", // or "plain" | "scram-sha-256"
|
|
78
|
+
username: process.env.KAFKA_USER!,
|
|
79
|
+
password: process.env.KAFKA_PASSWORD!,
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
### SASL/OAUTHBEARER (Azure Event Hubs, OIDC, MSK IAM)
|
|
85
|
+
|
|
86
|
+
```ts
|
|
87
|
+
new KafkaPublisher({
|
|
88
|
+
brokers: ["broker:9093"],
|
|
89
|
+
ssl: true,
|
|
90
|
+
sasl: {
|
|
91
|
+
mechanism: "oauthbearer",
|
|
92
|
+
oauthBearerProvider: async () => {
|
|
93
|
+
const token = await myTokenIssuer();
|
|
94
|
+
return {
|
|
95
|
+
value: token.value, // required for both drivers
|
|
96
|
+
principal: token.principal, // required for confluent driver
|
|
97
|
+
lifetime: token.expiresInMs, // required for confluent driver
|
|
98
|
+
extensions: token.extensions, // optional
|
|
99
|
+
};
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
});
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
> **Driver asymmetry:** `kafkajs` reads only `value`; `@confluentinc/kafka-javascript` requires `value` + `principal` + `lifetime` (in milliseconds) and accepts an optional `extensions` map. Cross-driver portable providers should populate all four fields.
|
|
106
|
+
|
|
107
|
+
## Producer tuning
|
|
108
|
+
|
|
109
|
+
The high-throughput recipe (confluent driver):
|
|
110
|
+
|
|
111
|
+
```ts
|
|
112
|
+
new KafkaPublisher({
|
|
113
|
+
driver: "confluent",
|
|
114
|
+
brokers: ["broker:9092"],
|
|
115
|
+
idempotent: true,
|
|
116
|
+
compression: "zstd",
|
|
117
|
+
lingerMs: 25, // batch up to 25ms for higher throughput
|
|
118
|
+
batchSize: 131_072, // 128 KB per partition batch
|
|
119
|
+
maxInFlightRequests: 5,
|
|
120
|
+
maxRequestSize: 2_000_000,
|
|
121
|
+
});
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Driver support matrix:
|
|
125
|
+
|
|
126
|
+
| Knob | `kafkajs` | `confluent` |
|
|
127
|
+
|---|:--:|:--:|
|
|
128
|
+
| `transactionTimeoutMs` | ✅ | ✅ |
|
|
129
|
+
| `requestTimeoutMs` | ✅ | ✅ |
|
|
130
|
+
| `maxInFlightRequests` | ✅ | ✅ |
|
|
131
|
+
| `lingerMs` | ⚠️ warn + ignore | ✅ |
|
|
132
|
+
| `batchSize` | ⚠️ warn + ignore | ✅ |
|
|
133
|
+
| `deliveryTimeoutMs` | ⚠️ warn + ignore | ✅ |
|
|
134
|
+
| `maxRequestSize` | ⚠️ warn + ignore | ✅ |
|
|
135
|
+
|
|
136
|
+
`kafkajs` has no equivalent producer-level config for the last four — its batching is sticky-partitioner + hardcoded internals. The typed API accepts them for portability; on the kafkajs driver they log a one-time warning and are otherwise ignored. Use the confluent driver when you need fine-grained tuning.
|
|
137
|
+
|
|
138
|
+
## Partitioning
|
|
139
|
+
|
|
140
|
+
### Default (key-based, java-compatible)
|
|
141
|
+
|
|
142
|
+
By default a record's `key` is hashed (murmur2, matching the Java client) and the partition derived from it. Same key → same partition → ordered stream per aggregate. No config needed.
|
|
143
|
+
|
|
144
|
+
### Explicit partition override
|
|
145
|
+
|
|
146
|
+
Pin a record to a specific partition by setting `partition` on the
|
|
147
|
+
`PublishableMessage` — for compacted topics with application-managed sharding, tenant-affinity routing, or geo-pinning:
|
|
148
|
+
|
|
149
|
+
```ts
|
|
150
|
+
const msg: PublishableMessage = {
|
|
151
|
+
topic: "orders.created",
|
|
152
|
+
key: "tenant-a:order-42",
|
|
153
|
+
value: encoded,
|
|
154
|
+
headers: {},
|
|
155
|
+
recordId: row.id,
|
|
156
|
+
messageId: row.message_id,
|
|
157
|
+
partition: 3, // ← pins this record to partition 3
|
|
158
|
+
};
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
### kafkajs partitioner choice
|
|
162
|
+
|
|
163
|
+
The kafkajs driver exposes the v2 partitioner selection (and silences the
|
|
164
|
+
`KafkaJSPartitionerNotSpecified` warning):
|
|
165
|
+
|
|
166
|
+
```ts
|
|
167
|
+
new KafkaPublisher({
|
|
168
|
+
driver: "kafkajs",
|
|
169
|
+
brokers: ["broker:9092"],
|
|
170
|
+
partitioner: "java-compatible", // (default) | "legacy" | "default"
|
|
171
|
+
});
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
- `"java-compatible"` — kafkajs's `JavaCompatiblePartitioner`; greenfield recommendation, matches the Java client's murmur2.
|
|
175
|
+
- `"legacy"` — pre-v2 hashing. Use when migrating an existing topic to keep hash continuity.
|
|
176
|
+
- `"default"` — kafkajs's current default. May change in future major versions.
|
|
177
|
+
|
|
38
178
|
📖 **Full documentation:** [github.com/SametGoktepe/eventferry](https://github.com/SametGoktepe/eventferry#readme)
|
|
39
179
|
|
|
40
180
|
## License
|
package/dist/index.cjs
CHANGED
|
@@ -32,11 +32,108 @@ var index_exports = {};
|
|
|
32
32
|
__export(index_exports, {
|
|
33
33
|
ConfluentDriver: () => ConfluentDriver,
|
|
34
34
|
KafkaJsDriver: () => KafkaJsDriver,
|
|
35
|
-
KafkaPublisher: () => KafkaPublisher
|
|
35
|
+
KafkaPublisher: () => KafkaPublisher,
|
|
36
|
+
_resetKafkajsWarnDedup: () => _resetKafkajsWarnDedup,
|
|
37
|
+
buildConfluentClientConfig: () => buildConfluentClientConfig,
|
|
38
|
+
classifyConfluentError: () => classifyConfluentError,
|
|
39
|
+
classifyKafkajsError: () => classifyKafkajsError
|
|
36
40
|
});
|
|
37
41
|
module.exports = __toCommonJS(index_exports);
|
|
38
42
|
|
|
43
|
+
// src/kafkajs-classifier.ts
|
|
44
|
+
function classifyKafkajsError(err) {
|
|
45
|
+
if (!err || typeof err !== "object") return "retriable";
|
|
46
|
+
const e = err;
|
|
47
|
+
if (e.name === "KafkaJSConnectionError") return "retriable";
|
|
48
|
+
if (e.name === "KafkaJSRequestTimeoutError") return "retriable";
|
|
49
|
+
if (e.name === "KafkaJSNonRetriableError") return "fatal";
|
|
50
|
+
const type = typeof e.type === "string" ? e.type : void 0;
|
|
51
|
+
if (type) {
|
|
52
|
+
if (RETRIABLE_TYPES.has(type)) return "retriable";
|
|
53
|
+
if (POISON_TYPES.has(type)) return "poison";
|
|
54
|
+
if (FATAL_TYPES.has(type)) return "fatal";
|
|
55
|
+
}
|
|
56
|
+
if (typeof e.code === "number") {
|
|
57
|
+
const k = CODE_TO_KIND.get(e.code);
|
|
58
|
+
if (k) return k;
|
|
59
|
+
}
|
|
60
|
+
return "retriable";
|
|
61
|
+
}
|
|
62
|
+
var RETRIABLE_TYPES = /* @__PURE__ */ new Set([
|
|
63
|
+
"NOT_LEADER_FOR_PARTITION",
|
|
64
|
+
"LEADER_NOT_AVAILABLE",
|
|
65
|
+
"UNKNOWN_TOPIC_OR_PARTITION",
|
|
66
|
+
"NETWORK_EXCEPTION",
|
|
67
|
+
"REQUEST_TIMED_OUT",
|
|
68
|
+
"REPLICA_NOT_AVAILABLE",
|
|
69
|
+
"NOT_ENOUGH_REPLICAS",
|
|
70
|
+
"NOT_ENOUGH_REPLICAS_AFTER_APPEND",
|
|
71
|
+
"FENCED_LEADER_EPOCH",
|
|
72
|
+
"UNKNOWN_LEADER_EPOCH",
|
|
73
|
+
"BROKER_NOT_AVAILABLE",
|
|
74
|
+
"COORDINATOR_LOAD_IN_PROGRESS",
|
|
75
|
+
"COORDINATOR_NOT_AVAILABLE"
|
|
76
|
+
]);
|
|
77
|
+
var POISON_TYPES = /* @__PURE__ */ new Set([
|
|
78
|
+
"CORRUPT_MESSAGE",
|
|
79
|
+
"MESSAGE_TOO_LARGE",
|
|
80
|
+
"INVALID_RECORD",
|
|
81
|
+
"UNSUPPORTED_COMPRESSION_TYPE",
|
|
82
|
+
"INVALID_REQUIRED_ACKS",
|
|
83
|
+
"INVALID_PARTITIONS"
|
|
84
|
+
]);
|
|
85
|
+
var FATAL_TYPES = /* @__PURE__ */ new Set([
|
|
86
|
+
"INVALID_PRODUCER_EPOCH",
|
|
87
|
+
"PRODUCER_FENCED",
|
|
88
|
+
"TOPIC_AUTHORIZATION_FAILED",
|
|
89
|
+
"CLUSTER_AUTHORIZATION_FAILED",
|
|
90
|
+
"TRANSACTIONAL_ID_AUTHORIZATION_FAILED",
|
|
91
|
+
"SASL_AUTHENTICATION_FAILED",
|
|
92
|
+
"INVALID_TRANSACTION_STATE",
|
|
93
|
+
"UNSUPPORTED_VERSION"
|
|
94
|
+
]);
|
|
95
|
+
var CODE_TO_KIND = /* @__PURE__ */ new Map([
|
|
96
|
+
[2, "poison"],
|
|
97
|
+
// CORRUPT_MESSAGE
|
|
98
|
+
[3, "retriable"],
|
|
99
|
+
// UNKNOWN_TOPIC_OR_PARTITION
|
|
100
|
+
[5, "retriable"],
|
|
101
|
+
// LEADER_NOT_AVAILABLE
|
|
102
|
+
[6, "retriable"],
|
|
103
|
+
// NOT_LEADER_FOR_PARTITION
|
|
104
|
+
[7, "retriable"],
|
|
105
|
+
// REQUEST_TIMED_OUT
|
|
106
|
+
[9, "retriable"],
|
|
107
|
+
// REPLICA_NOT_AVAILABLE
|
|
108
|
+
[10, "poison"],
|
|
109
|
+
// MESSAGE_TOO_LARGE
|
|
110
|
+
[13, "retriable"],
|
|
111
|
+
// NETWORK_EXCEPTION
|
|
112
|
+
[19, "retriable"],
|
|
113
|
+
// NOT_ENOUGH_REPLICAS
|
|
114
|
+
[29, "fatal"],
|
|
115
|
+
// TOPIC_AUTHORIZATION_FAILED
|
|
116
|
+
[31, "fatal"],
|
|
117
|
+
// CLUSTER_AUTHORIZATION_FAILED
|
|
118
|
+
[47, "fatal"],
|
|
119
|
+
// INVALID_PRODUCER_EPOCH
|
|
120
|
+
[58, "fatal"],
|
|
121
|
+
// SASL_AUTHENTICATION_FAILED
|
|
122
|
+
[74, "retriable"],
|
|
123
|
+
// FENCED_LEADER_EPOCH
|
|
124
|
+
[76, "poison"],
|
|
125
|
+
// UNSUPPORTED_COMPRESSION_TYPE
|
|
126
|
+
[87, "poison"]
|
|
127
|
+
// INVALID_RECORD
|
|
128
|
+
]);
|
|
129
|
+
|
|
39
130
|
// src/kafkajs-driver.ts
|
|
131
|
+
var UNSUPPORTED_BY_KAFKAJS = [
|
|
132
|
+
"lingerMs",
|
|
133
|
+
"batchSize",
|
|
134
|
+
"deliveryTimeoutMs",
|
|
135
|
+
"maxRequestSize"
|
|
136
|
+
];
|
|
40
137
|
var KafkaJsDriver = class {
|
|
41
138
|
transactional;
|
|
42
139
|
producer = null;
|
|
@@ -49,6 +146,7 @@ var KafkaJsDriver = class {
|
|
|
49
146
|
"KafkaJsDriver: transactionalId is required when transactional=true"
|
|
50
147
|
);
|
|
51
148
|
}
|
|
149
|
+
warnUnsupportedKafkajsOptions(opts);
|
|
52
150
|
}
|
|
53
151
|
async connect() {
|
|
54
152
|
this.producer = await this.createProducer();
|
|
@@ -63,13 +161,36 @@ var KafkaJsDriver = class {
|
|
|
63
161
|
const kafka = new mod.Kafka({
|
|
64
162
|
clientId: this.opts.clientId ?? "eventferry",
|
|
65
163
|
brokers: this.opts.brokers,
|
|
164
|
+
// kafkajs accepts `ssl: tls.ConnectionOptions` directly — Buffer + PEM
|
|
165
|
+
// string both supported. Our TlsConfig is a structural subset of that
|
|
166
|
+
// (`rejectUnauthorized` intentionally omitted; the cluster CA goes via
|
|
167
|
+
// `ca`). No translation needed.
|
|
66
168
|
ssl: this.opts.ssl,
|
|
169
|
+
// SASL: PLAIN / SCRAM-SHA-256 / SCRAM-SHA-512 / OAUTHBEARER. kafkajs's
|
|
170
|
+
// shape matches ours; for OAUTHBEARER kafkajs reads only `value` from
|
|
171
|
+
// the provider's returned token (other fields are ignored).
|
|
67
172
|
sasl: this.opts.sasl
|
|
68
173
|
});
|
|
174
|
+
const createPartitioner = resolveCreatePartitioner(
|
|
175
|
+
mod.Partitioners,
|
|
176
|
+
this.opts.partitioner,
|
|
177
|
+
this.transactional
|
|
178
|
+
);
|
|
69
179
|
return kafka.producer({
|
|
70
180
|
idempotent: this.opts.idempotent ?? true,
|
|
71
|
-
|
|
72
|
-
|
|
181
|
+
// Idempotent / transactional producers cap maxInFlight at 5. When the
|
|
182
|
+
// user picks transactional we force 1 to keep strict ordering across
|
|
183
|
+
// retries on classic (non-idempotent) clusters that haven't migrated
|
|
184
|
+
// to the broker-side fence.
|
|
185
|
+
maxInFlightRequests: this.transactional ? 1 : this.opts.maxInFlightRequests,
|
|
186
|
+
transactionalId: this.transactional ? this.opts.transactionalId : void 0,
|
|
187
|
+
// kafkajs accepts these directly when set; undefined falls through to
|
|
188
|
+
// the kafkajs default.
|
|
189
|
+
requestTimeout: this.opts.requestTimeoutMs,
|
|
190
|
+
transactionTimeout: this.opts.transactionTimeoutMs,
|
|
191
|
+
// Setting any partitioner choice silences kafkajs's
|
|
192
|
+
// KafkaJSPartitionerNotSpecified warning.
|
|
193
|
+
createPartitioner
|
|
73
194
|
});
|
|
74
195
|
}
|
|
75
196
|
async disconnect() {
|
|
@@ -87,19 +208,27 @@ var KafkaJsDriver = class {
|
|
|
87
208
|
return messages.map((m) => ({ recordId: m.recordId, ok: true }));
|
|
88
209
|
} catch (err) {
|
|
89
210
|
await txn.abort().catch(() => void 0);
|
|
90
|
-
|
|
91
|
-
return messages.map((m) => ({ recordId: m.recordId, ok: false, error }));
|
|
211
|
+
return failedResults(messages, err);
|
|
92
212
|
}
|
|
93
213
|
}
|
|
94
214
|
try {
|
|
95
215
|
await this.producer.sendBatch({ topicMessages, acks: this.opts.acks ?? -1 });
|
|
96
216
|
return messages.map((m) => ({ recordId: m.recordId, ok: true }));
|
|
97
217
|
} catch (err) {
|
|
98
|
-
|
|
99
|
-
return messages.map((m) => ({ recordId: m.recordId, ok: false, error }));
|
|
218
|
+
return failedResults(messages, err);
|
|
100
219
|
}
|
|
101
220
|
}
|
|
102
221
|
};
|
|
222
|
+
function failedResults(messages, err) {
|
|
223
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
224
|
+
const errorKind = classifyKafkajsError(err);
|
|
225
|
+
return messages.map((m) => ({
|
|
226
|
+
recordId: m.recordId,
|
|
227
|
+
ok: false,
|
|
228
|
+
error,
|
|
229
|
+
errorKind
|
|
230
|
+
}));
|
|
231
|
+
}
|
|
103
232
|
function groupByTopic(messages, compression) {
|
|
104
233
|
const byTopic = /* @__PURE__ */ new Map();
|
|
105
234
|
for (const m of messages) {
|
|
@@ -107,7 +236,12 @@ function groupByTopic(messages, compression) {
|
|
|
107
236
|
arr.push({
|
|
108
237
|
key: m.key,
|
|
109
238
|
value: m.value,
|
|
110
|
-
headers: m.headers
|
|
239
|
+
headers: m.headers,
|
|
240
|
+
// Per-message partition override. When set, kafkajs routes the record
|
|
241
|
+
// to this exact partition; when undefined, the configured partitioner
|
|
242
|
+
// chooses. We keep the key here too because compacted topics need it
|
|
243
|
+
// even when partition is pinned.
|
|
244
|
+
...m.partition !== void 0 ? { partition: m.partition } : {}
|
|
111
245
|
});
|
|
112
246
|
byTopic.set(m.topic, arr);
|
|
113
247
|
}
|
|
@@ -117,6 +251,32 @@ function groupByTopic(messages, compression) {
|
|
|
117
251
|
...compression && compression !== "none" ? { compression } : {}
|
|
118
252
|
}));
|
|
119
253
|
}
|
|
254
|
+
function resolveCreatePartitioner(partitioners, choice, transactional) {
|
|
255
|
+
if (!partitioners) return void 0;
|
|
256
|
+
const effective = choice ?? (transactional ? "default" : "java-compatible");
|
|
257
|
+
switch (effective) {
|
|
258
|
+
case "java-compatible":
|
|
259
|
+
return partitioners.JavaCompatiblePartitioner;
|
|
260
|
+
case "legacy":
|
|
261
|
+
return partitioners.LegacyPartitioner;
|
|
262
|
+
case "default":
|
|
263
|
+
return partitioners.DefaultPartitioner;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
var warnedKafkajsKeys = /* @__PURE__ */ new Set();
|
|
267
|
+
function warnUnsupportedKafkajsOptions(opts) {
|
|
268
|
+
for (const key of UNSUPPORTED_BY_KAFKAJS) {
|
|
269
|
+
if (opts[key] === void 0) continue;
|
|
270
|
+
if (warnedKafkajsKeys.has(key)) continue;
|
|
271
|
+
warnedKafkajsKeys.add(key);
|
|
272
|
+
console.warn(
|
|
273
|
+
`[@eventferry/kafka] '${key}' is not configurable on the kafkajs driver and was ignored. Switch to the confluent driver (driver: "confluent") for fine-grained tuning, or remove the option to silence this warning.`
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
function _resetKafkajsWarnDedup() {
|
|
278
|
+
warnedKafkajsKeys.clear();
|
|
279
|
+
}
|
|
120
280
|
async function importKafkaJs() {
|
|
121
281
|
try {
|
|
122
282
|
return await import("kafkajs");
|
|
@@ -127,6 +287,169 @@ async function importKafkaJs() {
|
|
|
127
287
|
}
|
|
128
288
|
}
|
|
129
289
|
|
|
290
|
+
// src/confluent-classifier.ts
|
|
291
|
+
function classifyConfluentError(err) {
|
|
292
|
+
if (!err || typeof err !== "object") return "retriable";
|
|
293
|
+
const e = err;
|
|
294
|
+
if (typeof e.code === "number") {
|
|
295
|
+
const k = CODE_TO_KIND2.get(e.code);
|
|
296
|
+
if (k) return k;
|
|
297
|
+
}
|
|
298
|
+
if (typeof e.name === "string") {
|
|
299
|
+
const k = NAME_TO_KIND.get(e.name);
|
|
300
|
+
if (k) return k;
|
|
301
|
+
}
|
|
302
|
+
return "retriable";
|
|
303
|
+
}
|
|
304
|
+
var CODE_TO_KIND2 = /* @__PURE__ */ new Map([
|
|
305
|
+
// Library-internal (negative codes)
|
|
306
|
+
[-184, "backpressure"],
|
|
307
|
+
// ERR__QUEUE_FULL — our outbound buffer is full
|
|
308
|
+
[-185, "retriable"],
|
|
309
|
+
// ERR__TIMED_OUT
|
|
310
|
+
[-187, "retriable"],
|
|
311
|
+
// ERR__ALL_BROKERS_DOWN
|
|
312
|
+
[-188, "poison"],
|
|
313
|
+
// ERR__UNKNOWN_TOPIC — topic doesn't exist on broker
|
|
314
|
+
[-190, "poison"],
|
|
315
|
+
// ERR__UNKNOWN_PARTITION
|
|
316
|
+
[-192, "retriable"],
|
|
317
|
+
// ERR__MSG_TIMED_OUT
|
|
318
|
+
[-195, "retriable"],
|
|
319
|
+
// ERR__TRANSPORT
|
|
320
|
+
[-198, "poison"],
|
|
321
|
+
// ERR__BAD_COMPRESSION
|
|
322
|
+
[-144, "fatal"],
|
|
323
|
+
// ERR__FENCED — producer fenced by another with same txn id
|
|
324
|
+
[-150, "fatal"],
|
|
325
|
+
// ERR__FATAL — unrecoverable librdkafka error
|
|
326
|
+
[-169, "fatal"],
|
|
327
|
+
// ERR__AUTHENTICATION
|
|
328
|
+
[-181, "fatal"],
|
|
329
|
+
// ERR__SSL
|
|
330
|
+
[-196, "retriable"],
|
|
331
|
+
// ERR__FAIL — catch-all, safe-default to retriable
|
|
332
|
+
// Wire-protocol (non-negative codes — Kafka error-code registry)
|
|
333
|
+
[2, "poison"],
|
|
334
|
+
// CORRUPT_MESSAGE
|
|
335
|
+
[3, "retriable"],
|
|
336
|
+
// UNKNOWN_TOPIC_OR_PARTITION
|
|
337
|
+
[5, "retriable"],
|
|
338
|
+
// LEADER_NOT_AVAILABLE
|
|
339
|
+
[6, "retriable"],
|
|
340
|
+
// NOT_LEADER_FOR_PARTITION
|
|
341
|
+
[7, "retriable"],
|
|
342
|
+
// REQUEST_TIMED_OUT
|
|
343
|
+
[9, "retriable"],
|
|
344
|
+
// REPLICA_NOT_AVAILABLE
|
|
345
|
+
[10, "poison"],
|
|
346
|
+
// MESSAGE_TOO_LARGE
|
|
347
|
+
[13, "retriable"],
|
|
348
|
+
// NETWORK_EXCEPTION
|
|
349
|
+
[19, "retriable"],
|
|
350
|
+
// NOT_ENOUGH_REPLICAS
|
|
351
|
+
[29, "fatal"],
|
|
352
|
+
// TOPIC_AUTHORIZATION_FAILED
|
|
353
|
+
[31, "fatal"],
|
|
354
|
+
// CLUSTER_AUTHORIZATION_FAILED
|
|
355
|
+
[47, "fatal"],
|
|
356
|
+
// INVALID_PRODUCER_EPOCH
|
|
357
|
+
[58, "fatal"],
|
|
358
|
+
// SASL_AUTHENTICATION_FAILED
|
|
359
|
+
[74, "retriable"],
|
|
360
|
+
// FENCED_LEADER_EPOCH
|
|
361
|
+
[76, "poison"],
|
|
362
|
+
// UNSUPPORTED_COMPRESSION_TYPE
|
|
363
|
+
[87, "poison"],
|
|
364
|
+
// INVALID_RECORD
|
|
365
|
+
[89, "quota"]
|
|
366
|
+
// THROTTLING_QUOTA_EXCEEDED
|
|
367
|
+
]);
|
|
368
|
+
var NAME_TO_KIND = /* @__PURE__ */ new Map([
|
|
369
|
+
["ERR__QUEUE_FULL", "backpressure"],
|
|
370
|
+
["ERR__FENCED", "fatal"],
|
|
371
|
+
["ERR__FATAL", "fatal"],
|
|
372
|
+
["ERR__AUTHENTICATION", "fatal"],
|
|
373
|
+
["ERR__SSL", "fatal"],
|
|
374
|
+
["ERR__UNKNOWN_TOPIC", "poison"],
|
|
375
|
+
["ERR__UNKNOWN_PARTITION", "poison"],
|
|
376
|
+
["ERR__BAD_COMPRESSION", "poison"],
|
|
377
|
+
["ERR_TOPIC_AUTHORIZATION_FAILED", "fatal"],
|
|
378
|
+
["ERR_CLUSTER_AUTHORIZATION_FAILED", "fatal"],
|
|
379
|
+
["ERR_INVALID_PRODUCER_EPOCH", "fatal"],
|
|
380
|
+
["ERR_SASL_AUTHENTICATION_FAILED", "fatal"],
|
|
381
|
+
["ERR_CORRUPT_MESSAGE", "poison"],
|
|
382
|
+
["ERR_MSG_SIZE_TOO_LARGE", "poison"],
|
|
383
|
+
["ERR_INVALID_RECORD", "poison"],
|
|
384
|
+
["ERR_UNSUPPORTED_COMPRESSION_TYPE", "poison"],
|
|
385
|
+
["ERR_THROTTLING_QUOTA_EXCEEDED", "quota"]
|
|
386
|
+
]);
|
|
387
|
+
|
|
388
|
+
// src/confluent-config.ts
|
|
389
|
+
function buildConfluentClientConfig(opts) {
|
|
390
|
+
const kafkaJS = {
|
|
391
|
+
clientId: opts.clientId ?? "eventferry",
|
|
392
|
+
brokers: opts.brokers
|
|
393
|
+
};
|
|
394
|
+
const librdkafka = {};
|
|
395
|
+
if (opts.lingerMs !== void 0) librdkafka["linger.ms"] = opts.lingerMs;
|
|
396
|
+
if (opts.batchSize !== void 0) librdkafka["batch.size"] = opts.batchSize;
|
|
397
|
+
if (opts.maxInFlightRequests !== void 0) {
|
|
398
|
+
librdkafka["max.in.flight.requests.per.connection"] = opts.maxInFlightRequests;
|
|
399
|
+
}
|
|
400
|
+
if (opts.requestTimeoutMs !== void 0) {
|
|
401
|
+
librdkafka["request.timeout.ms"] = opts.requestTimeoutMs;
|
|
402
|
+
}
|
|
403
|
+
if (opts.deliveryTimeoutMs !== void 0) {
|
|
404
|
+
librdkafka["delivery.timeout.ms"] = opts.deliveryTimeoutMs;
|
|
405
|
+
}
|
|
406
|
+
if (opts.maxRequestSize !== void 0) {
|
|
407
|
+
librdkafka["message.max.bytes"] = opts.maxRequestSize;
|
|
408
|
+
}
|
|
409
|
+
if (opts.transactionTimeoutMs !== void 0) {
|
|
410
|
+
librdkafka["transaction.timeout.ms"] = opts.transactionTimeoutMs;
|
|
411
|
+
}
|
|
412
|
+
const tlsRequested = opts.ssl === true || isTlsConfig(opts.ssl);
|
|
413
|
+
const saslRequested = !!opts.sasl;
|
|
414
|
+
if (saslRequested && tlsRequested) {
|
|
415
|
+
librdkafka["security.protocol"] = "sasl_ssl";
|
|
416
|
+
} else if (tlsRequested) {
|
|
417
|
+
librdkafka["security.protocol"] = "ssl";
|
|
418
|
+
} else if (saslRequested) {
|
|
419
|
+
librdkafka["security.protocol"] = "sasl_plaintext";
|
|
420
|
+
}
|
|
421
|
+
if (isTlsConfig(opts.ssl)) {
|
|
422
|
+
const tls = opts.ssl;
|
|
423
|
+
if (tls.ca !== void 0) {
|
|
424
|
+
librdkafka["ssl.ca.pem"] = stringifyPem(tls.ca);
|
|
425
|
+
}
|
|
426
|
+
if (tls.cert !== void 0) {
|
|
427
|
+
librdkafka["ssl.certificate.pem"] = stringifyPem(tls.cert);
|
|
428
|
+
}
|
|
429
|
+
if (tls.key !== void 0) {
|
|
430
|
+
librdkafka["ssl.key.pem"] = stringifyPem(tls.key);
|
|
431
|
+
}
|
|
432
|
+
if (tls.passphrase !== void 0) {
|
|
433
|
+
librdkafka["ssl.key.password"] = tls.passphrase;
|
|
434
|
+
}
|
|
435
|
+
} else if (opts.ssl === true) {
|
|
436
|
+
kafkaJS["ssl"] = true;
|
|
437
|
+
}
|
|
438
|
+
if (opts.sasl) {
|
|
439
|
+
kafkaJS["sasl"] = opts.sasl;
|
|
440
|
+
}
|
|
441
|
+
return { kafkaJS, librdkafka };
|
|
442
|
+
}
|
|
443
|
+
function isTlsConfig(v) {
|
|
444
|
+
return typeof v === "object" && v !== null;
|
|
445
|
+
}
|
|
446
|
+
function stringifyPem(input) {
|
|
447
|
+
if (Array.isArray(input)) {
|
|
448
|
+
return input.map((x) => typeof x === "string" ? x : x.toString("utf8")).join("\n");
|
|
449
|
+
}
|
|
450
|
+
return typeof input === "string" ? input : input.toString("utf8");
|
|
451
|
+
}
|
|
452
|
+
|
|
130
453
|
// src/confluent-driver.ts
|
|
131
454
|
var ConfluentDriver = class {
|
|
132
455
|
transactional;
|
|
@@ -151,13 +474,10 @@ var ConfluentDriver = class {
|
|
|
151
474
|
*/
|
|
152
475
|
async createProducer() {
|
|
153
476
|
const mod = await importConfluent();
|
|
477
|
+
const { kafkaJS, librdkafka } = buildConfluentClientConfig(this.opts);
|
|
154
478
|
const kafka = new mod.KafkaJS.Kafka({
|
|
155
|
-
kafkaJS
|
|
156
|
-
|
|
157
|
-
brokers: this.opts.brokers,
|
|
158
|
-
ssl: this.opts.ssl,
|
|
159
|
-
sasl: this.opts.sasl
|
|
160
|
-
}
|
|
479
|
+
kafkaJS,
|
|
480
|
+
...librdkafka
|
|
161
481
|
});
|
|
162
482
|
return kafka.producer({
|
|
163
483
|
kafkaJS: {
|
|
@@ -193,24 +513,39 @@ var ConfluentDriver = class {
|
|
|
193
513
|
return messages.map((m) => ({ recordId: m.recordId, ok: true }));
|
|
194
514
|
} catch (err) {
|
|
195
515
|
await txn.abort().catch(() => void 0);
|
|
196
|
-
|
|
197
|
-
return messages.map((m) => ({ recordId: m.recordId, ok: false, error }));
|
|
516
|
+
return failedResults2(messages, err);
|
|
198
517
|
}
|
|
199
518
|
}
|
|
200
519
|
try {
|
|
201
520
|
await doSends(this.producer);
|
|
202
521
|
return messages.map((m) => ({ recordId: m.recordId, ok: true }));
|
|
203
522
|
} catch (err) {
|
|
204
|
-
|
|
205
|
-
return messages.map((m) => ({ recordId: m.recordId, ok: false, error }));
|
|
523
|
+
return failedResults2(messages, err);
|
|
206
524
|
}
|
|
207
525
|
}
|
|
208
526
|
};
|
|
527
|
+
function failedResults2(messages, err) {
|
|
528
|
+
const error = err instanceof Error ? err : new Error(String(err));
|
|
529
|
+
const errorKind = classifyConfluentError(err);
|
|
530
|
+
return messages.map((m) => ({
|
|
531
|
+
recordId: m.recordId,
|
|
532
|
+
ok: false,
|
|
533
|
+
error,
|
|
534
|
+
errorKind
|
|
535
|
+
}));
|
|
536
|
+
}
|
|
209
537
|
function groupByTopic2(messages) {
|
|
210
538
|
const byTopic = /* @__PURE__ */ new Map();
|
|
211
539
|
for (const m of messages) {
|
|
212
540
|
const arr = byTopic.get(m.topic) ?? [];
|
|
213
|
-
arr.push({
|
|
541
|
+
arr.push({
|
|
542
|
+
key: m.key,
|
|
543
|
+
value: m.value,
|
|
544
|
+
headers: m.headers,
|
|
545
|
+
// Per-message partition override. librdkafka honors an explicit
|
|
546
|
+
// partition value; undefined leaves the default partitioner in charge.
|
|
547
|
+
...m.partition !== void 0 ? { partition: m.partition } : {}
|
|
548
|
+
});
|
|
214
549
|
byTopic.set(m.topic, arr);
|
|
215
550
|
}
|
|
216
551
|
return [...byTopic.entries()].map(([topic, msgs]) => ({
|
|
@@ -282,6 +617,10 @@ function selectDriver(opts) {
|
|
|
282
617
|
0 && (module.exports = {
|
|
283
618
|
ConfluentDriver,
|
|
284
619
|
KafkaJsDriver,
|
|
285
|
-
KafkaPublisher
|
|
620
|
+
KafkaPublisher,
|
|
621
|
+
_resetKafkajsWarnDedup,
|
|
622
|
+
buildConfluentClientConfig,
|
|
623
|
+
classifyConfluentError,
|
|
624
|
+
classifyKafkajsError
|
|
286
625
|
});
|
|
287
626
|
//# sourceMappingURL=index.cjs.map
|