@eventferry/kafka 3.3.0 → 3.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +441 -0
- package/README.md +184 -0
- package/dist/consume.cjs +115 -0
- package/dist/consume.cjs.map +1 -0
- package/dist/consume.d.cts +114 -0
- package/dist/consume.d.ts +114 -0
- package/dist/consume.js +88 -0
- package/dist/consume.js.map +1 -0
- package/dist/index.cjs +269 -7
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +218 -1
- package/dist/index.d.ts +218 -1
- package/dist/index.js +269 -7
- package/dist/index.js.map +1 -1
- package/package.json +10 -4
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,97 @@
|
|
|
1
1
|
import { PublishableMessage, PublishResult, Logger, PublishErrorKind, Publisher } from '@eventferry/core';
|
|
2
2
|
|
|
3
|
+
/**
|
|
4
|
+
* Admin surface for `KafkaPublisher`. A typed wrapper over each driver's
|
|
5
|
+
* underlying admin client — implementations live in `kafkajs-driver.ts` and
|
|
6
|
+
* `confluent-driver.ts`. The publisher exposes this via `publisher.admin()`
|
|
7
|
+
* (returns a connected admin) and `publisher.ensureTopics()` (idempotent
|
|
8
|
+
* topic provisioning built on top).
|
|
9
|
+
*
|
|
10
|
+
* Scope is deliberately the most-used subset of the Kafka AdminClient
|
|
11
|
+
* protocol — listing, describing, creating topics and partitions. ACL
|
|
12
|
+
* management, quota inspection, and consumer-group operations are left to
|
|
13
|
+
* the underlying client (reach for kafkajs's `Admin` directly if needed).
|
|
14
|
+
*/
|
|
15
|
+
/** Specification for creating one topic. */
|
|
16
|
+
interface TopicCreateSpec {
|
|
17
|
+
topic: string;
|
|
18
|
+
/** Default: cluster's `num.partitions` broker setting. */
|
|
19
|
+
numPartitions?: number;
|
|
20
|
+
/** Default: cluster's `default.replication.factor` broker setting. */
|
|
21
|
+
replicationFactor?: number;
|
|
22
|
+
/**
|
|
23
|
+
* Per-topic config entries (e.g. `{ "retention.ms": "604800000" }`).
|
|
24
|
+
* See Kafka broker docs for the full set.
|
|
25
|
+
*/
|
|
26
|
+
configEntries?: Record<string, string>;
|
|
27
|
+
}
|
|
28
|
+
/** Topic + partition descriptor returned by describeTopics. */
|
|
29
|
+
interface TopicMetadata {
|
|
30
|
+
topic: string;
|
|
31
|
+
/** Empty when the topic doesn't exist (so callers can detect absence cheaply). */
|
|
32
|
+
partitions: PartitionMetadata[];
|
|
33
|
+
}
|
|
34
|
+
interface PartitionMetadata {
|
|
35
|
+
partitionId: number;
|
|
36
|
+
/** Broker id of the partition leader; -1 if no leader is known. */
|
|
37
|
+
leader: number;
|
|
38
|
+
/** Replica broker ids. */
|
|
39
|
+
replicas: number[];
|
|
40
|
+
/** In-sync replica broker ids. */
|
|
41
|
+
isr: number[];
|
|
42
|
+
}
|
|
43
|
+
/** Specification for growing a topic's partition count (never shrink). */
|
|
44
|
+
interface PartitionGrowSpec {
|
|
45
|
+
topic: string;
|
|
46
|
+
/** Total partition count after the change. MUST be ≥ the current count. */
|
|
47
|
+
totalCount: number;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Typed admin surface exposed by {@link KafkaPublisher.admin}.
|
|
51
|
+
*
|
|
52
|
+
* The returned object is already connected — call `.close()` (or let the
|
|
53
|
+
* publisher's `disconnect()` cascade close it) when done.
|
|
54
|
+
*
|
|
55
|
+
* All methods may throw native errors from the underlying client. The
|
|
56
|
+
* publisher does not classify admin errors via the `errorKind` machinery
|
|
57
|
+
* because admin failures are operator-facing and don't flow through the
|
|
58
|
+
* relay's retry path.
|
|
59
|
+
*/
|
|
60
|
+
interface KafkaAdmin {
|
|
61
|
+
/** All topic names visible to this principal, including internal topics. */
|
|
62
|
+
listTopics(): Promise<string[]>;
|
|
63
|
+
/**
|
|
64
|
+
* Metadata for the given topics. Topics that don't exist on the cluster
|
|
65
|
+
* are returned with an empty `partitions` array — callers detect absence
|
|
66
|
+
* cheaply without a try/catch.
|
|
67
|
+
*/
|
|
68
|
+
describeTopics(topics: string[]): Promise<TopicMetadata[]>;
|
|
69
|
+
/**
|
|
70
|
+
* Create topics. Idempotent at this layer: topics that already exist are
|
|
71
|
+
* silently skipped (the underlying client may throw `TopicExistsError`;
|
|
72
|
+
* we swallow it). Use `ensureTopics` on the publisher for partition-count
|
|
73
|
+
* + replication-factor coherence checks.
|
|
74
|
+
*/
|
|
75
|
+
createTopics(specs: TopicCreateSpec[]): Promise<void>;
|
|
76
|
+
/**
|
|
77
|
+
* Grow each topic's partition count. Never shrinks (Kafka does not
|
|
78
|
+
* support shrinking partitions). Specs whose `totalCount` equals the
|
|
79
|
+
* current partition count are silently skipped.
|
|
80
|
+
*/
|
|
81
|
+
createPartitions(specs: PartitionGrowSpec[]): Promise<void>;
|
|
82
|
+
/** Disconnect the admin client. */
|
|
83
|
+
close(): Promise<void>;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Optional driver-level hook returned by {@link KafkaDriver.admin}. Drivers
|
|
87
|
+
* implement this thin contract; the publisher composes the higher-level
|
|
88
|
+
* `KafkaAdmin` and `ensureTopics` on top.
|
|
89
|
+
*/
|
|
90
|
+
interface KafkaDriverAdmin extends KafkaAdmin {
|
|
91
|
+
/** Called by the publisher before any method is invoked. */
|
|
92
|
+
connect(): Promise<void>;
|
|
93
|
+
}
|
|
94
|
+
|
|
3
95
|
/**
|
|
4
96
|
* Low-level driver contract. Each concrete driver (kafkajs, confluent)
|
|
5
97
|
* adapts its native client to this minimal surface. The KafkaPublisher
|
|
@@ -19,6 +111,14 @@ interface KafkaDriver {
|
|
|
19
111
|
* uses this to decide whether `sendBatch` is atomic.
|
|
20
112
|
*/
|
|
21
113
|
readonly transactional: boolean;
|
|
114
|
+
/**
|
|
115
|
+
* Construct a NEW admin client. The returned admin is not yet connected —
|
|
116
|
+
* the publisher calls `.connect()` before handing it to the user.
|
|
117
|
+
*
|
|
118
|
+
* Optional: drivers without an admin surface may omit this; the publisher
|
|
119
|
+
* throws a clear error when `publisher.admin()` is called on such a driver.
|
|
120
|
+
*/
|
|
121
|
+
admin?(): Promise<KafkaDriverAdmin>;
|
|
22
122
|
}
|
|
23
123
|
/**
|
|
24
124
|
* TLS configuration for client connections. Pass a full {@link TlsConfig}
|
|
@@ -149,6 +249,14 @@ interface ProducerBehaviorConfig {
|
|
|
149
249
|
acks?: number;
|
|
150
250
|
/** Compression codec. Driver maps to its native enum. */
|
|
151
251
|
compression?: "none" | "gzip" | "snappy" | "lz4" | "zstd";
|
|
252
|
+
/**
|
|
253
|
+
* (confluent only) Compression level for the chosen codec. Defaults vary
|
|
254
|
+
* per codec — librdkafka picks the broker-friendly default when unset.
|
|
255
|
+
* Common ranges: gzip 1–9, lz4 0–12, zstd 1–22 (higher = smaller + slower).
|
|
256
|
+
*
|
|
257
|
+
* No-op on the kafkajs driver (kafkajs does not expose codec levels).
|
|
258
|
+
*/
|
|
259
|
+
compressionLevel?: number;
|
|
152
260
|
/**
|
|
153
261
|
* (confluent only) How long the producer waits to accumulate records before
|
|
154
262
|
* flushing a partition batch. Default 0 (ship-immediately). Increase to
|
|
@@ -199,6 +307,36 @@ interface ProducerBehaviorConfig {
|
|
|
199
307
|
* whether this callback throws.
|
|
200
308
|
*/
|
|
201
309
|
onTransactionAbort?: (error: Error) => void;
|
|
310
|
+
/**
|
|
311
|
+
* (confluent only) Raw librdkafka producer-config keys merged on top of
|
|
312
|
+
* eventferry's translated config. Use for tuning surface area we don't
|
|
313
|
+
* expose typed (e.g. `queue.buffering.max.messages`, `socket.keepalive.enable`,
|
|
314
|
+
* `statistics.interval.ms`). Native keys win against the translated ones,
|
|
315
|
+
* so this can also be used to override defaults.
|
|
316
|
+
*
|
|
317
|
+
* Ignored by the kafkajs driver — log a one-time warning instead of
|
|
318
|
+
* silently dropping. Use `rawKafkaJsProducerConfig` for kafkajs-side tuning.
|
|
319
|
+
*/
|
|
320
|
+
rawProducerConfig?: Record<string, unknown>;
|
|
321
|
+
/**
|
|
322
|
+
* (kafkajs only) Raw producer-config keys merged into kafkajs's
|
|
323
|
+
* `kafka.producer({...})` call. Native keys win against the translated
|
|
324
|
+
* ones. Use for kafkajs-internal knobs like `retry`, `metadataMaxAge`,
|
|
325
|
+
* `idempotent` overrides, etc.
|
|
326
|
+
*
|
|
327
|
+
* No-op on the confluent driver — use `rawProducerConfig` there.
|
|
328
|
+
*/
|
|
329
|
+
rawKafkaJsProducerConfig?: Record<string, unknown>;
|
|
330
|
+
/**
|
|
331
|
+
* (kafkajs only) Custom partitioner factory passed straight to
|
|
332
|
+
* `kafka.producer({ createPartitioner })`. Overrides {@link partitioner}
|
|
333
|
+
* preset entirely. See kafkajs docs for the factory signature:
|
|
334
|
+
* `() => (args: { topic, partitionMetadata, message }) => number`.
|
|
335
|
+
*
|
|
336
|
+
* Ignored by the confluent driver — librdkafka's partitioner is a C
|
|
337
|
+
* extension point, not a JS callback.
|
|
338
|
+
*/
|
|
339
|
+
customPartitioner?: () => (args: unknown) => number;
|
|
202
340
|
}
|
|
203
341
|
type DriverKind = "kafkajs" | "confluent";
|
|
204
342
|
|
|
@@ -213,6 +351,11 @@ interface KjsTransaction {
|
|
|
213
351
|
commit(): Promise<void>;
|
|
214
352
|
abort(): Promise<void>;
|
|
215
353
|
}
|
|
354
|
+
interface KjsPartitionersNamespace {
|
|
355
|
+
DefaultPartitioner: () => unknown;
|
|
356
|
+
LegacyPartitioner: () => unknown;
|
|
357
|
+
JavaCompatiblePartitioner: () => unknown;
|
|
358
|
+
}
|
|
216
359
|
interface KafkaJsDriverOptions extends KafkaConnectionConfig, ProducerBehaviorConfig {
|
|
217
360
|
/**
|
|
218
361
|
* Optional logger for the driver's own diagnostics (e.g. warnings about
|
|
@@ -235,7 +378,19 @@ declare class KafkaJsDriver implements KafkaDriver {
|
|
|
235
378
|
* the send/transaction logic can be exercised without a real broker.
|
|
236
379
|
*/
|
|
237
380
|
protected createProducer(): Promise<KjsProducer>;
|
|
381
|
+
/**
|
|
382
|
+
* Compute the options object passed to `kafka.producer({...})`. Exposed
|
|
383
|
+
* as a test seam so power-user escape hatches (customPartitioner,
|
|
384
|
+
* rawKafkaJsProducerConfig) can be asserted without a live broker.
|
|
385
|
+
*/
|
|
386
|
+
protected buildProducerOptions(partitioners: KjsPartitionersNamespace | undefined): Promise<Record<string, unknown>>;
|
|
238
387
|
disconnect(): Promise<void>;
|
|
388
|
+
/**
|
|
389
|
+
* Construct a kafkajs admin client wrapped in the eventferry-facing
|
|
390
|
+
* `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
|
|
391
|
+
* returned object before exposing it via `publisher.admin()`.
|
|
392
|
+
*/
|
|
393
|
+
admin(): Promise<KafkaDriverAdmin>;
|
|
239
394
|
sendBatch(messages: PublishableMessage[]): Promise<PublishResult[]>;
|
|
240
395
|
}
|
|
241
396
|
/** Internal — used by tests. Resets the dedup so warnings can be observed in isolation. */
|
|
@@ -288,6 +443,12 @@ declare class ConfluentDriver implements KafkaDriver {
|
|
|
288
443
|
*/
|
|
289
444
|
protected createProducer(): Promise<CkProducer>;
|
|
290
445
|
disconnect(): Promise<void>;
|
|
446
|
+
/**
|
|
447
|
+
* Construct a librdkafka-backed admin client wrapped in the eventferry
|
|
448
|
+
* `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
|
|
449
|
+
* the admin reaches the user.
|
|
450
|
+
*/
|
|
451
|
+
admin(): Promise<KafkaDriverAdmin>;
|
|
291
452
|
sendBatch(messages: PublishableMessage[]): Promise<PublishResult[]>;
|
|
292
453
|
}
|
|
293
454
|
|
|
@@ -423,6 +584,21 @@ interface KafkaTracer {
|
|
|
423
584
|
* server.address/port).
|
|
424
585
|
*/
|
|
425
586
|
startPublishSpan(name: string, attributes: Record<string, SpanAttributeValue>): SpanLike;
|
|
587
|
+
/**
|
|
588
|
+
* OPTIONAL: inject the active trace context (W3C `traceparent` +
|
|
589
|
+
* `tracestate`) into a per-message header map. Called by the publisher
|
|
590
|
+
* AFTER the batch span is created and BEFORE the records hit the wire.
|
|
591
|
+
*
|
|
592
|
+
* Implementations typically wrap OpenTelemetry's `propagation.inject(...)`
|
|
593
|
+
* or your tracing SDK's equivalent. Mutate the `headers` object in
|
|
594
|
+
* place — the publisher allocates a fresh copy per message so this is
|
|
595
|
+
* safe and matches the propagation API of every major SDK.
|
|
596
|
+
*
|
|
597
|
+
* Tracers without distributed-context propagation (or that only care
|
|
598
|
+
* about local spans) may leave this off — consumers can still derive
|
|
599
|
+
* trace headers themselves by other means.
|
|
600
|
+
*/
|
|
601
|
+
inject?(span: SpanLike, headers: Record<string, string>): void;
|
|
426
602
|
}
|
|
427
603
|
/**
|
|
428
604
|
* No-op tracer. Used when the user does not configure one. Cheap allocation
|
|
@@ -457,6 +633,16 @@ interface KafkaPublisherOptions extends KafkaConnectionConfig, ProducerBehaviorC
|
|
|
457
633
|
* Use a thin adapter over your tracing SDK (see {@link KafkaTracer}).
|
|
458
634
|
*/
|
|
459
635
|
tracer?: KafkaTracer;
|
|
636
|
+
/**
|
|
637
|
+
* If set, `connect()` checks that every topic in this list exists on the
|
|
638
|
+
* cluster and throws a descriptive error if any are missing. Use this to
|
|
639
|
+
* fail-fast at startup instead of letting the first send-time error
|
|
640
|
+
* surprise you.
|
|
641
|
+
*
|
|
642
|
+
* Validation runs AFTER the producer connects but BEFORE `onConnect` hooks
|
|
643
|
+
* fire. Driver must implement `admin()` (the built-ins do).
|
|
644
|
+
*/
|
|
645
|
+
validateTopicsOnConnect?: string[];
|
|
460
646
|
}
|
|
461
647
|
/**
|
|
462
648
|
* The Publisher the Relay talks to. Wraps a pluggable KafkaDriver and adds
|
|
@@ -469,8 +655,39 @@ declare class KafkaPublisher implements Publisher {
|
|
|
469
655
|
private readonly logger;
|
|
470
656
|
private readonly hooks;
|
|
471
657
|
private readonly tracer;
|
|
658
|
+
private readonly validateTopicsOnConnect;
|
|
472
659
|
constructor(opts: KafkaPublisherOptions);
|
|
473
660
|
connect(): Promise<void>;
|
|
661
|
+
/**
|
|
662
|
+
* Borrow a new admin client from the driver. The returned admin is
|
|
663
|
+
* connected and ready to use; the CALLER must `close()` it. Throws if the
|
|
664
|
+
* driver does not implement admin (custom driver lacking the capability).
|
|
665
|
+
*/
|
|
666
|
+
admin(): Promise<KafkaAdmin>;
|
|
667
|
+
/**
|
|
668
|
+
* Idempotently provision topics. Each spec creates the topic if absent;
|
|
669
|
+
* existing topics are skipped without error. If `growPartitions: true`
|
|
670
|
+
* (default false), topics whose current partition count is below the
|
|
671
|
+
* requested `numPartitions` are grown via `createPartitions`.
|
|
672
|
+
*
|
|
673
|
+
* Replication factor and config entries on EXISTING topics are NOT
|
|
674
|
+
* reconciled — Kafka does not provide a safe in-place alter for those
|
|
675
|
+
* (changing replication requires reassignment; configs use alterConfigs).
|
|
676
|
+
* Reach for the raw admin if you need that.
|
|
677
|
+
*/
|
|
678
|
+
ensureTopics(specs: TopicCreateSpec[], opts?: {
|
|
679
|
+
growPartitions?: boolean;
|
|
680
|
+
}): Promise<void>;
|
|
681
|
+
/**
|
|
682
|
+
* Borrow a fresh admin from the driver and connect it. Throws when the
|
|
683
|
+
* driver does not implement admin (custom drivers without that capability).
|
|
684
|
+
*/
|
|
685
|
+
private openDriverAdmin;
|
|
686
|
+
/**
|
|
687
|
+
* Open an admin, list topics, throw if any required topic is missing.
|
|
688
|
+
* Always closes the admin (success or failure).
|
|
689
|
+
*/
|
|
690
|
+
private assertTopicsExist;
|
|
474
691
|
disconnect(): Promise<void>;
|
|
475
692
|
publish(messages: PublishableMessage[]): Promise<PublishResult[]>;
|
|
476
693
|
/**
|
|
@@ -491,4 +708,4 @@ declare class KafkaPublisher implements Publisher {
|
|
|
491
708
|
private startBatchSpan;
|
|
492
709
|
}
|
|
493
710
|
|
|
494
|
-
export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaConnectionConfig, type KafkaDriver, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };
|
|
711
|
+
export { type ConfluentClientConfig, ConfluentDriver, type ConfluentDriverOptions, type DriverKind, type KafkaAdmin, type KafkaConnectionConfig, type KafkaDriver, type KafkaDriverAdmin, KafkaJsDriver, type KafkaJsDriverOptions, type KafkaJsPartitionerChoice, KafkaPublisher, type KafkaPublisherHooks, type KafkaPublisherOptions, type KafkaTracer, NoopKafkaTracer, type OauthBearerToken, type PartitionGrowSpec, type PartitionMetadata, type ProducerBehaviorConfig, type SaslConfig, type SaslOauthbearerConfig, type SaslPasswordConfig, type SpanAttributeValue, type SpanLike, type TlsConfig, type TopicCreateSpec, type TopicMetadata, _resetKafkajsWarnDedup, buildConfluentClientConfig, classifyConfluentError, classifyKafkajsError, safeHook };
|
package/dist/index.js
CHANGED
|
@@ -104,7 +104,10 @@ var UNSUPPORTED_BY_KAFKAJS = [
|
|
|
104
104
|
"lingerMs",
|
|
105
105
|
"batchSize",
|
|
106
106
|
"deliveryTimeoutMs",
|
|
107
|
-
"maxRequestSize"
|
|
107
|
+
"maxRequestSize",
|
|
108
|
+
// Confluent-only escape hatches; ignored on kafkajs.
|
|
109
|
+
"compressionLevel",
|
|
110
|
+
"rawProducerConfig"
|
|
108
111
|
];
|
|
109
112
|
var KafkaJsDriver = class {
|
|
110
113
|
transactional;
|
|
@@ -143,13 +146,21 @@ var KafkaJsDriver = class {
|
|
|
143
146
|
// the provider's returned token (other fields are ignored).
|
|
144
147
|
sasl: this.opts.sasl
|
|
145
148
|
});
|
|
146
|
-
|
|
147
|
-
|
|
149
|
+
return kafka.producer(await this.buildProducerOptions(mod.Partitioners));
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Compute the options object passed to `kafka.producer({...})`. Exposed
|
|
153
|
+
* as a test seam so power-user escape hatches (customPartitioner,
|
|
154
|
+
* rawKafkaJsProducerConfig) can be asserted without a live broker.
|
|
155
|
+
*/
|
|
156
|
+
async buildProducerOptions(partitioners) {
|
|
157
|
+
const createPartitioner = this.opts.customPartitioner ?? resolveCreatePartitioner(
|
|
158
|
+
partitioners,
|
|
148
159
|
this.opts.partitioner,
|
|
149
160
|
this.transactional
|
|
150
161
|
);
|
|
151
162
|
const resolvedTxId = this.transactional ? await resolveTransactionalId(this.opts.transactionalId) : void 0;
|
|
152
|
-
return
|
|
163
|
+
return {
|
|
153
164
|
idempotent: this.opts.idempotent ?? true,
|
|
154
165
|
// Idempotent / transactional producers cap maxInFlight at 5. When the
|
|
155
166
|
// user picks transactional we force 1 to keep strict ordering across
|
|
@@ -163,13 +174,32 @@ var KafkaJsDriver = class {
|
|
|
163
174
|
transactionTimeout: this.opts.transactionTimeoutMs,
|
|
164
175
|
// Setting any partitioner choice silences kafkajs's
|
|
165
176
|
// KafkaJSPartitionerNotSpecified warning.
|
|
166
|
-
createPartitioner
|
|
167
|
-
|
|
177
|
+
createPartitioner,
|
|
178
|
+
// Power-user escape hatch — merged LAST so raw keys win against the
|
|
179
|
+
// translated ones. That's the contract: anything you put here is
|
|
180
|
+
// final, even if it overrides idempotent/transactionalId/etc.
|
|
181
|
+
...this.opts.rawKafkaJsProducerConfig ?? {}
|
|
182
|
+
};
|
|
168
183
|
}
|
|
169
184
|
async disconnect() {
|
|
170
185
|
await this.producer?.disconnect();
|
|
171
186
|
this.producer = null;
|
|
172
187
|
}
|
|
188
|
+
/**
|
|
189
|
+
* Construct a kafkajs admin client wrapped in the eventferry-facing
|
|
190
|
+
* `KafkaDriverAdmin` shape. The publisher calls `.connect()` on the
|
|
191
|
+
* returned object before exposing it via `publisher.admin()`.
|
|
192
|
+
*/
|
|
193
|
+
async admin() {
|
|
194
|
+
const mod = await importKafkaJs();
|
|
195
|
+
const kafka = new mod.Kafka({
|
|
196
|
+
clientId: this.opts.clientId ?? "eventferry-admin",
|
|
197
|
+
brokers: this.opts.brokers,
|
|
198
|
+
ssl: this.opts.ssl,
|
|
199
|
+
sasl: this.opts.sasl
|
|
200
|
+
});
|
|
201
|
+
return new KafkaJsAdmin(kafka.admin());
|
|
202
|
+
}
|
|
173
203
|
async sendBatch(messages) {
|
|
174
204
|
if (!this.producer) throw new Error("KafkaJsDriver not connected");
|
|
175
205
|
const topicMessages = groupByTopic(messages, this.opts.compression);
|
|
@@ -258,6 +288,69 @@ function warnUnsupportedKafkajsOptions(opts) {
|
|
|
258
288
|
function _resetKafkajsWarnDedup() {
|
|
259
289
|
warnedKafkajsKeys.clear();
|
|
260
290
|
}
|
|
291
|
+
var KafkaJsAdmin = class {
|
|
292
|
+
constructor(client) {
|
|
293
|
+
this.client = client;
|
|
294
|
+
}
|
|
295
|
+
client;
|
|
296
|
+
async connect() {
|
|
297
|
+
await this.client.connect();
|
|
298
|
+
}
|
|
299
|
+
async close() {
|
|
300
|
+
await this.client.disconnect();
|
|
301
|
+
}
|
|
302
|
+
async listTopics() {
|
|
303
|
+
return await this.client.listTopics();
|
|
304
|
+
}
|
|
305
|
+
async describeTopics(topics) {
|
|
306
|
+
if (topics.length === 0) return [];
|
|
307
|
+
const all = new Set(await this.client.listTopics());
|
|
308
|
+
const existing = topics.filter((t) => all.has(t));
|
|
309
|
+
const missing = topics.filter((t) => !all.has(t));
|
|
310
|
+
const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
|
|
311
|
+
const byName = new Map(meta.topics.map((t) => [t.name, t]));
|
|
312
|
+
return topics.map((topic) => {
|
|
313
|
+
if (missing.includes(topic)) return { topic, partitions: [] };
|
|
314
|
+
const found = byName.get(topic);
|
|
315
|
+
if (!found) return { topic, partitions: [] };
|
|
316
|
+
return {
|
|
317
|
+
topic,
|
|
318
|
+
partitions: found.partitions.map((p) => ({
|
|
319
|
+
partitionId: p.partitionId,
|
|
320
|
+
leader: p.leader,
|
|
321
|
+
replicas: p.replicas,
|
|
322
|
+
isr: p.isr
|
|
323
|
+
}))
|
|
324
|
+
};
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
async createTopics(specs) {
|
|
328
|
+
if (specs.length === 0) return;
|
|
329
|
+
const topics = specs.map((s) => ({
|
|
330
|
+
topic: s.topic,
|
|
331
|
+
numPartitions: s.numPartitions,
|
|
332
|
+
replicationFactor: s.replicationFactor,
|
|
333
|
+
configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
|
|
334
|
+
}));
|
|
335
|
+
try {
|
|
336
|
+
await this.client.createTopics({ topics, waitForLeaders: true });
|
|
337
|
+
} catch (err) {
|
|
338
|
+
const e = err;
|
|
339
|
+
if (e?.type === "TOPIC_ALREADY_EXISTS") return;
|
|
340
|
+
if (/already exists/i.test(e?.message ?? "")) return;
|
|
341
|
+
throw err;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
async createPartitions(specs) {
|
|
345
|
+
if (specs.length === 0) return;
|
|
346
|
+
await this.client.createPartitions({
|
|
347
|
+
topicPartitions: specs.map((s) => ({
|
|
348
|
+
topic: s.topic,
|
|
349
|
+
count: s.totalCount
|
|
350
|
+
}))
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
};
|
|
261
354
|
async function importKafkaJs() {
|
|
262
355
|
try {
|
|
263
356
|
return await import("kafkajs");
|
|
@@ -390,6 +483,9 @@ function buildConfluentClientConfig(opts) {
|
|
|
390
483
|
if (opts.transactionTimeoutMs !== void 0) {
|
|
391
484
|
librdkafka["transaction.timeout.ms"] = opts.transactionTimeoutMs;
|
|
392
485
|
}
|
|
486
|
+
if (opts.compressionLevel !== void 0) {
|
|
487
|
+
librdkafka["compression.level"] = opts.compressionLevel;
|
|
488
|
+
}
|
|
393
489
|
const tlsRequested = opts.ssl === true || isTlsConfig(opts.ssl);
|
|
394
490
|
const saslRequested = !!opts.sasl;
|
|
395
491
|
if (saslRequested && tlsRequested) {
|
|
@@ -419,6 +515,9 @@ function buildConfluentClientConfig(opts) {
|
|
|
419
515
|
if (opts.sasl) {
|
|
420
516
|
kafkaJS["sasl"] = opts.sasl;
|
|
421
517
|
}
|
|
518
|
+
if (opts.rawProducerConfig) {
|
|
519
|
+
Object.assign(librdkafka, opts.rawProducerConfig);
|
|
520
|
+
}
|
|
422
521
|
return { kafkaJS, librdkafka };
|
|
423
522
|
}
|
|
424
523
|
function isTlsConfig(v) {
|
|
@@ -472,6 +571,17 @@ var ConfluentDriver = class {
|
|
|
472
571
|
await this.producer?.disconnect();
|
|
473
572
|
this.producer = null;
|
|
474
573
|
}
|
|
574
|
+
/**
|
|
575
|
+
* Construct a librdkafka-backed admin client wrapped in the eventferry
|
|
576
|
+
* `KafkaDriverAdmin` shape. The publisher's `connect()` is called before
|
|
577
|
+
* the admin reaches the user.
|
|
578
|
+
*/
|
|
579
|
+
async admin() {
|
|
580
|
+
const mod = await importConfluent();
|
|
581
|
+
const { kafkaJS, librdkafka } = buildConfluentClientConfig(this.opts);
|
|
582
|
+
const kafka = new mod.KafkaJS.Kafka({ kafkaJS, ...librdkafka });
|
|
583
|
+
return new ConfluentAdmin(kafka.admin());
|
|
584
|
+
}
|
|
475
585
|
async sendBatch(messages) {
|
|
476
586
|
if (!this.producer) throw new Error("ConfluentDriver not connected");
|
|
477
587
|
const topicMessages = groupByTopic2(messages);
|
|
@@ -540,6 +650,69 @@ function groupByTopic2(messages) {
|
|
|
540
650
|
messages: msgs
|
|
541
651
|
}));
|
|
542
652
|
}
|
|
653
|
+
var ConfluentAdmin = class {
|
|
654
|
+
constructor(client) {
|
|
655
|
+
this.client = client;
|
|
656
|
+
}
|
|
657
|
+
client;
|
|
658
|
+
async connect() {
|
|
659
|
+
await this.client.connect();
|
|
660
|
+
}
|
|
661
|
+
async close() {
|
|
662
|
+
await this.client.disconnect();
|
|
663
|
+
}
|
|
664
|
+
async listTopics() {
|
|
665
|
+
return await this.client.listTopics();
|
|
666
|
+
}
|
|
667
|
+
async describeTopics(topics) {
|
|
668
|
+
if (topics.length === 0) return [];
|
|
669
|
+
const all = new Set(await this.client.listTopics());
|
|
670
|
+
const existing = topics.filter((t) => all.has(t));
|
|
671
|
+
const missing = topics.filter((t) => !all.has(t));
|
|
672
|
+
const meta = existing.length ? await this.client.fetchTopicMetadata({ topics: existing }) : { topics: [] };
|
|
673
|
+
const byName = new Map(meta.topics.map((t) => [t.name, t]));
|
|
674
|
+
return topics.map((topic) => {
|
|
675
|
+
if (missing.includes(topic)) return { topic, partitions: [] };
|
|
676
|
+
const found = byName.get(topic);
|
|
677
|
+
if (!found) return { topic, partitions: [] };
|
|
678
|
+
return {
|
|
679
|
+
topic,
|
|
680
|
+
partitions: found.partitions.map((p) => ({
|
|
681
|
+
partitionId: p.partitionId,
|
|
682
|
+
leader: p.leader,
|
|
683
|
+
replicas: p.replicas,
|
|
684
|
+
isr: p.isr
|
|
685
|
+
}))
|
|
686
|
+
};
|
|
687
|
+
});
|
|
688
|
+
}
|
|
689
|
+
async createTopics(specs) {
|
|
690
|
+
if (specs.length === 0) return;
|
|
691
|
+
const topics = specs.map((s) => ({
|
|
692
|
+
topic: s.topic,
|
|
693
|
+
numPartitions: s.numPartitions,
|
|
694
|
+
replicationFactor: s.replicationFactor,
|
|
695
|
+
configEntries: s.configEntries ? Object.entries(s.configEntries).map(([name, value]) => ({ name, value })) : void 0
|
|
696
|
+
}));
|
|
697
|
+
try {
|
|
698
|
+
await this.client.createTopics({ topics, waitForLeaders: true });
|
|
699
|
+
} catch (err) {
|
|
700
|
+
const e = err;
|
|
701
|
+
if (e?.code === 36 || e?.name === "TOPIC_ALREADY_EXISTS") return;
|
|
702
|
+
if (/already exists/i.test(e?.message ?? "")) return;
|
|
703
|
+
throw err;
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
async createPartitions(specs) {
|
|
707
|
+
if (specs.length === 0) return;
|
|
708
|
+
await this.client.createPartitions({
|
|
709
|
+
topicPartitions: specs.map((s) => ({
|
|
710
|
+
topic: s.topic,
|
|
711
|
+
count: s.totalCount
|
|
712
|
+
}))
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
};
|
|
543
716
|
async function importConfluent() {
|
|
544
717
|
try {
|
|
545
718
|
return await import("@confluentinc/kafka-javascript");
|
|
@@ -590,10 +763,12 @@ var KafkaPublisher = class {
|
|
|
590
763
|
logger;
|
|
591
764
|
hooks;
|
|
592
765
|
tracer;
|
|
766
|
+
validateTopicsOnConnect;
|
|
593
767
|
constructor(opts) {
|
|
594
768
|
this.logger = opts.logger;
|
|
595
769
|
this.hooks = opts.hooks ?? {};
|
|
596
770
|
this.tracer = opts.tracer ?? new NoopKafkaTracer();
|
|
771
|
+
this.validateTopicsOnConnect = opts.validateTopicsOnConnect ? Object.freeze([...opts.validateTopicsOnConnect]) : void 0;
|
|
597
772
|
const onTransactionAbort = this.hooks.onTransactionAbort ? (error) => {
|
|
598
773
|
void safeHook(
|
|
599
774
|
this.logger,
|
|
@@ -605,8 +780,90 @@ var KafkaPublisher = class {
|
|
|
605
780
|
}
|
|
606
781
|
async connect() {
|
|
607
782
|
await this.driver.connect();
|
|
783
|
+
if (this.validateTopicsOnConnect && this.validateTopicsOnConnect.length) {
|
|
784
|
+
await this.assertTopicsExist(this.validateTopicsOnConnect);
|
|
785
|
+
}
|
|
608
786
|
await safeHook(this.logger, "onConnect", () => this.hooks.onConnect?.());
|
|
609
787
|
}
|
|
788
|
+
/**
|
|
789
|
+
* Borrow a new admin client from the driver. The returned admin is
|
|
790
|
+
* connected and ready to use; the CALLER must `close()` it. Throws if the
|
|
791
|
+
* driver does not implement admin (custom driver lacking the capability).
|
|
792
|
+
*/
|
|
793
|
+
async admin() {
|
|
794
|
+
const driverAdmin = await this.openDriverAdmin();
|
|
795
|
+
return driverAdmin;
|
|
796
|
+
}
|
|
797
|
+
/**
|
|
798
|
+
* Idempotently provision topics. Each spec creates the topic if absent;
|
|
799
|
+
* existing topics are skipped without error. If `growPartitions: true`
|
|
800
|
+
* (default false), topics whose current partition count is below the
|
|
801
|
+
* requested `numPartitions` are grown via `createPartitions`.
|
|
802
|
+
*
|
|
803
|
+
* Replication factor and config entries on EXISTING topics are NOT
|
|
804
|
+
* reconciled — Kafka does not provide a safe in-place alter for those
|
|
805
|
+
* (changing replication requires reassignment; configs use alterConfigs).
|
|
806
|
+
* Reach for the raw admin if you need that.
|
|
807
|
+
*/
|
|
808
|
+
async ensureTopics(specs, opts = {}) {
|
|
809
|
+
if (specs.length === 0) return;
|
|
810
|
+
const admin = await this.openDriverAdmin();
|
|
811
|
+
try {
|
|
812
|
+
const topicNames = specs.map((s) => s.topic);
|
|
813
|
+
const existing = await admin.describeTopics(topicNames);
|
|
814
|
+
const existingByName = new Map(existing.map((t) => [t.topic, t]));
|
|
815
|
+
const toCreate = specs.filter(
|
|
816
|
+
(s) => (existingByName.get(s.topic)?.partitions.length ?? 0) === 0
|
|
817
|
+
);
|
|
818
|
+
if (toCreate.length) await admin.createTopics(toCreate);
|
|
819
|
+
if (opts.growPartitions) {
|
|
820
|
+
const grow = [];
|
|
821
|
+
for (const s of specs) {
|
|
822
|
+
if (s.numPartitions === void 0) continue;
|
|
823
|
+
const current = existingByName.get(s.topic);
|
|
824
|
+
const currentCount = current?.partitions.length ?? 0;
|
|
825
|
+
if (currentCount > 0 && currentCount < s.numPartitions) {
|
|
826
|
+
grow.push({ topic: s.topic, totalCount: s.numPartitions });
|
|
827
|
+
}
|
|
828
|
+
}
|
|
829
|
+
if (grow.length) await admin.createPartitions(grow);
|
|
830
|
+
}
|
|
831
|
+
} finally {
|
|
832
|
+
await admin.close();
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
/**
|
|
836
|
+
* Borrow a fresh admin from the driver and connect it. Throws when the
|
|
837
|
+
* driver does not implement admin (custom drivers without that capability).
|
|
838
|
+
*/
|
|
839
|
+
async openDriverAdmin() {
|
|
840
|
+
if (!this.driver.admin) {
|
|
841
|
+
throw new Error(
|
|
842
|
+
"KafkaPublisher: configured driver does not implement admin(). Use the built-in kafkajs or confluent driver, or extend your custom driver."
|
|
843
|
+
);
|
|
844
|
+
}
|
|
845
|
+
const admin = await this.driver.admin();
|
|
846
|
+
await admin.connect();
|
|
847
|
+
return admin;
|
|
848
|
+
}
|
|
849
|
+
/**
|
|
850
|
+
* Open an admin, list topics, throw if any required topic is missing.
|
|
851
|
+
* Always closes the admin (success or failure).
|
|
852
|
+
*/
|
|
853
|
+
async assertTopicsExist(required) {
|
|
854
|
+
const admin = await this.openDriverAdmin();
|
|
855
|
+
try {
|
|
856
|
+
const all = new Set(await admin.listTopics());
|
|
857
|
+
const missing = required.filter((t) => !all.has(t));
|
|
858
|
+
if (missing.length) {
|
|
859
|
+
throw new Error(
|
|
860
|
+
`KafkaPublisher: validateTopicsOnConnect failed \u2014 topics missing on cluster: ${missing.join(", ")}`
|
|
861
|
+
);
|
|
862
|
+
}
|
|
863
|
+
} finally {
|
|
864
|
+
await admin.close();
|
|
865
|
+
}
|
|
866
|
+
}
|
|
610
867
|
async disconnect() {
|
|
611
868
|
await this.driver.disconnect();
|
|
612
869
|
await safeHook(
|
|
@@ -618,9 +875,14 @@ var KafkaPublisher = class {
|
|
|
618
875
|
async publish(messages) {
|
|
619
876
|
if (messages.length === 0) return [];
|
|
620
877
|
const span = this.startBatchSpan(messages);
|
|
878
|
+
const outgoing = this.tracer.inject ? messages.map((m) => {
|
|
879
|
+
const headers = { ...m.headers };
|
|
880
|
+
this.tracer.inject(span, headers);
|
|
881
|
+
return { ...m, headers };
|
|
882
|
+
}) : messages;
|
|
621
883
|
let results;
|
|
622
884
|
try {
|
|
623
|
-
results = await this.driver.sendBatch(
|
|
885
|
+
results = await this.driver.sendBatch(outgoing);
|
|
624
886
|
} catch (err) {
|
|
625
887
|
const error = err instanceof Error ? err : new Error(String(err));
|
|
626
888
|
span.setStatus({ code: "error", message: error.message });
|