@drarzter/kafka-client 0.9.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +625 -8
- package/dist/chunk-CMO7SMVK.mjs +4814 -0
- package/dist/chunk-CMO7SMVK.mjs.map +1 -0
- package/dist/cli/dlq.d.ts +119 -0
- package/dist/cli/dlq.d.ts.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/{chunk-TPIP5VV7.mjs → cli/index.js} +965 -265
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/index.mjs +355 -0
- package/dist/cli/index.mjs.map +1 -0
- package/dist/client/config/from-env.d.ts +188 -0
- package/dist/client/config/from-env.d.ts.map +1 -0
- package/dist/client/config/index.d.ts +2 -0
- package/dist/client/config/index.d.ts.map +1 -0
- package/dist/client/errors.d.ts +67 -0
- package/dist/client/errors.d.ts.map +1 -0
- package/dist/client/kafka.client/admin/ops.d.ts +114 -0
- package/dist/client/kafka.client/admin/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts +24 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +52 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts +4 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts +10 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts +5 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/handler.d.ts +149 -0
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/ops.d.ts +51 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts +167 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/queue.d.ts +37 -0
- package/dist/client/kafka.client/consumer/queue.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +65 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/setup.d.ts +63 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/start.d.ts +7 -0
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/stop.d.ts +19 -0
- package/dist/client/kafka.client/consumer/stop.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts +4 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts.map +1 -0
- package/dist/client/kafka.client/context.d.ts +72 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -0
- package/dist/client/kafka.client/index.d.ts +155 -0
- package/dist/client/kafka.client/index.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts +61 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts +28 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts +22 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts +67 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts +41 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/ops.d.ts +70 -0
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/send.d.ts +21 -0
- package/dist/client/kafka.client/producer/send.d.ts.map +1 -0
- package/dist/client/kafka.client/validate-options.d.ts +11 -0
- package/dist/client/kafka.client/validate-options.d.ts.map +1 -0
- package/dist/client/message/envelope.d.ts +105 -0
- package/dist/client/message/envelope.d.ts.map +1 -0
- package/dist/client/message/schema-registry.d.ts +105 -0
- package/dist/client/message/schema-registry.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +138 -0
- package/dist/client/message/topic.d.ts.map +1 -0
- package/dist/client/message/versioned-schema.d.ts +53 -0
- package/dist/client/message/versioned-schema.d.ts.map +1 -0
- package/dist/client/outbox/index.d.ts +4 -0
- package/dist/client/outbox/index.d.ts.map +1 -0
- package/dist/client/outbox/outbox.relay.d.ts +90 -0
- package/dist/client/outbox/outbox.relay.d.ts.map +1 -0
- package/dist/client/outbox/outbox.store.d.ts +42 -0
- package/dist/client/outbox/outbox.store.d.ts.map +1 -0
- package/dist/client/outbox/outbox.types.d.ts +144 -0
- package/dist/client/outbox/outbox.types.d.ts.map +1 -0
- package/dist/client/security/acl.d.ts +108 -0
- package/dist/client/security/acl.d.ts.map +1 -0
- package/dist/client/security/index.d.ts +5 -0
- package/dist/client/security/index.d.ts.map +1 -0
- package/dist/client/security/providers.d.ts +88 -0
- package/dist/client/security/providers.d.ts.map +1 -0
- package/dist/client/security/resolve-security.d.ts +19 -0
- package/dist/client/security/resolve-security.d.ts.map +1 -0
- package/dist/client/security/security.types.d.ts +76 -0
- package/dist/client/security/security.types.d.ts.map +1 -0
- package/dist/client/transport/confluent.transport.d.ts +32 -0
- package/dist/client/transport/confluent.transport.d.ts.map +1 -0
- package/dist/client/transport/transport.interface.d.ts +216 -0
- package/dist/client/transport/transport.interface.d.ts.map +1 -0
- package/dist/client/types/admin.interface.d.ts +174 -0
- package/dist/client/types/admin.interface.d.ts.map +1 -0
- package/dist/client/types/admin.types.d.ts +140 -0
- package/dist/client/types/admin.types.d.ts.map +1 -0
- package/dist/client/types/client.d.ts +21 -0
- package/dist/client/types/client.d.ts.map +1 -0
- package/dist/client/types/common.d.ts +84 -0
- package/dist/client/types/common.d.ts.map +1 -0
- package/dist/client/types/config.types.d.ts +150 -0
- package/dist/client/types/config.types.d.ts.map +1 -0
- package/dist/client/types/consumer.interface.d.ts +115 -0
- package/dist/client/types/consumer.interface.d.ts.map +1 -0
- package/dist/{consumer.types-fFCag3VJ.d.mts → client/types/consumer.types.d.ts} +62 -383
- package/dist/client/types/consumer.types.d.ts.map +1 -0
- package/dist/client/types/dedup.types.d.ts +50 -0
- package/dist/client/types/dedup.types.d.ts.map +1 -0
- package/dist/client/types/lifecycle.interface.d.ts +72 -0
- package/dist/client/types/lifecycle.interface.d.ts.map +1 -0
- package/dist/client/types/producer.interface.d.ts +52 -0
- package/dist/client/types/producer.interface.d.ts.map +1 -0
- package/dist/client/types/producer.types.d.ts +90 -0
- package/dist/client/types/producer.types.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/core.d.ts +10 -314
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +1326 -74
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +39 -3
- package/dist/index.d.ts +7 -128
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1343 -74
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +56 -3
- package/dist/index.mjs.map +1 -1
- package/dist/nest/kafka.constants.d.ts +5 -0
- package/dist/nest/kafka.constants.d.ts.map +1 -0
- package/dist/nest/kafka.decorator.d.ts +49 -0
- package/dist/nest/kafka.decorator.d.ts.map +1 -0
- package/dist/nest/kafka.explorer.d.ts +17 -0
- package/dist/nest/kafka.explorer.d.ts.map +1 -0
- package/dist/nest/kafka.health.d.ts +7 -0
- package/dist/nest/kafka.health.d.ts.map +1 -0
- package/dist/nest/kafka.module.d.ts +61 -0
- package/dist/nest/kafka.module.d.ts.map +1 -0
- package/dist/otel.d.ts +83 -5
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +100 -6
- package/dist/otel.js.map +1 -1
- package/dist/otel.mjs +98 -5
- package/dist/otel.mjs.map +1 -1
- package/dist/testing/client.mock.d.ts +47 -0
- package/dist/testing/client.mock.d.ts.map +1 -0
- package/dist/testing/index.d.ts +4 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/test.container.d.ts +63 -0
- package/dist/testing/test.container.d.ts.map +1 -0
- package/dist/{testing.d.mts → testing/transport.fake.d.ts} +7 -111
- package/dist/testing/transport.fake.d.ts.map +1 -0
- package/dist/testing.d.ts +2 -318
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +28 -2
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +28 -2
- package/dist/testing.mjs.map +1 -1
- package/package.json +22 -9
- package/dist/chunk-TPIP5VV7.mjs.map +0 -1
- package/dist/client-CBBUDDtu.d.ts +0 -751
- package/dist/client-D-SxYV2b.d.mts +0 -751
- package/dist/consumer.types-fFCag3VJ.d.ts +0 -958
- package/dist/core.d.mts +0 -314
- package/dist/index.d.mts +0 -128
- package/dist/otel.d.mts +0 -27
package/dist/index.js
CHANGED
|
@@ -30,11 +30,15 @@ var __decorateParam = (index, decorator) => (target, key) => decorator(target, k
|
|
|
30
30
|
var index_exports = {};
|
|
31
31
|
__export(index_exports, {
|
|
32
32
|
HEADER_CORRELATION_ID: () => HEADER_CORRELATION_ID,
|
|
33
|
+
HEADER_DELAYED_TARGET: () => HEADER_DELAYED_TARGET,
|
|
34
|
+
HEADER_DELAYED_UNTIL: () => HEADER_DELAYED_UNTIL,
|
|
33
35
|
HEADER_EVENT_ID: () => HEADER_EVENT_ID,
|
|
34
36
|
HEADER_LAMPORT_CLOCK: () => HEADER_LAMPORT_CLOCK,
|
|
35
37
|
HEADER_SCHEMA_VERSION: () => HEADER_SCHEMA_VERSION,
|
|
36
38
|
HEADER_TIMESTAMP: () => HEADER_TIMESTAMP,
|
|
37
39
|
HEADER_TRACEPARENT: () => HEADER_TRACEPARENT,
|
|
40
|
+
InMemoryDedupStore: () => InMemoryDedupStore,
|
|
41
|
+
InMemoryOutboxStore: () => InMemoryOutboxStore,
|
|
38
42
|
InjectKafkaClient: () => InjectKafkaClient,
|
|
39
43
|
KAFKA_CLIENT: () => KAFKA_CLIENT,
|
|
40
44
|
KAFKA_SUBSCRIBER_METADATA: () => KAFKA_SUBSCRIBER_METADATA,
|
|
@@ -45,24 +49,39 @@ __export(index_exports, {
|
|
|
45
49
|
KafkaProcessingError: () => KafkaProcessingError,
|
|
46
50
|
KafkaRetryExhaustedError: () => KafkaRetryExhaustedError,
|
|
47
51
|
KafkaValidationError: () => KafkaValidationError,
|
|
52
|
+
SchemaRegistryClient: () => SchemaRegistryClient,
|
|
48
53
|
SubscribeTo: () => SubscribeTo,
|
|
54
|
+
awsMskIamProvider: () => awsMskIamProvider,
|
|
49
55
|
buildEnvelopeHeaders: () => buildEnvelopeHeaders,
|
|
56
|
+
consumerOptionsFromEnv: () => consumerOptionsFromEnv,
|
|
50
57
|
decodeHeaders: () => decodeHeaders,
|
|
58
|
+
describeRequiredAcls: () => describeRequiredAcls,
|
|
51
59
|
extractEnvelope: () => extractEnvelope,
|
|
60
|
+
gcpAccessTokenProvider: () => gcpAccessTokenProvider,
|
|
52
61
|
getEnvelopeContext: () => getEnvelopeContext,
|
|
53
62
|
getKafkaClientToken: () => getKafkaClientToken,
|
|
63
|
+
kafkaClientConfigFromEnv: () => kafkaClientConfigFromEnv,
|
|
64
|
+
mergeConsumerOptions: () => mergeConsumerOptions,
|
|
65
|
+
registrySchema: () => registrySchema,
|
|
66
|
+
resolveSecurityOptions: () => resolveSecurityOptions,
|
|
54
67
|
runWithEnvelopeContext: () => runWithEnvelopeContext,
|
|
55
|
-
|
|
68
|
+
startOutboxRelay: () => startOutboxRelay,
|
|
69
|
+
toError: () => toError,
|
|
70
|
+
toKafkaAclCommands: () => toKafkaAclCommands,
|
|
71
|
+
toMskIamPolicy: () => toMskIamPolicy,
|
|
72
|
+
topic: () => topic,
|
|
73
|
+
versionedSchema: () => versionedSchema
|
|
56
74
|
});
|
|
57
75
|
module.exports = __toCommonJS(index_exports);
|
|
58
76
|
|
|
59
|
-
// src/client/
|
|
77
|
+
// src/client/transport/confluent.transport.ts
|
|
60
78
|
var import_kafka_javascript = require("@confluentinc/kafka-javascript");
|
|
61
79
|
var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = import_kafka_javascript.KafkaJS;
|
|
62
80
|
var ConfluentTransaction = class {
|
|
63
81
|
constructor(tx) {
|
|
64
82
|
this.tx = tx;
|
|
65
83
|
}
|
|
84
|
+
tx;
|
|
66
85
|
async send(record) {
|
|
67
86
|
await this.tx.send(record);
|
|
68
87
|
}
|
|
@@ -84,10 +103,17 @@ var ConfluentProducer = class {
|
|
|
84
103
|
constructor(producer) {
|
|
85
104
|
this.producer = producer;
|
|
86
105
|
}
|
|
106
|
+
producer;
|
|
107
|
+
connectPromise;
|
|
87
108
|
async connect() {
|
|
88
|
-
|
|
109
|
+
this.connectPromise ??= this.producer.connect().catch((err) => {
|
|
110
|
+
this.connectPromise = void 0;
|
|
111
|
+
throw err;
|
|
112
|
+
});
|
|
113
|
+
return this.connectPromise;
|
|
89
114
|
}
|
|
90
115
|
async disconnect() {
|
|
116
|
+
this.connectPromise = void 0;
|
|
91
117
|
await this.producer.disconnect();
|
|
92
118
|
}
|
|
93
119
|
async send(record) {
|
|
@@ -102,6 +128,7 @@ var ConfluentConsumer = class {
|
|
|
102
128
|
constructor(consumer) {
|
|
103
129
|
this.consumer = consumer;
|
|
104
130
|
}
|
|
131
|
+
consumer;
|
|
105
132
|
/** Returns the underlying KafkaJS.Consumer — used by ConfluentTransaction.sendOffsets. */
|
|
106
133
|
getNative() {
|
|
107
134
|
return this.consumer;
|
|
@@ -141,6 +168,7 @@ var ConfluentAdmin = class {
|
|
|
141
168
|
constructor(admin) {
|
|
142
169
|
this.admin = admin;
|
|
143
170
|
}
|
|
171
|
+
admin;
|
|
144
172
|
async connect() {
|
|
145
173
|
await this.admin.connect();
|
|
146
174
|
}
|
|
@@ -154,7 +182,7 @@ var ConfluentAdmin = class {
|
|
|
154
182
|
return this.admin.fetchTopicOffsets(topic2);
|
|
155
183
|
}
|
|
156
184
|
async fetchTopicOffsetsByTimestamp(topic2, timestamp) {
|
|
157
|
-
return this.admin.
|
|
185
|
+
return this.admin.fetchTopicOffsetsByTimestamp(topic2, timestamp);
|
|
158
186
|
}
|
|
159
187
|
async fetchOffsets(options) {
|
|
160
188
|
return this.admin.fetchOffsets(options);
|
|
@@ -180,10 +208,29 @@ var ConfluentAdmin = class {
|
|
|
180
208
|
};
|
|
181
209
|
var ConfluentTransport = class {
|
|
182
210
|
kafka;
|
|
183
|
-
constructor(clientId, brokers) {
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
211
|
+
constructor(clientId, brokers, security) {
|
|
212
|
+
const kafkaJS = { clientId, brokers, logLevel: KafkaLogLevel.ERROR };
|
|
213
|
+
if (security?.ssl !== void 0) kafkaJS.ssl = security.ssl;
|
|
214
|
+
if (security?.sasl) {
|
|
215
|
+
if (security.sasl.mechanism === "oauthbearer") {
|
|
216
|
+
const provider = security.sasl.oauthBearerProvider;
|
|
217
|
+
kafkaJS.sasl = {
|
|
218
|
+
mechanism: "oauthbearer",
|
|
219
|
+
oauthBearerProvider: async () => {
|
|
220
|
+
const token = await provider();
|
|
221
|
+
return {
|
|
222
|
+
value: token.value,
|
|
223
|
+
principal: token.principal ?? "kafka-client",
|
|
224
|
+
lifetime: token.lifetimeMs ?? Date.now() + 15 * 6e4,
|
|
225
|
+
...token.extensions && { extensions: token.extensions }
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
} else {
|
|
230
|
+
kafkaJS.sasl = security.sasl;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
this.kafka = new KafkaClass({ kafkaJS });
|
|
187
234
|
}
|
|
188
235
|
producer(options) {
|
|
189
236
|
const native = this.kafka.producer({
|
|
@@ -210,6 +257,9 @@ var ConfluentTransport = class {
|
|
|
210
257
|
partitionAssigners: [assigner]
|
|
211
258
|
}
|
|
212
259
|
};
|
|
260
|
+
if (options.groupInstanceId) {
|
|
261
|
+
config["group.instance.id"] = options.groupInstanceId;
|
|
262
|
+
}
|
|
213
263
|
if (options.onRebalance) {
|
|
214
264
|
const cb = options.onRebalance;
|
|
215
265
|
config.rebalance_cb = (err, assignment) => {
|
|
@@ -227,6 +277,25 @@ var ConfluentTransport = class {
|
|
|
227
277
|
}
|
|
228
278
|
};
|
|
229
279
|
|
|
280
|
+
// src/client/kafka.client/infra/dedup.store.ts
|
|
281
|
+
var InMemoryDedupStore = class {
|
|
282
|
+
constructor(states) {
|
|
283
|
+
this.states = states;
|
|
284
|
+
}
|
|
285
|
+
states;
|
|
286
|
+
getLastClock(groupId, topicPartition) {
|
|
287
|
+
return this.states.get(groupId)?.get(topicPartition);
|
|
288
|
+
}
|
|
289
|
+
setLastClock(groupId, topicPartition, clock) {
|
|
290
|
+
let group = this.states.get(groupId);
|
|
291
|
+
if (!group) {
|
|
292
|
+
group = /* @__PURE__ */ new Map();
|
|
293
|
+
this.states.set(groupId, group);
|
|
294
|
+
}
|
|
295
|
+
group.set(topicPartition, clock);
|
|
296
|
+
}
|
|
297
|
+
};
|
|
298
|
+
|
|
230
299
|
// src/client/message/envelope.ts
|
|
231
300
|
var import_node_async_hooks = require("async_hooks");
|
|
232
301
|
var import_node_crypto = require("crypto");
|
|
@@ -236,6 +305,8 @@ var HEADER_TIMESTAMP = "x-timestamp";
|
|
|
236
305
|
var HEADER_SCHEMA_VERSION = "x-schema-version";
|
|
237
306
|
var HEADER_TRACEPARENT = "traceparent";
|
|
238
307
|
var HEADER_LAMPORT_CLOCK = "x-lamport-clock";
|
|
308
|
+
var HEADER_DELAYED_UNTIL = "x-delayed-until";
|
|
309
|
+
var HEADER_DELAYED_TARGET = "x-delayed-target";
|
|
239
310
|
var envelopeStorage = new import_node_async_hooks.AsyncLocalStorage();
|
|
240
311
|
function getEnvelopeContext() {
|
|
241
312
|
return envelopeStorage.getStore();
|
|
@@ -290,6 +361,9 @@ function extractEnvelope(payload, headers, topic2, partition, offset) {
|
|
|
290
361
|
}
|
|
291
362
|
|
|
292
363
|
// src/client/errors.ts
|
|
364
|
+
function toError(error) {
|
|
365
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
366
|
+
}
|
|
293
367
|
var KafkaProcessingError = class extends Error {
|
|
294
368
|
constructor(message, topic2, originalMessage, options) {
|
|
295
369
|
super(message, options);
|
|
@@ -298,6 +372,8 @@ var KafkaProcessingError = class extends Error {
|
|
|
298
372
|
this.name = "KafkaProcessingError";
|
|
299
373
|
if (options?.cause) this.cause = options.cause;
|
|
300
374
|
}
|
|
375
|
+
topic;
|
|
376
|
+
originalMessage;
|
|
301
377
|
};
|
|
302
378
|
var KafkaValidationError = class extends Error {
|
|
303
379
|
constructor(topic2, originalMessage, options) {
|
|
@@ -307,6 +383,8 @@ var KafkaValidationError = class extends Error {
|
|
|
307
383
|
this.name = "KafkaValidationError";
|
|
308
384
|
if (options?.cause) this.cause = options.cause;
|
|
309
385
|
}
|
|
386
|
+
topic;
|
|
387
|
+
originalMessage;
|
|
310
388
|
};
|
|
311
389
|
var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
312
390
|
constructor(topic2, originalMessage, attempts, options) {
|
|
@@ -319,6 +397,7 @@ var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
|
319
397
|
this.attempts = attempts;
|
|
320
398
|
this.name = "KafkaRetryExhaustedError";
|
|
321
399
|
}
|
|
400
|
+
attempts;
|
|
322
401
|
};
|
|
323
402
|
|
|
324
403
|
// src/client/kafka.client/producer/ops.ts
|
|
@@ -391,7 +470,9 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
391
470
|
value: JSON.stringify(
|
|
392
471
|
await validateMessage(topicOrDesc, m.value, deps, sendCtx)
|
|
393
472
|
),
|
|
394
|
-
key
|
|
473
|
+
// Explicit key wins; otherwise fall back to the descriptor's .key()
|
|
474
|
+
// extractor (runs on the original, pre-validation payload).
|
|
475
|
+
key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
|
|
395
476
|
headers: envelopeHeaders
|
|
396
477
|
};
|
|
397
478
|
})
|
|
@@ -400,7 +481,7 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
|
400
481
|
}
|
|
401
482
|
|
|
402
483
|
// src/client/kafka.client/consumer/ops.ts
|
|
403
|
-
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment) {
|
|
484
|
+
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment, groupInstanceId) {
|
|
404
485
|
const { consumers, consumerCreationOptions, transport, onRebalance, logger } = deps;
|
|
405
486
|
if (consumers.has(groupId)) {
|
|
406
487
|
const prev = consumerCreationOptions.get(groupId);
|
|
@@ -433,6 +514,7 @@ function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partition
|
|
|
433
514
|
fromBeginning,
|
|
434
515
|
autoCommit,
|
|
435
516
|
partitionAssigner: partitionAssigner ?? "cooperative-sticky",
|
|
517
|
+
groupInstanceId,
|
|
436
518
|
onRebalance: (type, assignments) => {
|
|
437
519
|
if (type === "assign") fireOnAssignment();
|
|
438
520
|
else if (type === "revoke") scheduleSettle();
|
|
@@ -478,6 +560,7 @@ var AdminOps = class {
|
|
|
478
560
|
constructor(deps) {
|
|
479
561
|
this.deps = deps;
|
|
480
562
|
}
|
|
563
|
+
deps;
|
|
481
564
|
isConnected = false;
|
|
482
565
|
/** Underlying admin client — used by index.ts for topic validation. */
|
|
483
566
|
get admin() {
|
|
@@ -584,7 +667,10 @@ var AdminOps = class {
|
|
|
584
667
|
const found = results.find(
|
|
585
668
|
(r) => r.partition === partition
|
|
586
669
|
);
|
|
587
|
-
return { partition, offset: found
|
|
670
|
+
if (found) return { partition, offset: found.offset };
|
|
671
|
+
const topicOffsets = await this.deps.admin.fetchTopicOffsets(topic2);
|
|
672
|
+
const po = topicOffsets.find((o) => o.partition === partition);
|
|
673
|
+
return { partition, offset: po?.high ?? "0" };
|
|
588
674
|
})
|
|
589
675
|
);
|
|
590
676
|
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
@@ -664,8 +750,9 @@ var AdminOps = class {
|
|
|
664
750
|
return result.topics.map((t) => ({
|
|
665
751
|
name: t.name,
|
|
666
752
|
partitions: t.partitions.map((p) => ({
|
|
667
|
-
partition: p.partitionId ?? p.partition,
|
|
668
|
-
leader
|
|
753
|
+
partition: p.partitionId ?? p.partition ?? 0,
|
|
754
|
+
// -1 is Kafka's own "no leader" sentinel; 0 is a valid broker id
|
|
755
|
+
leader: p.leader ?? -1,
|
|
669
756
|
replicas: (p.replicas ?? []).map(
|
|
670
757
|
(r) => typeof r === "number" ? r : r.nodeId
|
|
671
758
|
),
|
|
@@ -749,9 +836,6 @@ var AdminOps = class {
|
|
|
749
836
|
};
|
|
750
837
|
|
|
751
838
|
// src/client/kafka.client/consumer/pipeline.ts
|
|
752
|
-
function toError(error) {
|
|
753
|
-
return error instanceof Error ? error : new Error(String(error));
|
|
754
|
-
}
|
|
755
839
|
function sleep(ms) {
|
|
756
840
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
757
841
|
}
|
|
@@ -1035,6 +1119,7 @@ async function executeWithRetry(fn, ctx, deps) {
|
|
|
1035
1119
|
for (const env of envelopes) deps.onMessage?.(env);
|
|
1036
1120
|
return;
|
|
1037
1121
|
}
|
|
1122
|
+
deps.onFailure?.(envelopes[0]);
|
|
1038
1123
|
const isLastAttempt = attempt === maxAttempts;
|
|
1039
1124
|
const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
|
|
1040
1125
|
topic2,
|
|
@@ -1119,8 +1204,13 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
|
1119
1204
|
}
|
|
1120
1205
|
}
|
|
1121
1206
|
|
|
1122
|
-
// src/client/kafka.client/consumer/dlq-replay.ts
|
|
1207
|
+
// src/client/kafka.client/consumer/features/dlq-replay.ts
|
|
1123
1208
|
async function replayDlqTopic(topic2, deps, options = {}) {
|
|
1209
|
+
if (topic2.endsWith(".dlq")) {
|
|
1210
|
+
throw new Error(
|
|
1211
|
+
`replayDlq: pass the ORIGINAL topic name \u2014 "${topic2}" already ends in ".dlq" (the ".dlq" suffix is appended internally, so this would read "${topic2}.dlq")`
|
|
1212
|
+
);
|
|
1213
|
+
}
|
|
1124
1214
|
const dlqTopic = `${topic2}.dlq`;
|
|
1125
1215
|
const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
|
|
1126
1216
|
const activePartitions = partitionOffsets.filter(
|
|
@@ -1186,6 +1276,7 @@ var MetricsManager = class {
|
|
|
1186
1276
|
constructor(deps) {
|
|
1187
1277
|
this.deps = deps;
|
|
1188
1278
|
}
|
|
1279
|
+
deps;
|
|
1189
1280
|
topicMetrics = /* @__PURE__ */ new Map();
|
|
1190
1281
|
metricsFor(topic2) {
|
|
1191
1282
|
let m = this.topicMetrics.get(topic2);
|
|
@@ -1211,16 +1302,25 @@ var MetricsManager = class {
|
|
|
1211
1302
|
for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1212
1303
|
}
|
|
1213
1304
|
/**
|
|
1214
|
-
* Increment the DLQ counter for the envelope's topic
|
|
1215
|
-
*
|
|
1305
|
+
* Increment the DLQ counter for the envelope's topic and fire all `onDlq` instrumentation hooks.
|
|
1306
|
+
* Circuit breaker failures are recorded separately via `notifyFailure` at the
|
|
1307
|
+
* handler-error boundary — dead-lettering itself is not a circuit event.
|
|
1216
1308
|
* @param envelope The message envelope being sent to the DLQ.
|
|
1217
1309
|
* @param reason The reason the message is being dead-lettered.
|
|
1218
|
-
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1219
1310
|
*/
|
|
1220
|
-
notifyDlq(envelope, reason
|
|
1311
|
+
notifyDlq(envelope, reason) {
|
|
1221
1312
|
this.metricsFor(envelope.topic).dlqCount++;
|
|
1222
1313
|
for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
|
|
1223
|
-
|
|
1314
|
+
}
|
|
1315
|
+
/**
|
|
1316
|
+
* Notify the circuit breaker of a handler failure. Fired on every failed
|
|
1317
|
+
* handler attempt (in-process retries and retry-topic levels included),
|
|
1318
|
+
* independent of whether the message is ultimately dead-lettered.
|
|
1319
|
+
* @param envelope The message envelope whose handler failed.
|
|
1320
|
+
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1321
|
+
*/
|
|
1322
|
+
notifyFailure(envelope, gid) {
|
|
1323
|
+
this.deps.onCircuitFailure(envelope, gid);
|
|
1224
1324
|
}
|
|
1225
1325
|
/**
|
|
1226
1326
|
* Increment the deduplication counter for the envelope's topic and fire all `onDuplicate` hooks.
|
|
@@ -1279,6 +1379,7 @@ var InFlightTracker = class {
|
|
|
1279
1379
|
constructor(warn) {
|
|
1280
1380
|
this.warn = warn;
|
|
1281
1381
|
}
|
|
1382
|
+
warn;
|
|
1282
1383
|
inFlightTotal = 0;
|
|
1283
1384
|
drainResolvers = [];
|
|
1284
1385
|
/**
|
|
@@ -1289,10 +1390,16 @@ var InFlightTracker = class {
|
|
|
1289
1390
|
*/
|
|
1290
1391
|
track(fn) {
|
|
1291
1392
|
this.inFlightTotal++;
|
|
1292
|
-
|
|
1393
|
+
const done = () => {
|
|
1293
1394
|
this.inFlightTotal--;
|
|
1294
1395
|
if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
|
|
1295
|
-
}
|
|
1396
|
+
};
|
|
1397
|
+
try {
|
|
1398
|
+
return fn().finally(done);
|
|
1399
|
+
} catch (err) {
|
|
1400
|
+
done();
|
|
1401
|
+
throw err;
|
|
1402
|
+
}
|
|
1296
1403
|
}
|
|
1297
1404
|
/**
|
|
1298
1405
|
* Resolve when all tracked handlers have completed, or after `timeoutMs` elapses.
|
|
@@ -1326,6 +1433,7 @@ var CircuitBreakerManager = class {
|
|
|
1326
1433
|
constructor(deps) {
|
|
1327
1434
|
this.deps = deps;
|
|
1328
1435
|
}
|
|
1436
|
+
deps;
|
|
1329
1437
|
states = /* @__PURE__ */ new Map();
|
|
1330
1438
|
configs = /* @__PURE__ */ new Map();
|
|
1331
1439
|
/**
|
|
@@ -1470,6 +1578,9 @@ var AsyncQueue = class {
|
|
|
1470
1578
|
this.onFull = onFull;
|
|
1471
1579
|
this.onDrained = onDrained;
|
|
1472
1580
|
}
|
|
1581
|
+
highWaterMark;
|
|
1582
|
+
onFull;
|
|
1583
|
+
onDrained;
|
|
1473
1584
|
items = [];
|
|
1474
1585
|
waiting = [];
|
|
1475
1586
|
closed = false;
|
|
@@ -1481,6 +1592,7 @@ var AsyncQueue = class {
|
|
|
1481
1592
|
* @param item The value to enqueue.
|
|
1482
1593
|
*/
|
|
1483
1594
|
push(item) {
|
|
1595
|
+
if (this.closed) return;
|
|
1484
1596
|
if (this.waiting.length > 0) {
|
|
1485
1597
|
this.waiting.shift().resolve({ value: item, done: false });
|
|
1486
1598
|
} else {
|
|
@@ -1531,6 +1643,101 @@ var AsyncQueue = class {
|
|
|
1531
1643
|
}
|
|
1532
1644
|
};
|
|
1533
1645
|
|
|
1646
|
+
// src/client/kafka.client/validate-options.ts
|
|
1647
|
+
function validateClientOptions(clientId, groupId, brokers, options) {
|
|
1648
|
+
const problems = [];
|
|
1649
|
+
if (typeof clientId !== "string" || clientId.trim() === "") {
|
|
1650
|
+
problems.push("clientId must be a non-empty string");
|
|
1651
|
+
}
|
|
1652
|
+
if (typeof groupId !== "string" || groupId.trim() === "") {
|
|
1653
|
+
problems.push("groupId must be a non-empty string");
|
|
1654
|
+
}
|
|
1655
|
+
if (!Array.isArray(brokers) || brokers.length === 0 && !options?.transport) {
|
|
1656
|
+
problems.push("brokers must be a non-empty array of broker addresses");
|
|
1657
|
+
} else if (brokers.some((b) => typeof b !== "string" || b.trim() === "")) {
|
|
1658
|
+
problems.push("brokers must not contain empty entries");
|
|
1659
|
+
}
|
|
1660
|
+
if (options) {
|
|
1661
|
+
const {
|
|
1662
|
+
numPartitions,
|
|
1663
|
+
transactionalId,
|
|
1664
|
+
clockRecovery,
|
|
1665
|
+
lagThrottle
|
|
1666
|
+
} = options;
|
|
1667
|
+
if (numPartitions !== void 0 && (!Number.isInteger(numPartitions) || numPartitions < 1)) {
|
|
1668
|
+
problems.push(
|
|
1669
|
+
`numPartitions must be a positive integer (got ${numPartitions})`
|
|
1670
|
+
);
|
|
1671
|
+
}
|
|
1672
|
+
if (transactionalId !== void 0 && transactionalId.trim() === "") {
|
|
1673
|
+
problems.push("transactionalId must be a non-empty string when set");
|
|
1674
|
+
}
|
|
1675
|
+
if (clockRecovery) {
|
|
1676
|
+
if (!Array.isArray(clockRecovery.topics)) {
|
|
1677
|
+
problems.push("clockRecovery.topics must be an array of topic names");
|
|
1678
|
+
}
|
|
1679
|
+
if (clockRecovery.timeoutMs !== void 0 && !(clockRecovery.timeoutMs > 0)) {
|
|
1680
|
+
problems.push(
|
|
1681
|
+
`clockRecovery.timeoutMs must be > 0 (got ${clockRecovery.timeoutMs})`
|
|
1682
|
+
);
|
|
1683
|
+
}
|
|
1684
|
+
}
|
|
1685
|
+
if (lagThrottle) {
|
|
1686
|
+
if (!(lagThrottle.maxLag >= 0)) {
|
|
1687
|
+
problems.push(`lagThrottle.maxLag must be >= 0 (got ${lagThrottle.maxLag})`);
|
|
1688
|
+
}
|
|
1689
|
+
if (lagThrottle.pollIntervalMs !== void 0 && !(lagThrottle.pollIntervalMs > 0)) {
|
|
1690
|
+
problems.push(
|
|
1691
|
+
`lagThrottle.pollIntervalMs must be > 0 (got ${lagThrottle.pollIntervalMs})`
|
|
1692
|
+
);
|
|
1693
|
+
}
|
|
1694
|
+
if (lagThrottle.maxWaitMs !== void 0 && !(lagThrottle.maxWaitMs >= 0)) {
|
|
1695
|
+
problems.push(
|
|
1696
|
+
`lagThrottle.maxWaitMs must be >= 0 (got ${lagThrottle.maxWaitMs})`
|
|
1697
|
+
);
|
|
1698
|
+
}
|
|
1699
|
+
}
|
|
1700
|
+
}
|
|
1701
|
+
if (problems.length > 0) {
|
|
1702
|
+
throw new Error(
|
|
1703
|
+
`KafkaClient: invalid configuration:
|
|
1704
|
+
- ${problems.join("\n- ")}`
|
|
1705
|
+
);
|
|
1706
|
+
}
|
|
1707
|
+
}
|
|
1708
|
+
|
|
1709
|
+
// src/client/security/resolve-security.ts
|
|
1710
|
+
var LOCAL_HOST_PATTERNS = [
|
|
1711
|
+
/^localhost(:\d+)?$/i,
|
|
1712
|
+
/^127\.\d+\.\d+\.\d+(:\d+)?$/,
|
|
1713
|
+
/^\[?::1\]?(:\d+)?$/,
|
|
1714
|
+
/^0\.0\.0\.0(:\d+)?$/,
|
|
1715
|
+
/^host\.docker\.internal(:\d+)?$/i
|
|
1716
|
+
];
|
|
1717
|
+
function isLocalBroker(broker) {
|
|
1718
|
+
return LOCAL_HOST_PATTERNS.some((re) => re.test(broker.trim()));
|
|
1719
|
+
}
|
|
1720
|
+
function resolveSecurityOptions(security, brokers, logger) {
|
|
1721
|
+
const hasRemoteBroker = brokers.some((b) => !isLocalBroker(b));
|
|
1722
|
+
if (!security?.sasl && security?.ssl !== true) {
|
|
1723
|
+
if (hasRemoteBroker && !security?.allowInsecure) {
|
|
1724
|
+
logger.warn(
|
|
1725
|
+
"Connecting to non-local brokers without TLS or SASL \u2014 traffic and payloads travel in plaintext. Configure `security: { ssl, sasl }` for production clusters, or set `security: { allowInsecure: true }` to acknowledge an intentionally insecure setup."
|
|
1726
|
+
);
|
|
1727
|
+
}
|
|
1728
|
+
return security;
|
|
1729
|
+
}
|
|
1730
|
+
if (security.sasl && security.ssl === void 0) {
|
|
1731
|
+
return { ...security, ssl: true };
|
|
1732
|
+
}
|
|
1733
|
+
if (security.sasl && security.ssl === false) {
|
|
1734
|
+
logger.warn(
|
|
1735
|
+
"SASL credentials are configured with `ssl: false` \u2014 credentials will be sent over plaintext. This is only safe on fully trusted networks."
|
|
1736
|
+
);
|
|
1737
|
+
}
|
|
1738
|
+
return security;
|
|
1739
|
+
}
|
|
1740
|
+
|
|
1534
1741
|
// src/client/kafka.client/producer/lifecycle.ts
|
|
1535
1742
|
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1536
1743
|
async function ensureTopic(ctx, topic2) {
|
|
@@ -1664,6 +1871,7 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1664
1871
|
const remaining = new Set(
|
|
1665
1872
|
partitionsToRead.map((p) => `${p.topic}:${p.partition}`)
|
|
1666
1873
|
);
|
|
1874
|
+
let settled = false;
|
|
1667
1875
|
const cleanup = () => {
|
|
1668
1876
|
consumer.disconnect().catch(() => {
|
|
1669
1877
|
}).finally(() => {
|
|
@@ -1671,6 +1879,16 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1671
1879
|
});
|
|
1672
1880
|
});
|
|
1673
1881
|
};
|
|
1882
|
+
const timeoutTimer = setTimeout(() => {
|
|
1883
|
+
if (settled) return;
|
|
1884
|
+
settled = true;
|
|
1885
|
+
ctx.logger.warn(
|
|
1886
|
+
`Clock recovery: timed out after ${ctx.clockRecoveryTimeoutMs} ms with ${remaining.size} partition(s) unread \u2014 proceeding with partial result`
|
|
1887
|
+
);
|
|
1888
|
+
cleanup();
|
|
1889
|
+
resolve();
|
|
1890
|
+
}, ctx.clockRecoveryTimeoutMs);
|
|
1891
|
+
timeoutTimer.unref?.();
|
|
1674
1892
|
consumer.connect().then(async () => {
|
|
1675
1893
|
const uniqueTopics = [
|
|
1676
1894
|
...new Set(partitionsToRead.map((p) => p.topic))
|
|
@@ -1691,13 +1909,18 @@ async function recoverLamportClockImpl(ctx, topics) {
|
|
|
1691
1909
|
const clock = Number(raw);
|
|
1692
1910
|
if (!Number.isNaN(clock) && clock > maxClock) maxClock = clock;
|
|
1693
1911
|
}
|
|
1694
|
-
if (remaining.size === 0) {
|
|
1912
|
+
if (remaining.size === 0 && !settled) {
|
|
1913
|
+
settled = true;
|
|
1914
|
+
clearTimeout(timeoutTimer);
|
|
1695
1915
|
cleanup();
|
|
1696
1916
|
resolve();
|
|
1697
1917
|
}
|
|
1698
1918
|
}
|
|
1699
1919
|
})
|
|
1700
1920
|
).catch((err) => {
|
|
1921
|
+
if (settled) return;
|
|
1922
|
+
settled = true;
|
|
1923
|
+
clearTimeout(timeoutTimer);
|
|
1701
1924
|
cleanup();
|
|
1702
1925
|
reject(err);
|
|
1703
1926
|
});
|
|
@@ -1738,6 +1961,15 @@ async function preparePayload(ctx, topicOrDesc, messages, compression) {
|
|
|
1738
1961
|
await ensureTopic(ctx, payload.topic);
|
|
1739
1962
|
return payload;
|
|
1740
1963
|
}
|
|
1964
|
+
async function redirectToDelayed(ctx, payload, deliverAfterMs) {
|
|
1965
|
+
const until = String(Date.now() + deliverAfterMs);
|
|
1966
|
+
for (const m of payload.messages) {
|
|
1967
|
+
m.headers[HEADER_DELAYED_UNTIL] = until;
|
|
1968
|
+
m.headers[HEADER_DELAYED_TARGET] = payload.topic;
|
|
1969
|
+
}
|
|
1970
|
+
payload.topic = `${payload.topic}.delayed`;
|
|
1971
|
+
await ensureTopic(ctx, payload.topic);
|
|
1972
|
+
}
|
|
1741
1973
|
async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
|
|
1742
1974
|
await waitIfThrottled(ctx);
|
|
1743
1975
|
const payload = await preparePayload(
|
|
@@ -1755,6 +1987,9 @@ async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
|
|
|
1755
1987
|
],
|
|
1756
1988
|
options.compression
|
|
1757
1989
|
);
|
|
1990
|
+
if (options.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
1991
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
1992
|
+
}
|
|
1758
1993
|
await ctx.producer.send(payload);
|
|
1759
1994
|
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1760
1995
|
}
|
|
@@ -1766,6 +2001,9 @@ async function sendBatchImpl(ctx, topicOrDesc, messages, options) {
|
|
|
1766
2001
|
messages,
|
|
1767
2002
|
options?.compression
|
|
1768
2003
|
);
|
|
2004
|
+
if (options?.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
2005
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
2006
|
+
}
|
|
1769
2007
|
await ctx.producer.send(payload);
|
|
1770
2008
|
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1771
2009
|
}
|
|
@@ -1802,6 +2040,17 @@ async function transactionImpl(ctx, fn) {
|
|
|
1802
2040
|
});
|
|
1803
2041
|
}
|
|
1804
2042
|
ctx.txProducer = await ctx.txProducerInitPromise;
|
|
2043
|
+
const prev = ctx._txChain;
|
|
2044
|
+
let release;
|
|
2045
|
+
ctx._txChain = new Promise((r) => release = r);
|
|
2046
|
+
await prev;
|
|
2047
|
+
try {
|
|
2048
|
+
await runTransaction(ctx, fn);
|
|
2049
|
+
} finally {
|
|
2050
|
+
release();
|
|
2051
|
+
}
|
|
2052
|
+
}
|
|
2053
|
+
async function runTransaction(ctx, fn) {
|
|
1805
2054
|
const tx = await ctx.txProducer.transaction();
|
|
1806
2055
|
try {
|
|
1807
2056
|
const txCtx = {
|
|
@@ -1969,6 +2218,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
|
|
|
1969
2218
|
await consumer.commitOffsets([nextOffset]);
|
|
1970
2219
|
return;
|
|
1971
2220
|
}
|
|
2221
|
+
deps.onFailure?.(envelope);
|
|
1972
2222
|
const exhausted = level >= currentMaxRetries;
|
|
1973
2223
|
const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
|
|
1974
2224
|
originalTopic,
|
|
@@ -2184,7 +2434,8 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2184
2434
|
options.autoCommit ?? true,
|
|
2185
2435
|
ctx.consumerOpsDeps,
|
|
2186
2436
|
options.partitionAssigner,
|
|
2187
|
-
resolveReady
|
|
2437
|
+
resolveReady,
|
|
2438
|
+
options.groupInstanceId
|
|
2188
2439
|
);
|
|
2189
2440
|
const schemaMap = buildSchemaMap(
|
|
2190
2441
|
stringTopics,
|
|
@@ -2196,6 +2447,9 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2196
2447
|
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2197
2448
|
await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
|
|
2198
2449
|
await consumer.connect();
|
|
2450
|
+
if (dlq || options.retryTopics || options.deduplication) {
|
|
2451
|
+
await ctx.producer.connect();
|
|
2452
|
+
}
|
|
2199
2453
|
await subscribeWithRetry(
|
|
2200
2454
|
consumer,
|
|
2201
2455
|
subscribeTopics,
|
|
@@ -2210,9 +2464,8 @@ async function setupConsumer(ctx, topics, mode, options) {
|
|
|
2210
2464
|
}
|
|
2211
2465
|
function resolveDeduplicationContext(ctx, groupId, options) {
|
|
2212
2466
|
if (!options) return void 0;
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
return { options, state: ctx.dedupStates.get(groupId) };
|
|
2467
|
+
const store = options.store ?? new InMemoryDedupStore(ctx.dedupStates);
|
|
2468
|
+
return { options, store, groupId };
|
|
2216
2469
|
}
|
|
2217
2470
|
function messageDepsFor(ctx, gid, options) {
|
|
2218
2471
|
const notifyRetry = ctx.metrics.notifyRetry.bind(ctx.metrics);
|
|
@@ -2226,9 +2479,10 @@ function messageDepsFor(ctx, gid, options) {
|
|
|
2226
2479
|
notifyRetry(envelope, attempt, max);
|
|
2227
2480
|
return options.onRetry(envelope, attempt, max);
|
|
2228
2481
|
} : notifyRetry,
|
|
2229
|
-
onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason
|
|
2482
|
+
onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason),
|
|
2230
2483
|
onDuplicate: ctx.metrics.notifyDuplicate.bind(ctx.metrics),
|
|
2231
|
-
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid)
|
|
2484
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2485
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid)
|
|
2232
2486
|
};
|
|
2233
2487
|
}
|
|
2234
2488
|
function buildRetryTopicDeps(ctx) {
|
|
@@ -2267,6 +2521,11 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
|
2267
2521
|
schemaMap,
|
|
2268
2522
|
{
|
|
2269
2523
|
...ctx.retryTopicDeps,
|
|
2524
|
+
// Bind circuit breaker events to the MAIN consumer group so failures and
|
|
2525
|
+
// successes inside the retry chain drive the same breaker as the main
|
|
2526
|
+
// consumer (the retry chain has no breaker config of its own).
|
|
2527
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid),
|
|
2528
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2270
2529
|
onLevelStarted: (levelGroupId) => {
|
|
2271
2530
|
ctx.companionGroupIds.get(gid).push(levelGroupId);
|
|
2272
2531
|
}
|
|
@@ -2282,7 +2541,15 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2282
2541
|
const incomingClock = Number(clockRaw);
|
|
2283
2542
|
if (Number.isNaN(incomingClock)) return false;
|
|
2284
2543
|
const stateKey = `${envelope.topic}:${envelope.partition}`;
|
|
2285
|
-
|
|
2544
|
+
let lastProcessedClock;
|
|
2545
|
+
try {
|
|
2546
|
+
lastProcessedClock = await dedup.store.getLastClock(dedup.groupId, stateKey) ?? -1;
|
|
2547
|
+
} catch (err) {
|
|
2548
|
+
deps.logger.error(
|
|
2549
|
+
`Dedup store getLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 treating message as not a duplicate (fail-open): ${err.message}`
|
|
2550
|
+
);
|
|
2551
|
+
return false;
|
|
2552
|
+
}
|
|
2286
2553
|
if (incomingClock <= lastProcessedClock) {
|
|
2287
2554
|
const meta = {
|
|
2288
2555
|
incomingClock,
|
|
@@ -2312,7 +2579,13 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
|
2312
2579
|
}
|
|
2313
2580
|
return true;
|
|
2314
2581
|
}
|
|
2315
|
-
|
|
2582
|
+
try {
|
|
2583
|
+
await dedup.store.setLastClock(dedup.groupId, stateKey, incomingClock);
|
|
2584
|
+
} catch (err) {
|
|
2585
|
+
deps.logger.error(
|
|
2586
|
+
`Dedup store setLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 processing message anyway (fail-open): ${err.message}`
|
|
2587
|
+
);
|
|
2588
|
+
}
|
|
2316
2589
|
return false;
|
|
2317
2590
|
}
|
|
2318
2591
|
async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
|
|
@@ -2925,7 +3198,7 @@ function stopConsumerByGid(ctx, gid) {
|
|
|
2925
3198
|
return stopConsumerImpl(ctx, gid);
|
|
2926
3199
|
}
|
|
2927
3200
|
|
|
2928
|
-
// src/client/kafka.client/consumer/window.ts
|
|
3201
|
+
// src/client/kafka.client/consumer/features/window.ts
|
|
2929
3202
|
async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
2930
3203
|
const { maxMessages, maxMs, ...consumerOptions } = options;
|
|
2931
3204
|
if (maxMessages <= 0)
|
|
@@ -2939,6 +3212,7 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2939
3212
|
const buffer = [];
|
|
2940
3213
|
let flushTimer = null;
|
|
2941
3214
|
let windowStart = 0;
|
|
3215
|
+
const onLost = consumerOptions.onMessageLost ?? ctx.onMessageLost;
|
|
2942
3216
|
const flush = async (trigger) => {
|
|
2943
3217
|
if (flushTimer !== null) {
|
|
2944
3218
|
clearTimeout(flushTimer);
|
|
@@ -2946,17 +3220,32 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2946
3220
|
}
|
|
2947
3221
|
if (buffer.length === 0) return;
|
|
2948
3222
|
const envelopes = buffer.splice(0);
|
|
2949
|
-
|
|
3223
|
+
try {
|
|
3224
|
+
await handler(envelopes, { trigger, windowStart, windowEnd: Date.now() });
|
|
3225
|
+
} catch (err) {
|
|
3226
|
+
const error = toError(err);
|
|
3227
|
+
ctx.logger.error(
|
|
3228
|
+
`startWindowConsumer: ${trigger}-triggered flush failed \u2014 window of ${envelopes.length} message(s) lost:`,
|
|
3229
|
+
error.stack
|
|
3230
|
+
);
|
|
3231
|
+
for (const envelope of envelopes) {
|
|
3232
|
+
await Promise.resolve(
|
|
3233
|
+
onLost?.({
|
|
3234
|
+
topic: envelope.topic,
|
|
3235
|
+
error,
|
|
3236
|
+
attempt: 0,
|
|
3237
|
+
headers: envelope.headers
|
|
3238
|
+
})
|
|
3239
|
+
).catch(() => {
|
|
3240
|
+
});
|
|
3241
|
+
}
|
|
3242
|
+
}
|
|
2950
3243
|
};
|
|
2951
3244
|
const scheduleFlush = () => {
|
|
2952
3245
|
if (flushTimer !== null) return;
|
|
2953
3246
|
flushTimer = setTimeout(() => {
|
|
2954
3247
|
flushTimer = null;
|
|
2955
|
-
flush("time")
|
|
2956
|
-
ctx.logger.warn(
|
|
2957
|
-
`startWindowConsumer: time-triggered flush error \u2014 ${toError(err).message}`
|
|
2958
|
-
);
|
|
2959
|
-
});
|
|
3248
|
+
void flush("time");
|
|
2960
3249
|
}, maxMs);
|
|
2961
3250
|
};
|
|
2962
3251
|
const handle = await startConsumerImpl(
|
|
@@ -2972,40 +3261,13 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
|
2972
3261
|
);
|
|
2973
3262
|
const originalStop = handle.stop.bind(handle);
|
|
2974
3263
|
handle.stop = async () => {
|
|
2975
|
-
|
|
2976
|
-
clearTimeout(flushTimer);
|
|
2977
|
-
flushTimer = null;
|
|
2978
|
-
}
|
|
2979
|
-
if (buffer.length > 0) {
|
|
2980
|
-
const envelopes = buffer.splice(0);
|
|
2981
|
-
await handler(envelopes, {
|
|
2982
|
-
trigger: "time",
|
|
2983
|
-
windowStart,
|
|
2984
|
-
windowEnd: Date.now()
|
|
2985
|
-
}).catch(async (err) => {
|
|
2986
|
-
const error = toError(err);
|
|
2987
|
-
ctx.logger.warn(
|
|
2988
|
-
`startWindowConsumer: shutdown flush error \u2014 ${error.message}`
|
|
2989
|
-
);
|
|
2990
|
-
for (const envelope of envelopes) {
|
|
2991
|
-
await Promise.resolve(
|
|
2992
|
-
ctx.onMessageLost?.({
|
|
2993
|
-
topic: envelope.topic,
|
|
2994
|
-
error,
|
|
2995
|
-
attempt: 0,
|
|
2996
|
-
headers: envelope.headers
|
|
2997
|
-
})
|
|
2998
|
-
).catch(() => {
|
|
2999
|
-
});
|
|
3000
|
-
}
|
|
3001
|
-
});
|
|
3002
|
-
}
|
|
3264
|
+
await flush("time");
|
|
3003
3265
|
return originalStop();
|
|
3004
3266
|
};
|
|
3005
3267
|
return handle;
|
|
3006
3268
|
}
|
|
3007
3269
|
|
|
3008
|
-
// src/client/kafka.client/consumer/routed.ts
|
|
3270
|
+
// src/client/kafka.client/consumer/features/routed.ts
|
|
3009
3271
|
async function startRoutedConsumerImpl(ctx, topics, routing, options) {
|
|
3010
3272
|
const { header, routes, fallback } = routing;
|
|
3011
3273
|
const handleMessage = async (envelope) => {
|
|
@@ -3020,7 +3282,118 @@ async function startRoutedConsumerImpl(ctx, topics, routing, options) {
|
|
|
3020
3282
|
return startConsumerImpl(ctx, topics, handleMessage, options);
|
|
3021
3283
|
}
|
|
3022
3284
|
|
|
3023
|
-
// src/client/kafka.client/consumer/
|
|
3285
|
+
// src/client/kafka.client/consumer/features/delayed.ts
|
|
3286
|
+
function delayedTopicName(topic2) {
|
|
3287
|
+
return `${topic2}.delayed`;
|
|
3288
|
+
}
|
|
3289
|
+
async function startDelayedRelayImpl(ctx, topics, options) {
|
|
3290
|
+
if (topics.length === 0) {
|
|
3291
|
+
throw new Error("startDelayedRelay: at least one topic is required");
|
|
3292
|
+
}
|
|
3293
|
+
const gid = options?.groupId ?? `${ctx.defaultGroupId}-delayed-relay`;
|
|
3294
|
+
if (ctx.runningConsumers.has(gid)) {
|
|
3295
|
+
throw new Error(
|
|
3296
|
+
`startDelayedRelay("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
|
|
3297
|
+
);
|
|
3298
|
+
}
|
|
3299
|
+
const delayedTopics = topics.map(delayedTopicName);
|
|
3300
|
+
for (const t of delayedTopics) await ensureTopic(ctx, t);
|
|
3301
|
+
const txProducer = await createRetryTxProducer(ctx, `${gid}-tx`);
|
|
3302
|
+
let resolveReady;
|
|
3303
|
+
const readyPromise = new Promise((resolve) => {
|
|
3304
|
+
resolveReady = resolve;
|
|
3305
|
+
});
|
|
3306
|
+
const consumer = getOrCreateConsumer(
|
|
3307
|
+
gid,
|
|
3308
|
+
false,
|
|
3309
|
+
false,
|
|
3310
|
+
ctx.consumerOpsDeps,
|
|
3311
|
+
void 0,
|
|
3312
|
+
resolveReady
|
|
3313
|
+
);
|
|
3314
|
+
await consumer.connect();
|
|
3315
|
+
await subscribeWithRetry(consumer, delayedTopics, ctx.logger);
|
|
3316
|
+
await consumer.run({
|
|
3317
|
+
eachMessage: async ({ topic: stagingTopic, partition, message }) => {
|
|
3318
|
+
const nextOffset = {
|
|
3319
|
+
topic: stagingTopic,
|
|
3320
|
+
partition,
|
|
3321
|
+
offset: (parseInt(message.offset, 10) + 1).toString()
|
|
3322
|
+
};
|
|
3323
|
+
if (!message.value) {
|
|
3324
|
+
await consumer.commitOffsets([nextOffset]);
|
|
3325
|
+
return;
|
|
3326
|
+
}
|
|
3327
|
+
const headers = decodeHeaders(message.headers);
|
|
3328
|
+
const target = headers[HEADER_DELAYED_TARGET] ?? stagingTopic.replace(/\.delayed$/, "");
|
|
3329
|
+
const until = parseInt(
|
|
3330
|
+
headers[HEADER_DELAYED_UNTIL] ?? "0",
|
|
3331
|
+
10
|
|
3332
|
+
);
|
|
3333
|
+
const remaining = until - Date.now();
|
|
3334
|
+
if (remaining > 0) {
|
|
3335
|
+
consumer.pause([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3336
|
+
await sleep(remaining);
|
|
3337
|
+
consumer.resume([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3338
|
+
}
|
|
3339
|
+
const forwardHeaders = Object.fromEntries(
|
|
3340
|
+
Object.entries(headers).filter(
|
|
3341
|
+
([k]) => k !== HEADER_DELAYED_UNTIL && k !== HEADER_DELAYED_TARGET
|
|
3342
|
+
)
|
|
3343
|
+
);
|
|
3344
|
+
const tx = await txProducer.transaction();
|
|
3345
|
+
try {
|
|
3346
|
+
await tx.send({
|
|
3347
|
+
topic: target,
|
|
3348
|
+
messages: [
|
|
3349
|
+
{
|
|
3350
|
+
value: message.value.toString(),
|
|
3351
|
+
key: message.key ? message.key.toString() : null,
|
|
3352
|
+
headers: forwardHeaders
|
|
3353
|
+
}
|
|
3354
|
+
]
|
|
3355
|
+
});
|
|
3356
|
+
await tx.sendOffsets({
|
|
3357
|
+
consumer,
|
|
3358
|
+
topics: [
|
|
3359
|
+
{
|
|
3360
|
+
topic: nextOffset.topic,
|
|
3361
|
+
partitions: [
|
|
3362
|
+
{ partition: nextOffset.partition, offset: nextOffset.offset }
|
|
3363
|
+
]
|
|
3364
|
+
}
|
|
3365
|
+
]
|
|
3366
|
+
});
|
|
3367
|
+
await tx.commit();
|
|
3368
|
+
ctx.logger.debug?.(
|
|
3369
|
+
`Delayed message relayed to "${target}" (deadline ${new Date(until).toISOString()})`
|
|
3370
|
+
);
|
|
3371
|
+
} catch (txErr) {
|
|
3372
|
+
try {
|
|
3373
|
+
await tx.abort();
|
|
3374
|
+
} catch {
|
|
3375
|
+
}
|
|
3376
|
+
ctx.logger.error(
|
|
3377
|
+
`Delayed relay to "${target}" failed \u2014 message will be redelivered:`,
|
|
3378
|
+
toError(txErr).stack
|
|
3379
|
+
);
|
|
3380
|
+
}
|
|
3381
|
+
}
|
|
3382
|
+
});
|
|
3383
|
+
ctx.runningConsumers.set(gid, "eachMessage");
|
|
3384
|
+
ctx.logger.log(
|
|
3385
|
+
`Delayed relay started for: ${delayedTopics.join(", ")} (group: ${gid})`
|
|
3386
|
+
);
|
|
3387
|
+
return {
|
|
3388
|
+
groupId: gid,
|
|
3389
|
+
ready: () => readyPromise,
|
|
3390
|
+
stop: async () => {
|
|
3391
|
+
await stopConsumerImpl(ctx, gid);
|
|
3392
|
+
}
|
|
3393
|
+
};
|
|
3394
|
+
}
|
|
3395
|
+
|
|
3396
|
+
// src/client/kafka.client/consumer/features/snapshot.ts
|
|
3024
3397
|
async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
3025
3398
|
await ctx.adminOps.ensureConnected();
|
|
3026
3399
|
let offsets;
|
|
@@ -3286,6 +3659,7 @@ var KafkaClient = class {
|
|
|
3286
3659
|
* ```
|
|
3287
3660
|
*/
|
|
3288
3661
|
constructor(clientId, groupId, brokers, options) {
|
|
3662
|
+
validateClientOptions(clientId, groupId, brokers, options);
|
|
3289
3663
|
this.clientId = clientId;
|
|
3290
3664
|
const logger = options?.logger ?? {
|
|
3291
3665
|
log: (msg) => console.log(`[KafkaClient:${clientId}] ${msg}`),
|
|
@@ -3293,7 +3667,8 @@ var KafkaClient = class {
|
|
|
3293
3667
|
error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args),
|
|
3294
3668
|
debug: (msg, ...args) => console.debug(`[KafkaClient:${clientId}] ${msg}`, ...args)
|
|
3295
3669
|
};
|
|
3296
|
-
const
|
|
3670
|
+
const security = resolveSecurityOptions(options?.security, brokers, logger);
|
|
3671
|
+
const transport = options?.transport ?? new ConfluentTransport(clientId, brokers, security);
|
|
3297
3672
|
const producer = transport.producer();
|
|
3298
3673
|
const runningConsumers = /* @__PURE__ */ new Map();
|
|
3299
3674
|
const consumers = /* @__PURE__ */ new Map();
|
|
@@ -3327,6 +3702,7 @@ var KafkaClient = class {
|
|
|
3327
3702
|
numPartitions: options?.numPartitions ?? 1,
|
|
3328
3703
|
txId: options?.transactionalId ?? `${clientId}-tx`,
|
|
3329
3704
|
clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
|
|
3705
|
+
clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
|
|
3330
3706
|
lagThrottleOpts: options?.lagThrottle,
|
|
3331
3707
|
instrumentation: options?.instrumentation ?? [],
|
|
3332
3708
|
onMessageLost: options?.onMessageLost,
|
|
@@ -3336,6 +3712,7 @@ var KafkaClient = class {
|
|
|
3336
3712
|
producer,
|
|
3337
3713
|
txProducer: void 0,
|
|
3338
3714
|
txProducerInitPromise: void 0,
|
|
3715
|
+
_txChain: Promise.resolve(),
|
|
3339
3716
|
retryTxProducers: /* @__PURE__ */ new Map(),
|
|
3340
3717
|
consumers,
|
|
3341
3718
|
runningConsumers,
|
|
@@ -3459,6 +3836,31 @@ var KafkaClient = class {
|
|
|
3459
3836
|
startRoutedConsumer(topics, routing, options) {
|
|
3460
3837
|
return startRoutedConsumerImpl(this.ctx, topics, routing, options);
|
|
3461
3838
|
}
|
|
3839
|
+
// ── Consumer: delayed delivery relay ──────────────────────────────
|
|
3840
|
+
/**
|
|
3841
|
+
* Start a relay that delivers messages produced with
|
|
3842
|
+
* `SendOptions.deliverAfterMs` from `<topic>.delayed` to their target topic
|
|
3843
|
+
* once their deadline passes.
|
|
3844
|
+
*
|
|
3845
|
+
* Forwarding is transactional (produce + source-offset commit are atomic),
|
|
3846
|
+
* so no duplicates are relayed even if the relay crashes mid-forward.
|
|
3847
|
+
* Delivery time is a lower bound — the relay must be running for delayed
|
|
3848
|
+
* messages to be delivered at all.
|
|
3849
|
+
*
|
|
3850
|
+
* @param topics Target topic name(s) whose `<topic>.delayed` staging topics to relay.
|
|
3851
|
+
* @param options Optional `groupId` override (default: `<defaultGroupId>-delayed-relay`).
|
|
3852
|
+
*
|
|
3853
|
+
* @example
|
|
3854
|
+
* ```ts
|
|
3855
|
+
* await kafka.startDelayedRelay(['orders.reminder']);
|
|
3856
|
+
* await kafka.sendMessage('orders.reminder', payload, { deliverAfterMs: 60_000 });
|
|
3857
|
+
* // → delivered to orders.reminder ~60 s later
|
|
3858
|
+
* ```
|
|
3859
|
+
*/
|
|
3860
|
+
async startDelayedRelay(topics, options) {
|
|
3861
|
+
const list = Array.isArray(topics) ? topics : [topics];
|
|
3862
|
+
return startDelayedRelayImpl(this.ctx, list, options);
|
|
3863
|
+
}
|
|
3462
3864
|
// ── Consumer: transactional EOS ───────────────────────────────────
|
|
3463
3865
|
/** @inheritDoc */
|
|
3464
3866
|
async startTransactionalConsumer(topics, handler, options = {}) {
|
|
@@ -3604,17 +4006,849 @@ var KafkaClient = class {
|
|
|
3604
4006
|
function topic(name) {
|
|
3605
4007
|
return {
|
|
3606
4008
|
/** Provide an explicit message type without a runtime schema. */
|
|
3607
|
-
type: () => ({
|
|
4009
|
+
type: () => keyable({
|
|
3608
4010
|
__topic: name,
|
|
3609
4011
|
__type: void 0
|
|
3610
4012
|
}),
|
|
3611
|
-
schema: (schema) => ({
|
|
4013
|
+
schema: (schema) => keyable({
|
|
3612
4014
|
__topic: name,
|
|
3613
4015
|
__type: void 0,
|
|
3614
4016
|
__schema: schema
|
|
3615
4017
|
})
|
|
3616
4018
|
};
|
|
3617
4019
|
}
|
|
4020
|
+
function keyable(desc) {
|
|
4021
|
+
return {
|
|
4022
|
+
...desc,
|
|
4023
|
+
key: (extractor) => ({
|
|
4024
|
+
...desc,
|
|
4025
|
+
__key: extractor
|
|
4026
|
+
})
|
|
4027
|
+
};
|
|
4028
|
+
}
|
|
4029
|
+
|
|
4030
|
+
// src/client/message/versioned-schema.ts
|
|
4031
|
+
function versionedSchema(versions, options) {
|
|
4032
|
+
const registered = Object.keys(versions).map(Number).filter((v) => Number.isInteger(v) && v > 0).sort((a, b) => a - b);
|
|
4033
|
+
if (registered.length === 0) {
|
|
4034
|
+
throw new Error(
|
|
4035
|
+
"versionedSchema: at least one schema version must be registered (keys must be positive integers)"
|
|
4036
|
+
);
|
|
4037
|
+
}
|
|
4038
|
+
const latestVersion = registered[registered.length - 1];
|
|
4039
|
+
return {
|
|
4040
|
+
async parse(data, ctx) {
|
|
4041
|
+
const version = ctx?.version ?? latestVersion;
|
|
4042
|
+
const schema = versions[version];
|
|
4043
|
+
if (!schema) {
|
|
4044
|
+
throw new Error(
|
|
4045
|
+
`versionedSchema: no schema registered for version ${version}${ctx?.topic ? ` (topic "${ctx.topic}")` : ""} \u2014 registered versions: ${registered.join(", ")}`
|
|
4046
|
+
);
|
|
4047
|
+
}
|
|
4048
|
+
const parsed = await schema.parse(data, ctx);
|
|
4049
|
+
if (version < latestVersion && options?.migrate) {
|
|
4050
|
+
return options.migrate(parsed, version, latestVersion);
|
|
4051
|
+
}
|
|
4052
|
+
return parsed;
|
|
4053
|
+
}
|
|
4054
|
+
};
|
|
4055
|
+
}
|
|
4056
|
+
|
|
4057
|
+
// src/client/message/schema-registry.ts
|
|
4058
|
+
var SchemaRegistryClient = class {
|
|
4059
|
+
constructor(options) {
|
|
4060
|
+
this.options = options;
|
|
4061
|
+
if (!options.baseUrl) {
|
|
4062
|
+
throw new Error("SchemaRegistryClient: baseUrl is required");
|
|
4063
|
+
}
|
|
4064
|
+
this.fetchFn = options.fetchFn ?? fetch;
|
|
4065
|
+
this.cacheTtlMs = options.cacheTtlMs ?? 3e5;
|
|
4066
|
+
}
|
|
4067
|
+
options;
|
|
4068
|
+
fetchFn;
|
|
4069
|
+
cacheTtlMs;
|
|
4070
|
+
latestCache = /* @__PURE__ */ new Map();
|
|
4071
|
+
headers() {
|
|
4072
|
+
const h = {
|
|
4073
|
+
"Content-Type": "application/vnd.schemaregistry.v1+json"
|
|
4074
|
+
};
|
|
4075
|
+
if (this.options.auth) {
|
|
4076
|
+
const { username, password } = this.options.auth;
|
|
4077
|
+
h["Authorization"] = "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
|
|
4078
|
+
}
|
|
4079
|
+
return h;
|
|
4080
|
+
}
|
|
4081
|
+
async request(method, path, body) {
|
|
4082
|
+
const url = `${this.options.baseUrl.replace(/\/$/, "")}${path}`;
|
|
4083
|
+
const res = await this.fetchFn(url, {
|
|
4084
|
+
method,
|
|
4085
|
+
headers: this.headers(),
|
|
4086
|
+
...body !== void 0 && { body: JSON.stringify(body) }
|
|
4087
|
+
});
|
|
4088
|
+
if (!res.ok) {
|
|
4089
|
+
const text = await res.text().catch(() => "");
|
|
4090
|
+
throw new Error(
|
|
4091
|
+
`SchemaRegistry ${method} ${path} failed: ${res.status} ${res.statusText}${text ? ` \u2014 ${text}` : ""}`
|
|
4092
|
+
);
|
|
4093
|
+
}
|
|
4094
|
+
return await res.json();
|
|
4095
|
+
}
|
|
4096
|
+
/** Fetch the latest schema registered under `subject`. Cached for `cacheTtlMs`. */
|
|
4097
|
+
async getLatestSchema(subject) {
|
|
4098
|
+
const cached = this.latestCache.get(subject);
|
|
4099
|
+
if (cached && cached.expiresAt > Date.now()) return cached.value;
|
|
4100
|
+
const raw = await this.request("GET", `/subjects/${encodeURIComponent(subject)}/versions/latest`);
|
|
4101
|
+
const value = {
|
|
4102
|
+
id: raw.id,
|
|
4103
|
+
version: raw.version,
|
|
4104
|
+
schema: raw.schema
|
|
4105
|
+
};
|
|
4106
|
+
this.latestCache.set(subject, {
|
|
4107
|
+
value,
|
|
4108
|
+
expiresAt: Date.now() + this.cacheTtlMs
|
|
4109
|
+
});
|
|
4110
|
+
return value;
|
|
4111
|
+
}
|
|
4112
|
+
/** Fetch a specific schema version of a subject. */
|
|
4113
|
+
async getSchemaVersion(subject, version) {
|
|
4114
|
+
const raw = await this.request(
|
|
4115
|
+
"GET",
|
|
4116
|
+
`/subjects/${encodeURIComponent(subject)}/versions/${version}`
|
|
4117
|
+
);
|
|
4118
|
+
return { id: raw.id, version: raw.version, schema: raw.schema };
|
|
4119
|
+
}
|
|
4120
|
+
/**
|
|
4121
|
+
* Register a schema under `subject` (idempotent — re-registering the same
|
|
4122
|
+
* schema returns the existing id). Returns the registry-assigned schema id.
|
|
4123
|
+
*/
|
|
4124
|
+
async registerSchema(subject, schema, schemaType = "JSON") {
|
|
4125
|
+
this.latestCache.delete(subject);
|
|
4126
|
+
return this.request(
|
|
4127
|
+
"POST",
|
|
4128
|
+
`/subjects/${encodeURIComponent(subject)}/versions`,
|
|
4129
|
+
{ schema, schemaType }
|
|
4130
|
+
);
|
|
4131
|
+
}
|
|
4132
|
+
/**
|
|
4133
|
+
* Test `schema` against the subject's compatibility policy without registering.
|
|
4134
|
+
* Returns `true` when the registry reports the schema as compatible.
|
|
4135
|
+
*/
|
|
4136
|
+
async checkCompatibility(subject, schema, schemaType = "JSON") {
|
|
4137
|
+
const res = await this.request(
|
|
4138
|
+
"POST",
|
|
4139
|
+
`/compatibility/subjects/${encodeURIComponent(subject)}/versions/latest`,
|
|
4140
|
+
{ schema, schemaType }
|
|
4141
|
+
);
|
|
4142
|
+
return res.is_compatible;
|
|
4143
|
+
}
|
|
4144
|
+
};
|
|
4145
|
+
function registrySchema(client, subject, options) {
|
|
4146
|
+
const enforceVersion = options?.enforceVersion ?? true;
|
|
4147
|
+
return {
|
|
4148
|
+
async parse(data, ctx) {
|
|
4149
|
+
const latest = await client.getLatestSchema(subject);
|
|
4150
|
+
if (enforceVersion && ctx?.version !== void 0 && ctx.version > latest.version) {
|
|
4151
|
+
throw new Error(
|
|
4152
|
+
`registrySchema: message version ${ctx.version} for subject "${subject}" is newer than the latest registered version ${latest.version} \u2014 register the schema before producing with it`
|
|
4153
|
+
);
|
|
4154
|
+
}
|
|
4155
|
+
if (options?.validator) {
|
|
4156
|
+
return options.validator.parse(data, ctx);
|
|
4157
|
+
}
|
|
4158
|
+
return data;
|
|
4159
|
+
}
|
|
4160
|
+
};
|
|
4161
|
+
}
|
|
4162
|
+
|
|
4163
|
+
// src/client/outbox/outbox.store.ts
|
|
4164
|
+
var InMemoryOutboxStore = class {
|
|
4165
|
+
/** Insertion-ordered rows. `published` flips to true after `markPublished`. */
|
|
4166
|
+
rows = [];
|
|
4167
|
+
/**
|
|
4168
|
+
* Append a message to the outbox. In a real store this INSERT would run inside
|
|
4169
|
+
* the same DB transaction as the corresponding business write.
|
|
4170
|
+
*/
|
|
4171
|
+
add(message) {
|
|
4172
|
+
this.rows.push({ message, published: false });
|
|
4173
|
+
}
|
|
4174
|
+
async fetchUnpublished(limit) {
|
|
4175
|
+
const out = [];
|
|
4176
|
+
for (const row of this.rows) {
|
|
4177
|
+
if (row.published) continue;
|
|
4178
|
+
out.push(row.message);
|
|
4179
|
+
if (out.length >= limit) break;
|
|
4180
|
+
}
|
|
4181
|
+
return out;
|
|
4182
|
+
}
|
|
4183
|
+
async markPublished(ids) {
|
|
4184
|
+
const idSet = new Set(ids);
|
|
4185
|
+
for (const row of this.rows) {
|
|
4186
|
+
if (idSet.has(row.message.id)) row.published = true;
|
|
4187
|
+
}
|
|
4188
|
+
}
|
|
4189
|
+
/** Test helper: count of rows not yet marked published. */
|
|
4190
|
+
get pendingCount() {
|
|
4191
|
+
return this.rows.filter((r) => !r.published).length;
|
|
4192
|
+
}
|
|
4193
|
+
/** Test helper: count of rows marked published. */
|
|
4194
|
+
get publishedCount() {
|
|
4195
|
+
return this.rows.filter((r) => r.published).length;
|
|
4196
|
+
}
|
|
4197
|
+
};
|
|
4198
|
+
|
|
4199
|
+
// src/client/outbox/outbox.relay.ts
|
|
4200
|
+
function toError2(e) {
|
|
4201
|
+
return e instanceof Error ? e : new Error(String(e));
|
|
4202
|
+
}
|
|
4203
|
+
function startOutboxRelay(kafka, store, options = {}) {
|
|
4204
|
+
const pollIntervalMs = options.pollIntervalMs ?? 1e3;
|
|
4205
|
+
const batchSize = options.batchSize ?? 100;
|
|
4206
|
+
const onError = options.onError ?? ((error, batch) => {
|
|
4207
|
+
console.error(
|
|
4208
|
+
`[outbox] batch of ${batch.length} message(s) failed \u2014 will retry:`,
|
|
4209
|
+
error
|
|
4210
|
+
);
|
|
4211
|
+
});
|
|
4212
|
+
const onPublished = options.onPublished;
|
|
4213
|
+
let stopped = false;
|
|
4214
|
+
let running = false;
|
|
4215
|
+
let inFlight = Promise.resolve();
|
|
4216
|
+
const iterate = async () => {
|
|
4217
|
+
let batch = [];
|
|
4218
|
+
try {
|
|
4219
|
+
batch = await store.fetchUnpublished(batchSize);
|
|
4220
|
+
if (batch.length === 0) return;
|
|
4221
|
+
await kafka.transaction(async (tx) => {
|
|
4222
|
+
for (const msg of batch) {
|
|
4223
|
+
await tx.send(msg.topic, msg.payload, {
|
|
4224
|
+
key: msg.key,
|
|
4225
|
+
headers: msg.headers,
|
|
4226
|
+
correlationId: msg.correlationId,
|
|
4227
|
+
eventId: msg.eventId
|
|
4228
|
+
});
|
|
4229
|
+
}
|
|
4230
|
+
});
|
|
4231
|
+
await store.markPublished(batch.map((m) => m.id));
|
|
4232
|
+
onPublished?.(batch.length);
|
|
4233
|
+
} catch (err) {
|
|
4234
|
+
onError(toError2(err), batch);
|
|
4235
|
+
}
|
|
4236
|
+
};
|
|
4237
|
+
const tick = () => {
|
|
4238
|
+
if (stopped || running) return;
|
|
4239
|
+
running = true;
|
|
4240
|
+
inFlight = iterate().finally(() => {
|
|
4241
|
+
running = false;
|
|
4242
|
+
});
|
|
4243
|
+
};
|
|
4244
|
+
const timer = setInterval(tick, pollIntervalMs);
|
|
4245
|
+
timer.unref?.();
|
|
4246
|
+
return {
|
|
4247
|
+
stop: async () => {
|
|
4248
|
+
stopped = true;
|
|
4249
|
+
clearInterval(timer);
|
|
4250
|
+
await inFlight;
|
|
4251
|
+
}
|
|
4252
|
+
};
|
|
4253
|
+
}
|
|
4254
|
+
|
|
4255
|
+
// src/client/security/providers.ts
|
|
4256
|
+
var defaultImport = (specifier) => import(specifier);
|
|
4257
|
+
function awsMskIamProvider(options) {
|
|
4258
|
+
const importFn = options.importFn ?? defaultImport;
|
|
4259
|
+
return async () => {
|
|
4260
|
+
let signer;
|
|
4261
|
+
try {
|
|
4262
|
+
signer = await importFn("aws-msk-iam-sasl-signer-js");
|
|
4263
|
+
} catch {
|
|
4264
|
+
throw new Error(
|
|
4265
|
+
"awsMskIamProvider: package 'aws-msk-iam-sasl-signer-js' is not installed. Run `npm install aws-msk-iam-sasl-signer-js` to enable MSK IAM authentication."
|
|
4266
|
+
);
|
|
4267
|
+
}
|
|
4268
|
+
const { token, expiryTime } = await signer.generateAuthToken({
|
|
4269
|
+
region: options.region
|
|
4270
|
+
});
|
|
4271
|
+
return {
|
|
4272
|
+
value: token,
|
|
4273
|
+
principal: "msk-iam",
|
|
4274
|
+
// expiryTime is epoch ms per the signer's contract
|
|
4275
|
+
lifetimeMs: expiryTime
|
|
4276
|
+
};
|
|
4277
|
+
};
|
|
4278
|
+
}
|
|
4279
|
+
function gcpAccessTokenProvider(options = {}) {
|
|
4280
|
+
const importFn = options.importFn ?? defaultImport;
|
|
4281
|
+
const ttlMs = options.tokenTtlMs ?? 50 * 6e4;
|
|
4282
|
+
return async () => {
|
|
4283
|
+
let lib;
|
|
4284
|
+
try {
|
|
4285
|
+
lib = await importFn("google-auth-library");
|
|
4286
|
+
} catch {
|
|
4287
|
+
throw new Error(
|
|
4288
|
+
"gcpAccessTokenProvider: package 'google-auth-library' is not installed. Run `npm install google-auth-library` to enable GCP authentication."
|
|
4289
|
+
);
|
|
4290
|
+
}
|
|
4291
|
+
const auth = new lib.GoogleAuth({
|
|
4292
|
+
scopes: options.scopes ?? ["https://www.googleapis.com/auth/cloud-platform"]
|
|
4293
|
+
});
|
|
4294
|
+
const token = await auth.getAccessToken();
|
|
4295
|
+
if (!token) {
|
|
4296
|
+
throw new Error(
|
|
4297
|
+
"gcpAccessTokenProvider: google-auth-library returned no access token \u2014 check Application Default Credentials."
|
|
4298
|
+
);
|
|
4299
|
+
}
|
|
4300
|
+
return {
|
|
4301
|
+
value: token,
|
|
4302
|
+
principal: options.principal ?? "gcp",
|
|
4303
|
+
lifetimeMs: Date.now() + ttlMs
|
|
4304
|
+
};
|
|
4305
|
+
};
|
|
4306
|
+
}
|
|
4307
|
+
|
|
4308
|
+
// src/client/security/acl.ts
|
|
4309
|
+
function addResource(out, r) {
|
|
4310
|
+
const key = `${r.resourceType}:${r.patternType}:${r.name}`;
|
|
4311
|
+
const existing = out.get(key);
|
|
4312
|
+
if (existing) {
|
|
4313
|
+
for (const op of r.operations)
|
|
4314
|
+
if (!existing.operations.includes(op)) existing.operations.push(op);
|
|
4315
|
+
if (!existing.reason.includes(r.reason))
|
|
4316
|
+
existing.reason += `; ${r.reason}`;
|
|
4317
|
+
} else {
|
|
4318
|
+
out.set(key, { ...r, operations: [...r.operations] });
|
|
4319
|
+
}
|
|
4320
|
+
}
|
|
4321
|
+
function describeRequiredAcls(input) {
|
|
4322
|
+
const out = /* @__PURE__ */ new Map();
|
|
4323
|
+
const f = input.features ?? {};
|
|
4324
|
+
const produce = input.produceTopics ?? [];
|
|
4325
|
+
const consume = input.consumeTopics ?? [];
|
|
4326
|
+
const groups = input.groupIds ?? [];
|
|
4327
|
+
for (const t of produce) {
|
|
4328
|
+
addResource(out, {
|
|
4329
|
+
resourceType: "topic",
|
|
4330
|
+
patternType: "literal",
|
|
4331
|
+
name: t,
|
|
4332
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4333
|
+
reason: "sendMessage/sendBatch"
|
|
4334
|
+
});
|
|
4335
|
+
}
|
|
4336
|
+
for (const t of consume) {
|
|
4337
|
+
addResource(out, {
|
|
4338
|
+
resourceType: "topic",
|
|
4339
|
+
patternType: "literal",
|
|
4340
|
+
name: t,
|
|
4341
|
+
operations: ["READ", "DESCRIBE"],
|
|
4342
|
+
reason: "startConsumer"
|
|
4343
|
+
});
|
|
4344
|
+
}
|
|
4345
|
+
for (const g of groups) {
|
|
4346
|
+
addResource(out, {
|
|
4347
|
+
resourceType: "group",
|
|
4348
|
+
patternType: "literal",
|
|
4349
|
+
name: g,
|
|
4350
|
+
operations: ["READ", "DESCRIBE"],
|
|
4351
|
+
reason: "consumer group membership + offset commits"
|
|
4352
|
+
});
|
|
4353
|
+
}
|
|
4354
|
+
if (f.dlq) {
|
|
4355
|
+
for (const t of consume) {
|
|
4356
|
+
addResource(out, {
|
|
4357
|
+
resourceType: "topic",
|
|
4358
|
+
patternType: "literal",
|
|
4359
|
+
name: `${t}.dlq`,
|
|
4360
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4361
|
+
reason: "dlq: true \u2014 failed messages routed to DLQ"
|
|
4362
|
+
});
|
|
4363
|
+
}
|
|
4364
|
+
}
|
|
4365
|
+
if (f.retryTopics) {
|
|
4366
|
+
for (const t of consume) {
|
|
4367
|
+
for (let level = 1; level <= f.retryTopics.maxRetries; level++) {
|
|
4368
|
+
addResource(out, {
|
|
4369
|
+
resourceType: "topic",
|
|
4370
|
+
patternType: "literal",
|
|
4371
|
+
name: `${t}.retry.${level}`,
|
|
4372
|
+
operations: ["READ", "WRITE", "DESCRIBE"],
|
|
4373
|
+
reason: "retryTopics \u2014 retry chain produce + companion consume"
|
|
4374
|
+
});
|
|
4375
|
+
}
|
|
4376
|
+
}
|
|
4377
|
+
for (const g of groups) {
|
|
4378
|
+
addResource(out, {
|
|
4379
|
+
resourceType: "group",
|
|
4380
|
+
patternType: "prefixed",
|
|
4381
|
+
name: `${g}-retry.`,
|
|
4382
|
+
operations: ["READ", "DESCRIBE"],
|
|
4383
|
+
reason: "retryTopics \u2014 companion retry-level consumer groups"
|
|
4384
|
+
});
|
|
4385
|
+
addResource(out, {
|
|
4386
|
+
resourceType: "transactional-id",
|
|
4387
|
+
patternType: "prefixed",
|
|
4388
|
+
name: `${g}-`,
|
|
4389
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4390
|
+
reason: "retryTopics \u2014 EOS routing transactions per retry level"
|
|
4391
|
+
});
|
|
4392
|
+
}
|
|
4393
|
+
}
|
|
4394
|
+
if (f.delayedDelivery) {
|
|
4395
|
+
for (const t of [.../* @__PURE__ */ new Set([...produce, ...consume])]) {
|
|
4396
|
+
addResource(out, {
|
|
4397
|
+
resourceType: "topic",
|
|
4398
|
+
patternType: "literal",
|
|
4399
|
+
name: `${t}.delayed`,
|
|
4400
|
+
operations: ["READ", "WRITE", "DESCRIBE"],
|
|
4401
|
+
reason: "deliverAfterMs staging + startDelayedRelay consume"
|
|
4402
|
+
});
|
|
4403
|
+
}
|
|
4404
|
+
for (const g of groups) {
|
|
4405
|
+
addResource(out, {
|
|
4406
|
+
resourceType: "group",
|
|
4407
|
+
patternType: "literal",
|
|
4408
|
+
name: `${g}-delayed-relay`,
|
|
4409
|
+
operations: ["READ", "DESCRIBE"],
|
|
4410
|
+
reason: "startDelayedRelay consumer group"
|
|
4411
|
+
});
|
|
4412
|
+
addResource(out, {
|
|
4413
|
+
resourceType: "transactional-id",
|
|
4414
|
+
patternType: "literal",
|
|
4415
|
+
name: `${g}-delayed-relay-tx`,
|
|
4416
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4417
|
+
reason: "startDelayedRelay transactional forwarding"
|
|
4418
|
+
});
|
|
4419
|
+
}
|
|
4420
|
+
}
|
|
4421
|
+
if (f.duplicatesTopic) {
|
|
4422
|
+
if (typeof f.duplicatesTopic === "string") {
|
|
4423
|
+
addResource(out, {
|
|
4424
|
+
resourceType: "topic",
|
|
4425
|
+
patternType: "literal",
|
|
4426
|
+
name: f.duplicatesTopic,
|
|
4427
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4428
|
+
reason: "deduplication.strategy 'topic' \u2014 custom duplicates topic"
|
|
4429
|
+
});
|
|
4430
|
+
} else {
|
|
4431
|
+
for (const t of consume) {
|
|
4432
|
+
addResource(out, {
|
|
4433
|
+
resourceType: "topic",
|
|
4434
|
+
patternType: "literal",
|
|
4435
|
+
name: `${t}.duplicates`,
|
|
4436
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4437
|
+
reason: "deduplication.strategy 'topic'"
|
|
4438
|
+
});
|
|
4439
|
+
}
|
|
4440
|
+
}
|
|
4441
|
+
}
|
|
4442
|
+
if (f.dlqReplay) {
|
|
4443
|
+
for (const t of consume) {
|
|
4444
|
+
addResource(out, {
|
|
4445
|
+
resourceType: "group",
|
|
4446
|
+
patternType: "prefixed",
|
|
4447
|
+
name: `${t}.dlq-replay`,
|
|
4448
|
+
operations: ["READ", "DESCRIBE", "DELETE"],
|
|
4449
|
+
reason: "replayDlq \u2014 ephemeral/stable replay groups (deleted after use)"
|
|
4450
|
+
});
|
|
4451
|
+
addResource(out, {
|
|
4452
|
+
resourceType: "topic",
|
|
4453
|
+
patternType: "literal",
|
|
4454
|
+
name: `${t}.dlq`,
|
|
4455
|
+
operations: ["READ", "DESCRIBE"],
|
|
4456
|
+
reason: "replayDlq \u2014 reads the DLQ"
|
|
4457
|
+
});
|
|
4458
|
+
}
|
|
4459
|
+
}
|
|
4460
|
+
if (f.snapshots) {
|
|
4461
|
+
addResource(out, {
|
|
4462
|
+
resourceType: "group",
|
|
4463
|
+
patternType: "prefixed",
|
|
4464
|
+
name: `${input.clientId}-snapshot-`,
|
|
4465
|
+
operations: ["READ", "DESCRIBE", "DELETE"],
|
|
4466
|
+
reason: "readSnapshot \u2014 timestamped ephemeral groups (deleted after use)"
|
|
4467
|
+
});
|
|
4468
|
+
}
|
|
4469
|
+
if (f.clockRecovery) {
|
|
4470
|
+
addResource(out, {
|
|
4471
|
+
resourceType: "group",
|
|
4472
|
+
patternType: "prefixed",
|
|
4473
|
+
name: `${input.clientId}-clock-recovery-`,
|
|
4474
|
+
operations: ["READ", "DESCRIBE", "DELETE"],
|
|
4475
|
+
reason: "clockRecovery \u2014 timestamped ephemeral groups (deleted after use)"
|
|
4476
|
+
});
|
|
4477
|
+
}
|
|
4478
|
+
if (f.transactions) {
|
|
4479
|
+
addResource(out, {
|
|
4480
|
+
resourceType: "transactional-id",
|
|
4481
|
+
patternType: "literal",
|
|
4482
|
+
name: `${input.clientId}-tx`,
|
|
4483
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4484
|
+
reason: "transaction() \u2014 default transactionalId (override-aware: adjust if you set one)"
|
|
4485
|
+
});
|
|
4486
|
+
}
|
|
4487
|
+
if (f.autoCreateTopics) {
|
|
4488
|
+
addResource(out, {
|
|
4489
|
+
resourceType: "cluster",
|
|
4490
|
+
patternType: "literal",
|
|
4491
|
+
name: "kafka-cluster",
|
|
4492
|
+
operations: ["CREATE"],
|
|
4493
|
+
reason: "autoCreateTopics: true \u2014 not recommended in production"
|
|
4494
|
+
});
|
|
4495
|
+
}
|
|
4496
|
+
return [...out.values()];
|
|
4497
|
+
}
|
|
4498
|
+
function toKafkaAclCommands(resources, principal, bootstrapServer = "<bootstrap-server>") {
|
|
4499
|
+
return resources.map((r) => {
|
|
4500
|
+
const ops = r.operations.map((o) => `--operation ${o}`).join(" ");
|
|
4501
|
+
const resourceFlag = r.resourceType === "topic" ? `--topic '${r.name}'` : r.resourceType === "group" ? `--group '${r.name}'` : r.resourceType === "transactional-id" ? `--transactional-id '${r.name}'` : "--cluster";
|
|
4502
|
+
const pattern = r.patternType === "prefixed" ? " --resource-pattern-type prefixed" : "";
|
|
4503
|
+
return `kafka-acls.sh --bootstrap-server ${bootstrapServer} --add --allow-principal '${principal}' ${ops} ${resourceFlag}${pattern} # ${r.reason}`;
|
|
4504
|
+
});
|
|
4505
|
+
}
|
|
4506
|
+
var MSK_TOPIC_ACTIONS = {
|
|
4507
|
+
READ: ["kafka-cluster:ReadData", "kafka-cluster:DescribeTopic"],
|
|
4508
|
+
WRITE: ["kafka-cluster:WriteData", "kafka-cluster:DescribeTopic"],
|
|
4509
|
+
DESCRIBE: ["kafka-cluster:DescribeTopic"],
|
|
4510
|
+
CREATE: ["kafka-cluster:CreateTopic"],
|
|
4511
|
+
DELETE: ["kafka-cluster:DeleteTopic"]
|
|
4512
|
+
};
|
|
4513
|
+
var MSK_GROUP_ACTIONS = {
|
|
4514
|
+
READ: ["kafka-cluster:AlterGroup", "kafka-cluster:DescribeGroup"],
|
|
4515
|
+
DESCRIBE: ["kafka-cluster:DescribeGroup"],
|
|
4516
|
+
DELETE: ["kafka-cluster:DeleteGroup"]
|
|
4517
|
+
};
|
|
4518
|
+
var MSK_TX_ACTIONS = {
|
|
4519
|
+
WRITE: [
|
|
4520
|
+
"kafka-cluster:AlterTransactionalId",
|
|
4521
|
+
"kafka-cluster:DescribeTransactionalId"
|
|
4522
|
+
],
|
|
4523
|
+
DESCRIBE: ["kafka-cluster:DescribeTransactionalId"]
|
|
4524
|
+
};
|
|
4525
|
+
function toMskIamPolicy(resources, cluster) {
|
|
4526
|
+
const { region, accountId, clusterName, clusterUuid } = cluster;
|
|
4527
|
+
const arn = (type, name) => `arn:aws:kafka:${region}:${accountId}:${type}/${clusterName}/${clusterUuid}/${name}`;
|
|
4528
|
+
const statements = [
|
|
4529
|
+
{
|
|
4530
|
+
Sid: "Connect",
|
|
4531
|
+
Effect: "Allow",
|
|
4532
|
+
Action: ["kafka-cluster:Connect"],
|
|
4533
|
+
Resource: [
|
|
4534
|
+
`arn:aws:kafka:${region}:${accountId}:cluster/${clusterName}/${clusterUuid}`
|
|
4535
|
+
]
|
|
4536
|
+
}
|
|
4537
|
+
];
|
|
4538
|
+
let sid = 0;
|
|
4539
|
+
for (const r of resources) {
|
|
4540
|
+
const suffix = r.patternType === "prefixed" ? `${r.name}*` : r.name;
|
|
4541
|
+
let actions = [];
|
|
4542
|
+
let resource;
|
|
4543
|
+
if (r.resourceType === "topic") {
|
|
4544
|
+
actions = [...new Set(r.operations.flatMap((o) => MSK_TOPIC_ACTIONS[o] ?? []))];
|
|
4545
|
+
resource = arn("topic", suffix);
|
|
4546
|
+
} else if (r.resourceType === "group") {
|
|
4547
|
+
actions = [...new Set(r.operations.flatMap((o) => MSK_GROUP_ACTIONS[o] ?? []))];
|
|
4548
|
+
resource = arn("group", suffix);
|
|
4549
|
+
} else if (r.resourceType === "transactional-id") {
|
|
4550
|
+
actions = [...new Set(r.operations.flatMap((o) => MSK_TX_ACTIONS[o] ?? []))];
|
|
4551
|
+
resource = arn("transactional-id", suffix);
|
|
4552
|
+
} else {
|
|
4553
|
+
actions = ["kafka-cluster:CreateTopic"];
|
|
4554
|
+
resource = `arn:aws:kafka:${region}:${accountId}:topic/${clusterName}/${clusterUuid}/*`;
|
|
4555
|
+
}
|
|
4556
|
+
if (actions.length === 0 || !resource) continue;
|
|
4557
|
+
statements.push({
|
|
4558
|
+
Sid: `Acl${sid++}`,
|
|
4559
|
+
Effect: "Allow",
|
|
4560
|
+
Action: actions,
|
|
4561
|
+
Resource: [resource]
|
|
4562
|
+
});
|
|
4563
|
+
}
|
|
4564
|
+
return { Version: "2012-10-17", Statement: statements };
|
|
4565
|
+
}
|
|
4566
|
+
|
|
4567
|
+
// src/client/config/from-env.ts
|
|
4568
|
+
var TRUE_VALUES = /* @__PURE__ */ new Set(["true", "1", "yes"]);
|
|
4569
|
+
var FALSE_VALUES = /* @__PURE__ */ new Set(["false", "0", "no"]);
|
|
4570
|
+
function parseBool(name, raw) {
|
|
4571
|
+
const normalized = raw.trim().toLowerCase();
|
|
4572
|
+
if (TRUE_VALUES.has(normalized)) return true;
|
|
4573
|
+
if (FALSE_VALUES.has(normalized)) return false;
|
|
4574
|
+
throw new Error(
|
|
4575
|
+
`Invalid boolean for ${name}: "${raw}". Use one of true/false, 1/0, yes/no (case-insensitive).`
|
|
4576
|
+
);
|
|
4577
|
+
}
|
|
4578
|
+
function parseNum(name, raw) {
|
|
4579
|
+
const value = Number(raw.trim());
|
|
4580
|
+
if (Number.isNaN(value)) {
|
|
4581
|
+
throw new Error(`Invalid number for ${name}: "${raw}".`);
|
|
4582
|
+
}
|
|
4583
|
+
return value;
|
|
4584
|
+
}
|
|
4585
|
+
function parseList(raw) {
|
|
4586
|
+
return raw.split(",").map((part) => part.trim()).filter((part) => part.length > 0);
|
|
4587
|
+
}
|
|
4588
|
+
function parseEnum(name, raw, allowed) {
|
|
4589
|
+
const value = raw.trim();
|
|
4590
|
+
if (!allowed.includes(value)) {
|
|
4591
|
+
throw new Error(
|
|
4592
|
+
`Invalid value for ${name}: "${raw}". Allowed: ${allowed.join(", ")}.`
|
|
4593
|
+
);
|
|
4594
|
+
}
|
|
4595
|
+
return value;
|
|
4596
|
+
}
|
|
4597
|
+
function readVar(env, key, apply) {
|
|
4598
|
+
const raw = env[key];
|
|
4599
|
+
if (raw === void 0 || raw.trim() === "") return;
|
|
4600
|
+
apply(raw);
|
|
4601
|
+
}
|
|
4602
|
+
function kafkaClientConfigFromEnv(env = process.env, prefix = "KAFKA_") {
|
|
4603
|
+
const options = {};
|
|
4604
|
+
const result = { options };
|
|
4605
|
+
readVar(env, `${prefix}CLIENT_ID`, (raw) => {
|
|
4606
|
+
result.clientId = raw.trim();
|
|
4607
|
+
});
|
|
4608
|
+
readVar(env, `${prefix}GROUP_ID`, (raw) => {
|
|
4609
|
+
result.groupId = raw.trim();
|
|
4610
|
+
});
|
|
4611
|
+
readVar(env, `${prefix}BROKERS`, (raw) => {
|
|
4612
|
+
result.brokers = parseList(raw);
|
|
4613
|
+
});
|
|
4614
|
+
readVar(env, `${prefix}AUTO_CREATE_TOPICS`, (raw) => {
|
|
4615
|
+
options.autoCreateTopics = parseBool(`${prefix}AUTO_CREATE_TOPICS`, raw);
|
|
4616
|
+
});
|
|
4617
|
+
readVar(env, `${prefix}STRICT_SCHEMAS`, (raw) => {
|
|
4618
|
+
options.strictSchemas = parseBool(`${prefix}STRICT_SCHEMAS`, raw);
|
|
4619
|
+
});
|
|
4620
|
+
readVar(env, `${prefix}NUM_PARTITIONS`, (raw) => {
|
|
4621
|
+
options.numPartitions = parseNum(`${prefix}NUM_PARTITIONS`, raw);
|
|
4622
|
+
});
|
|
4623
|
+
readVar(env, `${prefix}TRANSACTIONAL_ID`, (raw) => {
|
|
4624
|
+
options.transactionalId = raw.trim();
|
|
4625
|
+
});
|
|
4626
|
+
readVar(env, `${prefix}CLOCK_RECOVERY_TOPICS`, (raw) => {
|
|
4627
|
+
const topics = parseList(raw);
|
|
4628
|
+
if (topics.length === 0) return;
|
|
4629
|
+
options.clockRecovery = { topics };
|
|
4630
|
+
});
|
|
4631
|
+
readVar(env, `${prefix}CLOCK_RECOVERY_TIMEOUT_MS`, (raw) => {
|
|
4632
|
+
const timeoutMs = parseNum(`${prefix}CLOCK_RECOVERY_TIMEOUT_MS`, raw);
|
|
4633
|
+
if (options.clockRecovery) {
|
|
4634
|
+
options.clockRecovery.timeoutMs = timeoutMs;
|
|
4635
|
+
}
|
|
4636
|
+
});
|
|
4637
|
+
readVar(env, `${prefix}LAG_THROTTLE_MAX_LAG`, (raw) => {
|
|
4638
|
+
options.lagThrottle = {
|
|
4639
|
+
maxLag: parseNum(`${prefix}LAG_THROTTLE_MAX_LAG`, raw)
|
|
4640
|
+
};
|
|
4641
|
+
});
|
|
4642
|
+
readVar(env, `${prefix}LAG_THROTTLE_GROUP_ID`, (raw) => {
|
|
4643
|
+
if (options.lagThrottle) options.lagThrottle.groupId = raw.trim();
|
|
4644
|
+
});
|
|
4645
|
+
readVar(env, `${prefix}LAG_THROTTLE_POLL_INTERVAL_MS`, (raw) => {
|
|
4646
|
+
if (options.lagThrottle) {
|
|
4647
|
+
options.lagThrottle.pollIntervalMs = parseNum(
|
|
4648
|
+
`${prefix}LAG_THROTTLE_POLL_INTERVAL_MS`,
|
|
4649
|
+
raw
|
|
4650
|
+
);
|
|
4651
|
+
}
|
|
4652
|
+
});
|
|
4653
|
+
readVar(env, `${prefix}LAG_THROTTLE_MAX_WAIT_MS`, (raw) => {
|
|
4654
|
+
if (options.lagThrottle) {
|
|
4655
|
+
options.lagThrottle.maxWaitMs = parseNum(
|
|
4656
|
+
`${prefix}LAG_THROTTLE_MAX_WAIT_MS`,
|
|
4657
|
+
raw
|
|
4658
|
+
);
|
|
4659
|
+
}
|
|
4660
|
+
});
|
|
4661
|
+
const security = securityFromEnv(env, prefix);
|
|
4662
|
+
if (security) options.security = security;
|
|
4663
|
+
return result;
|
|
4664
|
+
}
|
|
4665
|
+
function securityFromEnv(env, prefix) {
|
|
4666
|
+
let ssl;
|
|
4667
|
+
let allowInsecure;
|
|
4668
|
+
let mechanism;
|
|
4669
|
+
let username;
|
|
4670
|
+
let password;
|
|
4671
|
+
readVar(env, `${prefix}SSL`, (raw) => {
|
|
4672
|
+
ssl = parseBool(`${prefix}SSL`, raw);
|
|
4673
|
+
});
|
|
4674
|
+
readVar(env, `${prefix}ALLOW_INSECURE`, (raw) => {
|
|
4675
|
+
allowInsecure = parseBool(`${prefix}ALLOW_INSECURE`, raw);
|
|
4676
|
+
});
|
|
4677
|
+
readVar(env, `${prefix}SASL_MECHANISM`, (raw) => {
|
|
4678
|
+
mechanism = parseEnum(`${prefix}SASL_MECHANISM`, raw, [
|
|
4679
|
+
"plain",
|
|
4680
|
+
"scram-sha-256",
|
|
4681
|
+
"scram-sha-512"
|
|
4682
|
+
]);
|
|
4683
|
+
});
|
|
4684
|
+
readVar(env, `${prefix}SASL_USERNAME`, (raw) => {
|
|
4685
|
+
username = raw.trim();
|
|
4686
|
+
});
|
|
4687
|
+
readVar(env, `${prefix}SASL_PASSWORD`, (raw) => {
|
|
4688
|
+
password = raw;
|
|
4689
|
+
});
|
|
4690
|
+
if (ssl === void 0 && allowInsecure === void 0 && mechanism === void 0 && username === void 0 && password === void 0) {
|
|
4691
|
+
return void 0;
|
|
4692
|
+
}
|
|
4693
|
+
const security = {};
|
|
4694
|
+
if (ssl !== void 0) security.ssl = ssl;
|
|
4695
|
+
if (allowInsecure !== void 0) security.allowInsecure = allowInsecure;
|
|
4696
|
+
if (mechanism !== void 0 || username !== void 0 || password !== void 0) {
|
|
4697
|
+
if (mechanism === void 0 || username === void 0 || password === void 0) {
|
|
4698
|
+
throw new Error(
|
|
4699
|
+
`Incomplete SASL configuration: ${prefix}SASL_MECHANISM, ${prefix}SASL_USERNAME, and ${prefix}SASL_PASSWORD must all be set together (oauthbearer must be configured in code).`
|
|
4700
|
+
);
|
|
4701
|
+
}
|
|
4702
|
+
const sasl = { mechanism, username, password };
|
|
4703
|
+
security.sasl = sasl;
|
|
4704
|
+
}
|
|
4705
|
+
return security;
|
|
4706
|
+
}
|
|
4707
|
+
function consumerOptionsFromEnv(env = process.env, prefix = "KAFKA_CONSUMER_") {
|
|
4708
|
+
const options = {};
|
|
4709
|
+
readVar(env, `${prefix}GROUP_ID`, (raw) => {
|
|
4710
|
+
options.groupId = raw.trim();
|
|
4711
|
+
});
|
|
4712
|
+
readVar(env, `${prefix}FROM_BEGINNING`, (raw) => {
|
|
4713
|
+
options.fromBeginning = parseBool(`${prefix}FROM_BEGINNING`, raw);
|
|
4714
|
+
});
|
|
4715
|
+
readVar(env, `${prefix}AUTO_COMMIT`, (raw) => {
|
|
4716
|
+
options.autoCommit = parseBool(`${prefix}AUTO_COMMIT`, raw);
|
|
4717
|
+
});
|
|
4718
|
+
readVar(env, `${prefix}DLQ`, (raw) => {
|
|
4719
|
+
options.dlq = parseBool(`${prefix}DLQ`, raw);
|
|
4720
|
+
});
|
|
4721
|
+
readVar(env, `${prefix}RETRY_MAX_RETRIES`, (raw) => {
|
|
4722
|
+
const retry = {
|
|
4723
|
+
maxRetries: parseNum(`${prefix}RETRY_MAX_RETRIES`, raw)
|
|
4724
|
+
};
|
|
4725
|
+
options.retry = retry;
|
|
4726
|
+
});
|
|
4727
|
+
readVar(env, `${prefix}RETRY_BACKOFF_MS`, (raw) => {
|
|
4728
|
+
if (options.retry) {
|
|
4729
|
+
options.retry.backoffMs = parseNum(`${prefix}RETRY_BACKOFF_MS`, raw);
|
|
4730
|
+
}
|
|
4731
|
+
});
|
|
4732
|
+
readVar(env, `${prefix}RETRY_MAX_BACKOFF_MS`, (raw) => {
|
|
4733
|
+
if (options.retry) {
|
|
4734
|
+
options.retry.maxBackoffMs = parseNum(`${prefix}RETRY_MAX_BACKOFF_MS`, raw);
|
|
4735
|
+
}
|
|
4736
|
+
});
|
|
4737
|
+
readVar(env, `${prefix}RETRY_TOPICS`, (raw) => {
|
|
4738
|
+
options.retryTopics = parseBool(`${prefix}RETRY_TOPICS`, raw);
|
|
4739
|
+
});
|
|
4740
|
+
readVar(env, `${prefix}RETRY_TOPIC_ASSIGNMENT_TIMEOUT_MS`, (raw) => {
|
|
4741
|
+
options.retryTopicAssignmentTimeoutMs = parseNum(
|
|
4742
|
+
`${prefix}RETRY_TOPIC_ASSIGNMENT_TIMEOUT_MS`,
|
|
4743
|
+
raw
|
|
4744
|
+
);
|
|
4745
|
+
});
|
|
4746
|
+
readVar(env, `${prefix}HANDLER_TIMEOUT_MS`, (raw) => {
|
|
4747
|
+
options.handlerTimeoutMs = parseNum(`${prefix}HANDLER_TIMEOUT_MS`, raw);
|
|
4748
|
+
});
|
|
4749
|
+
readVar(env, `${prefix}MESSAGE_TTL_MS`, (raw) => {
|
|
4750
|
+
options.messageTtlMs = parseNum(`${prefix}MESSAGE_TTL_MS`, raw);
|
|
4751
|
+
});
|
|
4752
|
+
readVar(env, `${prefix}DEDUPLICATION_STRATEGY`, (raw) => {
|
|
4753
|
+
const strategy = parseEnum(`${prefix}DEDUPLICATION_STRATEGY`, raw, [
|
|
4754
|
+
"drop",
|
|
4755
|
+
"dlq",
|
|
4756
|
+
"topic"
|
|
4757
|
+
]);
|
|
4758
|
+
const dedup = { strategy };
|
|
4759
|
+
options.deduplication = dedup;
|
|
4760
|
+
});
|
|
4761
|
+
readVar(env, `${prefix}DEDUPLICATION_TOPIC`, (raw) => {
|
|
4762
|
+
if (options.deduplication) {
|
|
4763
|
+
options.deduplication.duplicatesTopic = raw.trim();
|
|
4764
|
+
}
|
|
4765
|
+
});
|
|
4766
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_THRESHOLD`, (raw) => {
|
|
4767
|
+
const cb = {
|
|
4768
|
+
threshold: parseNum(`${prefix}CIRCUIT_BREAKER_THRESHOLD`, raw)
|
|
4769
|
+
};
|
|
4770
|
+
options.circuitBreaker = cb;
|
|
4771
|
+
});
|
|
4772
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_RECOVERY_MS`, (raw) => {
|
|
4773
|
+
if (options.circuitBreaker) {
|
|
4774
|
+
options.circuitBreaker.recoveryMs = parseNum(
|
|
4775
|
+
`${prefix}CIRCUIT_BREAKER_RECOVERY_MS`,
|
|
4776
|
+
raw
|
|
4777
|
+
);
|
|
4778
|
+
}
|
|
4779
|
+
});
|
|
4780
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_WINDOW_SIZE`, (raw) => {
|
|
4781
|
+
if (options.circuitBreaker) {
|
|
4782
|
+
options.circuitBreaker.windowSize = parseNum(
|
|
4783
|
+
`${prefix}CIRCUIT_BREAKER_WINDOW_SIZE`,
|
|
4784
|
+
raw
|
|
4785
|
+
);
|
|
4786
|
+
}
|
|
4787
|
+
});
|
|
4788
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_HALF_OPEN_SUCCESSES`, (raw) => {
|
|
4789
|
+
if (options.circuitBreaker) {
|
|
4790
|
+
options.circuitBreaker.halfOpenSuccesses = parseNum(
|
|
4791
|
+
`${prefix}CIRCUIT_BREAKER_HALF_OPEN_SUCCESSES`,
|
|
4792
|
+
raw
|
|
4793
|
+
);
|
|
4794
|
+
}
|
|
4795
|
+
});
|
|
4796
|
+
readVar(env, `${prefix}QUEUE_HIGH_WATER_MARK`, (raw) => {
|
|
4797
|
+
options.queueHighWaterMark = parseNum(`${prefix}QUEUE_HIGH_WATER_MARK`, raw);
|
|
4798
|
+
});
|
|
4799
|
+
readVar(env, `${prefix}PARTITION_ASSIGNER`, (raw) => {
|
|
4800
|
+
options.partitionAssigner = parseEnum(`${prefix}PARTITION_ASSIGNER`, raw, [
|
|
4801
|
+
"roundrobin",
|
|
4802
|
+
"range",
|
|
4803
|
+
"cooperative-sticky"
|
|
4804
|
+
]);
|
|
4805
|
+
});
|
|
4806
|
+
readVar(env, `${prefix}GROUP_INSTANCE_ID`, (raw) => {
|
|
4807
|
+
options.groupInstanceId = raw.trim();
|
|
4808
|
+
});
|
|
4809
|
+
readVar(env, `${prefix}SUBSCRIBE_RETRY_RETRIES`, (raw) => {
|
|
4810
|
+
const subscribeRetry = {
|
|
4811
|
+
retries: parseNum(`${prefix}SUBSCRIBE_RETRY_RETRIES`, raw)
|
|
4812
|
+
};
|
|
4813
|
+
options.subscribeRetry = subscribeRetry;
|
|
4814
|
+
});
|
|
4815
|
+
readVar(env, `${prefix}SUBSCRIBE_RETRY_DELAY_MS`, (raw) => {
|
|
4816
|
+
if (options.subscribeRetry) {
|
|
4817
|
+
options.subscribeRetry.backoffMs = parseNum(
|
|
4818
|
+
`${prefix}SUBSCRIBE_RETRY_DELAY_MS`,
|
|
4819
|
+
raw
|
|
4820
|
+
);
|
|
4821
|
+
}
|
|
4822
|
+
});
|
|
4823
|
+
return options;
|
|
4824
|
+
}
|
|
4825
|
+
var NESTED_CONSUMER_KEYS = [
|
|
4826
|
+
"retry",
|
|
4827
|
+
"deduplication",
|
|
4828
|
+
"circuitBreaker",
|
|
4829
|
+
"subscribeRetry"
|
|
4830
|
+
];
|
|
4831
|
+
function isPlainObject(value) {
|
|
4832
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4833
|
+
}
|
|
4834
|
+
function mergeConsumerOptions(...layers) {
|
|
4835
|
+
const result = {};
|
|
4836
|
+
for (const layer of layers) {
|
|
4837
|
+
if (!layer) continue;
|
|
4838
|
+
for (const [key, value] of Object.entries(layer)) {
|
|
4839
|
+
if (value === void 0) continue;
|
|
4840
|
+
if (NESTED_CONSUMER_KEYS.includes(key) && isPlainObject(value) && isPlainObject(result[key])) {
|
|
4841
|
+
result[key] = {
|
|
4842
|
+
...result[key],
|
|
4843
|
+
...value
|
|
4844
|
+
};
|
|
4845
|
+
} else {
|
|
4846
|
+
result[key] = value;
|
|
4847
|
+
}
|
|
4848
|
+
}
|
|
4849
|
+
}
|
|
4850
|
+
return result;
|
|
4851
|
+
}
|
|
3618
4852
|
|
|
3619
4853
|
// src/nest/kafka.module.ts
|
|
3620
4854
|
var import_common3 = require("@nestjs/common");
|
|
@@ -3663,11 +4897,14 @@ var SubscribeTo = (topics, options) => {
|
|
|
3663
4897
|
};
|
|
3664
4898
|
|
|
3665
4899
|
// src/nest/kafka.explorer.ts
|
|
4900
|
+
var wiredSubscriptions = /* @__PURE__ */ new WeakMap();
|
|
3666
4901
|
var KafkaExplorer = class {
|
|
3667
4902
|
constructor(discoveryService, moduleRef) {
|
|
3668
4903
|
this.discoveryService = discoveryService;
|
|
3669
4904
|
this.moduleRef = moduleRef;
|
|
3670
4905
|
}
|
|
4906
|
+
discoveryService;
|
|
4907
|
+
moduleRef;
|
|
3671
4908
|
logger = new import_common2.Logger(KafkaExplorer.name);
|
|
3672
4909
|
/**
|
|
3673
4910
|
* Scan all NestJS providers for `@SubscribeTo()` metadata and wire each decorated
|
|
@@ -3687,6 +4924,14 @@ var KafkaExplorer = class {
|
|
|
3687
4924
|
if (!metadata || metadata.length === 0) continue;
|
|
3688
4925
|
for (const entry of metadata) {
|
|
3689
4926
|
const token = getKafkaClientToken(entry.clientName);
|
|
4927
|
+
const entryKey = `${token}:${String(entry.methodName)}`;
|
|
4928
|
+
let wired = wiredSubscriptions.get(instance);
|
|
4929
|
+
if (!wired) {
|
|
4930
|
+
wired = /* @__PURE__ */ new Set();
|
|
4931
|
+
wiredSubscriptions.set(instance, wired);
|
|
4932
|
+
}
|
|
4933
|
+
if (wired.has(entryKey)) continue;
|
|
4934
|
+
wired.add(entryKey);
|
|
3690
4935
|
let client;
|
|
3691
4936
|
try {
|
|
3692
4937
|
client = this.moduleRef.get(token, { strict: false });
|
|
@@ -3776,6 +5021,12 @@ var KafkaModule = class {
|
|
|
3776
5021
|
instrumentation: options.instrumentation,
|
|
3777
5022
|
onMessageLost: options.onMessageLost,
|
|
3778
5023
|
onRebalance: options.onRebalance,
|
|
5024
|
+
transactionalId: options.transactionalId,
|
|
5025
|
+
clockRecovery: options.clockRecovery,
|
|
5026
|
+
lagThrottle: options.lagThrottle,
|
|
5027
|
+
onTtlExpired: options.onTtlExpired,
|
|
5028
|
+
transport: options.transport,
|
|
5029
|
+
security: options.security,
|
|
3779
5030
|
logger: new import_common3.Logger(`KafkaClient:${options.clientId}`)
|
|
3780
5031
|
}
|
|
3781
5032
|
);
|
|
@@ -3800,11 +5051,15 @@ KafkaHealthIndicator = __decorateClass([
|
|
|
3800
5051
|
// Annotate the CommonJS export names for ESM import in node:
|
|
3801
5052
|
0 && (module.exports = {
|
|
3802
5053
|
HEADER_CORRELATION_ID,
|
|
5054
|
+
HEADER_DELAYED_TARGET,
|
|
5055
|
+
HEADER_DELAYED_UNTIL,
|
|
3803
5056
|
HEADER_EVENT_ID,
|
|
3804
5057
|
HEADER_LAMPORT_CLOCK,
|
|
3805
5058
|
HEADER_SCHEMA_VERSION,
|
|
3806
5059
|
HEADER_TIMESTAMP,
|
|
3807
5060
|
HEADER_TRACEPARENT,
|
|
5061
|
+
InMemoryDedupStore,
|
|
5062
|
+
InMemoryOutboxStore,
|
|
3808
5063
|
InjectKafkaClient,
|
|
3809
5064
|
KAFKA_CLIENT,
|
|
3810
5065
|
KAFKA_SUBSCRIBER_METADATA,
|
|
@@ -3815,13 +5070,27 @@ KafkaHealthIndicator = __decorateClass([
|
|
|
3815
5070
|
KafkaProcessingError,
|
|
3816
5071
|
KafkaRetryExhaustedError,
|
|
3817
5072
|
KafkaValidationError,
|
|
5073
|
+
SchemaRegistryClient,
|
|
3818
5074
|
SubscribeTo,
|
|
5075
|
+
awsMskIamProvider,
|
|
3819
5076
|
buildEnvelopeHeaders,
|
|
5077
|
+
consumerOptionsFromEnv,
|
|
3820
5078
|
decodeHeaders,
|
|
5079
|
+
describeRequiredAcls,
|
|
3821
5080
|
extractEnvelope,
|
|
5081
|
+
gcpAccessTokenProvider,
|
|
3822
5082
|
getEnvelopeContext,
|
|
3823
5083
|
getKafkaClientToken,
|
|
5084
|
+
kafkaClientConfigFromEnv,
|
|
5085
|
+
mergeConsumerOptions,
|
|
5086
|
+
registrySchema,
|
|
5087
|
+
resolveSecurityOptions,
|
|
3824
5088
|
runWithEnvelopeContext,
|
|
3825
|
-
|
|
5089
|
+
startOutboxRelay,
|
|
5090
|
+
toError,
|
|
5091
|
+
toKafkaAclCommands,
|
|
5092
|
+
toMskIamPolicy,
|
|
5093
|
+
topic,
|
|
5094
|
+
versionedSchema
|
|
3826
5095
|
});
|
|
3827
5096
|
//# sourceMappingURL=index.js.map
|