@drarzter/kafka-client 0.9.4 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (180) hide show
  1. package/README.md +693 -8
  2. package/dist/chunk-OR7TPAAE.mjs +4760 -0
  3. package/dist/chunk-OR7TPAAE.mjs.map +1 -0
  4. package/dist/chunk-PQVBRDNV.mjs +149 -0
  5. package/dist/chunk-PQVBRDNV.mjs.map +1 -0
  6. package/dist/cli/dlq.d.ts +119 -0
  7. package/dist/cli/dlq.d.ts.map +1 -0
  8. package/dist/cli/index.d.ts +3 -0
  9. package/dist/cli/index.d.ts.map +1 -0
  10. package/dist/{chunk-SM4FZKAZ.mjs → cli/index.js} +1073 -309
  11. package/dist/cli/index.js.map +1 -0
  12. package/dist/cli/index.mjs +356 -0
  13. package/dist/cli/index.mjs.map +1 -0
  14. package/dist/client/config/from-env.d.ts +188 -0
  15. package/dist/client/config/from-env.d.ts.map +1 -0
  16. package/dist/client/config/index.d.ts +2 -0
  17. package/dist/client/config/index.d.ts.map +1 -0
  18. package/dist/client/errors.d.ts +67 -0
  19. package/dist/client/errors.d.ts.map +1 -0
  20. package/dist/client/kafka.client/admin/ops.d.ts +114 -0
  21. package/dist/client/kafka.client/admin/ops.d.ts.map +1 -0
  22. package/dist/client/kafka.client/consumer/features/delayed.d.ts +24 -0
  23. package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -0
  24. package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +52 -0
  25. package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -0
  26. package/dist/client/kafka.client/consumer/features/routed.d.ts +4 -0
  27. package/dist/client/kafka.client/consumer/features/routed.d.ts.map +1 -0
  28. package/dist/client/kafka.client/consumer/features/snapshot.d.ts +10 -0
  29. package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -0
  30. package/dist/client/kafka.client/consumer/features/window.d.ts +5 -0
  31. package/dist/client/kafka.client/consumer/features/window.d.ts.map +1 -0
  32. package/dist/client/kafka.client/consumer/handler.d.ts +163 -0
  33. package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -0
  34. package/dist/client/kafka.client/consumer/ops.d.ts +64 -0
  35. package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -0
  36. package/dist/client/kafka.client/consumer/pipeline.d.ts +168 -0
  37. package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -0
  38. package/dist/client/kafka.client/consumer/queue.d.ts +37 -0
  39. package/dist/client/kafka.client/consumer/queue.d.ts.map +1 -0
  40. package/dist/client/kafka.client/consumer/retry-topic.d.ts +68 -0
  41. package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -0
  42. package/dist/client/kafka.client/consumer/setup.d.ts +66 -0
  43. package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -0
  44. package/dist/client/kafka.client/consumer/start.d.ts +7 -0
  45. package/dist/client/kafka.client/consumer/start.d.ts.map +1 -0
  46. package/dist/client/kafka.client/consumer/stop.d.ts +19 -0
  47. package/dist/client/kafka.client/consumer/stop.d.ts.map +1 -0
  48. package/dist/client/kafka.client/consumer/subscribe-retry.d.ts +4 -0
  49. package/dist/client/kafka.client/consumer/subscribe-retry.d.ts.map +1 -0
  50. package/dist/client/kafka.client/context.d.ts +75 -0
  51. package/dist/client/kafka.client/context.d.ts.map +1 -0
  52. package/dist/client/kafka.client/index.d.ts +155 -0
  53. package/dist/client/kafka.client/index.d.ts.map +1 -0
  54. package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts +61 -0
  55. package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts.map +1 -0
  56. package/dist/client/kafka.client/infra/dedup.store.d.ts +28 -0
  57. package/dist/client/kafka.client/infra/dedup.store.d.ts.map +1 -0
  58. package/dist/client/kafka.client/infra/inflight.tracker.d.ts +22 -0
  59. package/dist/client/kafka.client/infra/inflight.tracker.d.ts.map +1 -0
  60. package/dist/client/kafka.client/infra/metrics.manager.d.ts +67 -0
  61. package/dist/client/kafka.client/infra/metrics.manager.d.ts.map +1 -0
  62. package/dist/client/kafka.client/producer/lifecycle.d.ts +41 -0
  63. package/dist/client/kafka.client/producer/lifecycle.d.ts.map +1 -0
  64. package/dist/client/kafka.client/producer/ops.d.ts +79 -0
  65. package/dist/client/kafka.client/producer/ops.d.ts.map +1 -0
  66. package/dist/client/kafka.client/producer/send.d.ts +21 -0
  67. package/dist/client/kafka.client/producer/send.d.ts.map +1 -0
  68. package/dist/client/kafka.client/validate-options.d.ts +11 -0
  69. package/dist/client/kafka.client/validate-options.d.ts.map +1 -0
  70. package/dist/client/message/envelope.d.ts +105 -0
  71. package/dist/client/message/envelope.d.ts.map +1 -0
  72. package/dist/client/message/schema-registry.d.ts +124 -0
  73. package/dist/client/message/schema-registry.d.ts.map +1 -0
  74. package/dist/client/message/serde.d.ts +68 -0
  75. package/dist/client/message/serde.d.ts.map +1 -0
  76. package/dist/client/message/topic.d.ts +159 -0
  77. package/dist/client/message/topic.d.ts.map +1 -0
  78. package/dist/client/message/versioned-schema.d.ts +53 -0
  79. package/dist/client/message/versioned-schema.d.ts.map +1 -0
  80. package/dist/client/outbox/index.d.ts +4 -0
  81. package/dist/client/outbox/index.d.ts.map +1 -0
  82. package/dist/client/outbox/outbox.relay.d.ts +90 -0
  83. package/dist/client/outbox/outbox.relay.d.ts.map +1 -0
  84. package/dist/client/outbox/outbox.store.d.ts +42 -0
  85. package/dist/client/outbox/outbox.store.d.ts.map +1 -0
  86. package/dist/client/outbox/outbox.types.d.ts +144 -0
  87. package/dist/client/outbox/outbox.types.d.ts.map +1 -0
  88. package/dist/client/security/acl.d.ts +108 -0
  89. package/dist/client/security/acl.d.ts.map +1 -0
  90. package/dist/client/security/index.d.ts +5 -0
  91. package/dist/client/security/index.d.ts.map +1 -0
  92. package/dist/client/security/providers.d.ts +88 -0
  93. package/dist/client/security/providers.d.ts.map +1 -0
  94. package/dist/client/security/resolve-security.d.ts +19 -0
  95. package/dist/client/security/resolve-security.d.ts.map +1 -0
  96. package/dist/client/security/security.types.d.ts +76 -0
  97. package/dist/client/security/security.types.d.ts.map +1 -0
  98. package/dist/client/transport/confluent.transport.d.ts +32 -0
  99. package/dist/client/transport/confluent.transport.d.ts.map +1 -0
  100. package/dist/client/transport/transport.interface.d.ts +221 -0
  101. package/dist/client/transport/transport.interface.d.ts.map +1 -0
  102. package/dist/client/types/admin.interface.d.ts +174 -0
  103. package/dist/client/types/admin.interface.d.ts.map +1 -0
  104. package/dist/client/types/admin.types.d.ts +140 -0
  105. package/dist/client/types/admin.types.d.ts.map +1 -0
  106. package/dist/client/types/client.d.ts +21 -0
  107. package/dist/client/types/client.d.ts.map +1 -0
  108. package/dist/client/types/common.d.ts +84 -0
  109. package/dist/client/types/common.d.ts.map +1 -0
  110. package/dist/client/types/config.types.d.ts +167 -0
  111. package/dist/client/types/config.types.d.ts.map +1 -0
  112. package/dist/client/types/consumer.interface.d.ts +115 -0
  113. package/dist/client/types/consumer.interface.d.ts.map +1 -0
  114. package/dist/{consumer.types-fFCag3VJ.d.mts → client/types/consumer.types.d.ts} +62 -383
  115. package/dist/client/types/consumer.types.d.ts.map +1 -0
  116. package/dist/client/types/dedup.types.d.ts +50 -0
  117. package/dist/client/types/dedup.types.d.ts.map +1 -0
  118. package/dist/client/types/lifecycle.interface.d.ts +72 -0
  119. package/dist/client/types/lifecycle.interface.d.ts.map +1 -0
  120. package/dist/client/types/producer.interface.d.ts +52 -0
  121. package/dist/client/types/producer.interface.d.ts.map +1 -0
  122. package/dist/client/types/producer.types.d.ts +90 -0
  123. package/dist/client/types/producer.types.d.ts.map +1 -0
  124. package/dist/client/types.d.ts +8 -0
  125. package/dist/client/types.d.ts.map +1 -0
  126. package/dist/core.d.ts +13 -314
  127. package/dist/core.d.ts.map +1 -0
  128. package/dist/core.js +1466 -123
  129. package/dist/core.js.map +1 -1
  130. package/dist/core.mjs +45 -3
  131. package/dist/index.d.ts +7 -128
  132. package/dist/index.d.ts.map +1 -0
  133. package/dist/index.js +1483 -123
  134. package/dist/index.js.map +1 -1
  135. package/dist/index.mjs +62 -3
  136. package/dist/index.mjs.map +1 -1
  137. package/dist/nest/kafka.constants.d.ts +5 -0
  138. package/dist/nest/kafka.constants.d.ts.map +1 -0
  139. package/dist/nest/kafka.decorator.d.ts +49 -0
  140. package/dist/nest/kafka.decorator.d.ts.map +1 -0
  141. package/dist/nest/kafka.explorer.d.ts +17 -0
  142. package/dist/nest/kafka.explorer.d.ts.map +1 -0
  143. package/dist/nest/kafka.health.d.ts +7 -0
  144. package/dist/nest/kafka.health.d.ts.map +1 -0
  145. package/dist/nest/kafka.module.d.ts +61 -0
  146. package/dist/nest/kafka.module.d.ts.map +1 -0
  147. package/dist/otel.d.ts +83 -5
  148. package/dist/otel.d.ts.map +1 -0
  149. package/dist/otel.js +100 -6
  150. package/dist/otel.js.map +1 -1
  151. package/dist/otel.mjs +98 -5
  152. package/dist/otel.mjs.map +1 -1
  153. package/dist/serde.d.ts +157 -0
  154. package/dist/serde.d.ts.map +1 -0
  155. package/dist/serde.js +308 -0
  156. package/dist/serde.js.map +1 -0
  157. package/dist/serde.mjs +158 -0
  158. package/dist/serde.mjs.map +1 -0
  159. package/dist/testing/client.mock.d.ts +47 -0
  160. package/dist/testing/client.mock.d.ts.map +1 -0
  161. package/dist/testing/index.d.ts +4 -0
  162. package/dist/testing/index.d.ts.map +1 -0
  163. package/dist/testing/test.container.d.ts +63 -0
  164. package/dist/testing/test.container.d.ts.map +1 -0
  165. package/dist/{testing.d.mts → testing/transport.fake.d.ts} +7 -111
  166. package/dist/testing/transport.fake.d.ts.map +1 -0
  167. package/dist/testing.d.ts +2 -318
  168. package/dist/testing.d.ts.map +1 -0
  169. package/dist/testing.js +26 -0
  170. package/dist/testing.js.map +1 -1
  171. package/dist/testing.mjs +26 -0
  172. package/dist/testing.mjs.map +1 -1
  173. package/package.json +40 -8
  174. package/dist/chunk-SM4FZKAZ.mjs.map +0 -1
  175. package/dist/client-1irhGEu0.d.mts +0 -751
  176. package/dist/client-BpFjkHhr.d.ts +0 -751
  177. package/dist/consumer.types-fFCag3VJ.d.ts +0 -958
  178. package/dist/core.d.mts +0 -314
  179. package/dist/index.d.mts +0 -128
  180. package/dist/otel.d.mts +0 -27
@@ -1,10 +1,14 @@
1
- // src/client/kafka.client/confluent-transport.ts
2
- import { KafkaJS } from "@confluentinc/kafka-javascript";
3
- var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = KafkaJS;
1
+ #!/usr/bin/env node
2
+ "use strict";
3
+
4
+ // src/client/transport/confluent.transport.ts
5
+ var import_kafka_javascript = require("@confluentinc/kafka-javascript");
6
+ var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = import_kafka_javascript.KafkaJS;
4
7
  var ConfluentTransaction = class {
5
8
  constructor(tx) {
6
9
  this.tx = tx;
7
10
  }
11
+ tx;
8
12
  async send(record) {
9
13
  await this.tx.send(record);
10
14
  }
@@ -26,10 +30,17 @@ var ConfluentProducer = class {
26
30
  constructor(producer) {
27
31
  this.producer = producer;
28
32
  }
33
+ producer;
34
+ connectPromise;
29
35
  async connect() {
30
- await this.producer.connect();
36
+ this.connectPromise ??= this.producer.connect().catch((err) => {
37
+ this.connectPromise = void 0;
38
+ throw err;
39
+ });
40
+ return this.connectPromise;
31
41
  }
32
42
  async disconnect() {
43
+ this.connectPromise = void 0;
33
44
  await this.producer.disconnect();
34
45
  }
35
46
  async send(record) {
@@ -44,6 +55,7 @@ var ConfluentConsumer = class {
44
55
  constructor(consumer) {
45
56
  this.consumer = consumer;
46
57
  }
58
+ consumer;
47
59
  /** Returns the underlying KafkaJS.Consumer — used by ConfluentTransaction.sendOffsets. */
48
60
  getNative() {
49
61
  return this.consumer;
@@ -83,6 +95,7 @@ var ConfluentAdmin = class {
83
95
  constructor(admin) {
84
96
  this.admin = admin;
85
97
  }
98
+ admin;
86
99
  async connect() {
87
100
  await this.admin.connect();
88
101
  }
@@ -92,11 +105,11 @@ var ConfluentAdmin = class {
92
105
  async createTopics(options) {
93
106
  await this.admin.createTopics(options);
94
107
  }
95
- async fetchTopicOffsets(topic2) {
96
- return this.admin.fetchTopicOffsets(topic2);
108
+ async fetchTopicOffsets(topic) {
109
+ return this.admin.fetchTopicOffsets(topic);
97
110
  }
98
- async fetchTopicOffsetsByTimestamp(topic2, timestamp) {
99
- return this.admin.fetchTopicOffsetsByTime(topic2, timestamp);
111
+ async fetchTopicOffsetsByTimestamp(topic, timestamp) {
112
+ return this.admin.fetchTopicOffsetsByTimestamp(topic, timestamp);
100
113
  }
101
114
  async fetchOffsets(options) {
102
115
  return this.admin.fetchOffsets(options);
@@ -122,10 +135,29 @@ var ConfluentAdmin = class {
122
135
  };
123
136
  var ConfluentTransport = class {
124
137
  kafka;
125
- constructor(clientId, brokers) {
126
- this.kafka = new KafkaClass({
127
- kafkaJS: { clientId, brokers, logLevel: KafkaLogLevel.ERROR }
128
- });
138
+ constructor(clientId, brokers, security) {
139
+ const kafkaJS = { clientId, brokers, logLevel: KafkaLogLevel.ERROR };
140
+ if (security?.ssl !== void 0) kafkaJS.ssl = security.ssl;
141
+ if (security?.sasl) {
142
+ if (security.sasl.mechanism === "oauthbearer") {
143
+ const provider = security.sasl.oauthBearerProvider;
144
+ kafkaJS.sasl = {
145
+ mechanism: "oauthbearer",
146
+ oauthBearerProvider: async () => {
147
+ const token = await provider();
148
+ return {
149
+ value: token.value,
150
+ principal: token.principal ?? "kafka-client",
151
+ lifetime: token.lifetimeMs ?? Date.now() + 15 * 6e4,
152
+ ...token.extensions && { extensions: token.extensions }
153
+ };
154
+ }
155
+ };
156
+ } else {
157
+ kafkaJS.sasl = security.sasl;
158
+ }
159
+ }
160
+ this.kafka = new KafkaClass({ kafkaJS });
129
161
  }
130
162
  producer(options) {
131
163
  const native = this.kafka.producer({
@@ -152,6 +184,9 @@ var ConfluentTransport = class {
152
184
  partitionAssigners: [assigner]
153
185
  }
154
186
  };
187
+ if (options.groupInstanceId) {
188
+ config["group.instance.id"] = options.groupInstanceId;
189
+ }
155
190
  if (options.onRebalance) {
156
191
  const cb = options.onRebalance;
157
192
  config.rebalance_cb = (err, assignment) => {
@@ -169,16 +204,49 @@ var ConfluentTransport = class {
169
204
  }
170
205
  };
171
206
 
207
+ // src/client/message/serde.ts
208
+ var JsonSerde = class {
209
+ /** JSON-stringify the validated payload. Returns a UTF-8 string. */
210
+ serialize(value) {
211
+ return JSON.stringify(value);
212
+ }
213
+ /** JSON-parse UTF-8 wire bytes into an object. */
214
+ deserialize(data) {
215
+ return JSON.parse(data.toString("utf8"));
216
+ }
217
+ };
218
+
219
+ // src/client/kafka.client/infra/dedup.store.ts
220
+ var InMemoryDedupStore = class {
221
+ constructor(states) {
222
+ this.states = states;
223
+ }
224
+ states;
225
+ getLastClock(groupId, topicPartition) {
226
+ return this.states.get(groupId)?.get(topicPartition);
227
+ }
228
+ setLastClock(groupId, topicPartition, clock) {
229
+ let group = this.states.get(groupId);
230
+ if (!group) {
231
+ group = /* @__PURE__ */ new Map();
232
+ this.states.set(groupId, group);
233
+ }
234
+ group.set(topicPartition, clock);
235
+ }
236
+ };
237
+
172
238
  // src/client/message/envelope.ts
173
- import { AsyncLocalStorage } from "async_hooks";
174
- import { randomUUID } from "crypto";
239
+ var import_node_async_hooks = require("async_hooks");
240
+ var import_node_crypto = require("crypto");
175
241
  var HEADER_EVENT_ID = "x-event-id";
176
242
  var HEADER_CORRELATION_ID = "x-correlation-id";
177
243
  var HEADER_TIMESTAMP = "x-timestamp";
178
244
  var HEADER_SCHEMA_VERSION = "x-schema-version";
179
245
  var HEADER_TRACEPARENT = "traceparent";
180
246
  var HEADER_LAMPORT_CLOCK = "x-lamport-clock";
181
- var envelopeStorage = new AsyncLocalStorage();
247
+ var HEADER_DELAYED_UNTIL = "x-delayed-until";
248
+ var HEADER_DELAYED_TARGET = "x-delayed-target";
249
+ var envelopeStorage = new import_node_async_hooks.AsyncLocalStorage();
182
250
  function getEnvelopeContext() {
183
251
  return envelopeStorage.getStore();
184
252
  }
@@ -187,8 +255,8 @@ function runWithEnvelopeContext(ctx, fn) {
187
255
  }
188
256
  function buildEnvelopeHeaders(options = {}) {
189
257
  const ctx = getEnvelopeContext();
190
- const correlationId = options.correlationId ?? ctx?.correlationId ?? randomUUID();
191
- const eventId = options.eventId ?? randomUUID();
258
+ const correlationId = options.correlationId ?? ctx?.correlationId ?? (0, import_node_crypto.randomUUID)();
259
+ const eventId = options.eventId ?? (0, import_node_crypto.randomUUID)();
192
260
  const timestamp = (/* @__PURE__ */ new Date()).toISOString();
193
261
  const schemaVersion = String(options.schemaVersion ?? 1);
194
262
  const envelope = {
@@ -216,14 +284,14 @@ function decodeHeaders(raw) {
216
284
  }
217
285
  return result;
218
286
  }
219
- function extractEnvelope(payload, headers, topic2, partition, offset) {
287
+ function extractEnvelope(payload, headers, topic, partition, offset) {
220
288
  return {
221
289
  payload,
222
- topic: topic2,
290
+ topic,
223
291
  partition,
224
292
  offset,
225
- eventId: headers[HEADER_EVENT_ID] ?? randomUUID(),
226
- correlationId: headers[HEADER_CORRELATION_ID] ?? randomUUID(),
293
+ eventId: headers[HEADER_EVENT_ID] ?? (0, import_node_crypto.randomUUID)(),
294
+ correlationId: headers[HEADER_CORRELATION_ID] ?? (0, import_node_crypto.randomUUID)(),
227
295
  timestamp: headers[HEADER_TIMESTAMP] ?? (/* @__PURE__ */ new Date()).toISOString(),
228
296
  schemaVersion: Number(headers[HEADER_SCHEMA_VERSION] ?? 1),
229
297
  traceparent: headers[HEADER_TRACEPARENT],
@@ -232,38 +300,49 @@ function extractEnvelope(payload, headers, topic2, partition, offset) {
232
300
  }
233
301
 
234
302
  // src/client/errors.ts
303
+ function toError(error) {
304
+ return error instanceof Error ? error : new Error(String(error));
305
+ }
235
306
  var KafkaProcessingError = class extends Error {
236
- constructor(message, topic2, originalMessage, options) {
307
+ constructor(message, topic, originalMessage, options) {
237
308
  super(message, options);
238
- this.topic = topic2;
309
+ this.topic = topic;
239
310
  this.originalMessage = originalMessage;
240
311
  this.name = "KafkaProcessingError";
241
312
  if (options?.cause) this.cause = options.cause;
242
313
  }
314
+ topic;
315
+ originalMessage;
243
316
  };
244
317
  var KafkaValidationError = class extends Error {
245
- constructor(topic2, originalMessage, options) {
246
- super(`Schema validation failed for topic "${topic2}"`, options);
247
- this.topic = topic2;
318
+ constructor(topic, originalMessage, options) {
319
+ super(`Schema validation failed for topic "${topic}"`, options);
320
+ this.topic = topic;
248
321
  this.originalMessage = originalMessage;
249
322
  this.name = "KafkaValidationError";
250
323
  if (options?.cause) this.cause = options.cause;
251
324
  }
325
+ topic;
326
+ originalMessage;
252
327
  };
253
328
  var KafkaRetryExhaustedError = class extends KafkaProcessingError {
254
- constructor(topic2, originalMessage, attempts, options) {
329
+ constructor(topic, originalMessage, attempts, options) {
255
330
  super(
256
- `Message processing failed after ${attempts} attempts on topic "${topic2}"`,
257
- topic2,
331
+ `Message processing failed after ${attempts} attempts on topic "${topic}"`,
332
+ topic,
258
333
  originalMessage,
259
334
  options
260
335
  );
261
336
  this.attempts = attempts;
262
337
  this.name = "KafkaRetryExhaustedError";
263
338
  }
339
+ attempts;
264
340
  };
265
341
 
266
342
  // src/client/kafka.client/producer/ops.ts
343
+ function resolveSerde(topicOrDesc, clientSerde) {
344
+ return topicOrDesc?.__serde ?? clientSerde;
345
+ }
267
346
  function resolveTopicName(topicOrDescriptor) {
268
347
  if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
269
348
  if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
@@ -273,14 +352,14 @@ function resolveTopicName(topicOrDescriptor) {
273
352
  }
274
353
  function registerSchema(topicOrDesc, schemaRegistry, logger) {
275
354
  if (topicOrDesc?.__schema) {
276
- const topic2 = resolveTopicName(topicOrDesc);
277
- const existing = schemaRegistry.get(topic2);
355
+ const topic = resolveTopicName(topicOrDesc);
356
+ const existing = schemaRegistry.get(topic);
278
357
  if (existing && existing !== topicOrDesc.__schema) {
279
358
  logger?.warn(
280
- `Schema conflict for topic "${topic2}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
359
+ `Schema conflict for topic "${topic}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
281
360
  );
282
361
  }
283
- schemaRegistry.set(topic2, topicOrDesc.__schema);
362
+ schemaRegistry.set(topic, topicOrDesc.__schema);
284
363
  }
285
364
  }
286
365
  async function validateMessage(topicOrDesc, message, deps, ctx) {
@@ -309,7 +388,8 @@ async function validateMessage(topicOrDesc, message, deps, ctx) {
309
388
  return message;
310
389
  }
311
390
  async function buildSendPayload(topicOrDesc, messages, deps, compression) {
312
- const topic2 = resolveTopicName(topicOrDesc);
391
+ const topic = resolveTopicName(topicOrDesc);
392
+ const serde = resolveSerde(topicOrDesc, deps.serde);
313
393
  const builtMessages = await Promise.all(
314
394
  messages.map(async (m) => {
315
395
  const envelopeHeaders = buildEnvelopeHeaders({
@@ -322,27 +402,32 @@ async function buildSendPayload(topicOrDesc, messages, deps, compression) {
322
402
  envelopeHeaders[HEADER_LAMPORT_CLOCK] = String(deps.nextLamportClock());
323
403
  }
324
404
  for (const inst of deps.instrumentation) {
325
- inst.beforeSend?.(topic2, envelopeHeaders);
405
+ inst.beforeSend?.(topic, envelopeHeaders);
326
406
  }
327
407
  const sendCtx = {
328
- topic: topic2,
408
+ topic,
329
409
  headers: envelopeHeaders,
330
410
  version: m.schemaVersion ?? 1
331
411
  };
412
+ const validated = await validateMessage(topicOrDesc, m.value, deps, sendCtx);
332
413
  return {
333
- value: JSON.stringify(
334
- await validateMessage(topicOrDesc, m.value, deps, sendCtx)
335
- ),
336
- key: m.key ?? null,
414
+ value: await serde.serialize(validated, {
415
+ topic,
416
+ headers: envelopeHeaders,
417
+ isKey: false
418
+ }),
419
+ // Explicit key wins; otherwise fall back to the descriptor's .key()
420
+ // extractor (runs on the original, pre-validation payload).
421
+ key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
337
422
  headers: envelopeHeaders
338
423
  };
339
424
  })
340
425
  );
341
- return { topic: topic2, messages: builtMessages, ...compression && { compression } };
426
+ return { topic, messages: builtMessages, ...compression && { compression } };
342
427
  }
343
428
 
344
429
  // src/client/kafka.client/consumer/ops.ts
345
- function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment) {
430
+ function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment, groupInstanceId) {
346
431
  const { consumers, consumerCreationOptions, transport, onRebalance, logger } = deps;
347
432
  if (consumers.has(groupId)) {
348
433
  const prev = consumerCreationOptions.get(groupId);
@@ -375,6 +460,7 @@ function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partition
375
460
  fromBeginning,
376
461
  autoCommit,
377
462
  partitionAssigner: partitionAssigner ?? "cooperative-sticky",
463
+ groupInstanceId,
378
464
  onRebalance: (type, assignments) => {
379
465
  if (type === "assign") fireOnAssignment();
380
466
  else if (type === "revoke") scheduleSettle();
@@ -414,12 +500,22 @@ function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
414
500
  }
415
501
  return schemaMap;
416
502
  }
503
+ function buildSerdeMap(topics) {
504
+ let serdeMap;
505
+ for (const t of topics) {
506
+ if (t?.__serde) {
507
+ (serdeMap ??= /* @__PURE__ */ new Map()).set(resolveTopicName(t), t.__serde);
508
+ }
509
+ }
510
+ return serdeMap;
511
+ }
417
512
 
418
513
  // src/client/kafka.client/admin/ops.ts
419
514
  var AdminOps = class {
420
515
  constructor(deps) {
421
516
  this.deps = deps;
422
517
  }
518
+ deps;
423
519
  isConnected = false;
424
520
  /** Underlying admin client — used by index.ts for topic validation. */
425
521
  get admin() {
@@ -450,7 +546,7 @@ var AdminOps = class {
450
546
  await this.deps.admin.disconnect();
451
547
  this.isConnected = false;
452
548
  }
453
- async resetOffsets(groupId, topic2, position) {
549
+ async resetOffsets(groupId, topic, position) {
454
550
  const gid = groupId ?? this.deps.defaultGroupId;
455
551
  if (this.deps.runningConsumers.has(gid)) {
456
552
  throw new Error(
@@ -458,14 +554,14 @@ var AdminOps = class {
458
554
  );
459
555
  }
460
556
  await this.ensureConnected();
461
- const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic2);
557
+ const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic);
462
558
  const partitions = partitionOffsets.map(({ partition, low, high }) => ({
463
559
  partition,
464
560
  offset: position === "earliest" ? low : high
465
561
  }));
466
- await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
562
+ await this.deps.admin.setOffsets({ groupId: gid, topic, partitions });
467
563
  this.deps.logger.log(
468
- `Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
564
+ `Offsets reset to ${position} for group "${gid}" on topic "${topic}"`
469
565
  );
470
566
  }
471
567
  /**
@@ -482,15 +578,15 @@ var AdminOps = class {
482
578
  }
483
579
  await this.ensureConnected();
484
580
  const byTopic = /* @__PURE__ */ new Map();
485
- for (const { topic: topic2, partition, offset } of assignments) {
486
- const list = byTopic.get(topic2) ?? [];
581
+ for (const { topic, partition, offset } of assignments) {
582
+ const list = byTopic.get(topic) ?? [];
487
583
  list.push({ partition, offset });
488
- byTopic.set(topic2, list);
584
+ byTopic.set(topic, list);
489
585
  }
490
- for (const [topic2, partitions] of byTopic) {
491
- await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
586
+ for (const [topic, partitions] of byTopic) {
587
+ await this.deps.admin.setOffsets({ groupId: gid, topic, partitions });
492
588
  this.deps.logger.log(
493
- `Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
589
+ `Offsets set for group "${gid}" on "${topic}": ${JSON.stringify(partitions)}`
494
590
  );
495
591
  }
496
592
  }
@@ -511,27 +607,30 @@ var AdminOps = class {
511
607
  }
512
608
  await this.ensureConnected();
513
609
  const byTopic = /* @__PURE__ */ new Map();
514
- for (const { topic: topic2, partition, timestamp } of assignments) {
515
- const list = byTopic.get(topic2) ?? [];
610
+ for (const { topic, partition, timestamp } of assignments) {
611
+ const list = byTopic.get(topic) ?? [];
516
612
  list.push({ partition, timestamp });
517
- byTopic.set(topic2, list);
613
+ byTopic.set(topic, list);
518
614
  }
519
- for (const [topic2, parts] of byTopic) {
615
+ for (const [topic, parts] of byTopic) {
520
616
  const offsets = await Promise.all(
521
617
  parts.map(async ({ partition, timestamp }) => {
522
618
  const results = await this.deps.admin.fetchTopicOffsetsByTimestamp(
523
- topic2,
619
+ topic,
524
620
  timestamp
525
621
  );
526
622
  const found = results.find(
527
623
  (r) => r.partition === partition
528
624
  );
529
- return { partition, offset: found?.offset ?? "-1" };
625
+ if (found) return { partition, offset: found.offset };
626
+ const topicOffsets = await this.deps.admin.fetchTopicOffsets(topic);
627
+ const po = topicOffsets.find((o) => o.partition === partition);
628
+ return { partition, offset: po?.high ?? "0" };
530
629
  })
531
630
  );
532
- await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
631
+ await this.deps.admin.setOffsets({ groupId: gid, topic, partitions: offsets });
533
632
  this.deps.logger.log(
534
- `Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
633
+ `Offsets set by timestamp for group "${gid}" on "${topic}": ${JSON.stringify(offsets)}`
535
634
  );
536
635
  }
537
636
  }
@@ -551,11 +650,11 @@ var AdminOps = class {
551
650
  await this.ensureConnected();
552
651
  const committedByTopic = await this.deps.admin.fetchOffsets({ groupId: gid });
553
652
  const brokerOffsetsAll = await Promise.all(
554
- committedByTopic.map(({ topic: topic2 }) => this.deps.admin.fetchTopicOffsets(topic2))
653
+ committedByTopic.map(({ topic }) => this.deps.admin.fetchTopicOffsets(topic))
555
654
  );
556
655
  const result = [];
557
656
  for (let i = 0; i < committedByTopic.length; i++) {
558
- const { topic: topic2, partitions } = committedByTopic[i];
657
+ const { topic, partitions } = committedByTopic[i];
559
658
  const brokerOffsets = brokerOffsetsAll[i];
560
659
  for (const { partition, offset } of partitions) {
561
660
  const broker = brokerOffsets.find((o) => o.partition === partition);
@@ -563,7 +662,7 @@ var AdminOps = class {
563
662
  const committed = parseInt(offset, 10);
564
663
  const high = parseInt(broker.high, 10);
565
664
  const lag = committed === -1 ? high : Math.max(0, high - committed);
566
- result.push({ topic: topic2, partition, lag });
665
+ result.push({ topic, partition, lag });
567
666
  }
568
667
  }
569
668
  return result;
@@ -607,7 +706,8 @@ var AdminOps = class {
607
706
  name: t.name,
608
707
  partitions: t.partitions.map((p) => ({
609
708
  partition: p.partitionId ?? p.partition ?? 0,
610
- leader: p.leader ?? 0,
709
+ // -1 is Kafka's own "no leader" sentinel; 0 is a valid broker id
710
+ leader: p.leader ?? -1,
611
711
  replicas: (p.replicas ?? []).map(
612
712
  (r) => typeof r === "number" ? r : r.nodeId
613
713
  ),
@@ -632,9 +732,9 @@ var AdminOps = class {
632
732
  * Delete records from a topic up to (but not including) the given offsets.
633
733
  * All messages with offsets **before** the given offset are deleted.
634
734
  */
635
- async deleteRecords(topic2, partitions) {
735
+ async deleteRecords(topic, partitions) {
636
736
  await this.ensureConnected();
637
- await this.deps.admin.deleteTopicRecords({ topic: topic2, partitions });
737
+ await this.deps.admin.deleteTopicRecords({ topic, partitions });
638
738
  }
639
739
  /**
640
740
  * When `retryTopics: true` and `autoCreateTopics: false`, verify that every
@@ -691,28 +791,14 @@ var AdminOps = class {
691
791
  };
692
792
 
693
793
  // src/client/kafka.client/consumer/pipeline.ts
694
- function toError(error) {
695
- return error instanceof Error ? error : new Error(String(error));
696
- }
697
794
  function sleep(ms) {
698
795
  return new Promise((resolve) => setTimeout(resolve, ms));
699
796
  }
700
- function parseJsonMessage(raw, topic2, logger) {
701
- try {
702
- return JSON.parse(raw);
703
- } catch (error) {
704
- logger.error(
705
- `Failed to parse message from topic ${topic2}:`,
706
- toError(error).stack
707
- );
708
- return null;
709
- }
710
- }
711
- async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
712
- const schema = schemaMap.get(topic2);
797
+ async function validateWithSchema(message, raw, topic, schemaMap, interceptors, dlq, deps) {
798
+ const schema = schemaMap.get(topic);
713
799
  if (!schema) return message;
714
800
  const ctx = {
715
- topic: topic2,
801
+ topic,
716
802
  headers: deps.originalHeaders ?? {},
717
803
  version: Number(deps.originalHeaders?.["x-schema-version"] ?? 1)
718
804
  };
@@ -720,22 +806,22 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
720
806
  return await schema.parse(message, ctx);
721
807
  } catch (error) {
722
808
  const err = toError(error);
723
- const validationError = new KafkaValidationError(topic2, message, {
809
+ const validationError = new KafkaValidationError(topic, message, {
724
810
  cause: err
725
811
  });
726
812
  deps.logger.error(
727
- `Schema validation failed for topic ${topic2}:`,
813
+ `Schema validation failed for topic ${topic}:`,
728
814
  err.message
729
815
  );
730
816
  if (dlq) {
731
- await sendToDlq(topic2, raw, deps, {
817
+ await sendToDlq(topic, raw, deps, {
732
818
  error: validationError,
733
819
  attempt: 0,
734
820
  originalHeaders: deps.originalHeaders
735
821
  });
736
822
  } else {
737
823
  await deps.onMessageLost?.({
738
- topic: topic2,
824
+ topic,
739
825
  error: validationError,
740
826
  attempt: 0,
741
827
  headers: deps.originalHeaders ?? {}
@@ -744,7 +830,7 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
744
830
  const errorEnvelope = extractEnvelope(
745
831
  message,
746
832
  deps.originalHeaders ?? {},
747
- topic2,
833
+ topic,
748
834
  -1,
749
835
  ""
750
836
  );
@@ -757,11 +843,11 @@ async function validateWithSchema(message, raw, topic2, schemaMap, interceptors,
757
843
  return null;
758
844
  }
759
845
  }
760
- function buildDlqPayload(topic2, rawMessage, meta) {
761
- const dlqTopic = `${topic2}.dlq`;
846
+ function buildDlqPayload(topic, rawMessage, meta) {
847
+ const dlqTopic = `${topic}.dlq`;
762
848
  const headers = {
763
849
  ...meta?.originalHeaders ?? {},
764
- "x-dlq-original-topic": topic2,
850
+ "x-dlq-original-topic": topic,
765
851
  "x-dlq-failed-at": (/* @__PURE__ */ new Date()).toISOString(),
766
852
  "x-dlq-error-message": meta?.error.message ?? "unknown",
767
853
  "x-dlq-error-stack": meta?.error.stack?.slice(0, 2e3) ?? "",
@@ -769,8 +855,8 @@ function buildDlqPayload(topic2, rawMessage, meta) {
769
855
  };
770
856
  return { topic: dlqTopic, messages: [{ value: rawMessage, headers }] };
771
857
  }
772
- async function sendToDlq(topic2, rawMessage, deps, meta) {
773
- const payload = buildDlqPayload(topic2, rawMessage, meta);
858
+ async function sendToDlq(topic, rawMessage, deps, meta) {
859
+ const payload = buildDlqPayload(topic, rawMessage, meta);
774
860
  try {
775
861
  await deps.producer.send(payload);
776
862
  deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
@@ -781,7 +867,7 @@ async function sendToDlq(topic2, rawMessage, deps, meta) {
781
867
  err.stack
782
868
  );
783
869
  await deps.onMessageLost?.({
784
- topic: topic2,
870
+ topic,
785
871
  error: err,
786
872
  attempt: meta?.attempt ?? 0,
787
873
  headers: meta?.originalHeaders ?? {}
@@ -954,7 +1040,7 @@ async function executeWithRetry(fn, ctx, deps) {
954
1040
  const backoffMs = retry?.backoffMs ?? 1e3;
955
1041
  const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
956
1042
  const envelopes = Array.isArray(envelope) ? envelope : [envelope];
957
- const topic2 = envelopes[0]?.topic ?? "unknown";
1043
+ const topic = envelopes[0]?.topic ?? "unknown";
958
1044
  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
959
1045
  const error = await runHandlerWithPipeline(
960
1046
  fn,
@@ -977,16 +1063,17 @@ async function executeWithRetry(fn, ctx, deps) {
977
1063
  for (const env of envelopes) deps.onMessage?.(env);
978
1064
  return;
979
1065
  }
1066
+ deps.onFailure?.(envelopes[0]);
980
1067
  const isLastAttempt = attempt === maxAttempts;
981
1068
  const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
982
- topic2,
1069
+ topic,
983
1070
  envelopes.map((e) => e.payload),
984
1071
  maxAttempts,
985
1072
  { cause: error }
986
1073
  ) : error;
987
1074
  await notifyInterceptorsOnError(envelopes, interceptors, reportedError);
988
1075
  deps.logger.error(
989
- `Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
1076
+ `Error processing ${isBatch ? "batch" : "message"} from topic ${topic} (attempt ${attempt}/${maxAttempts}):`,
990
1077
  error.stack
991
1078
  );
992
1079
  if (retryTopics && retry) {
@@ -1004,7 +1091,7 @@ async function executeWithRetry(fn, ctx, deps) {
1004
1091
  }
1005
1092
  } else {
1006
1093
  await sendToRetryTopic(
1007
- topic2,
1094
+ topic,
1008
1095
  rawMessages,
1009
1096
  1,
1010
1097
  retry.maxRetries,
@@ -1017,7 +1104,7 @@ async function executeWithRetry(fn, ctx, deps) {
1017
1104
  } else if (isLastAttempt) {
1018
1105
  if (dlq) {
1019
1106
  for (let i = 0; i < rawMessages.length; i++) {
1020
- await sendToDlq(topic2, rawMessages[i], deps, {
1107
+ await sendToDlq(topic, rawMessages[i], deps, {
1021
1108
  error,
1022
1109
  attempt,
1023
1110
  originalHeaders: envelopes[i]?.headers
@@ -1026,7 +1113,7 @@ async function executeWithRetry(fn, ctx, deps) {
1026
1113
  }
1027
1114
  } else {
1028
1115
  await deps.onMessageLost?.({
1029
- topic: topic2,
1116
+ topic,
1030
1117
  error,
1031
1118
  attempt,
1032
1119
  headers: envelopes[0]?.headers ?? {}
@@ -1061,9 +1148,14 @@ async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
1061
1148
  }
1062
1149
  }
1063
1150
 
1064
- // src/client/kafka.client/consumer/dlq-replay.ts
1065
- async function replayDlqTopic(topic2, deps, options = {}) {
1066
- const dlqTopic = `${topic2}.dlq`;
1151
+ // src/client/kafka.client/consumer/features/dlq-replay.ts
1152
+ async function replayDlqTopic(topic, deps, options = {}) {
1153
+ if (topic.endsWith(".dlq")) {
1154
+ throw new Error(
1155
+ `replayDlq: pass the ORIGINAL topic name \u2014 "${topic}" already ends in ".dlq" (the ".dlq" suffix is appended internally, so this would read "${topic}.dlq")`
1156
+ );
1157
+ }
1158
+ const dlqTopic = `${topic}.dlq`;
1067
1159
  const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
1068
1160
  const activePartitions = partitionOffsets.filter(
1069
1161
  (p) => Number.parseInt(p.high, 10) > Number.parseInt(p.low, 10)
@@ -1094,15 +1186,15 @@ async function replayDlqTopic(topic2, deps, options = {}) {
1094
1186
  const originalHeaders = Object.fromEntries(
1095
1187
  Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
1096
1188
  );
1097
- const value = message.value.toString();
1098
- const shouldProcess = !options.filter || options.filter(headers, value);
1189
+ const bytes = message.value;
1190
+ const shouldProcess = !options.filter || options.filter(headers, bytes.toString("utf8"));
1099
1191
  if (!targetTopic || !shouldProcess) {
1100
1192
  skipped++;
1101
1193
  } else if (options.dryRun) {
1102
1194
  deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
1103
1195
  replayed++;
1104
1196
  } else {
1105
- await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
1197
+ await deps.send(targetTopic, [{ value: bytes, headers: originalHeaders }]);
1106
1198
  replayed++;
1107
1199
  }
1108
1200
  const allDone = Array.from(highWatermarks.entries()).every(
@@ -1128,19 +1220,20 @@ var MetricsManager = class {
1128
1220
  constructor(deps) {
1129
1221
  this.deps = deps;
1130
1222
  }
1223
+ deps;
1131
1224
  topicMetrics = /* @__PURE__ */ new Map();
1132
- metricsFor(topic2) {
1133
- let m = this.topicMetrics.get(topic2);
1225
+ metricsFor(topic) {
1226
+ let m = this.topicMetrics.get(topic);
1134
1227
  if (!m) {
1135
1228
  m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
1136
- this.topicMetrics.set(topic2, m);
1229
+ this.topicMetrics.set(topic, m);
1137
1230
  }
1138
1231
  return m;
1139
1232
  }
1140
1233
  /** Fire `afterSend` instrumentation hooks for each message in a batch. */
1141
- notifyAfterSend(topic2, count) {
1234
+ notifyAfterSend(topic, count) {
1142
1235
  for (let i = 0; i < count; i++)
1143
- for (const inst of this.deps.instrumentation) inst.afterSend?.(topic2);
1236
+ for (const inst of this.deps.instrumentation) inst.afterSend?.(topic);
1144
1237
  }
1145
1238
  /**
1146
1239
  * Increment the retry counter for the envelope's topic and fire all `onRetry` instrumentation hooks.
@@ -1153,16 +1246,25 @@ var MetricsManager = class {
1153
1246
  for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
1154
1247
  }
1155
1248
  /**
1156
- * Increment the DLQ counter for the envelope's topic, fire all `onDlq` instrumentation hooks,
1157
- * and notify the circuit breaker of a failure (when `gid` is provided).
1249
+ * Increment the DLQ counter for the envelope's topic and fire all `onDlq` instrumentation hooks.
1250
+ * Circuit breaker failures are recorded separately via `notifyFailure` at the
1251
+ * handler-error boundary — dead-lettering itself is not a circuit event.
1158
1252
  * @param envelope The message envelope being sent to the DLQ.
1159
1253
  * @param reason The reason the message is being dead-lettered.
1160
- * @param gid Consumer group ID — used to drive circuit breaker state.
1161
1254
  */
1162
- notifyDlq(envelope, reason, gid) {
1255
+ notifyDlq(envelope, reason) {
1163
1256
  this.metricsFor(envelope.topic).dlqCount++;
1164
1257
  for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
1165
- if (gid) this.deps.onCircuitFailure(envelope, gid);
1258
+ }
1259
+ /**
1260
+ * Notify the circuit breaker of a handler failure. Fired on every failed
1261
+ * handler attempt (in-process retries and retry-topic levels included),
1262
+ * independent of whether the message is ultimately dead-lettered.
1263
+ * @param envelope The message envelope whose handler failed.
1264
+ * @param gid Consumer group ID — used to drive circuit breaker state.
1265
+ */
1266
+ notifyFailure(envelope, gid) {
1267
+ this.deps.onCircuitFailure(envelope, gid);
1166
1268
  }
1167
1269
  /**
1168
1270
  * Increment the deduplication counter for the envelope's topic and fire all `onDuplicate` hooks.
@@ -1189,9 +1291,9 @@ var MetricsManager = class {
1189
1291
  * @param topic When provided, returns counters for that topic only; otherwise aggregates all topics.
1190
1292
  * @returns Read-only `KafkaMetrics` snapshot. Returns zero-valued counters if the topic has no events.
1191
1293
  */
1192
- getMetrics(topic2) {
1193
- if (topic2 !== void 0) {
1194
- const m = this.topicMetrics.get(topic2);
1294
+ getMetrics(topic) {
1295
+ if (topic !== void 0) {
1296
+ const m = this.topicMetrics.get(topic);
1195
1297
  return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
1196
1298
  }
1197
1299
  const agg = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
@@ -1207,9 +1309,9 @@ var MetricsManager = class {
1207
1309
  * Reset event counters to zero.
1208
1310
  * @param topic When provided, clears counters for that topic only; otherwise clears all topics.
1209
1311
  */
1210
- resetMetrics(topic2) {
1211
- if (topic2 !== void 0) {
1212
- this.topicMetrics.delete(topic2);
1312
+ resetMetrics(topic) {
1313
+ if (topic !== void 0) {
1314
+ this.topicMetrics.delete(topic);
1213
1315
  return;
1214
1316
  }
1215
1317
  this.topicMetrics.clear();
@@ -1221,6 +1323,7 @@ var InFlightTracker = class {
1221
1323
  constructor(warn) {
1222
1324
  this.warn = warn;
1223
1325
  }
1326
+ warn;
1224
1327
  inFlightTotal = 0;
1225
1328
  drainResolvers = [];
1226
1329
  /**
@@ -1231,10 +1334,16 @@ var InFlightTracker = class {
1231
1334
  */
1232
1335
  track(fn) {
1233
1336
  this.inFlightTotal++;
1234
- return fn().finally(() => {
1337
+ const done = () => {
1235
1338
  this.inFlightTotal--;
1236
1339
  if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
1237
- });
1340
+ };
1341
+ try {
1342
+ return fn().finally(done);
1343
+ } catch (err) {
1344
+ done();
1345
+ throw err;
1346
+ }
1238
1347
  }
1239
1348
  /**
1240
1349
  * Resolve when all tracked handlers have completed, or after `timeoutMs` elapses.
@@ -1268,6 +1377,7 @@ var CircuitBreakerManager = class {
1268
1377
  constructor(deps) {
1269
1378
  this.deps = deps;
1270
1379
  }
1380
+ deps;
1271
1381
  states = /* @__PURE__ */ new Map();
1272
1382
  configs = /* @__PURE__ */ new Map();
1273
1383
  /**
@@ -1283,8 +1393,8 @@ var CircuitBreakerManager = class {
1283
1393
  * Returns a snapshot of the circuit breaker state for a given topic-partition.
1284
1394
  * Returns `undefined` when no state exists for the key.
1285
1395
  */
1286
- getState(topic2, partition, gid) {
1287
- const state = this.states.get(`${gid}:${topic2}:${partition}`);
1396
+ getState(topic, partition, gid) {
1397
+ const state = this.states.get(`${gid}:${topic}:${partition}`);
1288
1398
  if (!state) return void 0;
1289
1399
  return {
1290
1400
  status: state.status,
@@ -1412,6 +1522,9 @@ var AsyncQueue = class {
1412
1522
  this.onFull = onFull;
1413
1523
  this.onDrained = onDrained;
1414
1524
  }
1525
+ highWaterMark;
1526
+ onFull;
1527
+ onDrained;
1415
1528
  items = [];
1416
1529
  waiting = [];
1417
1530
  closed = false;
@@ -1423,6 +1536,7 @@ var AsyncQueue = class {
1423
1536
  * @param item The value to enqueue.
1424
1537
  */
1425
1538
  push(item) {
1539
+ if (this.closed) return;
1426
1540
  if (this.waiting.length > 0) {
1427
1541
  this.waiting.shift().resolve({ value: item, done: false });
1428
1542
  } else {
@@ -1473,20 +1587,115 @@ var AsyncQueue = class {
1473
1587
  }
1474
1588
  };
1475
1589
 
1590
+ // src/client/kafka.client/validate-options.ts
1591
+ function validateClientOptions(clientId, groupId, brokers, options) {
1592
+ const problems = [];
1593
+ if (typeof clientId !== "string" || clientId.trim() === "") {
1594
+ problems.push("clientId must be a non-empty string");
1595
+ }
1596
+ if (typeof groupId !== "string" || groupId.trim() === "") {
1597
+ problems.push("groupId must be a non-empty string");
1598
+ }
1599
+ if (!Array.isArray(brokers) || brokers.length === 0 && !options?.transport) {
1600
+ problems.push("brokers must be a non-empty array of broker addresses");
1601
+ } else if (brokers.some((b) => typeof b !== "string" || b.trim() === "")) {
1602
+ problems.push("brokers must not contain empty entries");
1603
+ }
1604
+ if (options) {
1605
+ const {
1606
+ numPartitions,
1607
+ transactionalId,
1608
+ clockRecovery,
1609
+ lagThrottle
1610
+ } = options;
1611
+ if (numPartitions !== void 0 && (!Number.isInteger(numPartitions) || numPartitions < 1)) {
1612
+ problems.push(
1613
+ `numPartitions must be a positive integer (got ${numPartitions})`
1614
+ );
1615
+ }
1616
+ if (transactionalId !== void 0 && transactionalId.trim() === "") {
1617
+ problems.push("transactionalId must be a non-empty string when set");
1618
+ }
1619
+ if (clockRecovery) {
1620
+ if (!Array.isArray(clockRecovery.topics)) {
1621
+ problems.push("clockRecovery.topics must be an array of topic names");
1622
+ }
1623
+ if (clockRecovery.timeoutMs !== void 0 && !(clockRecovery.timeoutMs > 0)) {
1624
+ problems.push(
1625
+ `clockRecovery.timeoutMs must be > 0 (got ${clockRecovery.timeoutMs})`
1626
+ );
1627
+ }
1628
+ }
1629
+ if (lagThrottle) {
1630
+ if (!(lagThrottle.maxLag >= 0)) {
1631
+ problems.push(`lagThrottle.maxLag must be >= 0 (got ${lagThrottle.maxLag})`);
1632
+ }
1633
+ if (lagThrottle.pollIntervalMs !== void 0 && !(lagThrottle.pollIntervalMs > 0)) {
1634
+ problems.push(
1635
+ `lagThrottle.pollIntervalMs must be > 0 (got ${lagThrottle.pollIntervalMs})`
1636
+ );
1637
+ }
1638
+ if (lagThrottle.maxWaitMs !== void 0 && !(lagThrottle.maxWaitMs >= 0)) {
1639
+ problems.push(
1640
+ `lagThrottle.maxWaitMs must be >= 0 (got ${lagThrottle.maxWaitMs})`
1641
+ );
1642
+ }
1643
+ }
1644
+ }
1645
+ if (problems.length > 0) {
1646
+ throw new Error(
1647
+ `KafkaClient: invalid configuration:
1648
+ - ${problems.join("\n- ")}`
1649
+ );
1650
+ }
1651
+ }
1652
+
1653
+ // src/client/security/resolve-security.ts
1654
+ var LOCAL_HOST_PATTERNS = [
1655
+ /^localhost(:\d+)?$/i,
1656
+ /^127\.\d+\.\d+\.\d+(:\d+)?$/,
1657
+ /^\[?::1\]?(:\d+)?$/,
1658
+ /^0\.0\.0\.0(:\d+)?$/,
1659
+ /^host\.docker\.internal(:\d+)?$/i
1660
+ ];
1661
+ function isLocalBroker(broker) {
1662
+ return LOCAL_HOST_PATTERNS.some((re) => re.test(broker.trim()));
1663
+ }
1664
+ function resolveSecurityOptions(security, brokers, logger) {
1665
+ const hasRemoteBroker = brokers.some((b) => !isLocalBroker(b));
1666
+ if (!security?.sasl && security?.ssl !== true) {
1667
+ if (hasRemoteBroker && !security?.allowInsecure) {
1668
+ logger.warn(
1669
+ "Connecting to non-local brokers without TLS or SASL \u2014 traffic and payloads travel in plaintext. Configure `security: { ssl, sasl }` for production clusters, or set `security: { allowInsecure: true }` to acknowledge an intentionally insecure setup."
1670
+ );
1671
+ }
1672
+ return security;
1673
+ }
1674
+ if (security.sasl && security.ssl === void 0) {
1675
+ return { ...security, ssl: true };
1676
+ }
1677
+ if (security.sasl && security.ssl === false) {
1678
+ logger.warn(
1679
+ "SASL credentials are configured with `ssl: false` \u2014 credentials will be sent over plaintext. This is only safe on fully trusted networks."
1680
+ );
1681
+ }
1682
+ return security;
1683
+ }
1684
+
1476
1685
  // src/client/kafka.client/producer/lifecycle.ts
1477
1686
  var _activeTransactionalIds = /* @__PURE__ */ new Set();
1478
- async function ensureTopic(ctx, topic2) {
1479
- if (!ctx.autoCreateTopicsEnabled || ctx.ensuredTopics.has(topic2)) return;
1480
- let p = ctx.ensureTopicPromises.get(topic2);
1687
+ async function ensureTopic(ctx, topic) {
1688
+ if (!ctx.autoCreateTopicsEnabled || ctx.ensuredTopics.has(topic)) return;
1689
+ let p = ctx.ensureTopicPromises.get(topic);
1481
1690
  if (!p) {
1482
1691
  p = (async () => {
1483
1692
  await ctx.adminOps.ensureConnected();
1484
1693
  await ctx.adminOps.admin.createTopics({
1485
- topics: [{ topic: topic2, numPartitions: ctx.numPartitions }]
1694
+ topics: [{ topic, numPartitions: ctx.numPartitions }]
1486
1695
  });
1487
- ctx.ensuredTopics.add(topic2);
1488
- })().finally(() => ctx.ensureTopicPromises.delete(topic2));
1489
- ctx.ensureTopicPromises.set(topic2, p);
1696
+ ctx.ensuredTopics.add(topic);
1697
+ })().finally(() => ctx.ensureTopicPromises.delete(topic));
1698
+ ctx.ensureTopicPromises.set(topic, p);
1490
1699
  }
1491
1700
  await p;
1492
1701
  }
@@ -1606,6 +1815,7 @@ async function recoverLamportClockImpl(ctx, topics) {
1606
1815
  const remaining = new Set(
1607
1816
  partitionsToRead.map((p) => `${p.topic}:${p.partition}`)
1608
1817
  );
1818
+ let settled = false;
1609
1819
  const cleanup = () => {
1610
1820
  consumer.disconnect().catch(() => {
1611
1821
  }).finally(() => {
@@ -1613,6 +1823,16 @@ async function recoverLamportClockImpl(ctx, topics) {
1613
1823
  });
1614
1824
  });
1615
1825
  };
1826
+ const timeoutTimer = setTimeout(() => {
1827
+ if (settled) return;
1828
+ settled = true;
1829
+ ctx.logger.warn(
1830
+ `Clock recovery: timed out after ${ctx.clockRecoveryTimeoutMs} ms with ${remaining.size} partition(s) unread \u2014 proceeding with partial result`
1831
+ );
1832
+ cleanup();
1833
+ resolve();
1834
+ }, ctx.clockRecoveryTimeoutMs);
1835
+ timeoutTimer.unref?.();
1616
1836
  consumer.connect().then(async () => {
1617
1837
  const uniqueTopics = [
1618
1838
  ...new Set(partitionsToRead.map((p) => p.topic))
@@ -1633,13 +1853,18 @@ async function recoverLamportClockImpl(ctx, topics) {
1633
1853
  const clock = Number(raw);
1634
1854
  if (!Number.isNaN(clock) && clock > maxClock) maxClock = clock;
1635
1855
  }
1636
- if (remaining.size === 0) {
1856
+ if (remaining.size === 0 && !settled) {
1857
+ settled = true;
1858
+ clearTimeout(timeoutTimer);
1637
1859
  cleanup();
1638
1860
  resolve();
1639
1861
  }
1640
1862
  }
1641
1863
  })
1642
1864
  ).catch((err) => {
1865
+ if (settled) return;
1866
+ settled = true;
1867
+ clearTimeout(timeoutTimer);
1643
1868
  cleanup();
1644
1869
  reject(err);
1645
1870
  });
@@ -1655,14 +1880,14 @@ async function recoverLamportClockImpl(ctx, topics) {
1655
1880
  );
1656
1881
  }
1657
1882
  }
1658
- function wrapWithTimeoutWarning(logger, fn, timeoutMs, topic2) {
1883
+ function wrapWithTimeoutWarning(logger, fn, timeoutMs, topic) {
1659
1884
  let timer;
1660
1885
  const promise = fn().finally(() => {
1661
1886
  if (timer !== void 0) clearTimeout(timer);
1662
1887
  });
1663
1888
  timer = setTimeout(() => {
1664
1889
  logger.warn(
1665
- `Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
1890
+ `Handler for topic "${topic}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
1666
1891
  );
1667
1892
  }, timeoutMs);
1668
1893
  return promise;
@@ -1680,6 +1905,15 @@ async function preparePayload(ctx, topicOrDesc, messages, compression) {
1680
1905
  await ensureTopic(ctx, payload.topic);
1681
1906
  return payload;
1682
1907
  }
1908
+ async function redirectToDelayed(ctx, payload, deliverAfterMs) {
1909
+ const until = String(Date.now() + deliverAfterMs);
1910
+ for (const m of payload.messages) {
1911
+ m.headers[HEADER_DELAYED_UNTIL] = until;
1912
+ m.headers[HEADER_DELAYED_TARGET] = payload.topic;
1913
+ }
1914
+ payload.topic = `${payload.topic}.delayed`;
1915
+ await ensureTopic(ctx, payload.topic);
1916
+ }
1683
1917
  async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
1684
1918
  await waitIfThrottled(ctx);
1685
1919
  const payload = await preparePayload(
@@ -1697,6 +1931,9 @@ async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
1697
1931
  ],
1698
1932
  options.compression
1699
1933
  );
1934
+ if (options.deliverAfterMs && options.deliverAfterMs > 0) {
1935
+ await redirectToDelayed(ctx, payload, options.deliverAfterMs);
1936
+ }
1700
1937
  await ctx.producer.send(payload);
1701
1938
  ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
1702
1939
  }
@@ -1708,19 +1945,22 @@ async function sendBatchImpl(ctx, topicOrDesc, messages, options) {
1708
1945
  messages,
1709
1946
  options?.compression
1710
1947
  );
1948
+ if (options?.deliverAfterMs && options.deliverAfterMs > 0) {
1949
+ await redirectToDelayed(ctx, payload, options.deliverAfterMs);
1950
+ }
1711
1951
  await ctx.producer.send(payload);
1712
1952
  ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
1713
1953
  }
1714
- async function sendTombstoneImpl(ctx, topic2, key, headers) {
1954
+ async function sendTombstoneImpl(ctx, topic, key, headers) {
1715
1955
  await waitIfThrottled(ctx);
1716
1956
  const hdrs = { ...headers };
1717
- for (const inst of ctx.instrumentation) inst.beforeSend?.(topic2, hdrs);
1718
- await ensureTopic(ctx, topic2);
1957
+ for (const inst of ctx.instrumentation) inst.beforeSend?.(topic, hdrs);
1958
+ await ensureTopic(ctx, topic);
1719
1959
  await ctx.producer.send({
1720
- topic: topic2,
1960
+ topic,
1721
1961
  messages: [{ value: null, key, headers: hdrs }]
1722
1962
  });
1723
- for (const inst of ctx.instrumentation) inst.afterSend?.(topic2);
1963
+ for (const inst of ctx.instrumentation) inst.afterSend?.(topic);
1724
1964
  }
1725
1965
  async function transactionImpl(ctx, fn) {
1726
1966
  if (!ctx.txProducerInitPromise) {
@@ -1744,6 +1984,17 @@ async function transactionImpl(ctx, fn) {
1744
1984
  });
1745
1985
  }
1746
1986
  ctx.txProducer = await ctx.txProducerInitPromise;
1987
+ const prev = ctx._txChain;
1988
+ let release;
1989
+ ctx._txChain = new Promise((r) => release = r);
1990
+ await prev;
1991
+ try {
1992
+ await runTransaction(ctx, fn);
1993
+ } finally {
1994
+ release();
1995
+ }
1996
+ }
1997
+ async function runTransaction(ctx, fn) {
1747
1998
  const tx = await ctx.txProducer.transaction();
1748
1999
  try {
1749
2000
  const txCtx = {
@@ -1817,7 +2068,7 @@ async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs =
1817
2068
  `Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
1818
2069
  );
1819
2070
  }
1820
- async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
2071
+ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
1821
2072
  const {
1822
2073
  logger,
1823
2074
  producer,
@@ -1863,20 +2114,35 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
1863
2114
  await sleep(remaining);
1864
2115
  consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
1865
2116
  }
1866
- const raw = message.value.toString();
1867
- const parsed = parseJsonMessage(raw, levelTopic, logger);
1868
- if (parsed === null) {
1869
- await consumer.commitOffsets([nextOffset]);
1870
- return;
1871
- }
2117
+ const rawBytes = message.value;
1872
2118
  const currentMaxRetries = parseInt(
1873
2119
  headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
1874
2120
  10
1875
2121
  );
1876
2122
  const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
2123
+ const serde = serdeMap?.get(originalTopic) ?? deps.serde;
2124
+ let parsed;
2125
+ try {
2126
+ parsed = await serde.deserialize(rawBytes, {
2127
+ topic: originalTopic,
2128
+ headers,
2129
+ isKey: false
2130
+ });
2131
+ } catch (err) {
2132
+ logger.error(
2133
+ `Failed to deserialize retry message from topic ${levelTopic}:`,
2134
+ toError(err).stack
2135
+ );
2136
+ await consumer.commitOffsets([nextOffset]);
2137
+ return;
2138
+ }
2139
+ if (parsed === null) {
2140
+ await consumer.commitOffsets([nextOffset]);
2141
+ return;
2142
+ }
1877
2143
  const validated = await validateWithSchema(
1878
2144
  parsed,
1879
- raw,
2145
+ rawBytes,
1880
2146
  originalTopic,
1881
2147
  schemaMap,
1882
2148
  interceptors,
@@ -1911,6 +2177,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
1911
2177
  await consumer.commitOffsets([nextOffset]);
1912
2178
  return;
1913
2179
  }
2180
+ deps.onFailure?.(envelope);
1914
2181
  const exhausted = level >= currentMaxRetries;
1915
2182
  const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
1916
2183
  originalTopic,
@@ -1929,7 +2196,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
1929
2196
  const delay = Math.floor(Math.random() * cap);
1930
2197
  const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
1931
2198
  originalTopic,
1932
- [raw],
2199
+ [rawBytes],
1933
2200
  nextLevel,
1934
2201
  currentMaxRetries,
1935
2202
  delay,
@@ -1971,7 +2238,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
1971
2238
  } else if (dlq) {
1972
2239
  const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
1973
2240
  originalTopic,
1974
- raw,
2241
+ rawBytes,
1975
2242
  {
1976
2243
  error,
1977
2244
  // +1 to account for the main consumer's initial attempt before routing.
@@ -2033,7 +2300,7 @@ async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopi
2033
2300
  `Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
2034
2301
  );
2035
2302
  }
2036
- async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
2303
+ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs, serdeMap) {
2037
2304
  const levelGroupIds = new Array(retry.maxRetries);
2038
2305
  await Promise.all(
2039
2306
  Array.from({ length: retry.maxRetries }, async (_, i) => {
@@ -2051,7 +2318,8 @@ async function startRetryTopicConsumers(originalTopics, originalGroupId, handleM
2051
2318
  interceptors,
2052
2319
  schemaMap,
2053
2320
  deps,
2054
- assignmentTimeoutMs
2321
+ assignmentTimeoutMs,
2322
+ serdeMap
2055
2323
  );
2056
2324
  levelGroupIds[i] = levelGroupId;
2057
2325
  })
@@ -2126,7 +2394,8 @@ async function setupConsumer(ctx, topics, mode, options) {
2126
2394
  options.autoCommit ?? true,
2127
2395
  ctx.consumerOpsDeps,
2128
2396
  options.partitionAssigner,
2129
- resolveReady
2397
+ resolveReady,
2398
+ options.groupInstanceId
2130
2399
  );
2131
2400
  const schemaMap = buildSchemaMap(
2132
2401
  stringTopics,
@@ -2134,10 +2403,14 @@ async function setupConsumer(ctx, topics, mode, options) {
2134
2403
  optionSchemas,
2135
2404
  ctx.logger
2136
2405
  );
2406
+ const serdeMap = buildSerdeMap(stringTopics);
2137
2407
  const topicNames = stringTopics.map((t) => resolveTopicName(t));
2138
2408
  const subscribeTopics = [...topicNames, ...regexTopics];
2139
2409
  await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
2140
2410
  await consumer.connect();
2411
+ if (dlq || options.retryTopics || options.deduplication) {
2412
+ await ctx.producer.connect();
2413
+ }
2141
2414
  await subscribeWithRetry(
2142
2415
  consumer,
2143
2416
  subscribeTopics,
@@ -2148,19 +2421,19 @@ async function setupConsumer(ctx, topics, mode, options) {
2148
2421
  ctx.logger.log(
2149
2422
  `${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
2150
2423
  );
2151
- return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
2424
+ return { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
2152
2425
  }
2153
2426
  function resolveDeduplicationContext(ctx, groupId, options) {
2154
2427
  if (!options) return void 0;
2155
- if (!ctx.dedupStates.has(groupId))
2156
- ctx.dedupStates.set(groupId, /* @__PURE__ */ new Map());
2157
- return { options, state: ctx.dedupStates.get(groupId) };
2428
+ const store = options.store ?? new InMemoryDedupStore(ctx.dedupStates);
2429
+ return { options, store, groupId };
2158
2430
  }
2159
2431
  function messageDepsFor(ctx, gid, options) {
2160
2432
  const notifyRetry = ctx.metrics.notifyRetry.bind(ctx.metrics);
2161
2433
  return {
2162
2434
  logger: ctx.logger,
2163
2435
  producer: ctx.producer,
2436
+ serde: ctx.serde,
2164
2437
  instrumentation: ctx.instrumentation,
2165
2438
  onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
2166
2439
  onTtlExpired: ctx.onTtlExpired,
@@ -2168,15 +2441,17 @@ function messageDepsFor(ctx, gid, options) {
2168
2441
  notifyRetry(envelope, attempt, max);
2169
2442
  return options.onRetry(envelope, attempt, max);
2170
2443
  } : notifyRetry,
2171
- onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason, gid),
2444
+ onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason),
2172
2445
  onDuplicate: ctx.metrics.notifyDuplicate.bind(ctx.metrics),
2173
- onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid)
2446
+ onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
2447
+ onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid)
2174
2448
  };
2175
2449
  }
2176
2450
  function buildRetryTopicDeps(ctx) {
2177
2451
  return {
2178
2452
  logger: ctx.logger,
2179
2453
  producer: ctx.producer,
2454
+ serde: ctx.serde,
2180
2455
  instrumentation: ctx.instrumentation,
2181
2456
  onMessageLost: ctx.onMessageLost,
2182
2457
  onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
@@ -2194,7 +2469,7 @@ async function makeEosMainContext(ctx, gid, consumer, options) {
2194
2469
  return { txProducer, consumer };
2195
2470
  }
2196
2471
  async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
2197
- const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
2472
+ const { retry, dlq, interceptors, schemaMap, serdeMap, assignmentTimeoutMs } = opts;
2198
2473
  if (!ctx.autoCreateTopicsEnabled) {
2199
2474
  await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
2200
2475
  }
@@ -2209,11 +2484,17 @@ async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
2209
2484
  schemaMap,
2210
2485
  {
2211
2486
  ...ctx.retryTopicDeps,
2487
+ // Bind circuit breaker events to the MAIN consumer group so failures and
2488
+ // successes inside the retry chain drive the same breaker as the main
2489
+ // consumer (the retry chain has no breaker config of its own).
2490
+ onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid),
2491
+ onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
2212
2492
  onLevelStarted: (levelGroupId) => {
2213
2493
  ctx.companionGroupIds.get(gid).push(levelGroupId);
2214
2494
  }
2215
2495
  },
2216
- assignmentTimeoutMs
2496
+ assignmentTimeoutMs,
2497
+ serdeMap
2217
2498
  );
2218
2499
  }
2219
2500
 
@@ -2224,7 +2505,15 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
2224
2505
  const incomingClock = Number(clockRaw);
2225
2506
  if (Number.isNaN(incomingClock)) return false;
2226
2507
  const stateKey = `${envelope.topic}:${envelope.partition}`;
2227
- const lastProcessedClock = dedup.state.get(stateKey) ?? -1;
2508
+ let lastProcessedClock;
2509
+ try {
2510
+ lastProcessedClock = await dedup.store.getLastClock(dedup.groupId, stateKey) ?? -1;
2511
+ } catch (err) {
2512
+ deps.logger.error(
2513
+ `Dedup store getLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 treating message as not a duplicate (fail-open): ${err.message}`
2514
+ );
2515
+ return false;
2516
+ }
2228
2517
  if (incomingClock <= lastProcessedClock) {
2229
2518
  const meta = {
2230
2519
  incomingClock,
@@ -2254,34 +2543,52 @@ async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
2254
2543
  }
2255
2544
  return true;
2256
2545
  }
2257
- dedup.state.set(stateKey, incomingClock);
2546
+ try {
2547
+ await dedup.store.setLastClock(dedup.groupId, stateKey, incomingClock);
2548
+ } catch (err) {
2549
+ deps.logger.error(
2550
+ `Dedup store setLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 processing message anyway (fail-open): ${err.message}`
2551
+ );
2552
+ }
2258
2553
  return false;
2259
2554
  }
2260
- async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
2555
+ async function parseSingleMessage(message, topic, partition, schemaMap, interceptors, dlq, deps, serdeMap) {
2261
2556
  if (!message.value) {
2262
- deps.logger.warn(`Received empty message from topic ${topic2}`);
2557
+ deps.logger.warn(`Received empty message from topic ${topic}`);
2263
2558
  return null;
2264
2559
  }
2265
- const raw = message.value.toString();
2266
- const parsed = parseJsonMessage(raw, topic2, deps.logger);
2267
- if (parsed === null) return null;
2560
+ const bytes = message.value;
2268
2561
  const headers = decodeHeaders(message.headers);
2562
+ const serde = serdeMap?.get(topic) ?? deps.serde;
2563
+ let parsed;
2564
+ try {
2565
+ parsed = await serde.deserialize(bytes, { topic, headers, isKey: false });
2566
+ } catch (error) {
2567
+ deps.logger.error(
2568
+ `Failed to deserialize message from topic ${topic}:`,
2569
+ toError(error).stack
2570
+ );
2571
+ return null;
2572
+ }
2573
+ if (parsed === null) return null;
2269
2574
  const validated = await validateWithSchema(
2270
2575
  parsed,
2271
- raw,
2272
- topic2,
2576
+ // Forward the ORIGINAL bytes to DLQ on validation failure (binary-safe).
2577
+ bytes,
2578
+ topic,
2273
2579
  schemaMap,
2274
2580
  interceptors,
2275
2581
  dlq,
2276
2582
  { ...deps, originalHeaders: headers }
2277
2583
  );
2278
2584
  if (validated === null) return null;
2279
- return extractEnvelope(validated, headers, topic2, partition, message.offset);
2585
+ return extractEnvelope(validated, headers, topic, partition, message.offset);
2280
2586
  }
2281
2587
  async function handleEachMessage(payload, opts, deps) {
2282
- const { topic: topic2, partition, message } = payload;
2588
+ const { topic, partition, message } = payload;
2283
2589
  const {
2284
2590
  schemaMap,
2591
+ serdeMap,
2285
2592
  handleMessage,
2286
2593
  interceptors,
2287
2594
  dlq,
@@ -2290,16 +2597,17 @@ async function handleEachMessage(payload, opts, deps) {
2290
2597
  timeoutMs,
2291
2598
  wrapWithTimeout
2292
2599
  } = opts;
2600
+ const rawBytes = message.value;
2293
2601
  const eos = opts.eosMainContext;
2294
2602
  const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
2295
2603
  const commitOffset = eos ? async () => {
2296
2604
  await eos.consumer.commitOffsets([
2297
- { topic: topic2, partition, offset: nextOffsetStr }
2605
+ { topic, partition, offset: nextOffsetStr }
2298
2606
  ]);
2299
2607
  } : void 0;
2300
2608
  const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
2301
2609
  const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
2302
- topic2,
2610
+ topic,
2303
2611
  rawMsgs,
2304
2612
  1,
2305
2613
  retry.maxRetries,
@@ -2313,7 +2621,7 @@ async function handleEachMessage(payload, opts, deps) {
2313
2621
  consumer: eos.consumer,
2314
2622
  topics: [
2315
2623
  {
2316
- topic: topic2,
2624
+ topic,
2317
2625
  partitions: [{ partition, offset: nextOffsetStr }]
2318
2626
  }
2319
2627
  ]
@@ -2329,12 +2637,13 @@ async function handleEachMessage(payload, opts, deps) {
2329
2637
  } : void 0;
2330
2638
  const envelope = await parseSingleMessage(
2331
2639
  message,
2332
- topic2,
2640
+ topic,
2333
2641
  partition,
2334
2642
  schemaMap,
2335
2643
  interceptors,
2336
2644
  dlq,
2337
- deps
2645
+ deps,
2646
+ serdeMap
2338
2647
  );
2339
2648
  if (envelope === null) {
2340
2649
  await commitOffset?.();
@@ -2343,7 +2652,7 @@ async function handleEachMessage(payload, opts, deps) {
2343
2652
  if (opts.deduplication) {
2344
2653
  const isDuplicate = await applyDeduplication(
2345
2654
  envelope,
2346
- message.value.toString(),
2655
+ rawBytes,
2347
2656
  opts.deduplication,
2348
2657
  dlq,
2349
2658
  deps
@@ -2357,10 +2666,10 @@ async function handleEachMessage(payload, opts, deps) {
2357
2666
  const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
2358
2667
  if (ageMs > opts.messageTtlMs) {
2359
2668
  deps.logger.warn(
2360
- `[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
2669
+ `[KafkaClient] TTL expired on ${topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
2361
2670
  );
2362
2671
  if (dlq) {
2363
- await sendToDlq(topic2, message.value.toString(), deps, {
2672
+ await sendToDlq(topic, rawBytes, deps, {
2364
2673
  error: new Error(`Message TTL expired: age ${ageMs}ms`),
2365
2674
  attempt: 0,
2366
2675
  originalHeaders: envelope.headers
@@ -2369,7 +2678,7 @@ async function handleEachMessage(payload, opts, deps) {
2369
2678
  } else {
2370
2679
  const ttlHandler = opts.onTtlExpired ?? deps.onTtlExpired;
2371
2680
  await ttlHandler?.({
2372
- topic: topic2,
2681
+ topic,
2373
2682
  ageMs,
2374
2683
  messageTtlMs: opts.messageTtlMs,
2375
2684
  headers: envelope.headers
@@ -2388,11 +2697,11 @@ async function handleEachMessage(payload, opts, deps) {
2388
2697
  },
2389
2698
  () => handleMessage(envelope)
2390
2699
  );
2391
- return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic2) : fn();
2700
+ return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic) : fn();
2392
2701
  },
2393
2702
  {
2394
2703
  envelope,
2395
- rawMessages: [message.value.toString()],
2704
+ rawMessages: [rawBytes],
2396
2705
  interceptors,
2397
2706
  dlq,
2398
2707
  retry,
@@ -2405,6 +2714,7 @@ async function handleEachBatch(payload, opts, deps) {
2405
2714
  const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
2406
2715
  const {
2407
2716
  schemaMap,
2717
+ serdeMap,
2408
2718
  handleBatch,
2409
2719
  interceptors,
2410
2720
  dlq,
@@ -2460,6 +2770,7 @@ async function handleEachBatch(payload, opts, deps) {
2460
2770
  const envelopes = [];
2461
2771
  const rawMessages = [];
2462
2772
  for (const message of batch.messages) {
2773
+ const rawBytes = message.value;
2463
2774
  const envelope = await parseSingleMessage(
2464
2775
  message,
2465
2776
  batch.topic,
@@ -2467,14 +2778,14 @@ async function handleEachBatch(payload, opts, deps) {
2467
2778
  schemaMap,
2468
2779
  interceptors,
2469
2780
  dlq,
2470
- deps
2781
+ deps,
2782
+ serdeMap
2471
2783
  );
2472
2784
  if (envelope === null) continue;
2473
2785
  if (opts.deduplication) {
2474
- const raw = message.value.toString();
2475
2786
  const isDuplicate = await applyDeduplication(
2476
2787
  envelope,
2477
- raw,
2788
+ rawBytes,
2478
2789
  opts.deduplication,
2479
2790
  dlq,
2480
2791
  deps
@@ -2488,7 +2799,7 @@ async function handleEachBatch(payload, opts, deps) {
2488
2799
  `[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
2489
2800
  );
2490
2801
  if (dlq) {
2491
- await sendToDlq(batch.topic, message.value.toString(), deps, {
2802
+ await sendToDlq(batch.topic, rawBytes, deps, {
2492
2803
  error: new Error(`Message TTL expired: age ${ageMs}ms`),
2493
2804
  attempt: 0,
2494
2805
  originalHeaders: envelope.headers
@@ -2507,7 +2818,7 @@ async function handleEachBatch(payload, opts, deps) {
2507
2818
  }
2508
2819
  }
2509
2820
  envelopes.push(envelope);
2510
- rawMessages.push(message.value.toString());
2821
+ rawMessages.push(rawBytes);
2511
2822
  }
2512
2823
  if (envelopes.length === 0) {
2513
2824
  await commitBatchOffset?.();
@@ -2632,7 +2943,7 @@ function pauseConsumerImpl(ctx, groupId, assignments) {
2632
2943
  }
2633
2944
  consumer.pause(
2634
2945
  assignments.flatMap(
2635
- ({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
2946
+ ({ topic, partitions }) => partitions.map((p) => ({ topic, partitions: [p] }))
2636
2947
  )
2637
2948
  );
2638
2949
  }
@@ -2645,32 +2956,32 @@ function resumeConsumerImpl(ctx, groupId, assignments) {
2645
2956
  }
2646
2957
  consumer.resume(
2647
2958
  assignments.flatMap(
2648
- ({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
2959
+ ({ topic, partitions }) => partitions.map((p) => ({ topic, partitions: [p] }))
2649
2960
  )
2650
2961
  );
2651
2962
  }
2652
- function pauseTopicAllPartitions(ctx, gid, topic2) {
2963
+ function pauseTopicAllPartitions(ctx, gid, topic) {
2653
2964
  const consumer = ctx.consumers.get(gid);
2654
2965
  if (!consumer) return;
2655
2966
  const assignment = consumer.assignment();
2656
- const partitions = assignment.filter((a) => a.topic === topic2).map((a) => a.partition);
2967
+ const partitions = assignment.filter((a) => a.topic === topic).map((a) => a.partition);
2657
2968
  if (partitions.length > 0)
2658
- consumer.pause(partitions.map((p) => ({ topic: topic2, partitions: [p] })));
2969
+ consumer.pause(partitions.map((p) => ({ topic, partitions: [p] })));
2659
2970
  }
2660
- function resumeTopicAllPartitions(ctx, gid, topic2) {
2971
+ function resumeTopicAllPartitions(ctx, gid, topic) {
2661
2972
  const consumer = ctx.consumers.get(gid);
2662
2973
  if (!consumer) return;
2663
2974
  const assignment = consumer.assignment();
2664
- const partitions = assignment.filter((a) => a.topic === topic2).map((a) => a.partition);
2975
+ const partitions = assignment.filter((a) => a.topic === topic).map((a) => a.partition);
2665
2976
  if (partitions.length > 0)
2666
- consumer.resume(partitions.map((p) => ({ topic: topic2, partitions: [p] })));
2977
+ consumer.resume(partitions.map((p) => ({ topic, partitions: [p] })));
2667
2978
  }
2668
2979
 
2669
2980
  // src/client/kafka.client/consumer/start.ts
2670
2981
  async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
2671
2982
  validateTopicConsumerOpts(topics, options);
2672
2983
  const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
2673
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
2984
+ const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
2674
2985
  if (options.circuitBreaker)
2675
2986
  ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
2676
2987
  const deps = messageDepsFor(ctx, gid, options);
@@ -2681,13 +2992,14 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
2681
2992
  payload,
2682
2993
  {
2683
2994
  schemaMap,
2995
+ serdeMap,
2684
2996
  handleMessage,
2685
2997
  interceptors,
2686
2998
  dlq,
2687
2999
  retry,
2688
3000
  retryTopics: options.retryTopics,
2689
3001
  timeoutMs: options.handlerTimeoutMs,
2690
- wrapWithTimeout: (fn, ms, topic2) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic2),
3002
+ wrapWithTimeout: (fn, ms, topic) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic),
2691
3003
  deduplication: resolveDeduplicationContext(
2692
3004
  ctx,
2693
3005
  gid,
@@ -2708,6 +3020,7 @@ async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
2708
3020
  dlq,
2709
3021
  interceptors,
2710
3022
  schemaMap,
3023
+ serdeMap,
2711
3024
  assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
2712
3025
  });
2713
3026
  }
@@ -2721,7 +3034,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
2721
3034
  );
2722
3035
  }
2723
3036
  const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
2724
- const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
3037
+ const { consumer, schemaMap, serdeMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
2725
3038
  if (options.circuitBreaker)
2726
3039
  ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
2727
3040
  const deps = messageDepsFor(ctx, gid, options);
@@ -2732,13 +3045,14 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
2732
3045
  payload,
2733
3046
  {
2734
3047
  schemaMap,
3048
+ serdeMap,
2735
3049
  handleBatch,
2736
3050
  interceptors,
2737
3051
  dlq,
2738
3052
  retry,
2739
3053
  retryTopics: options.retryTopics,
2740
3054
  timeoutMs: options.handlerTimeoutMs,
2741
- wrapWithTimeout: (fn, ms, topic2) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic2),
3055
+ wrapWithTimeout: (fn, ms, topic) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic),
2742
3056
  deduplication: resolveDeduplicationContext(
2743
3057
  ctx,
2744
3058
  gid,
@@ -2769,6 +3083,7 @@ async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
2769
3083
  dlq,
2770
3084
  interceptors,
2771
3085
  schemaMap,
3086
+ serdeMap,
2772
3087
  assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
2773
3088
  });
2774
3089
  }
@@ -2781,7 +3096,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
2781
3096
  );
2782
3097
  }
2783
3098
  const setupOptions = { ...options, autoCommit: false };
2784
- const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
3099
+ const { consumer, schemaMap, serdeMap, gid, readyPromise } = await setupConsumer(
2785
3100
  ctx,
2786
3101
  topics,
2787
3102
  "eachMessage",
@@ -2790,20 +3105,21 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
2790
3105
  const txProducer = await createRetryTxProducer(ctx, `${gid}-txc`);
2791
3106
  const deps = messageDepsFor(ctx, gid);
2792
3107
  await consumer.run({
2793
- eachMessage: ({ topic: topic2, partition, message }) => ctx.inFlight.track(async () => {
3108
+ eachMessage: ({ topic, partition, message }) => ctx.inFlight.track(async () => {
2794
3109
  const envelope = await parseSingleMessage(
2795
3110
  message,
2796
- topic2,
3111
+ topic,
2797
3112
  partition,
2798
3113
  schemaMap,
2799
3114
  options.interceptors ?? [],
2800
3115
  false,
2801
- deps
3116
+ deps,
3117
+ serdeMap
2802
3118
  );
2803
3119
  const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
2804
3120
  if (envelope === null) {
2805
3121
  await consumer.commitOffsets([
2806
- { topic: topic2, partition, offset: nextOffset }
3122
+ { topic, partition, offset: nextOffset }
2807
3123
  ]);
2808
3124
  return;
2809
3125
  }
@@ -2843,7 +3159,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
2843
3159
  await tx.sendOffsets({
2844
3160
  consumer,
2845
3161
  topics: [
2846
- { topic: topic2, partitions: [{ partition, offset: nextOffset }] }
3162
+ { topic, partitions: [{ partition, offset: nextOffset }] }
2847
3163
  ]
2848
3164
  });
2849
3165
  await tx.commit();
@@ -2854,7 +3170,7 @@ async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}
2854
3170
  } catch {
2855
3171
  }
2856
3172
  ctx.logger.warn(
2857
- `startTransactionalConsumer: handler failed on ${topic2}[${partition}]@${message.offset} \u2014 tx aborted, message will be redelivered (${toError(err).message})`
3173
+ `startTransactionalConsumer: handler failed on ${topic}[${partition}]@${message.offset} \u2014 tx aborted, message will be redelivered (${toError(err).message})`
2858
3174
  );
2859
3175
  throw err;
2860
3176
  }
@@ -2867,8 +3183,8 @@ function stopConsumerByGid(ctx, gid) {
2867
3183
  return stopConsumerImpl(ctx, gid);
2868
3184
  }
2869
3185
 
2870
- // src/client/kafka.client/consumer/window.ts
2871
- async function startWindowConsumerImpl(ctx, topic2, handler, options) {
3186
+ // src/client/kafka.client/consumer/features/window.ts
3187
+ async function startWindowConsumerImpl(ctx, topic, handler, options) {
2872
3188
  const { maxMessages, maxMs, ...consumerOptions } = options;
2873
3189
  if (maxMessages <= 0)
2874
3190
  throw new Error("startWindowConsumer: maxMessages must be > 0");
@@ -2881,6 +3197,7 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
2881
3197
  const buffer = [];
2882
3198
  let flushTimer = null;
2883
3199
  let windowStart = 0;
3200
+ const onLost = consumerOptions.onMessageLost ?? ctx.onMessageLost;
2884
3201
  const flush = async (trigger) => {
2885
3202
  if (flushTimer !== null) {
2886
3203
  clearTimeout(flushTimer);
@@ -2888,22 +3205,37 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
2888
3205
  }
2889
3206
  if (buffer.length === 0) return;
2890
3207
  const envelopes = buffer.splice(0);
2891
- await handler(envelopes, { trigger, windowStart, windowEnd: Date.now() });
3208
+ try {
3209
+ await handler(envelopes, { trigger, windowStart, windowEnd: Date.now() });
3210
+ } catch (err) {
3211
+ const error = toError(err);
3212
+ ctx.logger.error(
3213
+ `startWindowConsumer: ${trigger}-triggered flush failed \u2014 window of ${envelopes.length} message(s) lost:`,
3214
+ error.stack
3215
+ );
3216
+ for (const envelope of envelopes) {
3217
+ await Promise.resolve(
3218
+ onLost?.({
3219
+ topic: envelope.topic,
3220
+ error,
3221
+ attempt: 0,
3222
+ headers: envelope.headers
3223
+ })
3224
+ ).catch(() => {
3225
+ });
3226
+ }
3227
+ }
2892
3228
  };
2893
3229
  const scheduleFlush = () => {
2894
3230
  if (flushTimer !== null) return;
2895
3231
  flushTimer = setTimeout(() => {
2896
3232
  flushTimer = null;
2897
- flush("time").catch((err) => {
2898
- ctx.logger.warn(
2899
- `startWindowConsumer: time-triggered flush error \u2014 ${toError(err).message}`
2900
- );
2901
- });
3233
+ void flush("time");
2902
3234
  }, maxMs);
2903
3235
  };
2904
3236
  const handle = await startConsumerImpl(
2905
3237
  ctx,
2906
- [topic2],
3238
+ [topic],
2907
3239
  async (envelope) => {
2908
3240
  if (buffer.length === 0) windowStart = Date.now();
2909
3241
  buffer.push(envelope);
@@ -2914,40 +3246,13 @@ async function startWindowConsumerImpl(ctx, topic2, handler, options) {
2914
3246
  );
2915
3247
  const originalStop = handle.stop.bind(handle);
2916
3248
  handle.stop = async () => {
2917
- if (flushTimer !== null) {
2918
- clearTimeout(flushTimer);
2919
- flushTimer = null;
2920
- }
2921
- if (buffer.length > 0) {
2922
- const envelopes = buffer.splice(0);
2923
- await handler(envelopes, {
2924
- trigger: "time",
2925
- windowStart,
2926
- windowEnd: Date.now()
2927
- }).catch(async (err) => {
2928
- const error = toError(err);
2929
- ctx.logger.warn(
2930
- `startWindowConsumer: shutdown flush error \u2014 ${error.message}`
2931
- );
2932
- for (const envelope of envelopes) {
2933
- await Promise.resolve(
2934
- ctx.onMessageLost?.({
2935
- topic: envelope.topic,
2936
- error,
2937
- attempt: 0,
2938
- headers: envelope.headers
2939
- })
2940
- ).catch(() => {
2941
- });
2942
- }
2943
- });
2944
- }
3249
+ await flush("time");
2945
3250
  return originalStop();
2946
3251
  };
2947
3252
  return handle;
2948
3253
  }
2949
3254
 
2950
- // src/client/kafka.client/consumer/routed.ts
3255
+ // src/client/kafka.client/consumer/features/routed.ts
2951
3256
  async function startRoutedConsumerImpl(ctx, topics, routing, options) {
2952
3257
  const { header, routes, fallback } = routing;
2953
3258
  const handleMessage = async (envelope) => {
@@ -2962,15 +3267,128 @@ async function startRoutedConsumerImpl(ctx, topics, routing, options) {
2962
3267
  return startConsumerImpl(ctx, topics, handleMessage, options);
2963
3268
  }
2964
3269
 
2965
- // src/client/kafka.client/consumer/snapshot.ts
2966
- async function readSnapshotImpl(ctx, topic2, options = {}) {
3270
+ // src/client/kafka.client/consumer/features/delayed.ts
3271
+ function delayedTopicName(topic) {
3272
+ return `${topic}.delayed`;
3273
+ }
3274
+ async function startDelayedRelayImpl(ctx, topics, options) {
3275
+ if (topics.length === 0) {
3276
+ throw new Error("startDelayedRelay: at least one topic is required");
3277
+ }
3278
+ const gid = options?.groupId ?? `${ctx.defaultGroupId}-delayed-relay`;
3279
+ if (ctx.runningConsumers.has(gid)) {
3280
+ throw new Error(
3281
+ `startDelayedRelay("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
3282
+ );
3283
+ }
3284
+ const delayedTopics = topics.map(delayedTopicName);
3285
+ for (const t of delayedTopics) await ensureTopic(ctx, t);
3286
+ const txProducer = await createRetryTxProducer(ctx, `${gid}-tx`);
3287
+ let resolveReady;
3288
+ const readyPromise = new Promise((resolve) => {
3289
+ resolveReady = resolve;
3290
+ });
3291
+ const consumer = getOrCreateConsumer(
3292
+ gid,
3293
+ false,
3294
+ false,
3295
+ ctx.consumerOpsDeps,
3296
+ void 0,
3297
+ resolveReady
3298
+ );
3299
+ await consumer.connect();
3300
+ await subscribeWithRetry(consumer, delayedTopics, ctx.logger);
3301
+ await consumer.run({
3302
+ eachMessage: async ({ topic: stagingTopic, partition, message }) => {
3303
+ const nextOffset = {
3304
+ topic: stagingTopic,
3305
+ partition,
3306
+ offset: (parseInt(message.offset, 10) + 1).toString()
3307
+ };
3308
+ if (!message.value) {
3309
+ await consumer.commitOffsets([nextOffset]);
3310
+ return;
3311
+ }
3312
+ const headers = decodeHeaders(message.headers);
3313
+ const target = headers[HEADER_DELAYED_TARGET] ?? stagingTopic.replace(/\.delayed$/, "");
3314
+ const until = parseInt(
3315
+ headers[HEADER_DELAYED_UNTIL] ?? "0",
3316
+ 10
3317
+ );
3318
+ const remaining = until - Date.now();
3319
+ if (remaining > 0) {
3320
+ consumer.pause([{ topic: stagingTopic, partitions: [partition] }]);
3321
+ await sleep(remaining);
3322
+ consumer.resume([{ topic: stagingTopic, partitions: [partition] }]);
3323
+ }
3324
+ const forwardHeaders = Object.fromEntries(
3325
+ Object.entries(headers).filter(
3326
+ ([k]) => k !== HEADER_DELAYED_UNTIL && k !== HEADER_DELAYED_TARGET
3327
+ )
3328
+ );
3329
+ const tx = await txProducer.transaction();
3330
+ try {
3331
+ await tx.send({
3332
+ topic: target,
3333
+ messages: [
3334
+ {
3335
+ // Forward the ORIGINAL wire bytes unchanged — no re-serialization,
3336
+ // so binary payloads (Avro/Protobuf) are relayed losslessly.
3337
+ value: message.value,
3338
+ key: message.key ? message.key.toString() : null,
3339
+ headers: forwardHeaders
3340
+ }
3341
+ ]
3342
+ });
3343
+ await tx.sendOffsets({
3344
+ consumer,
3345
+ topics: [
3346
+ {
3347
+ topic: nextOffset.topic,
3348
+ partitions: [
3349
+ { partition: nextOffset.partition, offset: nextOffset.offset }
3350
+ ]
3351
+ }
3352
+ ]
3353
+ });
3354
+ await tx.commit();
3355
+ ctx.logger.debug?.(
3356
+ `Delayed message relayed to "${target}" (deadline ${new Date(until).toISOString()})`
3357
+ );
3358
+ } catch (txErr) {
3359
+ try {
3360
+ await tx.abort();
3361
+ } catch {
3362
+ }
3363
+ ctx.logger.error(
3364
+ `Delayed relay to "${target}" failed \u2014 message will be redelivered:`,
3365
+ toError(txErr).stack
3366
+ );
3367
+ }
3368
+ }
3369
+ });
3370
+ ctx.runningConsumers.set(gid, "eachMessage");
3371
+ ctx.logger.log(
3372
+ `Delayed relay started for: ${delayedTopics.join(", ")} (group: ${gid})`
3373
+ );
3374
+ return {
3375
+ groupId: gid,
3376
+ ready: () => readyPromise,
3377
+ stop: async () => {
3378
+ await stopConsumerImpl(ctx, gid);
3379
+ }
3380
+ };
3381
+ }
3382
+
3383
+ // src/client/kafka.client/consumer/features/snapshot.ts
3384
+ async function readSnapshotImpl(ctx, topic, options = {}) {
2967
3385
  await ctx.adminOps.ensureConnected();
2968
3386
  let offsets;
2969
3387
  try {
2970
- offsets = await ctx.adminOps.admin.fetchTopicOffsets(topic2);
3388
+ offsets = await ctx.adminOps.admin.fetchTopicOffsets(topic);
2971
3389
  } catch {
2972
3390
  ctx.logger.warn(
2973
- `readSnapshot: could not fetch offsets for "${String(topic2)}", returning empty snapshot`
3391
+ `readSnapshot: could not fetch offsets for "${String(topic)}", returning empty snapshot`
2974
3392
  );
2975
3393
  return /* @__PURE__ */ new Map();
2976
3394
  }
@@ -2982,7 +3400,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
2982
3400
  }
2983
3401
  if (targets.size === 0) {
2984
3402
  ctx.logger.debug?.(
2985
- `readSnapshot: topic "${String(topic2)}" is empty \u2014 returning empty snapshot`
3403
+ `readSnapshot: topic "${String(topic)}" is empty \u2014 returning empty snapshot`
2986
3404
  );
2987
3405
  return /* @__PURE__ */ new Map();
2988
3406
  }
@@ -3001,7 +3419,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
3001
3419
  });
3002
3420
  });
3003
3421
  };
3004
- consumer.connect().then(() => consumer.subscribe({ topics: [topic2] })).then(
3422
+ consumer.connect().then(() => consumer.subscribe({ topics: [topic] })).then(
3005
3423
  () => consumer.run({
3006
3424
  eachMessage: async ({ topic: t, partition, message }) => {
3007
3425
  if (!remaining.has(partition)) return;
@@ -3022,7 +3440,7 @@ async function readSnapshotImpl(ctx, topic2, options = {}) {
3022
3440
  });
3023
3441
  });
3024
3442
  ctx.logger.log(
3025
- `readSnapshot: ${snapshot.size} key(s) from "${String(topic2)}"`
3443
+ `readSnapshot: ${snapshot.size} key(s) from "${String(topic)}"`
3026
3444
  );
3027
3445
  return snapshot;
3028
3446
  }
@@ -3059,9 +3477,9 @@ async function checkpointOffsetsImpl(ctx, groupId, checkpointTopic) {
3059
3477
  await ctx.adminOps.ensureConnected();
3060
3478
  const committed = await ctx.adminOps.admin.fetchOffsets({ groupId: gid });
3061
3479
  const offsets = [];
3062
- for (const { topic: topic2, partitions } of committed) {
3480
+ for (const { topic, partitions } of committed) {
3063
3481
  for (const { partition, offset } of partitions) {
3064
- offsets.push({ topic: topic2, partition, offset });
3482
+ offsets.push({ topic, partition, offset });
3065
3483
  }
3066
3484
  }
3067
3485
  const savedAt = Date.now();
@@ -3228,6 +3646,7 @@ var KafkaClient = class {
3228
3646
  * ```
3229
3647
  */
3230
3648
  constructor(clientId, groupId, brokers, options) {
3649
+ validateClientOptions(clientId, groupId, brokers, options);
3231
3650
  this.clientId = clientId;
3232
3651
  const logger = options?.logger ?? {
3233
3652
  log: (msg) => console.log(`[KafkaClient:${clientId}] ${msg}`),
@@ -3235,12 +3654,14 @@ var KafkaClient = class {
3235
3654
  error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args),
3236
3655
  debug: (msg, ...args) => console.debug(`[KafkaClient:${clientId}] ${msg}`, ...args)
3237
3656
  };
3238
- const transport = options?.transport ?? new ConfluentTransport(clientId, brokers);
3657
+ const security = resolveSecurityOptions(options?.security, brokers, logger);
3658
+ const transport = options?.transport ?? new ConfluentTransport(clientId, brokers, security);
3239
3659
  const producer = transport.producer();
3240
3660
  const runningConsumers = /* @__PURE__ */ new Map();
3241
3661
  const consumers = /* @__PURE__ */ new Map();
3242
3662
  const consumerCreationOptions = /* @__PURE__ */ new Map();
3243
3663
  const schemaRegistry = /* @__PURE__ */ new Map();
3664
+ const serde = options?.serde ?? new JsonSerde();
3244
3665
  const adminOps = new AdminOps({
3245
3666
  admin: transport.admin(),
3246
3667
  logger,
@@ -3267,8 +3688,10 @@ var KafkaClient = class {
3267
3688
  autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
3268
3689
  strictSchemasEnabled: options?.strictSchemas ?? true,
3269
3690
  numPartitions: options?.numPartitions ?? 1,
3691
+ serde,
3270
3692
  txId: options?.transactionalId ?? `${clientId}-tx`,
3271
3693
  clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
3694
+ clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
3272
3695
  lagThrottleOpts: options?.lagThrottle,
3273
3696
  instrumentation: options?.instrumentation ?? [],
3274
3697
  onMessageLost: options?.onMessageLost,
@@ -3278,6 +3701,7 @@ var KafkaClient = class {
3278
3701
  producer,
3279
3702
  txProducer: void 0,
3280
3703
  txProducerInitPromise: void 0,
3704
+ _txChain: Promise.resolve(),
3281
3705
  retryTxProducers: /* @__PURE__ */ new Map(),
3282
3706
  consumers,
3283
3707
  runningConsumers,
@@ -3299,6 +3723,7 @@ var KafkaClient = class {
3299
3723
  strictSchemasEnabled: options?.strictSchemas ?? true,
3300
3724
  instrumentation: options?.instrumentation ?? [],
3301
3725
  logger,
3726
+ serde,
3302
3727
  nextLamportClock: () => 0
3303
3728
  // patched below
3304
3729
  },
@@ -3317,6 +3742,7 @@ var KafkaClient = class {
3317
3742
  strictSchemasEnabled: options?.strictSchemas ?? true,
3318
3743
  instrumentation: options?.instrumentation ?? [],
3319
3744
  logger,
3745
+ serde,
3320
3746
  nextLamportClock: () => ++ctx._lamportClock
3321
3747
  };
3322
3748
  ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
@@ -3326,8 +3752,8 @@ var KafkaClient = class {
3326
3752
  return sendMessageImpl(this.ctx, topicOrDesc, message, options);
3327
3753
  }
3328
3754
  /** @inheritDoc */
3329
- async sendTombstone(topic2, key, headers) {
3330
- return sendTombstoneImpl(this.ctx, topic2, key, headers);
3755
+ async sendTombstone(topic, key, headers) {
3756
+ return sendTombstoneImpl(this.ctx, topic, key, headers);
3331
3757
  }
3332
3758
  async sendBatch(topicOrDesc, messages, options) {
3333
3759
  return sendBatchImpl(this.ctx, topicOrDesc, messages, options);
@@ -3358,7 +3784,7 @@ var KafkaClient = class {
3358
3784
  }
3359
3785
  // ── Consumer: AsyncIterableIterator ──────────────────────────────
3360
3786
  /** @inheritDoc */
3361
- consume(topic2, options) {
3787
+ consume(topic, options) {
3362
3788
  if (options?.retryTopics) {
3363
3789
  throw new Error(
3364
3790
  "consume() does not support retryTopics (EOS retry chains). Use startConsumer() with retryTopics: true for guaranteed retry delivery."
@@ -3367,11 +3793,11 @@ var KafkaClient = class {
3367
3793
  const gid = options?.groupId ?? this.ctx.defaultGroupId;
3368
3794
  const queue = new AsyncQueue(
3369
3795
  options?.queueHighWaterMark,
3370
- () => pauseTopicAllPartitions(this.ctx, gid, topic2),
3371
- () => resumeTopicAllPartitions(this.ctx, gid, topic2)
3796
+ () => pauseTopicAllPartitions(this.ctx, gid, topic),
3797
+ () => resumeTopicAllPartitions(this.ctx, gid, topic)
3372
3798
  );
3373
3799
  const handlePromise = this.startConsumer(
3374
- [topic2],
3800
+ [topic],
3375
3801
  async (envelope) => {
3376
3802
  queue.push(envelope);
3377
3803
  },
@@ -3393,14 +3819,39 @@ var KafkaClient = class {
3393
3819
  }
3394
3820
  // ── Consumer: windowed ────────────────────────────────────────────
3395
3821
  /** @inheritDoc */
3396
- startWindowConsumer(topic2, handler, options) {
3397
- return startWindowConsumerImpl(this.ctx, topic2, handler, options);
3822
+ startWindowConsumer(topic, handler, options) {
3823
+ return startWindowConsumerImpl(this.ctx, topic, handler, options);
3398
3824
  }
3399
3825
  // ── Consumer: header routing ──────────────────────────────────────
3400
3826
  /** @inheritDoc */
3401
3827
  startRoutedConsumer(topics, routing, options) {
3402
3828
  return startRoutedConsumerImpl(this.ctx, topics, routing, options);
3403
3829
  }
3830
+ // ── Consumer: delayed delivery relay ──────────────────────────────
3831
+ /**
3832
+ * Start a relay that delivers messages produced with
3833
+ * `SendOptions.deliverAfterMs` from `<topic>.delayed` to their target topic
3834
+ * once their deadline passes.
3835
+ *
3836
+ * Forwarding is transactional (produce + source-offset commit are atomic),
3837
+ * so no duplicates are relayed even if the relay crashes mid-forward.
3838
+ * Delivery time is a lower bound — the relay must be running for delayed
3839
+ * messages to be delivered at all.
3840
+ *
3841
+ * @param topics Target topic name(s) whose `<topic>.delayed` staging topics to relay.
3842
+ * @param options Optional `groupId` override (default: `<defaultGroupId>-delayed-relay`).
3843
+ *
3844
+ * @example
3845
+ * ```ts
3846
+ * await kafka.startDelayedRelay(['orders.reminder']);
3847
+ * await kafka.sendMessage('orders.reminder', payload, { deliverAfterMs: 60_000 });
3848
+ * // → delivered to orders.reminder ~60 s later
3849
+ * ```
3850
+ */
3851
+ async startDelayedRelay(topics, options) {
3852
+ const list = Array.isArray(topics) ? topics : [topics];
3853
+ return startDelayedRelayImpl(this.ctx, list, options);
3854
+ }
3404
3855
  // ── Consumer: transactional EOS ───────────────────────────────────
3405
3856
  /** @inheritDoc */
3406
3857
  async startTransactionalConsumer(topics, handler, options = {}) {
@@ -3421,10 +3872,10 @@ var KafkaClient = class {
3421
3872
  }
3422
3873
  // ── DLQ replay ────────────────────────────────────────────────────
3423
3874
  /** @inheritDoc */
3424
- async replayDlq(topic2, options = {}) {
3875
+ async replayDlq(topic, options = {}) {
3425
3876
  await this.ctx.adminOps.ensureConnected();
3426
3877
  return replayDlqTopic(
3427
- topic2,
3878
+ topic,
3428
3879
  {
3429
3880
  logger: this.ctx.logger,
3430
3881
  fetchTopicOffsets: (t) => this.ctx.adminOps.admin.fetchTopicOffsets(t),
@@ -3451,8 +3902,8 @@ var KafkaClient = class {
3451
3902
  }
3452
3903
  // ── Snapshot & checkpoint ─────────────────────────────────────────
3453
3904
  /** @inheritDoc */
3454
- async readSnapshot(topic2, options = {}) {
3455
- return readSnapshotImpl(this.ctx, topic2, options);
3905
+ async readSnapshot(topic, options = {}) {
3906
+ return readSnapshotImpl(this.ctx, topic, options);
3456
3907
  }
3457
3908
  /** @inheritDoc */
3458
3909
  async checkpointOffsets(groupId, checkpointTopic) {
@@ -3464,8 +3915,8 @@ var KafkaClient = class {
3464
3915
  }
3465
3916
  // ── Admin ─────────────────────────────────────────────────────────
3466
3917
  /** @inheritDoc */
3467
- async resetOffsets(groupId, topic2, position) {
3468
- return this.ctx.adminOps.resetOffsets(groupId, topic2, position);
3918
+ async resetOffsets(groupId, topic, position) {
3919
+ return this.ctx.adminOps.resetOffsets(groupId, topic, position);
3469
3920
  }
3470
3921
  /** @inheritDoc */
3471
3922
  async seekToOffset(groupId, assignments) {
@@ -3492,26 +3943,26 @@ var KafkaClient = class {
3492
3943
  return this.ctx.adminOps.describeTopics(topics);
3493
3944
  }
3494
3945
  /** @inheritDoc */
3495
- async deleteRecords(topic2, partitions) {
3496
- return this.ctx.adminOps.deleteRecords(topic2, partitions);
3946
+ async deleteRecords(topic, partitions) {
3947
+ return this.ctx.adminOps.deleteRecords(topic, partitions);
3497
3948
  }
3498
3949
  // ── Circuit breaker ───────────────────────────────────────────────
3499
3950
  /** @inheritDoc */
3500
- getCircuitState(topic2, partition, groupId) {
3951
+ getCircuitState(topic, partition, groupId) {
3501
3952
  return this.ctx.circuitBreaker.getState(
3502
- topic2,
3953
+ topic,
3503
3954
  partition,
3504
3955
  groupId ?? this.ctx.defaultGroupId
3505
3956
  );
3506
3957
  }
3507
3958
  // ── Metrics ───────────────────────────────────────────────────────
3508
3959
  /** @inheritDoc */
3509
- getMetrics(topic2) {
3510
- return this.ctx.metrics.getMetrics(topic2);
3960
+ getMetrics(topic) {
3961
+ return this.ctx.metrics.getMetrics(topic);
3511
3962
  }
3512
3963
  /** @inheritDoc */
3513
- resetMetrics(topic2) {
3514
- this.ctx.metrics.resetMetrics(topic2);
3964
+ resetMetrics(topic) {
3965
+ this.ctx.metrics.resetMetrics(topic);
3515
3966
  }
3516
3967
  getClientId() {
3517
3968
  return this.clientId;
@@ -3542,38 +3993,351 @@ var KafkaClient = class {
3542
3993
  }
3543
3994
  };
3544
3995
 
3545
- // src/client/message/topic.ts
3546
- function topic(name) {
3996
+ // src/cli/dlq.ts
3997
+ var DLQ_SUFFIX = ".dlq";
3998
+ var DlqUsageError = class extends Error {
3999
+ constructor(message) {
4000
+ super(message);
4001
+ this.name = "DlqUsageError";
4002
+ }
4003
+ };
4004
+ var USAGE = `kafka-client-dlq \u2014 dead-letter queue operations
4005
+
4006
+ Usage:
4007
+ kafka-client-dlq ls --brokers <b1,b2> [--prefix <name>]
4008
+ kafka-client-dlq peek --brokers <b1,b2> --topic <name> [--limit <n>]
4009
+ kafka-client-dlq replay --brokers <b1,b2> --topic <name> [--target <t>] [--dry-run] [--from-beginning | --incremental]
4010
+
4011
+ Commands:
4012
+ ls List DLQ topics (ending in .dlq) with per-topic message counts.
4013
+ peek Print up to N messages from <topic>.dlq (offset, x-dlq-* headers, value).
4014
+ replay Re-publish <topic>.dlq messages to their original topic (or --target).
4015
+
4016
+ Options:
4017
+ --brokers <list> Comma-separated broker addresses (required). e.g. localhost:9092
4018
+ --prefix <name> ls: only show DLQ topics whose base name starts with <name>.
4019
+ --topic <name> peek/replay: base topic name (the CLI uses <name>.dlq).
4020
+ --limit <n> peek: max messages to print (default 10).
4021
+ --target <t> replay: override destination topic.
4022
+ --dry-run replay: log without publishing.
4023
+ --from-beginning replay: full replay every call (default).
4024
+ --incremental replay: only messages added since the previous replay.
4025
+ -h, --help Show this help.
4026
+
4027
+ Examples:
4028
+ kafka-client-dlq ls --brokers localhost:9092
4029
+ kafka-client-dlq ls --brokers localhost:9092 --prefix orders
4030
+ kafka-client-dlq peek --brokers localhost:9092 --topic orders.created --limit 5
4031
+ kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --dry-run
4032
+ kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --target orders.manual --incremental
4033
+ `;
4034
+ var VALUE_FLAGS = /* @__PURE__ */ new Set([
4035
+ "--brokers",
4036
+ "--prefix",
4037
+ "--topic",
4038
+ "--limit",
4039
+ "--target"
4040
+ ]);
4041
+ var BOOL_FLAGS = /* @__PURE__ */ new Set(["--dry-run", "--from-beginning", "--incremental"]);
4042
+ function parseFlags(args) {
4043
+ const values = {};
4044
+ const bools = /* @__PURE__ */ new Set();
4045
+ for (let i = 0; i < args.length; i++) {
4046
+ const arg = args[i];
4047
+ if (!arg.startsWith("--")) {
4048
+ throw new DlqUsageError(`Unexpected argument: "${arg}"`);
4049
+ }
4050
+ if (VALUE_FLAGS.has(arg)) {
4051
+ const value = args[i + 1];
4052
+ if (value === void 0 || value.startsWith("--")) {
4053
+ throw new DlqUsageError(`Flag "${arg}" requires a value.`);
4054
+ }
4055
+ values[arg] = value;
4056
+ i++;
4057
+ } else if (BOOL_FLAGS.has(arg)) {
4058
+ bools.add(arg);
4059
+ } else {
4060
+ throw new DlqUsageError(`Unknown flag: "${arg}"`);
4061
+ }
4062
+ }
4063
+ return { values, bools };
4064
+ }
4065
+ function requireBrokers(flags) {
4066
+ const raw = flags.values["--brokers"];
4067
+ if (raw === void 0) {
4068
+ throw new DlqUsageError("Missing required flag: --brokers");
4069
+ }
4070
+ const brokers = raw.split(",").map((b) => b.trim()).filter((b) => b.length > 0);
4071
+ if (brokers.length === 0) {
4072
+ throw new DlqUsageError("--brokers must list at least one broker address.");
4073
+ }
4074
+ return brokers;
4075
+ }
4076
+ function requireTopic(flags) {
4077
+ const topic = flags.values["--topic"];
4078
+ if (topic === void 0 || topic.length === 0) {
4079
+ throw new DlqUsageError("Missing required flag: --topic");
4080
+ }
4081
+ return topic;
4082
+ }
4083
+ function parseArgs(argv) {
4084
+ const [command, ...rest] = argv;
4085
+ if (command === void 0 || command === "-h" || command === "--help" || command === "help") {
4086
+ return { command: "help" };
4087
+ }
4088
+ switch (command) {
4089
+ case "ls": {
4090
+ const flags = parseFlags(rest);
4091
+ const brokers = requireBrokers(flags);
4092
+ const prefix = flags.values["--prefix"];
4093
+ return { command: "ls", brokers, prefix };
4094
+ }
4095
+ case "peek": {
4096
+ const flags = parseFlags(rest);
4097
+ const brokers = requireBrokers(flags);
4098
+ const topic = requireTopic(flags);
4099
+ const limit = parseLimit(flags.values["--limit"]);
4100
+ return { command: "peek", brokers, topic, limit };
4101
+ }
4102
+ case "replay": {
4103
+ const flags = parseFlags(rest);
4104
+ const brokers = requireBrokers(flags);
4105
+ const topic = requireTopic(flags);
4106
+ const target = flags.values["--target"];
4107
+ const dryRun = flags.bools.has("--dry-run");
4108
+ if (flags.bools.has("--from-beginning") && flags.bools.has("--incremental")) {
4109
+ throw new DlqUsageError(
4110
+ "--from-beginning and --incremental are mutually exclusive."
4111
+ );
4112
+ }
4113
+ const fromBeginning = !flags.bools.has("--incremental");
4114
+ return { command: "replay", brokers, topic, target, dryRun, fromBeginning };
4115
+ }
4116
+ default:
4117
+ throw new DlqUsageError(`Unknown command: "${command}"`);
4118
+ }
4119
+ }
4120
+ function parseLimit(raw) {
4121
+ if (raw === void 0) return 10;
4122
+ const n = Number(raw);
4123
+ if (!Number.isInteger(n) || n <= 0) {
4124
+ throw new DlqUsageError(`--limit must be a positive integer, got "${raw}".`);
4125
+ }
4126
+ return n;
4127
+ }
4128
+ function countFromWatermarks(watermarks) {
4129
+ let total = 0;
4130
+ for (const { low, high } of watermarks) {
4131
+ const width = Number(high) - Number(low);
4132
+ total += width > 0 ? width : 0;
4133
+ }
4134
+ return total;
4135
+ }
4136
+ function truncate(value, max = 200) {
4137
+ if (value.length <= max) return value;
4138
+ return `${value.slice(0, max)}\u2026 (${value.length} chars)`;
4139
+ }
4140
+ async function runDlqCommand(cmd, deps) {
4141
+ if (cmd.command === "help") {
4142
+ deps.out(USAGE);
4143
+ return { command: "help" };
4144
+ }
4145
+ const client = await deps.createClient(cmd.brokers);
4146
+ try {
4147
+ switch (cmd.command) {
4148
+ case "ls":
4149
+ return await runLs(cmd, client, deps);
4150
+ case "peek":
4151
+ return await runPeek(cmd, client, deps);
4152
+ case "replay":
4153
+ return await runReplay(cmd, client, deps);
4154
+ }
4155
+ } finally {
4156
+ await client.close();
4157
+ }
4158
+ }
4159
+ async function runLs(cmd, client, deps) {
4160
+ const allTopics = await client.listTopics();
4161
+ let dlqTopics = allTopics.filter((t) => t.endsWith(DLQ_SUFFIX));
4162
+ if (cmd.prefix) {
4163
+ const prefix = cmd.prefix;
4164
+ dlqTopics = dlqTopics.filter(
4165
+ (t) => t.slice(0, -DLQ_SUFFIX.length).startsWith(prefix)
4166
+ );
4167
+ }
4168
+ dlqTopics.sort();
4169
+ const counts = [];
4170
+ for (const dlqTopic of dlqTopics) {
4171
+ const watermarks = await client.fetchTopicOffsets(dlqTopic);
4172
+ counts.push({
4173
+ dlqTopic,
4174
+ baseTopic: dlqTopic.slice(0, -DLQ_SUFFIX.length),
4175
+ count: countFromWatermarks(watermarks)
4176
+ });
4177
+ }
4178
+ if (counts.length === 0) {
4179
+ deps.out(
4180
+ cmd.prefix ? `No DLQ topics found matching prefix "${cmd.prefix}".` : "No DLQ topics found."
4181
+ );
4182
+ } else {
4183
+ const width = Math.max(...counts.map((c) => c.dlqTopic.length));
4184
+ deps.out(`${"TOPIC".padEnd(width)} MESSAGES`);
4185
+ for (const c of counts) {
4186
+ deps.out(`${c.dlqTopic.padEnd(width)} ${c.count}`);
4187
+ }
4188
+ const total = counts.reduce((s, c) => s + c.count, 0);
4189
+ deps.out(`${counts.length} DLQ topic(s), ${total} message(s) total.`);
4190
+ }
4191
+ return { command: "ls", topics: counts };
4192
+ }
4193
+ async function runPeek(cmd, client, deps) {
4194
+ const dlqTopic = `${cmd.topic}${DLQ_SUFFIX}`;
4195
+ const messages = await client.peekMessages(dlqTopic, cmd.limit);
4196
+ if (messages.length === 0) {
4197
+ deps.out(`No messages in ${dlqTopic}.`);
4198
+ return { command: "peek", printed: 0 };
4199
+ }
4200
+ deps.out(`Peeking up to ${cmd.limit} message(s) from ${dlqTopic}:`);
4201
+ let printed = 0;
4202
+ for (const env of messages) {
4203
+ if (printed >= cmd.limit) break;
4204
+ deps.out("");
4205
+ deps.out(
4206
+ `\u2500 offset ${env.offset} \xB7 partition ${env.partition} \xB7 ${env.timestamp}`
4207
+ );
4208
+ const dlqHeaders = Object.entries(env.headers).filter(([k]) => k.startsWith("x-dlq-")).sort(([a], [b]) => a.localeCompare(b));
4209
+ for (const [k, v] of dlqHeaders) {
4210
+ deps.out(` ${k}: ${truncate(String(v), 500)}`);
4211
+ }
4212
+ deps.out(` value: ${truncate(JSON.stringify(env.payload))}`);
4213
+ printed++;
4214
+ }
4215
+ deps.out("");
4216
+ deps.out(`Printed ${printed} message(s).`);
4217
+ return { command: "peek", printed };
4218
+ }
4219
+ async function runReplay(cmd, client, deps) {
4220
+ const options = {
4221
+ dryRun: cmd.dryRun,
4222
+ fromBeginning: cmd.fromBeginning
4223
+ };
4224
+ if (cmd.target !== void 0) options.targetTopic = cmd.target;
4225
+ const mode = cmd.fromBeginning ? "full" : "incremental";
4226
+ const targetDesc = cmd.target ? ` \u2192 ${cmd.target}` : " \u2192 original topic";
4227
+ deps.out(
4228
+ `Replaying ${cmd.topic}${DLQ_SUFFIX}${targetDesc} (${mode}${cmd.dryRun ? ", dry-run" : ""})\u2026`
4229
+ );
4230
+ const { replayed, skipped } = await client.replayDlq(cmd.topic, options);
4231
+ deps.out(
4232
+ cmd.dryRun ? `Dry-run: ${replayed} message(s) would be replayed, ${skipped} skipped.` : `Replayed ${replayed} message(s), ${skipped} skipped.`
4233
+ );
4234
+ return { command: "replay", replayed, skipped, dryRun: cmd.dryRun };
4235
+ }
4236
+
4237
+ // src/cli/index.ts
4238
+ var CLIENT_ID = `dlq-cli-${process.pid}`;
4239
+ var GROUP_ID = `${CLIENT_ID}-peek`;
4240
+ function createRealClient(brokers) {
4241
+ const transport = new ConfluentTransport(CLIENT_ID, brokers);
4242
+ const kafka = new KafkaClient(
4243
+ CLIENT_ID,
4244
+ GROUP_ID,
4245
+ brokers,
4246
+ { transport, autoCreateTopics: false, strictSchemas: false }
4247
+ );
4248
+ const admin = transport.admin();
4249
+ let adminConnected = false;
4250
+ async function ensureAdmin() {
4251
+ if (!adminConnected) {
4252
+ await admin.connect();
4253
+ adminConnected = true;
4254
+ }
4255
+ }
3547
4256
  return {
3548
- /** Provide an explicit message type without a runtime schema. */
3549
- type: () => ({
3550
- __topic: name,
3551
- __type: void 0
3552
- }),
3553
- schema: (schema) => ({
3554
- __topic: name,
3555
- __type: void 0,
3556
- __schema: schema
3557
- })
4257
+ async listTopics() {
4258
+ const status = await kafka.checkStatus();
4259
+ if (status.status === "down") {
4260
+ throw new Error(`Broker unreachable: ${status.error}`);
4261
+ }
4262
+ return status.topics;
4263
+ },
4264
+ async fetchTopicOffsets(topic) {
4265
+ await ensureAdmin();
4266
+ return admin.fetchTopicOffsets(topic);
4267
+ },
4268
+ async peekMessages(dlqTopic, limit) {
4269
+ const collected = [];
4270
+ const iterator = kafka.consume(dlqTopic, {
4271
+ groupId: `${dlqTopic}.dlq-peek-${Date.now()}`,
4272
+ fromBeginning: true
4273
+ });
4274
+ await ensureAdmin();
4275
+ const watermarks = await admin.fetchTopicOffsets(dlqTopic);
4276
+ const available = watermarks.reduce(
4277
+ (sum, w) => sum + Math.max(0, Number(w.high) - Number(w.low)),
4278
+ 0
4279
+ );
4280
+ const target = Math.min(limit, available);
4281
+ if (target === 0) {
4282
+ await iterator.return?.();
4283
+ return collected;
4284
+ }
4285
+ try {
4286
+ for await (const env of iterator) {
4287
+ collected.push(env);
4288
+ if (collected.length >= target) break;
4289
+ }
4290
+ } finally {
4291
+ await iterator.return?.();
4292
+ }
4293
+ return collected;
4294
+ },
4295
+ replayDlq(topic, options) {
4296
+ return kafka.replayDlq(topic, options);
4297
+ },
4298
+ async close() {
4299
+ if (adminConnected) {
4300
+ await admin.disconnect().catch(() => {
4301
+ });
4302
+ }
4303
+ await kafka.disconnect().catch(() => {
4304
+ });
4305
+ }
3558
4306
  };
3559
4307
  }
4308
+ async function main() {
4309
+ let cmd;
4310
+ try {
4311
+ cmd = parseArgs(process.argv.slice(2));
4312
+ } catch (err) {
4313
+ if (err instanceof DlqUsageError) {
4314
+ process.stderr.write(`Error: ${err.message}
3560
4315
 
3561
- export {
3562
- HEADER_EVENT_ID,
3563
- HEADER_CORRELATION_ID,
3564
- HEADER_TIMESTAMP,
3565
- HEADER_SCHEMA_VERSION,
3566
- HEADER_TRACEPARENT,
3567
- HEADER_LAMPORT_CLOCK,
3568
- getEnvelopeContext,
3569
- runWithEnvelopeContext,
3570
- buildEnvelopeHeaders,
3571
- decodeHeaders,
3572
- extractEnvelope,
3573
- KafkaProcessingError,
3574
- KafkaValidationError,
3575
- KafkaRetryExhaustedError,
3576
- KafkaClient,
3577
- topic
3578
- };
3579
- //# sourceMappingURL=chunk-SM4FZKAZ.mjs.map
4316
+ `);
4317
+ process.stderr.write(USAGE);
4318
+ return 2;
4319
+ }
4320
+ throw err;
4321
+ }
4322
+ try {
4323
+ await runDlqCommand(cmd, {
4324
+ createClient: createRealClient,
4325
+ out: (line) => process.stdout.write(`${line}
4326
+ `)
4327
+ });
4328
+ return 0;
4329
+ } catch (err) {
4330
+ const message = err instanceof Error ? err.message : String(err);
4331
+ process.stderr.write(`Error: ${message}
4332
+ `);
4333
+ return 1;
4334
+ }
4335
+ }
4336
+ main().then((code) => {
4337
+ process.exitCode = code;
4338
+ }).catch((err) => {
4339
+ process.stderr.write(`Fatal: ${err?.stack ?? err}
4340
+ `);
4341
+ process.exitCode = 1;
4342
+ });
4343
+ //# sourceMappingURL=index.js.map