@drarzter/kafka-client 0.9.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +625 -8
- package/dist/chunk-CMO7SMVK.mjs +4814 -0
- package/dist/chunk-CMO7SMVK.mjs.map +1 -0
- package/dist/cli/dlq.d.ts +119 -0
- package/dist/cli/dlq.d.ts.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/{chunk-SM4FZKAZ.mjs → cli/index.js} +964 -264
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/index.mjs +355 -0
- package/dist/cli/index.mjs.map +1 -0
- package/dist/client/config/from-env.d.ts +188 -0
- package/dist/client/config/from-env.d.ts.map +1 -0
- package/dist/client/config/index.d.ts +2 -0
- package/dist/client/config/index.d.ts.map +1 -0
- package/dist/client/errors.d.ts +67 -0
- package/dist/client/errors.d.ts.map +1 -0
- package/dist/client/kafka.client/admin/ops.d.ts +114 -0
- package/dist/client/kafka.client/admin/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts +24 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +52 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts +4 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts +10 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts +5 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/handler.d.ts +149 -0
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/ops.d.ts +51 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts +167 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/queue.d.ts +37 -0
- package/dist/client/kafka.client/consumer/queue.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +65 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/setup.d.ts +63 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/start.d.ts +7 -0
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/stop.d.ts +19 -0
- package/dist/client/kafka.client/consumer/stop.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts +4 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts.map +1 -0
- package/dist/client/kafka.client/context.d.ts +72 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -0
- package/dist/client/kafka.client/index.d.ts +155 -0
- package/dist/client/kafka.client/index.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts +61 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts +28 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts +22 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts +67 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts +41 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/ops.d.ts +70 -0
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/send.d.ts +21 -0
- package/dist/client/kafka.client/producer/send.d.ts.map +1 -0
- package/dist/client/kafka.client/validate-options.d.ts +11 -0
- package/dist/client/kafka.client/validate-options.d.ts.map +1 -0
- package/dist/client/message/envelope.d.ts +105 -0
- package/dist/client/message/envelope.d.ts.map +1 -0
- package/dist/client/message/schema-registry.d.ts +105 -0
- package/dist/client/message/schema-registry.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +138 -0
- package/dist/client/message/topic.d.ts.map +1 -0
- package/dist/client/message/versioned-schema.d.ts +53 -0
- package/dist/client/message/versioned-schema.d.ts.map +1 -0
- package/dist/client/outbox/index.d.ts +4 -0
- package/dist/client/outbox/index.d.ts.map +1 -0
- package/dist/client/outbox/outbox.relay.d.ts +90 -0
- package/dist/client/outbox/outbox.relay.d.ts.map +1 -0
- package/dist/client/outbox/outbox.store.d.ts +42 -0
- package/dist/client/outbox/outbox.store.d.ts.map +1 -0
- package/dist/client/outbox/outbox.types.d.ts +144 -0
- package/dist/client/outbox/outbox.types.d.ts.map +1 -0
- package/dist/client/security/acl.d.ts +108 -0
- package/dist/client/security/acl.d.ts.map +1 -0
- package/dist/client/security/index.d.ts +5 -0
- package/dist/client/security/index.d.ts.map +1 -0
- package/dist/client/security/providers.d.ts +88 -0
- package/dist/client/security/providers.d.ts.map +1 -0
- package/dist/client/security/resolve-security.d.ts +19 -0
- package/dist/client/security/resolve-security.d.ts.map +1 -0
- package/dist/client/security/security.types.d.ts +76 -0
- package/dist/client/security/security.types.d.ts.map +1 -0
- package/dist/client/transport/confluent.transport.d.ts +32 -0
- package/dist/client/transport/confluent.transport.d.ts.map +1 -0
- package/dist/client/transport/transport.interface.d.ts +216 -0
- package/dist/client/transport/transport.interface.d.ts.map +1 -0
- package/dist/client/types/admin.interface.d.ts +174 -0
- package/dist/client/types/admin.interface.d.ts.map +1 -0
- package/dist/client/types/admin.types.d.ts +140 -0
- package/dist/client/types/admin.types.d.ts.map +1 -0
- package/dist/client/types/client.d.ts +21 -0
- package/dist/client/types/client.d.ts.map +1 -0
- package/dist/client/types/common.d.ts +84 -0
- package/dist/client/types/common.d.ts.map +1 -0
- package/dist/client/types/config.types.d.ts +150 -0
- package/dist/client/types/config.types.d.ts.map +1 -0
- package/dist/client/types/consumer.interface.d.ts +115 -0
- package/dist/client/types/consumer.interface.d.ts.map +1 -0
- package/dist/{consumer.types-fFCag3VJ.d.mts → client/types/consumer.types.d.ts} +62 -383
- package/dist/client/types/consumer.types.d.ts.map +1 -0
- package/dist/client/types/dedup.types.d.ts +50 -0
- package/dist/client/types/dedup.types.d.ts.map +1 -0
- package/dist/client/types/lifecycle.interface.d.ts +72 -0
- package/dist/client/types/lifecycle.interface.d.ts.map +1 -0
- package/dist/client/types/producer.interface.d.ts +52 -0
- package/dist/client/types/producer.interface.d.ts.map +1 -0
- package/dist/client/types/producer.types.d.ts +90 -0
- package/dist/client/types/producer.types.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/core.d.ts +10 -314
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +1325 -73
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +39 -3
- package/dist/index.d.ts +7 -128
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1342 -73
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +56 -3
- package/dist/index.mjs.map +1 -1
- package/dist/nest/kafka.constants.d.ts +5 -0
- package/dist/nest/kafka.constants.d.ts.map +1 -0
- package/dist/nest/kafka.decorator.d.ts +49 -0
- package/dist/nest/kafka.decorator.d.ts.map +1 -0
- package/dist/nest/kafka.explorer.d.ts +17 -0
- package/dist/nest/kafka.explorer.d.ts.map +1 -0
- package/dist/nest/kafka.health.d.ts +7 -0
- package/dist/nest/kafka.health.d.ts.map +1 -0
- package/dist/nest/kafka.module.d.ts +61 -0
- package/dist/nest/kafka.module.d.ts.map +1 -0
- package/dist/otel.d.ts +83 -5
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +100 -6
- package/dist/otel.js.map +1 -1
- package/dist/otel.mjs +98 -5
- package/dist/otel.mjs.map +1 -1
- package/dist/testing/client.mock.d.ts +47 -0
- package/dist/testing/client.mock.d.ts.map +1 -0
- package/dist/testing/index.d.ts +4 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/test.container.d.ts +63 -0
- package/dist/testing/test.container.d.ts.map +1 -0
- package/dist/{testing.d.mts → testing/transport.fake.d.ts} +7 -111
- package/dist/testing/transport.fake.d.ts.map +1 -0
- package/dist/testing.d.ts +2 -318
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +26 -0
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +26 -0
- package/dist/testing.mjs.map +1 -1
- package/package.json +21 -8
- package/dist/chunk-SM4FZKAZ.mjs.map +0 -1
- package/dist/client-1irhGEu0.d.mts +0 -751
- package/dist/client-BpFjkHhr.d.ts +0 -751
- package/dist/consumer.types-fFCag3VJ.d.ts +0 -958
- package/dist/core.d.mts +0 -314
- package/dist/index.d.mts +0 -128
- package/dist/otel.d.mts +0 -27
|
@@ -0,0 +1,4814 @@
|
|
|
1
|
+
// src/client/transport/confluent.transport.ts
|
|
2
|
+
import { KafkaJS } from "@confluentinc/kafka-javascript";
|
|
3
|
+
var { Kafka: KafkaClass, logLevel: KafkaLogLevel, PartitionAssigners } = KafkaJS;
|
|
4
|
+
var ConfluentTransaction = class {
|
|
5
|
+
constructor(tx) {
|
|
6
|
+
this.tx = tx;
|
|
7
|
+
}
|
|
8
|
+
tx;
|
|
9
|
+
async send(record) {
|
|
10
|
+
await this.tx.send(record);
|
|
11
|
+
}
|
|
12
|
+
async sendOffsets(options) {
|
|
13
|
+
const nativeConsumer = options.consumer.getNative();
|
|
14
|
+
await this.tx.sendOffsets({
|
|
15
|
+
consumer: nativeConsumer,
|
|
16
|
+
topics: options.topics
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
async commit() {
|
|
20
|
+
await this.tx.commit();
|
|
21
|
+
}
|
|
22
|
+
async abort() {
|
|
23
|
+
await this.tx.abort();
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
var ConfluentProducer = class {
|
|
27
|
+
constructor(producer) {
|
|
28
|
+
this.producer = producer;
|
|
29
|
+
}
|
|
30
|
+
producer;
|
|
31
|
+
connectPromise;
|
|
32
|
+
async connect() {
|
|
33
|
+
this.connectPromise ??= this.producer.connect().catch((err) => {
|
|
34
|
+
this.connectPromise = void 0;
|
|
35
|
+
throw err;
|
|
36
|
+
});
|
|
37
|
+
return this.connectPromise;
|
|
38
|
+
}
|
|
39
|
+
async disconnect() {
|
|
40
|
+
this.connectPromise = void 0;
|
|
41
|
+
await this.producer.disconnect();
|
|
42
|
+
}
|
|
43
|
+
async send(record) {
|
|
44
|
+
await this.producer.send(record);
|
|
45
|
+
}
|
|
46
|
+
async transaction() {
|
|
47
|
+
const tx = await this.producer.transaction();
|
|
48
|
+
return new ConfluentTransaction(tx);
|
|
49
|
+
}
|
|
50
|
+
};
|
|
51
|
+
var ConfluentConsumer = class {
|
|
52
|
+
constructor(consumer) {
|
|
53
|
+
this.consumer = consumer;
|
|
54
|
+
}
|
|
55
|
+
consumer;
|
|
56
|
+
/** Returns the underlying KafkaJS.Consumer — used by ConfluentTransaction.sendOffsets. */
|
|
57
|
+
getNative() {
|
|
58
|
+
return this.consumer;
|
|
59
|
+
}
|
|
60
|
+
async connect() {
|
|
61
|
+
await this.consumer.connect();
|
|
62
|
+
}
|
|
63
|
+
async disconnect() {
|
|
64
|
+
await this.consumer.disconnect();
|
|
65
|
+
}
|
|
66
|
+
async subscribe(options) {
|
|
67
|
+
await this.consumer.subscribe(options);
|
|
68
|
+
}
|
|
69
|
+
async run(config) {
|
|
70
|
+
await this.consumer.run(config);
|
|
71
|
+
}
|
|
72
|
+
pause(assignments) {
|
|
73
|
+
this.consumer.pause(assignments);
|
|
74
|
+
}
|
|
75
|
+
resume(assignments) {
|
|
76
|
+
this.consumer.resume(assignments);
|
|
77
|
+
}
|
|
78
|
+
seek(options) {
|
|
79
|
+
this.consumer.seek(options);
|
|
80
|
+
}
|
|
81
|
+
assignment() {
|
|
82
|
+
return this.consumer.assignment();
|
|
83
|
+
}
|
|
84
|
+
async commitOffsets(offsets) {
|
|
85
|
+
await this.consumer.commitOffsets(offsets);
|
|
86
|
+
}
|
|
87
|
+
async stop() {
|
|
88
|
+
await this.consumer.stop?.();
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
var ConfluentAdmin = class {
|
|
92
|
+
constructor(admin) {
|
|
93
|
+
this.admin = admin;
|
|
94
|
+
}
|
|
95
|
+
admin;
|
|
96
|
+
async connect() {
|
|
97
|
+
await this.admin.connect();
|
|
98
|
+
}
|
|
99
|
+
async disconnect() {
|
|
100
|
+
await this.admin.disconnect();
|
|
101
|
+
}
|
|
102
|
+
async createTopics(options) {
|
|
103
|
+
await this.admin.createTopics(options);
|
|
104
|
+
}
|
|
105
|
+
async fetchTopicOffsets(topic2) {
|
|
106
|
+
return this.admin.fetchTopicOffsets(topic2);
|
|
107
|
+
}
|
|
108
|
+
async fetchTopicOffsetsByTimestamp(topic2, timestamp) {
|
|
109
|
+
return this.admin.fetchTopicOffsetsByTimestamp(topic2, timestamp);
|
|
110
|
+
}
|
|
111
|
+
async fetchOffsets(options) {
|
|
112
|
+
return this.admin.fetchOffsets(options);
|
|
113
|
+
}
|
|
114
|
+
async setOffsets(options) {
|
|
115
|
+
await this.admin.setOffsets(options);
|
|
116
|
+
}
|
|
117
|
+
async listTopics() {
|
|
118
|
+
return this.admin.listTopics();
|
|
119
|
+
}
|
|
120
|
+
async listGroups() {
|
|
121
|
+
return this.admin.listGroups();
|
|
122
|
+
}
|
|
123
|
+
async fetchTopicMetadata(options) {
|
|
124
|
+
return this.admin.fetchTopicMetadata(options);
|
|
125
|
+
}
|
|
126
|
+
async deleteGroups(groupIds) {
|
|
127
|
+
await this.admin.deleteGroups(groupIds);
|
|
128
|
+
}
|
|
129
|
+
async deleteTopicRecords(options) {
|
|
130
|
+
await this.admin.deleteTopicRecords(options);
|
|
131
|
+
}
|
|
132
|
+
};
|
|
133
|
+
var ConfluentTransport = class {
|
|
134
|
+
kafka;
|
|
135
|
+
constructor(clientId, brokers, security) {
|
|
136
|
+
const kafkaJS = { clientId, brokers, logLevel: KafkaLogLevel.ERROR };
|
|
137
|
+
if (security?.ssl !== void 0) kafkaJS.ssl = security.ssl;
|
|
138
|
+
if (security?.sasl) {
|
|
139
|
+
if (security.sasl.mechanism === "oauthbearer") {
|
|
140
|
+
const provider = security.sasl.oauthBearerProvider;
|
|
141
|
+
kafkaJS.sasl = {
|
|
142
|
+
mechanism: "oauthbearer",
|
|
143
|
+
oauthBearerProvider: async () => {
|
|
144
|
+
const token = await provider();
|
|
145
|
+
return {
|
|
146
|
+
value: token.value,
|
|
147
|
+
principal: token.principal ?? "kafka-client",
|
|
148
|
+
lifetime: token.lifetimeMs ?? Date.now() + 15 * 6e4,
|
|
149
|
+
...token.extensions && { extensions: token.extensions }
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
};
|
|
153
|
+
} else {
|
|
154
|
+
kafkaJS.sasl = security.sasl;
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
this.kafka = new KafkaClass({ kafkaJS });
|
|
158
|
+
}
|
|
159
|
+
producer(options) {
|
|
160
|
+
const native = this.kafka.producer({
|
|
161
|
+
kafkaJS: {
|
|
162
|
+
acks: -1,
|
|
163
|
+
...options?.idempotent !== void 0 && {
|
|
164
|
+
idempotent: options.idempotent
|
|
165
|
+
},
|
|
166
|
+
...options?.transactionalId !== void 0 && {
|
|
167
|
+
transactionalId: options.transactionalId,
|
|
168
|
+
maxInFlightRequests: 1
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
return new ConfluentProducer(native);
|
|
173
|
+
}
|
|
174
|
+
consumer(options) {
|
|
175
|
+
const assigner = options.partitionAssigner === "roundrobin" ? PartitionAssigners.roundRobin : options.partitionAssigner === "range" ? PartitionAssigners.range : PartitionAssigners.cooperativeSticky;
|
|
176
|
+
const config = {
|
|
177
|
+
kafkaJS: {
|
|
178
|
+
groupId: options.groupId,
|
|
179
|
+
fromBeginning: options.fromBeginning,
|
|
180
|
+
autoCommit: options.autoCommit,
|
|
181
|
+
partitionAssigners: [assigner]
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
if (options.groupInstanceId) {
|
|
185
|
+
config["group.instance.id"] = options.groupInstanceId;
|
|
186
|
+
}
|
|
187
|
+
if (options.onRebalance) {
|
|
188
|
+
const cb = options.onRebalance;
|
|
189
|
+
config.rebalance_cb = (err, assignment) => {
|
|
190
|
+
const type = err.code === -175 ? "assign" : "revoke";
|
|
191
|
+
cb(
|
|
192
|
+
type,
|
|
193
|
+
assignment.map((p) => ({ topic: p.topic, partition: p.partition }))
|
|
194
|
+
);
|
|
195
|
+
};
|
|
196
|
+
}
|
|
197
|
+
return new ConfluentConsumer(this.kafka.consumer(config));
|
|
198
|
+
}
|
|
199
|
+
admin() {
|
|
200
|
+
return new ConfluentAdmin(this.kafka.admin());
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
// src/client/kafka.client/infra/dedup.store.ts
|
|
205
|
+
var InMemoryDedupStore = class {
|
|
206
|
+
constructor(states) {
|
|
207
|
+
this.states = states;
|
|
208
|
+
}
|
|
209
|
+
states;
|
|
210
|
+
getLastClock(groupId, topicPartition) {
|
|
211
|
+
return this.states.get(groupId)?.get(topicPartition);
|
|
212
|
+
}
|
|
213
|
+
setLastClock(groupId, topicPartition, clock) {
|
|
214
|
+
let group = this.states.get(groupId);
|
|
215
|
+
if (!group) {
|
|
216
|
+
group = /* @__PURE__ */ new Map();
|
|
217
|
+
this.states.set(groupId, group);
|
|
218
|
+
}
|
|
219
|
+
group.set(topicPartition, clock);
|
|
220
|
+
}
|
|
221
|
+
};
|
|
222
|
+
|
|
223
|
+
// src/client/message/envelope.ts
|
|
224
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
225
|
+
import { randomUUID } from "crypto";
|
|
226
|
+
var HEADER_EVENT_ID = "x-event-id";
|
|
227
|
+
var HEADER_CORRELATION_ID = "x-correlation-id";
|
|
228
|
+
var HEADER_TIMESTAMP = "x-timestamp";
|
|
229
|
+
var HEADER_SCHEMA_VERSION = "x-schema-version";
|
|
230
|
+
var HEADER_TRACEPARENT = "traceparent";
|
|
231
|
+
var HEADER_LAMPORT_CLOCK = "x-lamport-clock";
|
|
232
|
+
var HEADER_DELAYED_UNTIL = "x-delayed-until";
|
|
233
|
+
var HEADER_DELAYED_TARGET = "x-delayed-target";
|
|
234
|
+
var envelopeStorage = new AsyncLocalStorage();
|
|
235
|
+
function getEnvelopeContext() {
|
|
236
|
+
return envelopeStorage.getStore();
|
|
237
|
+
}
|
|
238
|
+
function runWithEnvelopeContext(ctx, fn) {
|
|
239
|
+
return envelopeStorage.run(ctx, fn);
|
|
240
|
+
}
|
|
241
|
+
function buildEnvelopeHeaders(options = {}) {
|
|
242
|
+
const ctx = getEnvelopeContext();
|
|
243
|
+
const correlationId = options.correlationId ?? ctx?.correlationId ?? randomUUID();
|
|
244
|
+
const eventId = options.eventId ?? randomUUID();
|
|
245
|
+
const timestamp = (/* @__PURE__ */ new Date()).toISOString();
|
|
246
|
+
const schemaVersion = String(options.schemaVersion ?? 1);
|
|
247
|
+
const envelope = {
|
|
248
|
+
[HEADER_EVENT_ID]: eventId,
|
|
249
|
+
[HEADER_CORRELATION_ID]: correlationId,
|
|
250
|
+
[HEADER_TIMESTAMP]: timestamp,
|
|
251
|
+
[HEADER_SCHEMA_VERSION]: schemaVersion
|
|
252
|
+
};
|
|
253
|
+
if (ctx?.traceparent) {
|
|
254
|
+
envelope[HEADER_TRACEPARENT] = ctx.traceparent;
|
|
255
|
+
}
|
|
256
|
+
return { ...envelope, ...options.headers };
|
|
257
|
+
}
|
|
258
|
+
function decodeHeaders(raw) {
|
|
259
|
+
if (!raw) return {};
|
|
260
|
+
const result = {};
|
|
261
|
+
for (const [key, value] of Object.entries(raw)) {
|
|
262
|
+
if (value === void 0) continue;
|
|
263
|
+
if (Array.isArray(value)) {
|
|
264
|
+
const items = value.map((v) => Buffer.isBuffer(v) ? v.toString() : v);
|
|
265
|
+
result[key] = items[items.length - 1] ?? "";
|
|
266
|
+
} else {
|
|
267
|
+
result[key] = Buffer.isBuffer(value) ? value.toString() : value;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
return result;
|
|
271
|
+
}
|
|
272
|
+
function extractEnvelope(payload, headers, topic2, partition, offset) {
|
|
273
|
+
return {
|
|
274
|
+
payload,
|
|
275
|
+
topic: topic2,
|
|
276
|
+
partition,
|
|
277
|
+
offset,
|
|
278
|
+
eventId: headers[HEADER_EVENT_ID] ?? randomUUID(),
|
|
279
|
+
correlationId: headers[HEADER_CORRELATION_ID] ?? randomUUID(),
|
|
280
|
+
timestamp: headers[HEADER_TIMESTAMP] ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
281
|
+
schemaVersion: Number(headers[HEADER_SCHEMA_VERSION] ?? 1),
|
|
282
|
+
traceparent: headers[HEADER_TRACEPARENT],
|
|
283
|
+
headers
|
|
284
|
+
};
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// src/client/errors.ts
|
|
288
|
+
function toError(error) {
|
|
289
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
290
|
+
}
|
|
291
|
+
var KafkaProcessingError = class extends Error {
|
|
292
|
+
constructor(message, topic2, originalMessage, options) {
|
|
293
|
+
super(message, options);
|
|
294
|
+
this.topic = topic2;
|
|
295
|
+
this.originalMessage = originalMessage;
|
|
296
|
+
this.name = "KafkaProcessingError";
|
|
297
|
+
if (options?.cause) this.cause = options.cause;
|
|
298
|
+
}
|
|
299
|
+
topic;
|
|
300
|
+
originalMessage;
|
|
301
|
+
};
|
|
302
|
+
var KafkaValidationError = class extends Error {
|
|
303
|
+
constructor(topic2, originalMessage, options) {
|
|
304
|
+
super(`Schema validation failed for topic "${topic2}"`, options);
|
|
305
|
+
this.topic = topic2;
|
|
306
|
+
this.originalMessage = originalMessage;
|
|
307
|
+
this.name = "KafkaValidationError";
|
|
308
|
+
if (options?.cause) this.cause = options.cause;
|
|
309
|
+
}
|
|
310
|
+
topic;
|
|
311
|
+
originalMessage;
|
|
312
|
+
};
|
|
313
|
+
var KafkaRetryExhaustedError = class extends KafkaProcessingError {
|
|
314
|
+
constructor(topic2, originalMessage, attempts, options) {
|
|
315
|
+
super(
|
|
316
|
+
`Message processing failed after ${attempts} attempts on topic "${topic2}"`,
|
|
317
|
+
topic2,
|
|
318
|
+
originalMessage,
|
|
319
|
+
options
|
|
320
|
+
);
|
|
321
|
+
this.attempts = attempts;
|
|
322
|
+
this.name = "KafkaRetryExhaustedError";
|
|
323
|
+
}
|
|
324
|
+
attempts;
|
|
325
|
+
};
|
|
326
|
+
|
|
327
|
+
// src/client/kafka.client/producer/ops.ts
|
|
328
|
+
function resolveTopicName(topicOrDescriptor) {
|
|
329
|
+
if (typeof topicOrDescriptor === "string") return topicOrDescriptor;
|
|
330
|
+
if (topicOrDescriptor && typeof topicOrDescriptor === "object" && "__topic" in topicOrDescriptor) {
|
|
331
|
+
return topicOrDescriptor.__topic;
|
|
332
|
+
}
|
|
333
|
+
return String(topicOrDescriptor);
|
|
334
|
+
}
|
|
335
|
+
function registerSchema(topicOrDesc, schemaRegistry, logger) {
|
|
336
|
+
if (topicOrDesc?.__schema) {
|
|
337
|
+
const topic2 = resolveTopicName(topicOrDesc);
|
|
338
|
+
const existing = schemaRegistry.get(topic2);
|
|
339
|
+
if (existing && existing !== topicOrDesc.__schema) {
|
|
340
|
+
logger?.warn(
|
|
341
|
+
`Schema conflict for topic "${topic2}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
schemaRegistry.set(topic2, topicOrDesc.__schema);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
async function validateMessage(topicOrDesc, message, deps, ctx) {
|
|
348
|
+
const topicName = resolveTopicName(topicOrDesc);
|
|
349
|
+
if (topicOrDesc?.__schema) {
|
|
350
|
+
try {
|
|
351
|
+
return await topicOrDesc.__schema.parse(message, ctx);
|
|
352
|
+
} catch (error) {
|
|
353
|
+
throw new KafkaValidationError(topicName, message, {
|
|
354
|
+
cause: error instanceof Error ? error : new Error(String(error))
|
|
355
|
+
});
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
if (deps.strictSchemasEnabled && typeof topicOrDesc === "string") {
|
|
359
|
+
const schema = deps.schemaRegistry.get(topicOrDesc);
|
|
360
|
+
if (schema) {
|
|
361
|
+
try {
|
|
362
|
+
return await schema.parse(message, ctx);
|
|
363
|
+
} catch (error) {
|
|
364
|
+
throw new KafkaValidationError(topicName, message, {
|
|
365
|
+
cause: error instanceof Error ? error : new Error(String(error))
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
return message;
|
|
371
|
+
}
|
|
372
|
+
async function buildSendPayload(topicOrDesc, messages, deps, compression) {
|
|
373
|
+
const topic2 = resolveTopicName(topicOrDesc);
|
|
374
|
+
const builtMessages = await Promise.all(
|
|
375
|
+
messages.map(async (m) => {
|
|
376
|
+
const envelopeHeaders = buildEnvelopeHeaders({
|
|
377
|
+
correlationId: m.correlationId,
|
|
378
|
+
schemaVersion: m.schemaVersion,
|
|
379
|
+
eventId: m.eventId,
|
|
380
|
+
headers: m.headers
|
|
381
|
+
});
|
|
382
|
+
if (deps.nextLamportClock) {
|
|
383
|
+
envelopeHeaders[HEADER_LAMPORT_CLOCK] = String(deps.nextLamportClock());
|
|
384
|
+
}
|
|
385
|
+
for (const inst of deps.instrumentation) {
|
|
386
|
+
inst.beforeSend?.(topic2, envelopeHeaders);
|
|
387
|
+
}
|
|
388
|
+
const sendCtx = {
|
|
389
|
+
topic: topic2,
|
|
390
|
+
headers: envelopeHeaders,
|
|
391
|
+
version: m.schemaVersion ?? 1
|
|
392
|
+
};
|
|
393
|
+
return {
|
|
394
|
+
value: JSON.stringify(
|
|
395
|
+
await validateMessage(topicOrDesc, m.value, deps, sendCtx)
|
|
396
|
+
),
|
|
397
|
+
// Explicit key wins; otherwise fall back to the descriptor's .key()
|
|
398
|
+
// extractor (runs on the original, pre-validation payload).
|
|
399
|
+
key: m.key ?? topicOrDesc?.__key?.(m.value) ?? null,
|
|
400
|
+
headers: envelopeHeaders
|
|
401
|
+
};
|
|
402
|
+
})
|
|
403
|
+
);
|
|
404
|
+
return { topic: topic2, messages: builtMessages, ...compression && { compression } };
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
// src/client/kafka.client/consumer/ops.ts
|
|
408
|
+
function getOrCreateConsumer(groupId, fromBeginning, autoCommit, deps, partitionAssigner, onFirstAssignment, groupInstanceId) {
|
|
409
|
+
const { consumers, consumerCreationOptions, transport, onRebalance, logger } = deps;
|
|
410
|
+
if (consumers.has(groupId)) {
|
|
411
|
+
const prev = consumerCreationOptions.get(groupId);
|
|
412
|
+
if (prev.fromBeginning !== fromBeginning || prev.autoCommit !== autoCommit) {
|
|
413
|
+
logger.warn(
|
|
414
|
+
`Consumer group "${groupId}" already exists with options (fromBeginning: ${prev.fromBeginning}, autoCommit: ${prev.autoCommit}) \u2014 new options (fromBeginning: ${fromBeginning}, autoCommit: ${autoCommit}) ignored. Use a different groupId to apply different options.`
|
|
415
|
+
);
|
|
416
|
+
}
|
|
417
|
+
onFirstAssignment?.();
|
|
418
|
+
return consumers.get(groupId);
|
|
419
|
+
}
|
|
420
|
+
consumerCreationOptions.set(groupId, { fromBeginning, autoCommit });
|
|
421
|
+
const SETTLE_MS = fromBeginning ? 0 : 500;
|
|
422
|
+
let hasAssignment = false;
|
|
423
|
+
let settleTimer;
|
|
424
|
+
const scheduleSettle = () => {
|
|
425
|
+
if (!hasAssignment) return;
|
|
426
|
+
if (settleTimer) clearTimeout(settleTimer);
|
|
427
|
+
settleTimer = setTimeout(() => {
|
|
428
|
+
settleTimer = void 0;
|
|
429
|
+
onFirstAssignment?.();
|
|
430
|
+
}, SETTLE_MS);
|
|
431
|
+
};
|
|
432
|
+
const fireOnAssignment = () => {
|
|
433
|
+
hasAssignment = true;
|
|
434
|
+
scheduleSettle();
|
|
435
|
+
};
|
|
436
|
+
const consumer = transport.consumer({
|
|
437
|
+
groupId,
|
|
438
|
+
fromBeginning,
|
|
439
|
+
autoCommit,
|
|
440
|
+
partitionAssigner: partitionAssigner ?? "cooperative-sticky",
|
|
441
|
+
groupInstanceId,
|
|
442
|
+
onRebalance: (type, assignments) => {
|
|
443
|
+
if (type === "assign") fireOnAssignment();
|
|
444
|
+
else if (type === "revoke") scheduleSettle();
|
|
445
|
+
if (onRebalance) {
|
|
446
|
+
try {
|
|
447
|
+
onRebalance(type, assignments);
|
|
448
|
+
} catch (e) {
|
|
449
|
+
logger.warn(`onRebalance callback threw: ${e.message}`);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
});
|
|
454
|
+
consumers.set(groupId, consumer);
|
|
455
|
+
return consumer;
|
|
456
|
+
}
|
|
457
|
+
function buildSchemaMap(topics, schemaRegistry, optionSchemas, logger) {
|
|
458
|
+
const schemaMap = /* @__PURE__ */ new Map();
|
|
459
|
+
const registerChecked = (name, schema) => {
|
|
460
|
+
const existing = schemaRegistry.get(name);
|
|
461
|
+
if (existing && existing !== schema) {
|
|
462
|
+
logger?.warn(
|
|
463
|
+
`Schema conflict for topic "${name}": a different schema is already registered. Using the new schema \u2014 ensure consistent schemas to avoid silent validation mismatches.`
|
|
464
|
+
);
|
|
465
|
+
}
|
|
466
|
+
schemaMap.set(name, schema);
|
|
467
|
+
schemaRegistry.set(name, schema);
|
|
468
|
+
};
|
|
469
|
+
for (const t of topics) {
|
|
470
|
+
if (t?.__schema) {
|
|
471
|
+
registerChecked(resolveTopicName(t), t.__schema);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
if (optionSchemas) {
|
|
475
|
+
for (const [k, v] of optionSchemas) {
|
|
476
|
+
registerChecked(k, v);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
return schemaMap;
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
// src/client/kafka.client/admin/ops.ts
|
|
483
|
+
var AdminOps = class {
|
|
484
|
+
constructor(deps) {
|
|
485
|
+
this.deps = deps;
|
|
486
|
+
}
|
|
487
|
+
deps;
|
|
488
|
+
isConnected = false;
|
|
489
|
+
/** Underlying admin client — used by index.ts for topic validation. */
|
|
490
|
+
get admin() {
|
|
491
|
+
return this.deps.admin;
|
|
492
|
+
}
|
|
493
|
+
/** Whether the admin client is currently connected. */
|
|
494
|
+
get connected() {
|
|
495
|
+
return this.isConnected;
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* Connect the admin client if not already connected.
|
|
499
|
+
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
500
|
+
* throws the flag remains `false` so the next call will retry the connection.
|
|
501
|
+
*/
|
|
502
|
+
async ensureConnected() {
|
|
503
|
+
if (this.isConnected) return;
|
|
504
|
+
try {
|
|
505
|
+
await this.deps.admin.connect();
|
|
506
|
+
this.isConnected = true;
|
|
507
|
+
} catch (err) {
|
|
508
|
+
this.isConnected = false;
|
|
509
|
+
throw err;
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
/** Disconnect admin if connected. Resets the connected flag. */
|
|
513
|
+
async disconnect() {
|
|
514
|
+
if (!this.isConnected) return;
|
|
515
|
+
await this.deps.admin.disconnect();
|
|
516
|
+
this.isConnected = false;
|
|
517
|
+
}
|
|
518
|
+
async resetOffsets(groupId, topic2, position) {
|
|
519
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
520
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
521
|
+
throw new Error(
|
|
522
|
+
`resetOffsets: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before resetting offsets.`
|
|
523
|
+
);
|
|
524
|
+
}
|
|
525
|
+
await this.ensureConnected();
|
|
526
|
+
const partitionOffsets = await this.deps.admin.fetchTopicOffsets(topic2);
|
|
527
|
+
const partitions = partitionOffsets.map(({ partition, low, high }) => ({
|
|
528
|
+
partition,
|
|
529
|
+
offset: position === "earliest" ? low : high
|
|
530
|
+
}));
|
|
531
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
532
|
+
this.deps.logger.log(
|
|
533
|
+
`Offsets reset to ${position} for group "${gid}" on topic "${topic2}"`
|
|
534
|
+
);
|
|
535
|
+
}
|
|
536
|
+
/**
|
|
537
|
+
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
538
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
539
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
540
|
+
*/
|
|
541
|
+
async seekToOffset(groupId, assignments) {
|
|
542
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
543
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
544
|
+
throw new Error(
|
|
545
|
+
`seekToOffset: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
546
|
+
);
|
|
547
|
+
}
|
|
548
|
+
await this.ensureConnected();
|
|
549
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
550
|
+
for (const { topic: topic2, partition, offset } of assignments) {
|
|
551
|
+
const list = byTopic.get(topic2) ?? [];
|
|
552
|
+
list.push({ partition, offset });
|
|
553
|
+
byTopic.set(topic2, list);
|
|
554
|
+
}
|
|
555
|
+
for (const [topic2, partitions] of byTopic) {
|
|
556
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions });
|
|
557
|
+
this.deps.logger.log(
|
|
558
|
+
`Offsets set for group "${gid}" on "${topic2}": ${JSON.stringify(partitions)}`
|
|
559
|
+
);
|
|
560
|
+
}
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
564
|
+
* (in milliseconds) for a stopped consumer group.
|
|
565
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
566
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
567
|
+
* If no offset exists at the requested timestamp (e.g. empty partition or
|
|
568
|
+
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
569
|
+
*/
|
|
570
|
+
async seekToTimestamp(groupId, assignments) {
|
|
571
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
572
|
+
if (this.deps.runningConsumers.has(gid)) {
|
|
573
|
+
throw new Error(
|
|
574
|
+
`seekToTimestamp: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before seeking offsets.`
|
|
575
|
+
);
|
|
576
|
+
}
|
|
577
|
+
await this.ensureConnected();
|
|
578
|
+
const byTopic = /* @__PURE__ */ new Map();
|
|
579
|
+
for (const { topic: topic2, partition, timestamp } of assignments) {
|
|
580
|
+
const list = byTopic.get(topic2) ?? [];
|
|
581
|
+
list.push({ partition, timestamp });
|
|
582
|
+
byTopic.set(topic2, list);
|
|
583
|
+
}
|
|
584
|
+
for (const [topic2, parts] of byTopic) {
|
|
585
|
+
const offsets = await Promise.all(
|
|
586
|
+
parts.map(async ({ partition, timestamp }) => {
|
|
587
|
+
const results = await this.deps.admin.fetchTopicOffsetsByTimestamp(
|
|
588
|
+
topic2,
|
|
589
|
+
timestamp
|
|
590
|
+
);
|
|
591
|
+
const found = results.find(
|
|
592
|
+
(r) => r.partition === partition
|
|
593
|
+
);
|
|
594
|
+
if (found) return { partition, offset: found.offset };
|
|
595
|
+
const topicOffsets = await this.deps.admin.fetchTopicOffsets(topic2);
|
|
596
|
+
const po = topicOffsets.find((o) => o.partition === partition);
|
|
597
|
+
return { partition, offset: po?.high ?? "0" };
|
|
598
|
+
})
|
|
599
|
+
);
|
|
600
|
+
await this.deps.admin.setOffsets({ groupId: gid, topic: topic2, partitions: offsets });
|
|
601
|
+
this.deps.logger.log(
|
|
602
|
+
`Offsets set by timestamp for group "${gid}" on "${topic2}": ${JSON.stringify(offsets)}`
|
|
603
|
+
);
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
/**
|
|
607
|
+
* Query consumer group lag per partition.
|
|
608
|
+
* Lag = broker high-watermark − last committed offset.
|
|
609
|
+
* A committed offset of -1 (nothing committed yet) counts as full lag.
|
|
610
|
+
*
|
|
611
|
+
* Returns an empty array when the consumer group has never committed any
|
|
612
|
+
* offsets (freshly created group, `autoCommit: false` with no manual commits,
|
|
613
|
+
* or group not yet assigned). This is a Kafka protocol limitation:
|
|
614
|
+
* `fetchOffsets` only returns data for topic-partitions that have at least one
|
|
615
|
+
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
616
|
+
*/
|
|
617
|
+
async getConsumerLag(groupId) {
|
|
618
|
+
const gid = groupId ?? this.deps.defaultGroupId;
|
|
619
|
+
await this.ensureConnected();
|
|
620
|
+
const committedByTopic = await this.deps.admin.fetchOffsets({ groupId: gid });
|
|
621
|
+
const brokerOffsetsAll = await Promise.all(
|
|
622
|
+
committedByTopic.map(({ topic: topic2 }) => this.deps.admin.fetchTopicOffsets(topic2))
|
|
623
|
+
);
|
|
624
|
+
const result = [];
|
|
625
|
+
for (let i = 0; i < committedByTopic.length; i++) {
|
|
626
|
+
const { topic: topic2, partitions } = committedByTopic[i];
|
|
627
|
+
const brokerOffsets = brokerOffsetsAll[i];
|
|
628
|
+
for (const { partition, offset } of partitions) {
|
|
629
|
+
const broker = brokerOffsets.find((o) => o.partition === partition);
|
|
630
|
+
if (!broker) continue;
|
|
631
|
+
const committed = parseInt(offset, 10);
|
|
632
|
+
const high = parseInt(broker.high, 10);
|
|
633
|
+
const lag = committed === -1 ? high : Math.max(0, high - committed);
|
|
634
|
+
result.push({ topic: topic2, partition, lag });
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
return result;
|
|
638
|
+
}
|
|
639
|
+
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
640
|
+
async checkStatus() {
|
|
641
|
+
try {
|
|
642
|
+
await this.ensureConnected();
|
|
643
|
+
const topics = await this.deps.admin.listTopics();
|
|
644
|
+
return { status: "up", clientId: this.deps.clientId, topics };
|
|
645
|
+
} catch (error) {
|
|
646
|
+
return {
|
|
647
|
+
status: "down",
|
|
648
|
+
clientId: this.deps.clientId,
|
|
649
|
+
error: error instanceof Error ? error.message : String(error)
|
|
650
|
+
};
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
/**
|
|
654
|
+
* List all consumer groups known to the broker.
|
|
655
|
+
* Useful for monitoring which groups are active and their current state.
|
|
656
|
+
*/
|
|
657
|
+
async listConsumerGroups() {
|
|
658
|
+
await this.ensureConnected();
|
|
659
|
+
const result = await this.deps.admin.listGroups();
|
|
660
|
+
return result.groups.map((g) => ({
|
|
661
|
+
groupId: g.groupId,
|
|
662
|
+
state: g.state ?? "Unknown"
|
|
663
|
+
}));
|
|
664
|
+
}
|
|
665
|
+
/**
|
|
666
|
+
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
667
|
+
* @param topics Topic names to describe. Omit to describe all topics.
|
|
668
|
+
*/
|
|
669
|
+
async describeTopics(topics) {
|
|
670
|
+
await this.ensureConnected();
|
|
671
|
+
const result = await this.deps.admin.fetchTopicMetadata(
|
|
672
|
+
topics ? { topics } : void 0
|
|
673
|
+
);
|
|
674
|
+
return result.topics.map((t) => ({
|
|
675
|
+
name: t.name,
|
|
676
|
+
partitions: t.partitions.map((p) => ({
|
|
677
|
+
partition: p.partitionId ?? p.partition ?? 0,
|
|
678
|
+
// -1 is Kafka's own "no leader" sentinel; 0 is a valid broker id
|
|
679
|
+
leader: p.leader ?? -1,
|
|
680
|
+
replicas: (p.replicas ?? []).map(
|
|
681
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
682
|
+
),
|
|
683
|
+
isr: (p.isr ?? []).map(
|
|
684
|
+
(r) => typeof r === "number" ? r : r.nodeId
|
|
685
|
+
)
|
|
686
|
+
}))
|
|
687
|
+
}));
|
|
688
|
+
}
|
|
689
|
+
/**
|
|
690
|
+
* Delete consumer groups from the broker.
|
|
691
|
+
* Groups must be empty (no active members) before deletion.
|
|
692
|
+
* Silently skips unknown group IDs.
|
|
693
|
+
* @param groupIds Consumer group IDs to delete.
|
|
694
|
+
*/
|
|
695
|
+
async deleteGroups(groupIds) {
|
|
696
|
+
if (groupIds.length === 0) return;
|
|
697
|
+
await this.ensureConnected();
|
|
698
|
+
await this.deps.admin.deleteGroups(groupIds);
|
|
699
|
+
}
|
|
700
|
+
/**
|
|
701
|
+
* Delete records from a topic up to (but not including) the given offsets.
|
|
702
|
+
* All messages with offsets **before** the given offset are deleted.
|
|
703
|
+
*/
|
|
704
|
+
async deleteRecords(topic2, partitions) {
|
|
705
|
+
await this.ensureConnected();
|
|
706
|
+
await this.deps.admin.deleteTopicRecords({ topic: topic2, partitions });
|
|
707
|
+
}
|
|
708
|
+
/**
|
|
709
|
+
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
710
|
+
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
711
|
+
* rather than silently discovering missing topics on the first handler failure.
|
|
712
|
+
*/
|
|
713
|
+
async validateRetryTopicsExist(topicNames, maxRetries) {
|
|
714
|
+
await this.ensureConnected();
|
|
715
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
716
|
+
const missing = [];
|
|
717
|
+
for (const t of topicNames) {
|
|
718
|
+
for (let level = 1; level <= maxRetries; level++) {
|
|
719
|
+
const retryTopic = `${t}.retry.${level}`;
|
|
720
|
+
if (!existing.has(retryTopic)) missing.push(retryTopic);
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
if (missing.length > 0) {
|
|
724
|
+
throw new Error(
|
|
725
|
+
`retryTopics: true but the following retry topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
726
|
+
);
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
/**
|
|
730
|
+
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
731
|
+
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
732
|
+
* missing DLQ topics on the first handler failure.
|
|
733
|
+
*/
|
|
734
|
+
async validateDlqTopicsExist(topicNames) {
|
|
735
|
+
await this.ensureConnected();
|
|
736
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
737
|
+
const missing = topicNames.filter((t) => !existing.has(`${t}.dlq`)).map((t) => `${t}.dlq`);
|
|
738
|
+
if (missing.length > 0) {
|
|
739
|
+
throw new Error(
|
|
740
|
+
`dlq: true but the following DLQ topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
741
|
+
);
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
/**
|
|
745
|
+
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
746
|
+
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
747
|
+
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
748
|
+
*/
|
|
749
|
+
async validateDuplicatesTopicsExist(topicNames, customDestination) {
|
|
750
|
+
await this.ensureConnected();
|
|
751
|
+
const existing = new Set(await this.deps.admin.listTopics());
|
|
752
|
+
const toCheck = customDestination ? [customDestination] : topicNames.map((t) => `${t}.duplicates`);
|
|
753
|
+
const missing = toCheck.filter((t) => !existing.has(t));
|
|
754
|
+
if (missing.length > 0) {
|
|
755
|
+
throw new Error(
|
|
756
|
+
`deduplication.strategy: 'topic' but the following duplicate-routing topics do not exist: ${missing.join(", ")}. Create them manually or set autoCreateTopics: true.`
|
|
757
|
+
);
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
};
|
|
761
|
+
|
|
762
|
+
// src/client/kafka.client/consumer/pipeline.ts
|
|
763
|
+
function sleep(ms) {
|
|
764
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
765
|
+
}
|
|
766
|
+
function parseJsonMessage(raw, topic2, logger) {
|
|
767
|
+
try {
|
|
768
|
+
return JSON.parse(raw);
|
|
769
|
+
} catch (error) {
|
|
770
|
+
logger.error(
|
|
771
|
+
`Failed to parse message from topic ${topic2}:`,
|
|
772
|
+
toError(error).stack
|
|
773
|
+
);
|
|
774
|
+
return null;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
async function validateWithSchema(message, raw, topic2, schemaMap, interceptors, dlq, deps) {
|
|
778
|
+
const schema = schemaMap.get(topic2);
|
|
779
|
+
if (!schema) return message;
|
|
780
|
+
const ctx = {
|
|
781
|
+
topic: topic2,
|
|
782
|
+
headers: deps.originalHeaders ?? {},
|
|
783
|
+
version: Number(deps.originalHeaders?.["x-schema-version"] ?? 1)
|
|
784
|
+
};
|
|
785
|
+
try {
|
|
786
|
+
return await schema.parse(message, ctx);
|
|
787
|
+
} catch (error) {
|
|
788
|
+
const err = toError(error);
|
|
789
|
+
const validationError = new KafkaValidationError(topic2, message, {
|
|
790
|
+
cause: err
|
|
791
|
+
});
|
|
792
|
+
deps.logger.error(
|
|
793
|
+
`Schema validation failed for topic ${topic2}:`,
|
|
794
|
+
err.message
|
|
795
|
+
);
|
|
796
|
+
if (dlq) {
|
|
797
|
+
await sendToDlq(topic2, raw, deps, {
|
|
798
|
+
error: validationError,
|
|
799
|
+
attempt: 0,
|
|
800
|
+
originalHeaders: deps.originalHeaders
|
|
801
|
+
});
|
|
802
|
+
} else {
|
|
803
|
+
await deps.onMessageLost?.({
|
|
804
|
+
topic: topic2,
|
|
805
|
+
error: validationError,
|
|
806
|
+
attempt: 0,
|
|
807
|
+
headers: deps.originalHeaders ?? {}
|
|
808
|
+
});
|
|
809
|
+
}
|
|
810
|
+
const errorEnvelope = extractEnvelope(
|
|
811
|
+
message,
|
|
812
|
+
deps.originalHeaders ?? {},
|
|
813
|
+
topic2,
|
|
814
|
+
-1,
|
|
815
|
+
""
|
|
816
|
+
);
|
|
817
|
+
for (const inst of deps.instrumentation ?? []) {
|
|
818
|
+
inst.onConsumeError?.(errorEnvelope, validationError);
|
|
819
|
+
}
|
|
820
|
+
for (const interceptor of interceptors) {
|
|
821
|
+
await interceptor.onError?.(errorEnvelope, validationError);
|
|
822
|
+
}
|
|
823
|
+
return null;
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
function buildDlqPayload(topic2, rawMessage, meta) {
|
|
827
|
+
const dlqTopic = `${topic2}.dlq`;
|
|
828
|
+
const headers = {
|
|
829
|
+
...meta?.originalHeaders ?? {},
|
|
830
|
+
"x-dlq-original-topic": topic2,
|
|
831
|
+
"x-dlq-failed-at": (/* @__PURE__ */ new Date()).toISOString(),
|
|
832
|
+
"x-dlq-error-message": meta?.error.message ?? "unknown",
|
|
833
|
+
"x-dlq-error-stack": meta?.error.stack?.slice(0, 2e3) ?? "",
|
|
834
|
+
"x-dlq-attempt-count": String(meta?.attempt ?? 0)
|
|
835
|
+
};
|
|
836
|
+
return { topic: dlqTopic, messages: [{ value: rawMessage, headers }] };
|
|
837
|
+
}
|
|
838
|
+
async function sendToDlq(topic2, rawMessage, deps, meta) {
|
|
839
|
+
const payload = buildDlqPayload(topic2, rawMessage, meta);
|
|
840
|
+
try {
|
|
841
|
+
await deps.producer.send(payload);
|
|
842
|
+
deps.logger.warn(`Message sent to DLQ: ${payload.topic}`);
|
|
843
|
+
} catch (error) {
|
|
844
|
+
const err = toError(error);
|
|
845
|
+
deps.logger.error(
|
|
846
|
+
`Failed to send message to DLQ ${payload.topic}:`,
|
|
847
|
+
err.stack
|
|
848
|
+
);
|
|
849
|
+
await deps.onMessageLost?.({
|
|
850
|
+
topic: topic2,
|
|
851
|
+
error: err,
|
|
852
|
+
attempt: meta?.attempt ?? 0,
|
|
853
|
+
headers: meta?.originalHeaders ?? {}
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
}
|
|
857
|
+
var RETRY_HEADER_ATTEMPT = "x-retry-attempt";
|
|
858
|
+
var RETRY_HEADER_AFTER = "x-retry-after";
|
|
859
|
+
var RETRY_HEADER_MAX_RETRIES = "x-retry-max-retries";
|
|
860
|
+
var RETRY_HEADER_ORIGINAL_TOPIC = "x-retry-original-topic";
|
|
861
|
+
function buildRetryTopicPayload(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders) {
|
|
862
|
+
const retryTopic = `${originalTopic}.retry.${attempt}`;
|
|
863
|
+
const STRIP = /* @__PURE__ */ new Set([
|
|
864
|
+
RETRY_HEADER_ATTEMPT,
|
|
865
|
+
RETRY_HEADER_AFTER,
|
|
866
|
+
RETRY_HEADER_MAX_RETRIES,
|
|
867
|
+
RETRY_HEADER_ORIGINAL_TOPIC
|
|
868
|
+
]);
|
|
869
|
+
function buildHeaders(hdr) {
|
|
870
|
+
const userHeaders = Object.fromEntries(
|
|
871
|
+
Object.entries(hdr).filter(([k]) => !STRIP.has(k))
|
|
872
|
+
);
|
|
873
|
+
return {
|
|
874
|
+
...userHeaders,
|
|
875
|
+
[RETRY_HEADER_ATTEMPT]: String(attempt),
|
|
876
|
+
[RETRY_HEADER_AFTER]: String(Date.now() + delayMs),
|
|
877
|
+
[RETRY_HEADER_MAX_RETRIES]: String(maxRetries),
|
|
878
|
+
[RETRY_HEADER_ORIGINAL_TOPIC]: originalTopic
|
|
879
|
+
};
|
|
880
|
+
}
|
|
881
|
+
return {
|
|
882
|
+
topic: retryTopic,
|
|
883
|
+
messages: rawMessages.map((value, i) => ({
|
|
884
|
+
value,
|
|
885
|
+
headers: buildHeaders(
|
|
886
|
+
Array.isArray(originalHeaders) ? originalHeaders[i] ?? {} : originalHeaders
|
|
887
|
+
)
|
|
888
|
+
}))
|
|
889
|
+
};
|
|
890
|
+
}
|
|
891
|
+
async function sendToRetryTopic(originalTopic, rawMessages, attempt, maxRetries, delayMs, originalHeaders, deps) {
|
|
892
|
+
const payload = buildRetryTopicPayload(
|
|
893
|
+
originalTopic,
|
|
894
|
+
rawMessages,
|
|
895
|
+
attempt,
|
|
896
|
+
maxRetries,
|
|
897
|
+
delayMs,
|
|
898
|
+
originalHeaders
|
|
899
|
+
);
|
|
900
|
+
try {
|
|
901
|
+
await deps.producer.send(payload);
|
|
902
|
+
deps.logger.warn(
|
|
903
|
+
`Message queued in retry topic ${payload.topic} (attempt ${attempt}/${maxRetries})`
|
|
904
|
+
);
|
|
905
|
+
} catch (error) {
|
|
906
|
+
const err = toError(error);
|
|
907
|
+
deps.logger.error(
|
|
908
|
+
`Failed to send message to retry topic ${payload.topic}:`,
|
|
909
|
+
err.stack
|
|
910
|
+
);
|
|
911
|
+
await deps.onMessageLost?.({
|
|
912
|
+
topic: originalTopic,
|
|
913
|
+
error: err,
|
|
914
|
+
attempt,
|
|
915
|
+
headers: Array.isArray(originalHeaders) ? originalHeaders[0] ?? {} : originalHeaders
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
function buildDuplicateTopicPayload(sourceTopic, rawMessage, destinationTopic, meta) {
|
|
920
|
+
const headers = {
|
|
921
|
+
...meta?.originalHeaders ?? {},
|
|
922
|
+
"x-duplicate-original-topic": sourceTopic,
|
|
923
|
+
"x-duplicate-detected-at": (/* @__PURE__ */ new Date()).toISOString(),
|
|
924
|
+
"x-duplicate-reason": "lamport-clock-duplicate",
|
|
925
|
+
"x-duplicate-incoming-clock": String(meta?.incomingClock ?? 0),
|
|
926
|
+
"x-duplicate-last-processed-clock": String(meta?.lastProcessedClock ?? 0)
|
|
927
|
+
};
|
|
928
|
+
return {
|
|
929
|
+
topic: destinationTopic,
|
|
930
|
+
messages: [{ value: rawMessage, headers }]
|
|
931
|
+
};
|
|
932
|
+
}
|
|
933
|
+
async function sendToDuplicatesTopic(sourceTopic, rawMessage, destinationTopic, deps, meta) {
|
|
934
|
+
const payload = buildDuplicateTopicPayload(
|
|
935
|
+
sourceTopic,
|
|
936
|
+
rawMessage,
|
|
937
|
+
destinationTopic,
|
|
938
|
+
meta
|
|
939
|
+
);
|
|
940
|
+
try {
|
|
941
|
+
await deps.producer.send(payload);
|
|
942
|
+
deps.logger.warn(`Duplicate message forwarded to ${destinationTopic}`);
|
|
943
|
+
} catch (error) {
|
|
944
|
+
deps.logger.error(
|
|
945
|
+
`Failed to forward duplicate to ${destinationTopic}:`,
|
|
946
|
+
toError(error).stack
|
|
947
|
+
);
|
|
948
|
+
}
|
|
949
|
+
}
|
|
950
|
+
async function broadcastToInterceptors(envelopes, interceptors, cb) {
|
|
951
|
+
for (const env of envelopes) {
|
|
952
|
+
for (const interceptor of interceptors) {
|
|
953
|
+
await cb(interceptor, env);
|
|
954
|
+
}
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
async function runHandlerWithPipeline(fn, envelopes, interceptors, instrumentation) {
|
|
958
|
+
const cleanups = [];
|
|
959
|
+
const wraps = [];
|
|
960
|
+
try {
|
|
961
|
+
for (const env of envelopes) {
|
|
962
|
+
for (const inst of instrumentation) {
|
|
963
|
+
const result = inst.beforeConsume?.(env);
|
|
964
|
+
if (typeof result === "function") {
|
|
965
|
+
cleanups.push(result);
|
|
966
|
+
} else if (result) {
|
|
967
|
+
if (result.cleanup) cleanups.push(result.cleanup);
|
|
968
|
+
if (result.wrap) wraps.push(result.wrap);
|
|
969
|
+
}
|
|
970
|
+
}
|
|
971
|
+
}
|
|
972
|
+
for (const env of envelopes) {
|
|
973
|
+
for (const interceptor of interceptors) {
|
|
974
|
+
await interceptor.before?.(env);
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
let runFn = fn;
|
|
978
|
+
for (let i = wraps.length - 1; i >= 0; i--) {
|
|
979
|
+
const wrap = wraps[i];
|
|
980
|
+
const inner = runFn;
|
|
981
|
+
runFn = () => wrap(inner);
|
|
982
|
+
}
|
|
983
|
+
await runFn();
|
|
984
|
+
for (const env of envelopes) {
|
|
985
|
+
for (const interceptor of interceptors) {
|
|
986
|
+
await interceptor.after?.(env);
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
for (const cleanup of cleanups) cleanup();
|
|
990
|
+
return null;
|
|
991
|
+
} catch (error) {
|
|
992
|
+
const err = toError(error);
|
|
993
|
+
for (const env of envelopes) {
|
|
994
|
+
for (const inst of instrumentation) {
|
|
995
|
+
inst.onConsumeError?.(env, err);
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
for (const cleanup of cleanups) cleanup();
|
|
999
|
+
return err;
|
|
1000
|
+
}
|
|
1001
|
+
}
|
|
1002
|
+
async function notifyInterceptorsOnError(envelopes, interceptors, error) {
|
|
1003
|
+
await broadcastToInterceptors(
|
|
1004
|
+
envelopes,
|
|
1005
|
+
interceptors,
|
|
1006
|
+
(i, env) => i.onError?.(env, error)
|
|
1007
|
+
);
|
|
1008
|
+
}
|
|
1009
|
+
async function executeWithRetry(fn, ctx, deps) {
|
|
1010
|
+
const {
|
|
1011
|
+
envelope,
|
|
1012
|
+
rawMessages,
|
|
1013
|
+
interceptors,
|
|
1014
|
+
dlq,
|
|
1015
|
+
retry,
|
|
1016
|
+
isBatch,
|
|
1017
|
+
retryTopics
|
|
1018
|
+
} = ctx;
|
|
1019
|
+
const maxAttempts = retryTopics ? 1 : retry ? retry.maxRetries + 1 : 1;
|
|
1020
|
+
const backoffMs = retry?.backoffMs ?? 1e3;
|
|
1021
|
+
const maxBackoffMs = retry?.maxBackoffMs ?? 3e4;
|
|
1022
|
+
const envelopes = Array.isArray(envelope) ? envelope : [envelope];
|
|
1023
|
+
const topic2 = envelopes[0]?.topic ?? "unknown";
|
|
1024
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
1025
|
+
const error = await runHandlerWithPipeline(
|
|
1026
|
+
fn,
|
|
1027
|
+
envelopes,
|
|
1028
|
+
interceptors,
|
|
1029
|
+
deps.instrumentation
|
|
1030
|
+
);
|
|
1031
|
+
if (!error) {
|
|
1032
|
+
if (deps.eosCommitOnSuccess) {
|
|
1033
|
+
try {
|
|
1034
|
+
await deps.eosCommitOnSuccess();
|
|
1035
|
+
} catch (commitErr) {
|
|
1036
|
+
deps.logger.error(
|
|
1037
|
+
`EOS offset commit failed after successful handler \u2014 message will be redelivered:`,
|
|
1038
|
+
toError(commitErr).stack
|
|
1039
|
+
);
|
|
1040
|
+
return;
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
for (const env of envelopes) deps.onMessage?.(env);
|
|
1044
|
+
return;
|
|
1045
|
+
}
|
|
1046
|
+
deps.onFailure?.(envelopes[0]);
|
|
1047
|
+
const isLastAttempt = attempt === maxAttempts;
|
|
1048
|
+
const reportedError = isLastAttempt && maxAttempts > 1 ? new KafkaRetryExhaustedError(
|
|
1049
|
+
topic2,
|
|
1050
|
+
envelopes.map((e) => e.payload),
|
|
1051
|
+
maxAttempts,
|
|
1052
|
+
{ cause: error }
|
|
1053
|
+
) : error;
|
|
1054
|
+
await notifyInterceptorsOnError(envelopes, interceptors, reportedError);
|
|
1055
|
+
deps.logger.error(
|
|
1056
|
+
`Error processing ${isBatch ? "batch" : "message"} from topic ${topic2} (attempt ${attempt}/${maxAttempts}):`,
|
|
1057
|
+
error.stack
|
|
1058
|
+
);
|
|
1059
|
+
if (retryTopics && retry) {
|
|
1060
|
+
const cap = Math.min(backoffMs, maxBackoffMs);
|
|
1061
|
+
const delay = Math.floor(Math.random() * cap);
|
|
1062
|
+
if (deps.eosRouteToRetry) {
|
|
1063
|
+
try {
|
|
1064
|
+
await deps.eosRouteToRetry(rawMessages, envelopes, delay);
|
|
1065
|
+
deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
|
|
1066
|
+
} catch (txErr) {
|
|
1067
|
+
deps.logger.error(
|
|
1068
|
+
`EOS routing to retry topic failed \u2014 message will be redelivered:`,
|
|
1069
|
+
toError(txErr).stack
|
|
1070
|
+
);
|
|
1071
|
+
}
|
|
1072
|
+
} else {
|
|
1073
|
+
await sendToRetryTopic(
|
|
1074
|
+
topic2,
|
|
1075
|
+
rawMessages,
|
|
1076
|
+
1,
|
|
1077
|
+
retry.maxRetries,
|
|
1078
|
+
delay,
|
|
1079
|
+
isBatch ? envelopes.map((e) => e.headers) : envelopes[0]?.headers ?? {},
|
|
1080
|
+
deps
|
|
1081
|
+
);
|
|
1082
|
+
deps.onRetry?.(envelopes[0], 1, retry.maxRetries);
|
|
1083
|
+
}
|
|
1084
|
+
} else if (isLastAttempt) {
|
|
1085
|
+
if (dlq) {
|
|
1086
|
+
for (let i = 0; i < rawMessages.length; i++) {
|
|
1087
|
+
await sendToDlq(topic2, rawMessages[i], deps, {
|
|
1088
|
+
error,
|
|
1089
|
+
attempt,
|
|
1090
|
+
originalHeaders: envelopes[i]?.headers
|
|
1091
|
+
});
|
|
1092
|
+
deps.onDlq?.(envelopes[i] ?? envelopes[0], "handler-error");
|
|
1093
|
+
}
|
|
1094
|
+
} else {
|
|
1095
|
+
await deps.onMessageLost?.({
|
|
1096
|
+
topic: topic2,
|
|
1097
|
+
error,
|
|
1098
|
+
attempt,
|
|
1099
|
+
headers: envelopes[0]?.headers ?? {}
|
|
1100
|
+
});
|
|
1101
|
+
}
|
|
1102
|
+
} else {
|
|
1103
|
+
const cap = Math.min(backoffMs * 2 ** (attempt - 1), maxBackoffMs);
|
|
1104
|
+
deps.onRetry?.(envelopes[0], attempt, maxAttempts - 1);
|
|
1105
|
+
await sleep(Math.floor(Math.random() * cap));
|
|
1106
|
+
}
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
// src/client/kafka.client/consumer/subscribe-retry.ts
|
|
1111
|
+
async function subscribeWithRetry(consumer, topics, logger, retryOpts) {
|
|
1112
|
+
const maxAttempts = retryOpts?.retries ?? 5;
|
|
1113
|
+
const backoffMs = retryOpts?.backoffMs ?? 5e3;
|
|
1114
|
+
const displayTopics = topics.map((t) => t instanceof RegExp ? t.toString() : t).join(", ");
|
|
1115
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
|
1116
|
+
try {
|
|
1117
|
+
await consumer.subscribe({ topics });
|
|
1118
|
+
return;
|
|
1119
|
+
} catch (error) {
|
|
1120
|
+
if (attempt === maxAttempts) throw error;
|
|
1121
|
+
const msg = toError(error).message;
|
|
1122
|
+
const delay = Math.floor(Math.random() * backoffMs);
|
|
1123
|
+
logger.warn(
|
|
1124
|
+
`Failed to subscribe to [${displayTopics}] (attempt ${attempt}/${maxAttempts}): ${msg}. Retrying in ${delay}ms...`
|
|
1125
|
+
);
|
|
1126
|
+
await sleep(delay);
|
|
1127
|
+
}
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
|
|
1131
|
+
// src/client/kafka.client/consumer/features/dlq-replay.ts
|
|
1132
|
+
async function replayDlqTopic(topic2, deps, options = {}) {
|
|
1133
|
+
if (topic2.endsWith(".dlq")) {
|
|
1134
|
+
throw new Error(
|
|
1135
|
+
`replayDlq: pass the ORIGINAL topic name \u2014 "${topic2}" already ends in ".dlq" (the ".dlq" suffix is appended internally, so this would read "${topic2}.dlq")`
|
|
1136
|
+
);
|
|
1137
|
+
}
|
|
1138
|
+
const dlqTopic = `${topic2}.dlq`;
|
|
1139
|
+
const partitionOffsets = await deps.fetchTopicOffsets(dlqTopic);
|
|
1140
|
+
const activePartitions = partitionOffsets.filter(
|
|
1141
|
+
(p) => Number.parseInt(p.high, 10) > Number.parseInt(p.low, 10)
|
|
1142
|
+
);
|
|
1143
|
+
if (activePartitions.length === 0) {
|
|
1144
|
+
deps.logger.log(`replayDlq: "${dlqTopic}" is empty \u2014 nothing to replay`);
|
|
1145
|
+
return { replayed: 0, skipped: 0 };
|
|
1146
|
+
}
|
|
1147
|
+
const highWatermarks = new Map(
|
|
1148
|
+
activePartitions.map(({ partition, high }) => [partition, Number.parseInt(high, 10)])
|
|
1149
|
+
);
|
|
1150
|
+
const processedOffsets = /* @__PURE__ */ new Map();
|
|
1151
|
+
let replayed = 0;
|
|
1152
|
+
let skipped = 0;
|
|
1153
|
+
const fromBeginning = options.fromBeginning ?? true;
|
|
1154
|
+
const groupId = fromBeginning ? `${dlqTopic}-replay-${Date.now()}` : `${dlqTopic}-replay`;
|
|
1155
|
+
await new Promise((resolve, reject) => {
|
|
1156
|
+
const consumer = deps.createConsumer(groupId, fromBeginning);
|
|
1157
|
+
const cleanup = () => deps.cleanupConsumer(consumer, groupId, fromBeginning);
|
|
1158
|
+
consumer.connect().then(() => subscribeWithRetry(consumer, [dlqTopic], deps.logger)).then(
|
|
1159
|
+
() => consumer.run({
|
|
1160
|
+
eachMessage: async ({ partition, message }) => {
|
|
1161
|
+
if (!message.value) return;
|
|
1162
|
+
const offset = Number.parseInt(message.offset, 10);
|
|
1163
|
+
processedOffsets.set(partition, offset);
|
|
1164
|
+
const headers = decodeHeaders(message.headers);
|
|
1165
|
+
const targetTopic = options.targetTopic ?? headers["x-dlq-original-topic"];
|
|
1166
|
+
const originalHeaders = Object.fromEntries(
|
|
1167
|
+
Object.entries(headers).filter(([k]) => !deps.dlqHeaderKeys.has(k))
|
|
1168
|
+
);
|
|
1169
|
+
const value = message.value.toString();
|
|
1170
|
+
const shouldProcess = !options.filter || options.filter(headers, value);
|
|
1171
|
+
if (!targetTopic || !shouldProcess) {
|
|
1172
|
+
skipped++;
|
|
1173
|
+
} else if (options.dryRun) {
|
|
1174
|
+
deps.logger.log(`[DLQ replay dry-run] Would replay to "${targetTopic}"`);
|
|
1175
|
+
replayed++;
|
|
1176
|
+
} else {
|
|
1177
|
+
await deps.send(targetTopic, [{ value, headers: originalHeaders }]);
|
|
1178
|
+
replayed++;
|
|
1179
|
+
}
|
|
1180
|
+
const allDone = Array.from(highWatermarks.entries()).every(
|
|
1181
|
+
([p, hwm]) => (processedOffsets.get(p) ?? -1) >= hwm - 1
|
|
1182
|
+
);
|
|
1183
|
+
if (allDone) {
|
|
1184
|
+
cleanup();
|
|
1185
|
+
resolve();
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
})
|
|
1189
|
+
).catch((err) => {
|
|
1190
|
+
cleanup();
|
|
1191
|
+
reject(err);
|
|
1192
|
+
});
|
|
1193
|
+
});
|
|
1194
|
+
deps.logger.log(`replayDlq: replayed ${replayed}, skipped ${skipped} from "${dlqTopic}"`);
|
|
1195
|
+
return { replayed, skipped };
|
|
1196
|
+
}
|
|
1197
|
+
|
|
1198
|
+
// src/client/kafka.client/infra/metrics.manager.ts
|
|
1199
|
+
var MetricsManager = class {
|
|
1200
|
+
constructor(deps) {
|
|
1201
|
+
this.deps = deps;
|
|
1202
|
+
}
|
|
1203
|
+
deps;
|
|
1204
|
+
topicMetrics = /* @__PURE__ */ new Map();
|
|
1205
|
+
metricsFor(topic2) {
|
|
1206
|
+
let m = this.topicMetrics.get(topic2);
|
|
1207
|
+
if (!m) {
|
|
1208
|
+
m = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1209
|
+
this.topicMetrics.set(topic2, m);
|
|
1210
|
+
}
|
|
1211
|
+
return m;
|
|
1212
|
+
}
|
|
1213
|
+
/** Fire `afterSend` instrumentation hooks for each message in a batch. */
|
|
1214
|
+
notifyAfterSend(topic2, count) {
|
|
1215
|
+
for (let i = 0; i < count; i++)
|
|
1216
|
+
for (const inst of this.deps.instrumentation) inst.afterSend?.(topic2);
|
|
1217
|
+
}
|
|
1218
|
+
/**
|
|
1219
|
+
* Increment the retry counter for the envelope's topic and fire all `onRetry` instrumentation hooks.
|
|
1220
|
+
* @param envelope The message envelope being retried.
|
|
1221
|
+
* @param attempt Current retry attempt number (1-based).
|
|
1222
|
+
* @param maxRetries Maximum number of retries configured for this consumer.
|
|
1223
|
+
*/
|
|
1224
|
+
notifyRetry(envelope, attempt, maxRetries) {
|
|
1225
|
+
this.metricsFor(envelope.topic).retryCount++;
|
|
1226
|
+
for (const inst of this.deps.instrumentation) inst.onRetry?.(envelope, attempt, maxRetries);
|
|
1227
|
+
}
|
|
1228
|
+
/**
|
|
1229
|
+
* Increment the DLQ counter for the envelope's topic and fire all `onDlq` instrumentation hooks.
|
|
1230
|
+
* Circuit breaker failures are recorded separately via `notifyFailure` at the
|
|
1231
|
+
* handler-error boundary — dead-lettering itself is not a circuit event.
|
|
1232
|
+
* @param envelope The message envelope being sent to the DLQ.
|
|
1233
|
+
* @param reason The reason the message is being dead-lettered.
|
|
1234
|
+
*/
|
|
1235
|
+
notifyDlq(envelope, reason) {
|
|
1236
|
+
this.metricsFor(envelope.topic).dlqCount++;
|
|
1237
|
+
for (const inst of this.deps.instrumentation) inst.onDlq?.(envelope, reason);
|
|
1238
|
+
}
|
|
1239
|
+
/**
|
|
1240
|
+
* Notify the circuit breaker of a handler failure. Fired on every failed
|
|
1241
|
+
* handler attempt (in-process retries and retry-topic levels included),
|
|
1242
|
+
* independent of whether the message is ultimately dead-lettered.
|
|
1243
|
+
* @param envelope The message envelope whose handler failed.
|
|
1244
|
+
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1245
|
+
*/
|
|
1246
|
+
notifyFailure(envelope, gid) {
|
|
1247
|
+
this.deps.onCircuitFailure(envelope, gid);
|
|
1248
|
+
}
|
|
1249
|
+
/**
|
|
1250
|
+
* Increment the deduplication counter for the envelope's topic and fire all `onDuplicate` hooks.
|
|
1251
|
+
* @param envelope The duplicate message envelope.
|
|
1252
|
+
* @param strategy The deduplication strategy applied (`"drop"`, `"dlq"`, or `"topic"`).
|
|
1253
|
+
*/
|
|
1254
|
+
notifyDuplicate(envelope, strategy) {
|
|
1255
|
+
this.metricsFor(envelope.topic).dedupCount++;
|
|
1256
|
+
for (const inst of this.deps.instrumentation) inst.onDuplicate?.(envelope, strategy);
|
|
1257
|
+
}
|
|
1258
|
+
/**
|
|
1259
|
+
* Increment the processed counter for the envelope's topic, fire all `onMessage` hooks,
|
|
1260
|
+
* and notify the circuit breaker of a success (when `gid` is provided).
|
|
1261
|
+
* @param envelope The successfully processed message envelope.
|
|
1262
|
+
* @param gid Consumer group ID — used to drive circuit breaker state.
|
|
1263
|
+
*/
|
|
1264
|
+
notifyMessage(envelope, gid) {
|
|
1265
|
+
this.metricsFor(envelope.topic).processedCount++;
|
|
1266
|
+
for (const inst of this.deps.instrumentation) inst.onMessage?.(envelope);
|
|
1267
|
+
if (gid) this.deps.onCircuitSuccess(envelope, gid);
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* Return a snapshot of event counters.
|
|
1271
|
+
* @param topic When provided, returns counters for that topic only; otherwise aggregates all topics.
|
|
1272
|
+
* @returns Read-only `KafkaMetrics` snapshot. Returns zero-valued counters if the topic has no events.
|
|
1273
|
+
*/
|
|
1274
|
+
getMetrics(topic2) {
|
|
1275
|
+
if (topic2 !== void 0) {
|
|
1276
|
+
const m = this.topicMetrics.get(topic2);
|
|
1277
|
+
return m ? { ...m } : { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1278
|
+
}
|
|
1279
|
+
const agg = { processedCount: 0, retryCount: 0, dlqCount: 0, dedupCount: 0 };
|
|
1280
|
+
for (const m of this.topicMetrics.values()) {
|
|
1281
|
+
agg.processedCount += m.processedCount;
|
|
1282
|
+
agg.retryCount += m.retryCount;
|
|
1283
|
+
agg.dlqCount += m.dlqCount;
|
|
1284
|
+
agg.dedupCount += m.dedupCount;
|
|
1285
|
+
}
|
|
1286
|
+
return agg;
|
|
1287
|
+
}
|
|
1288
|
+
/**
|
|
1289
|
+
* Reset event counters to zero.
|
|
1290
|
+
* @param topic When provided, clears counters for that topic only; otherwise clears all topics.
|
|
1291
|
+
*/
|
|
1292
|
+
resetMetrics(topic2) {
|
|
1293
|
+
if (topic2 !== void 0) {
|
|
1294
|
+
this.topicMetrics.delete(topic2);
|
|
1295
|
+
return;
|
|
1296
|
+
}
|
|
1297
|
+
this.topicMetrics.clear();
|
|
1298
|
+
}
|
|
1299
|
+
};
|
|
1300
|
+
|
|
1301
|
+
// src/client/kafka.client/infra/inflight.tracker.ts
|
|
1302
|
+
var InFlightTracker = class {
|
|
1303
|
+
constructor(warn) {
|
|
1304
|
+
this.warn = warn;
|
|
1305
|
+
}
|
|
1306
|
+
warn;
|
|
1307
|
+
inFlightTotal = 0;
|
|
1308
|
+
drainResolvers = [];
|
|
1309
|
+
/**
|
|
1310
|
+
* Wrap an async handler so its lifetime is counted against the in-flight total.
|
|
1311
|
+
* Resolvers registered with `waitForDrain` are notified when the count reaches zero.
|
|
1312
|
+
* @param fn The async function to track.
|
|
1313
|
+
* @returns The same promise returned by `fn`.
|
|
1314
|
+
*/
|
|
1315
|
+
track(fn) {
|
|
1316
|
+
this.inFlightTotal++;
|
|
1317
|
+
const done = () => {
|
|
1318
|
+
this.inFlightTotal--;
|
|
1319
|
+
if (this.inFlightTotal === 0) this.drainResolvers.splice(0).forEach((r) => r());
|
|
1320
|
+
};
|
|
1321
|
+
try {
|
|
1322
|
+
return fn().finally(done);
|
|
1323
|
+
} catch (err) {
|
|
1324
|
+
done();
|
|
1325
|
+
throw err;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
/**
|
|
1329
|
+
* Resolve when all tracked handlers have completed, or after `timeoutMs` elapses.
|
|
1330
|
+
* Logs a warning (via the injected `warn` callback) if the timeout is hit before draining.
|
|
1331
|
+
* Returns immediately if there are no in-flight handlers.
|
|
1332
|
+
* @param timeoutMs Maximum time to wait in milliseconds before resolving anyway.
|
|
1333
|
+
*/
|
|
1334
|
+
waitForDrain(timeoutMs) {
|
|
1335
|
+
if (this.inFlightTotal === 0) return Promise.resolve();
|
|
1336
|
+
return new Promise((resolve) => {
|
|
1337
|
+
let handle;
|
|
1338
|
+
const onDrain = () => {
|
|
1339
|
+
clearTimeout(handle);
|
|
1340
|
+
resolve();
|
|
1341
|
+
};
|
|
1342
|
+
this.drainResolvers.push(onDrain);
|
|
1343
|
+
handle = setTimeout(() => {
|
|
1344
|
+
const idx = this.drainResolvers.indexOf(onDrain);
|
|
1345
|
+
if (idx !== -1) this.drainResolvers.splice(idx, 1);
|
|
1346
|
+
this.warn(
|
|
1347
|
+
`Drain timed out after ${timeoutMs}ms \u2014 ${this.inFlightTotal} handler(s) still in flight`
|
|
1348
|
+
);
|
|
1349
|
+
resolve();
|
|
1350
|
+
}, timeoutMs);
|
|
1351
|
+
});
|
|
1352
|
+
}
|
|
1353
|
+
};
|
|
1354
|
+
|
|
1355
|
+
// src/client/kafka.client/infra/circuit-breaker.manager.ts
|
|
1356
|
+
var CircuitBreakerManager = class {
|
|
1357
|
+
constructor(deps) {
|
|
1358
|
+
this.deps = deps;
|
|
1359
|
+
}
|
|
1360
|
+
deps;
|
|
1361
|
+
states = /* @__PURE__ */ new Map();
|
|
1362
|
+
configs = /* @__PURE__ */ new Map();
|
|
1363
|
+
/**
|
|
1364
|
+
* Register or update circuit breaker configuration for a consumer group.
|
|
1365
|
+
* Must be called before the group starts consuming for the config to take effect.
|
|
1366
|
+
* @param gid Consumer group ID.
|
|
1367
|
+
* @param options Circuit breaker thresholds and timing configuration.
|
|
1368
|
+
*/
|
|
1369
|
+
setConfig(gid, options) {
|
|
1370
|
+
this.configs.set(gid, options);
|
|
1371
|
+
}
|
|
1372
|
+
/**
|
|
1373
|
+
* Returns a snapshot of the circuit breaker state for a given topic-partition.
|
|
1374
|
+
* Returns `undefined` when no state exists for the key.
|
|
1375
|
+
*/
|
|
1376
|
+
getState(topic2, partition, gid) {
|
|
1377
|
+
const state = this.states.get(`${gid}:${topic2}:${partition}`);
|
|
1378
|
+
if (!state) return void 0;
|
|
1379
|
+
return {
|
|
1380
|
+
status: state.status,
|
|
1381
|
+
failures: state.window.filter((v) => !v).length,
|
|
1382
|
+
windowSize: state.window.length
|
|
1383
|
+
};
|
|
1384
|
+
}
|
|
1385
|
+
/**
|
|
1386
|
+
* Record a failure for the given envelope and group.
|
|
1387
|
+
* Drives the CLOSED → OPEN and HALF-OPEN → OPEN transitions.
|
|
1388
|
+
*/
|
|
1389
|
+
onFailure(envelope, gid) {
|
|
1390
|
+
const cfg = this.configs.get(gid);
|
|
1391
|
+
if (!cfg) return;
|
|
1392
|
+
const threshold = cfg.threshold ?? 5;
|
|
1393
|
+
const recoveryMs = cfg.recoveryMs ?? 3e4;
|
|
1394
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1395
|
+
let state = this.states.get(stateKey);
|
|
1396
|
+
if (!state) {
|
|
1397
|
+
state = { status: "closed", window: [], successes: 0 };
|
|
1398
|
+
this.states.set(stateKey, state);
|
|
1399
|
+
}
|
|
1400
|
+
if (state.status === "open") return;
|
|
1401
|
+
const openCircuit = () => {
|
|
1402
|
+
state.status = "open";
|
|
1403
|
+
state.window = [];
|
|
1404
|
+
state.successes = 0;
|
|
1405
|
+
clearTimeout(state.timer);
|
|
1406
|
+
for (const inst of this.deps.instrumentation)
|
|
1407
|
+
inst.onCircuitOpen?.(envelope.topic, envelope.partition);
|
|
1408
|
+
this.deps.pauseConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1409
|
+
state.timer = setTimeout(() => {
|
|
1410
|
+
state.status = "half-open";
|
|
1411
|
+
state.successes = 0;
|
|
1412
|
+
this.deps.logger.log(
|
|
1413
|
+
`[CircuitBreaker] HALF-OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1414
|
+
);
|
|
1415
|
+
for (const inst of this.deps.instrumentation)
|
|
1416
|
+
inst.onCircuitHalfOpen?.(envelope.topic, envelope.partition);
|
|
1417
|
+
this.deps.resumeConsumer(gid, [{ topic: envelope.topic, partitions: [envelope.partition] }]);
|
|
1418
|
+
}, recoveryMs);
|
|
1419
|
+
};
|
|
1420
|
+
if (state.status === "half-open") {
|
|
1421
|
+
clearTimeout(state.timer);
|
|
1422
|
+
this.deps.logger.warn(
|
|
1423
|
+
`[CircuitBreaker] OPEN (half-open failure) \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1424
|
+
);
|
|
1425
|
+
openCircuit();
|
|
1426
|
+
return;
|
|
1427
|
+
}
|
|
1428
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1429
|
+
state.window = [...state.window, false];
|
|
1430
|
+
if (state.window.length > windowSize) {
|
|
1431
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1432
|
+
}
|
|
1433
|
+
const failures = state.window.filter((v) => !v).length;
|
|
1434
|
+
if (failures >= threshold) {
|
|
1435
|
+
this.deps.logger.warn(
|
|
1436
|
+
`[CircuitBreaker] OPEN \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition} (${failures}/${state.window.length} failures, threshold=${threshold})`
|
|
1437
|
+
);
|
|
1438
|
+
openCircuit();
|
|
1439
|
+
}
|
|
1440
|
+
}
|
|
1441
|
+
/**
|
|
1442
|
+
* Record a success for the given envelope and group.
|
|
1443
|
+
* Drives the HALF-OPEN → CLOSED transition and updates the success window.
|
|
1444
|
+
*/
|
|
1445
|
+
onSuccess(envelope, gid) {
|
|
1446
|
+
const cfg = this.configs.get(gid);
|
|
1447
|
+
if (!cfg) return;
|
|
1448
|
+
const stateKey = `${gid}:${envelope.topic}:${envelope.partition}`;
|
|
1449
|
+
const state = this.states.get(stateKey);
|
|
1450
|
+
if (!state) return;
|
|
1451
|
+
const halfOpenSuccesses = cfg.halfOpenSuccesses ?? 1;
|
|
1452
|
+
if (state.status === "half-open") {
|
|
1453
|
+
state.successes++;
|
|
1454
|
+
if (state.successes >= halfOpenSuccesses) {
|
|
1455
|
+
clearTimeout(state.timer);
|
|
1456
|
+
state.timer = void 0;
|
|
1457
|
+
state.status = "closed";
|
|
1458
|
+
state.window = [];
|
|
1459
|
+
state.successes = 0;
|
|
1460
|
+
this.deps.logger.log(
|
|
1461
|
+
`[CircuitBreaker] CLOSED \u2014 group="${gid}" topic="${envelope.topic}" partition=${envelope.partition}`
|
|
1462
|
+
);
|
|
1463
|
+
for (const inst of this.deps.instrumentation)
|
|
1464
|
+
inst.onCircuitClose?.(envelope.topic, envelope.partition);
|
|
1465
|
+
}
|
|
1466
|
+
} else if (state.status === "closed") {
|
|
1467
|
+
const threshold = cfg.threshold ?? 5;
|
|
1468
|
+
const windowSize = cfg.windowSize ?? Math.max(threshold * 2, 10);
|
|
1469
|
+
state.window = [...state.window, true];
|
|
1470
|
+
if (state.window.length > windowSize) {
|
|
1471
|
+
state.window = state.window.slice(state.window.length - windowSize);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
/**
|
|
1476
|
+
* Remove all circuit state and config for the given group.
|
|
1477
|
+
* Called when a consumer is stopped via `stopConsumer(groupId)`.
|
|
1478
|
+
*/
|
|
1479
|
+
removeGroup(gid) {
|
|
1480
|
+
for (const key of [...this.states.keys()]) {
|
|
1481
|
+
if (key.startsWith(`${gid}:`)) {
|
|
1482
|
+
clearTimeout(this.states.get(key).timer);
|
|
1483
|
+
this.states.delete(key);
|
|
1484
|
+
}
|
|
1485
|
+
}
|
|
1486
|
+
this.configs.delete(gid);
|
|
1487
|
+
}
|
|
1488
|
+
/** Clear all circuit state and config. Called on `disconnect()`. */
|
|
1489
|
+
clear() {
|
|
1490
|
+
for (const state of this.states.values()) clearTimeout(state.timer);
|
|
1491
|
+
this.states.clear();
|
|
1492
|
+
this.configs.clear();
|
|
1493
|
+
}
|
|
1494
|
+
};
|
|
1495
|
+
|
|
1496
|
+
// src/client/kafka.client/consumer/queue.ts
|
|
1497
|
+
var AsyncQueue = class {
|
|
1498
|
+
constructor(highWaterMark = Infinity, onFull = () => {
|
|
1499
|
+
}, onDrained = () => {
|
|
1500
|
+
}) {
|
|
1501
|
+
this.highWaterMark = highWaterMark;
|
|
1502
|
+
this.onFull = onFull;
|
|
1503
|
+
this.onDrained = onDrained;
|
|
1504
|
+
}
|
|
1505
|
+
highWaterMark;
|
|
1506
|
+
onFull;
|
|
1507
|
+
onDrained;
|
|
1508
|
+
items = [];
|
|
1509
|
+
waiting = [];
|
|
1510
|
+
closed = false;
|
|
1511
|
+
error;
|
|
1512
|
+
paused = false;
|
|
1513
|
+
/**
|
|
1514
|
+
* Enqueue an item. If a consumer is already awaiting the next item, delivers it immediately.
|
|
1515
|
+
* When the internal buffer reaches `highWaterMark`, calls `onFull` to signal backpressure.
|
|
1516
|
+
* @param item The value to enqueue.
|
|
1517
|
+
*/
|
|
1518
|
+
push(item) {
|
|
1519
|
+
if (this.closed) return;
|
|
1520
|
+
if (this.waiting.length > 0) {
|
|
1521
|
+
this.waiting.shift().resolve({ value: item, done: false });
|
|
1522
|
+
} else {
|
|
1523
|
+
this.items.push(item);
|
|
1524
|
+
if (!this.paused && this.items.length >= this.highWaterMark) {
|
|
1525
|
+
this.paused = true;
|
|
1526
|
+
this.onFull();
|
|
1527
|
+
}
|
|
1528
|
+
}
|
|
1529
|
+
}
|
|
1530
|
+
/**
|
|
1531
|
+
* Terminate the queue with an error. All pending and future `next()` calls will reject.
|
|
1532
|
+
* @param err The error to propagate to all waiting consumers.
|
|
1533
|
+
*/
|
|
1534
|
+
fail(err) {
|
|
1535
|
+
this.closed = true;
|
|
1536
|
+
this.error = err;
|
|
1537
|
+
for (const { reject } of this.waiting.splice(0)) reject(err);
|
|
1538
|
+
}
|
|
1539
|
+
/**
|
|
1540
|
+
* Signal end-of-stream. All pending `next()` calls resolve with `{ done: true }`,
|
|
1541
|
+
* and future `next()` calls resolve immediately with `{ done: true }`.
|
|
1542
|
+
*/
|
|
1543
|
+
close() {
|
|
1544
|
+
this.closed = true;
|
|
1545
|
+
for (const { resolve } of this.waiting.splice(0))
|
|
1546
|
+
resolve({ value: void 0, done: true });
|
|
1547
|
+
}
|
|
1548
|
+
/**
|
|
1549
|
+
* Pull the next item from the queue, conforming to the `AsyncIterator` protocol.
|
|
1550
|
+
* Rejects if the queue has been failed. Resolves with `{ done: true }` if it has been closed.
|
|
1551
|
+
* Suspends (returns a pending Promise) when the queue is empty and not yet closed.
|
|
1552
|
+
* When items drain below `highWaterMark / 2`, calls `onDrained` to resume backpressure.
|
|
1553
|
+
* @returns A Promise resolving to an `IteratorResult<V>`.
|
|
1554
|
+
*/
|
|
1555
|
+
next() {
|
|
1556
|
+
if (this.error) return Promise.reject(this.error);
|
|
1557
|
+
if (this.items.length > 0) {
|
|
1558
|
+
const value = this.items.shift();
|
|
1559
|
+
if (this.paused && this.items.length <= Math.floor(this.highWaterMark / 2)) {
|
|
1560
|
+
this.paused = false;
|
|
1561
|
+
this.onDrained();
|
|
1562
|
+
}
|
|
1563
|
+
return Promise.resolve({ value, done: false });
|
|
1564
|
+
}
|
|
1565
|
+
if (this.closed) return Promise.resolve({ value: void 0, done: true });
|
|
1566
|
+
return new Promise((resolve, reject) => this.waiting.push({ resolve, reject }));
|
|
1567
|
+
}
|
|
1568
|
+
};
|
|
1569
|
+
|
|
1570
|
+
// src/client/kafka.client/validate-options.ts
|
|
1571
|
+
function validateClientOptions(clientId, groupId, brokers, options) {
|
|
1572
|
+
const problems = [];
|
|
1573
|
+
if (typeof clientId !== "string" || clientId.trim() === "") {
|
|
1574
|
+
problems.push("clientId must be a non-empty string");
|
|
1575
|
+
}
|
|
1576
|
+
if (typeof groupId !== "string" || groupId.trim() === "") {
|
|
1577
|
+
problems.push("groupId must be a non-empty string");
|
|
1578
|
+
}
|
|
1579
|
+
if (!Array.isArray(brokers) || brokers.length === 0 && !options?.transport) {
|
|
1580
|
+
problems.push("brokers must be a non-empty array of broker addresses");
|
|
1581
|
+
} else if (brokers.some((b) => typeof b !== "string" || b.trim() === "")) {
|
|
1582
|
+
problems.push("brokers must not contain empty entries");
|
|
1583
|
+
}
|
|
1584
|
+
if (options) {
|
|
1585
|
+
const {
|
|
1586
|
+
numPartitions,
|
|
1587
|
+
transactionalId,
|
|
1588
|
+
clockRecovery,
|
|
1589
|
+
lagThrottle
|
|
1590
|
+
} = options;
|
|
1591
|
+
if (numPartitions !== void 0 && (!Number.isInteger(numPartitions) || numPartitions < 1)) {
|
|
1592
|
+
problems.push(
|
|
1593
|
+
`numPartitions must be a positive integer (got ${numPartitions})`
|
|
1594
|
+
);
|
|
1595
|
+
}
|
|
1596
|
+
if (transactionalId !== void 0 && transactionalId.trim() === "") {
|
|
1597
|
+
problems.push("transactionalId must be a non-empty string when set");
|
|
1598
|
+
}
|
|
1599
|
+
if (clockRecovery) {
|
|
1600
|
+
if (!Array.isArray(clockRecovery.topics)) {
|
|
1601
|
+
problems.push("clockRecovery.topics must be an array of topic names");
|
|
1602
|
+
}
|
|
1603
|
+
if (clockRecovery.timeoutMs !== void 0 && !(clockRecovery.timeoutMs > 0)) {
|
|
1604
|
+
problems.push(
|
|
1605
|
+
`clockRecovery.timeoutMs must be > 0 (got ${clockRecovery.timeoutMs})`
|
|
1606
|
+
);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
if (lagThrottle) {
|
|
1610
|
+
if (!(lagThrottle.maxLag >= 0)) {
|
|
1611
|
+
problems.push(`lagThrottle.maxLag must be >= 0 (got ${lagThrottle.maxLag})`);
|
|
1612
|
+
}
|
|
1613
|
+
if (lagThrottle.pollIntervalMs !== void 0 && !(lagThrottle.pollIntervalMs > 0)) {
|
|
1614
|
+
problems.push(
|
|
1615
|
+
`lagThrottle.pollIntervalMs must be > 0 (got ${lagThrottle.pollIntervalMs})`
|
|
1616
|
+
);
|
|
1617
|
+
}
|
|
1618
|
+
if (lagThrottle.maxWaitMs !== void 0 && !(lagThrottle.maxWaitMs >= 0)) {
|
|
1619
|
+
problems.push(
|
|
1620
|
+
`lagThrottle.maxWaitMs must be >= 0 (got ${lagThrottle.maxWaitMs})`
|
|
1621
|
+
);
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
}
|
|
1625
|
+
if (problems.length > 0) {
|
|
1626
|
+
throw new Error(
|
|
1627
|
+
`KafkaClient: invalid configuration:
|
|
1628
|
+
- ${problems.join("\n- ")}`
|
|
1629
|
+
);
|
|
1630
|
+
}
|
|
1631
|
+
}
|
|
1632
|
+
|
|
1633
|
+
// src/client/security/resolve-security.ts
|
|
1634
|
+
var LOCAL_HOST_PATTERNS = [
|
|
1635
|
+
/^localhost(:\d+)?$/i,
|
|
1636
|
+
/^127\.\d+\.\d+\.\d+(:\d+)?$/,
|
|
1637
|
+
/^\[?::1\]?(:\d+)?$/,
|
|
1638
|
+
/^0\.0\.0\.0(:\d+)?$/,
|
|
1639
|
+
/^host\.docker\.internal(:\d+)?$/i
|
|
1640
|
+
];
|
|
1641
|
+
function isLocalBroker(broker) {
|
|
1642
|
+
return LOCAL_HOST_PATTERNS.some((re) => re.test(broker.trim()));
|
|
1643
|
+
}
|
|
1644
|
+
function resolveSecurityOptions(security, brokers, logger) {
|
|
1645
|
+
const hasRemoteBroker = brokers.some((b) => !isLocalBroker(b));
|
|
1646
|
+
if (!security?.sasl && security?.ssl !== true) {
|
|
1647
|
+
if (hasRemoteBroker && !security?.allowInsecure) {
|
|
1648
|
+
logger.warn(
|
|
1649
|
+
"Connecting to non-local brokers without TLS or SASL \u2014 traffic and payloads travel in plaintext. Configure `security: { ssl, sasl }` for production clusters, or set `security: { allowInsecure: true }` to acknowledge an intentionally insecure setup."
|
|
1650
|
+
);
|
|
1651
|
+
}
|
|
1652
|
+
return security;
|
|
1653
|
+
}
|
|
1654
|
+
if (security.sasl && security.ssl === void 0) {
|
|
1655
|
+
return { ...security, ssl: true };
|
|
1656
|
+
}
|
|
1657
|
+
if (security.sasl && security.ssl === false) {
|
|
1658
|
+
logger.warn(
|
|
1659
|
+
"SASL credentials are configured with `ssl: false` \u2014 credentials will be sent over plaintext. This is only safe on fully trusted networks."
|
|
1660
|
+
);
|
|
1661
|
+
}
|
|
1662
|
+
return security;
|
|
1663
|
+
}
|
|
1664
|
+
|
|
1665
|
+
// src/client/kafka.client/producer/lifecycle.ts
|
|
1666
|
+
var _activeTransactionalIds = /* @__PURE__ */ new Set();
|
|
1667
|
+
async function ensureTopic(ctx, topic2) {
|
|
1668
|
+
if (!ctx.autoCreateTopicsEnabled || ctx.ensuredTopics.has(topic2)) return;
|
|
1669
|
+
let p = ctx.ensureTopicPromises.get(topic2);
|
|
1670
|
+
if (!p) {
|
|
1671
|
+
p = (async () => {
|
|
1672
|
+
await ctx.adminOps.ensureConnected();
|
|
1673
|
+
await ctx.adminOps.admin.createTopics({
|
|
1674
|
+
topics: [{ topic: topic2, numPartitions: ctx.numPartitions }]
|
|
1675
|
+
});
|
|
1676
|
+
ctx.ensuredTopics.add(topic2);
|
|
1677
|
+
})().finally(() => ctx.ensureTopicPromises.delete(topic2));
|
|
1678
|
+
ctx.ensureTopicPromises.set(topic2, p);
|
|
1679
|
+
}
|
|
1680
|
+
await p;
|
|
1681
|
+
}
|
|
1682
|
+
async function createRetryTxProducer(ctx, transactionalId) {
|
|
1683
|
+
if (_activeTransactionalIds.has(transactionalId)) {
|
|
1684
|
+
ctx.logger.warn(
|
|
1685
|
+
`transactionalId "${transactionalId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
|
|
1686
|
+
);
|
|
1687
|
+
}
|
|
1688
|
+
const p = ctx.transport.producer({
|
|
1689
|
+
idempotent: true,
|
|
1690
|
+
transactionalId
|
|
1691
|
+
});
|
|
1692
|
+
await p.connect();
|
|
1693
|
+
_activeTransactionalIds.add(transactionalId);
|
|
1694
|
+
ctx.retryTxProducers.set(transactionalId, p);
|
|
1695
|
+
return p;
|
|
1696
|
+
}
|
|
1697
|
+
async function connectProducerImpl(ctx) {
|
|
1698
|
+
await ctx.producer.connect();
|
|
1699
|
+
await recoverLamportClockImpl(ctx, ctx.clockRecoveryTopics);
|
|
1700
|
+
if (ctx.lagThrottleOpts) startLagThrottlePoller(ctx);
|
|
1701
|
+
ctx.logger.log("Producer connected");
|
|
1702
|
+
}
|
|
1703
|
+
async function disconnectImpl(ctx, drainTimeoutMs = 3e4) {
|
|
1704
|
+
if (ctx._lagThrottleTimer) {
|
|
1705
|
+
clearInterval(ctx._lagThrottleTimer);
|
|
1706
|
+
ctx._lagThrottleTimer = void 0;
|
|
1707
|
+
}
|
|
1708
|
+
await ctx.inFlight.waitForDrain(drainTimeoutMs);
|
|
1709
|
+
const tasks = [ctx.producer.disconnect()];
|
|
1710
|
+
if (ctx.txProducer) {
|
|
1711
|
+
tasks.push(ctx.txProducer.disconnect());
|
|
1712
|
+
_activeTransactionalIds.delete(ctx.txId);
|
|
1713
|
+
ctx.txProducer = void 0;
|
|
1714
|
+
ctx.txProducerInitPromise = void 0;
|
|
1715
|
+
}
|
|
1716
|
+
for (const txId of ctx.retryTxProducers.keys())
|
|
1717
|
+
_activeTransactionalIds.delete(txId);
|
|
1718
|
+
for (const p of ctx.retryTxProducers.values()) tasks.push(p.disconnect());
|
|
1719
|
+
ctx.retryTxProducers.clear();
|
|
1720
|
+
for (const consumer of ctx.consumers.values())
|
|
1721
|
+
tasks.push(consumer.disconnect());
|
|
1722
|
+
tasks.push(ctx.adminOps.disconnect());
|
|
1723
|
+
await Promise.allSettled(tasks);
|
|
1724
|
+
ctx.consumers.clear();
|
|
1725
|
+
ctx.runningConsumers.clear();
|
|
1726
|
+
ctx.consumerCreationOptions.clear();
|
|
1727
|
+
ctx.companionGroupIds.clear();
|
|
1728
|
+
ctx.circuitBreaker.clear();
|
|
1729
|
+
ctx.logger.log("All connections closed");
|
|
1730
|
+
}
|
|
1731
|
+
function startLagThrottlePoller(ctx) {
|
|
1732
|
+
const opts = ctx.lagThrottleOpts;
|
|
1733
|
+
const { maxLag, pollIntervalMs = 5e3 } = opts;
|
|
1734
|
+
const groupId = opts.groupId;
|
|
1735
|
+
const poll = async () => {
|
|
1736
|
+
try {
|
|
1737
|
+
const lags = await ctx.adminOps.getConsumerLag(groupId);
|
|
1738
|
+
const total = lags.reduce((sum, e) => sum + e.lag, 0);
|
|
1739
|
+
if (total > maxLag && !ctx._lagThrottled) {
|
|
1740
|
+
ctx._lagThrottled = true;
|
|
1741
|
+
ctx.logger.warn(
|
|
1742
|
+
`lagThrottle: lag ${total} > ${maxLag} \u2014 producer sends will be delayed`
|
|
1743
|
+
);
|
|
1744
|
+
} else if (total <= maxLag && ctx._lagThrottled) {
|
|
1745
|
+
ctx._lagThrottled = false;
|
|
1746
|
+
ctx.logger.log(
|
|
1747
|
+
`lagThrottle: lag ${total} \u2264 ${maxLag} \u2014 producer sends resumed`
|
|
1748
|
+
);
|
|
1749
|
+
}
|
|
1750
|
+
} catch {
|
|
1751
|
+
}
|
|
1752
|
+
};
|
|
1753
|
+
ctx._lagThrottleTimer = setInterval(() => {
|
|
1754
|
+
void poll();
|
|
1755
|
+
}, pollIntervalMs);
|
|
1756
|
+
ctx._lagThrottleTimer.unref?.();
|
|
1757
|
+
}
|
|
1758
|
+
async function recoverLamportClockImpl(ctx, topics) {
|
|
1759
|
+
if (topics.length === 0) return;
|
|
1760
|
+
ctx.logger.log(
|
|
1761
|
+
`Clock recovery: scanning ${topics.length} topic(s) for Lamport clock...`
|
|
1762
|
+
);
|
|
1763
|
+
await ctx.adminOps.ensureConnected();
|
|
1764
|
+
const partitionsToRead = [];
|
|
1765
|
+
for (const t of topics) {
|
|
1766
|
+
let offsets;
|
|
1767
|
+
try {
|
|
1768
|
+
offsets = await ctx.adminOps.admin.fetchTopicOffsets(t);
|
|
1769
|
+
} catch {
|
|
1770
|
+
ctx.logger.warn(
|
|
1771
|
+
`Clock recovery: could not fetch offsets for "${t}", skipping`
|
|
1772
|
+
);
|
|
1773
|
+
continue;
|
|
1774
|
+
}
|
|
1775
|
+
for (const { partition, high, low } of offsets) {
|
|
1776
|
+
if (Number.parseInt(high, 10) > Number.parseInt(low, 10)) {
|
|
1777
|
+
partitionsToRead.push({
|
|
1778
|
+
topic: t,
|
|
1779
|
+
partition,
|
|
1780
|
+
lastOffset: String(Number.parseInt(high, 10) - 1)
|
|
1781
|
+
});
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
}
|
|
1785
|
+
if (partitionsToRead.length === 0) {
|
|
1786
|
+
ctx.logger.log(
|
|
1787
|
+
"Clock recovery: all topics empty \u2014 keeping Lamport clock at 0"
|
|
1788
|
+
);
|
|
1789
|
+
return;
|
|
1790
|
+
}
|
|
1791
|
+
const recoveryGroupId = `${ctx.clientId}-clock-recovery-${Date.now()}`;
|
|
1792
|
+
let maxClock = -1;
|
|
1793
|
+
await new Promise((resolve, reject) => {
|
|
1794
|
+
const consumer = ctx.transport.consumer({ groupId: recoveryGroupId });
|
|
1795
|
+
const remaining = new Set(
|
|
1796
|
+
partitionsToRead.map((p) => `${p.topic}:${p.partition}`)
|
|
1797
|
+
);
|
|
1798
|
+
let settled = false;
|
|
1799
|
+
const cleanup = () => {
|
|
1800
|
+
consumer.disconnect().catch(() => {
|
|
1801
|
+
}).finally(() => {
|
|
1802
|
+
ctx.adminOps.deleteGroups([recoveryGroupId]).catch(() => {
|
|
1803
|
+
});
|
|
1804
|
+
});
|
|
1805
|
+
};
|
|
1806
|
+
const timeoutTimer = setTimeout(() => {
|
|
1807
|
+
if (settled) return;
|
|
1808
|
+
settled = true;
|
|
1809
|
+
ctx.logger.warn(
|
|
1810
|
+
`Clock recovery: timed out after ${ctx.clockRecoveryTimeoutMs} ms with ${remaining.size} partition(s) unread \u2014 proceeding with partial result`
|
|
1811
|
+
);
|
|
1812
|
+
cleanup();
|
|
1813
|
+
resolve();
|
|
1814
|
+
}, ctx.clockRecoveryTimeoutMs);
|
|
1815
|
+
timeoutTimer.unref?.();
|
|
1816
|
+
consumer.connect().then(async () => {
|
|
1817
|
+
const uniqueTopics = [
|
|
1818
|
+
...new Set(partitionsToRead.map((p) => p.topic))
|
|
1819
|
+
];
|
|
1820
|
+
await consumer.subscribe({ topics: uniqueTopics });
|
|
1821
|
+
for (const { topic: t, partition, lastOffset } of partitionsToRead) {
|
|
1822
|
+
consumer.seek({ topic: t, partition, offset: lastOffset });
|
|
1823
|
+
}
|
|
1824
|
+
}).then(
|
|
1825
|
+
() => consumer.run({
|
|
1826
|
+
eachMessage: async ({ topic: t, partition, message }) => {
|
|
1827
|
+
const key = `${t}:${partition}`;
|
|
1828
|
+
if (!remaining.has(key)) return;
|
|
1829
|
+
remaining.delete(key);
|
|
1830
|
+
const clockHeader = message.headers?.[HEADER_LAMPORT_CLOCK];
|
|
1831
|
+
if (clockHeader !== void 0) {
|
|
1832
|
+
const raw = Buffer.isBuffer(clockHeader) ? clockHeader.toString() : String(clockHeader);
|
|
1833
|
+
const clock = Number(raw);
|
|
1834
|
+
if (!Number.isNaN(clock) && clock > maxClock) maxClock = clock;
|
|
1835
|
+
}
|
|
1836
|
+
if (remaining.size === 0 && !settled) {
|
|
1837
|
+
settled = true;
|
|
1838
|
+
clearTimeout(timeoutTimer);
|
|
1839
|
+
cleanup();
|
|
1840
|
+
resolve();
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
})
|
|
1844
|
+
).catch((err) => {
|
|
1845
|
+
if (settled) return;
|
|
1846
|
+
settled = true;
|
|
1847
|
+
clearTimeout(timeoutTimer);
|
|
1848
|
+
cleanup();
|
|
1849
|
+
reject(err);
|
|
1850
|
+
});
|
|
1851
|
+
});
|
|
1852
|
+
if (maxClock >= 0) {
|
|
1853
|
+
ctx._lamportClock = maxClock;
|
|
1854
|
+
ctx.logger.log(
|
|
1855
|
+
`Clock recovery: Lamport clock restored \u2014 next clock will be ${maxClock + 1}`
|
|
1856
|
+
);
|
|
1857
|
+
} else {
|
|
1858
|
+
ctx.logger.log(
|
|
1859
|
+
"Clock recovery: no x-lamport-clock headers found \u2014 keeping clock at 0"
|
|
1860
|
+
);
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
function wrapWithTimeoutWarning(logger, fn, timeoutMs, topic2) {
|
|
1864
|
+
let timer;
|
|
1865
|
+
const promise = fn().finally(() => {
|
|
1866
|
+
if (timer !== void 0) clearTimeout(timer);
|
|
1867
|
+
});
|
|
1868
|
+
timer = setTimeout(() => {
|
|
1869
|
+
logger.warn(
|
|
1870
|
+
`Handler for topic "${topic2}" has not resolved after ${timeoutMs}ms \u2014 possible stuck handler`
|
|
1871
|
+
);
|
|
1872
|
+
}, timeoutMs);
|
|
1873
|
+
return promise;
|
|
1874
|
+
}
|
|
1875
|
+
|
|
1876
|
+
// src/client/kafka.client/producer/send.ts
|
|
1877
|
+
async function preparePayload(ctx, topicOrDesc, messages, compression) {
|
|
1878
|
+
registerSchema(topicOrDesc, ctx.schemaRegistry, ctx.logger);
|
|
1879
|
+
const payload = await buildSendPayload(
|
|
1880
|
+
topicOrDesc,
|
|
1881
|
+
messages,
|
|
1882
|
+
ctx.producerOpsDeps,
|
|
1883
|
+
compression
|
|
1884
|
+
);
|
|
1885
|
+
await ensureTopic(ctx, payload.topic);
|
|
1886
|
+
return payload;
|
|
1887
|
+
}
|
|
1888
|
+
async function redirectToDelayed(ctx, payload, deliverAfterMs) {
|
|
1889
|
+
const until = String(Date.now() + deliverAfterMs);
|
|
1890
|
+
for (const m of payload.messages) {
|
|
1891
|
+
m.headers[HEADER_DELAYED_UNTIL] = until;
|
|
1892
|
+
m.headers[HEADER_DELAYED_TARGET] = payload.topic;
|
|
1893
|
+
}
|
|
1894
|
+
payload.topic = `${payload.topic}.delayed`;
|
|
1895
|
+
await ensureTopic(ctx, payload.topic);
|
|
1896
|
+
}
|
|
1897
|
+
async function sendMessageImpl(ctx, topicOrDesc, message, options = {}) {
|
|
1898
|
+
await waitIfThrottled(ctx);
|
|
1899
|
+
const payload = await preparePayload(
|
|
1900
|
+
ctx,
|
|
1901
|
+
topicOrDesc,
|
|
1902
|
+
[
|
|
1903
|
+
{
|
|
1904
|
+
value: message,
|
|
1905
|
+
key: options.key,
|
|
1906
|
+
headers: options.headers,
|
|
1907
|
+
correlationId: options.correlationId,
|
|
1908
|
+
schemaVersion: options.schemaVersion,
|
|
1909
|
+
eventId: options.eventId
|
|
1910
|
+
}
|
|
1911
|
+
],
|
|
1912
|
+
options.compression
|
|
1913
|
+
);
|
|
1914
|
+
if (options.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
1915
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
1916
|
+
}
|
|
1917
|
+
await ctx.producer.send(payload);
|
|
1918
|
+
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1919
|
+
}
|
|
1920
|
+
async function sendBatchImpl(ctx, topicOrDesc, messages, options) {
|
|
1921
|
+
await waitIfThrottled(ctx);
|
|
1922
|
+
const payload = await preparePayload(
|
|
1923
|
+
ctx,
|
|
1924
|
+
topicOrDesc,
|
|
1925
|
+
messages,
|
|
1926
|
+
options?.compression
|
|
1927
|
+
);
|
|
1928
|
+
if (options?.deliverAfterMs && options.deliverAfterMs > 0) {
|
|
1929
|
+
await redirectToDelayed(ctx, payload, options.deliverAfterMs);
|
|
1930
|
+
}
|
|
1931
|
+
await ctx.producer.send(payload);
|
|
1932
|
+
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1933
|
+
}
|
|
1934
|
+
async function sendTombstoneImpl(ctx, topic2, key, headers) {
|
|
1935
|
+
await waitIfThrottled(ctx);
|
|
1936
|
+
const hdrs = { ...headers };
|
|
1937
|
+
for (const inst of ctx.instrumentation) inst.beforeSend?.(topic2, hdrs);
|
|
1938
|
+
await ensureTopic(ctx, topic2);
|
|
1939
|
+
await ctx.producer.send({
|
|
1940
|
+
topic: topic2,
|
|
1941
|
+
messages: [{ value: null, key, headers: hdrs }]
|
|
1942
|
+
});
|
|
1943
|
+
for (const inst of ctx.instrumentation) inst.afterSend?.(topic2);
|
|
1944
|
+
}
|
|
1945
|
+
async function transactionImpl(ctx, fn) {
|
|
1946
|
+
if (!ctx.txProducerInitPromise) {
|
|
1947
|
+
if (_activeTransactionalIds.has(ctx.txId)) {
|
|
1948
|
+
ctx.logger.warn(
|
|
1949
|
+
`transactionalId "${ctx.txId}" is already in use by another KafkaClient in this process. Kafka will fence one of the producers. Set a unique \`transactionalId\` (or distinct \`clientId\`) per instance.`
|
|
1950
|
+
);
|
|
1951
|
+
}
|
|
1952
|
+
const initPromise = (async () => {
|
|
1953
|
+
const p = ctx.transport.producer({
|
|
1954
|
+
idempotent: true,
|
|
1955
|
+
transactionalId: ctx.txId
|
|
1956
|
+
});
|
|
1957
|
+
await p.connect();
|
|
1958
|
+
_activeTransactionalIds.add(ctx.txId);
|
|
1959
|
+
return p;
|
|
1960
|
+
})();
|
|
1961
|
+
ctx.txProducerInitPromise = initPromise.catch((err) => {
|
|
1962
|
+
ctx.txProducerInitPromise = void 0;
|
|
1963
|
+
throw err;
|
|
1964
|
+
});
|
|
1965
|
+
}
|
|
1966
|
+
ctx.txProducer = await ctx.txProducerInitPromise;
|
|
1967
|
+
const prev = ctx._txChain;
|
|
1968
|
+
let release;
|
|
1969
|
+
ctx._txChain = new Promise((r) => release = r);
|
|
1970
|
+
await prev;
|
|
1971
|
+
try {
|
|
1972
|
+
await runTransaction(ctx, fn);
|
|
1973
|
+
} finally {
|
|
1974
|
+
release();
|
|
1975
|
+
}
|
|
1976
|
+
}
|
|
1977
|
+
async function runTransaction(ctx, fn) {
|
|
1978
|
+
const tx = await ctx.txProducer.transaction();
|
|
1979
|
+
try {
|
|
1980
|
+
const txCtx = {
|
|
1981
|
+
send: async (topicOrDesc, message, sendOpts = {}) => {
|
|
1982
|
+
const payload = await preparePayload(ctx, topicOrDesc, [
|
|
1983
|
+
{
|
|
1984
|
+
value: message,
|
|
1985
|
+
key: sendOpts.key,
|
|
1986
|
+
headers: sendOpts.headers,
|
|
1987
|
+
correlationId: sendOpts.correlationId,
|
|
1988
|
+
schemaVersion: sendOpts.schemaVersion,
|
|
1989
|
+
eventId: sendOpts.eventId
|
|
1990
|
+
}
|
|
1991
|
+
]);
|
|
1992
|
+
await tx.send(payload);
|
|
1993
|
+
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
1994
|
+
},
|
|
1995
|
+
sendBatch: async (topicOrDesc, messages, batchOpts) => {
|
|
1996
|
+
const payload = await preparePayload(
|
|
1997
|
+
ctx,
|
|
1998
|
+
topicOrDesc,
|
|
1999
|
+
messages,
|
|
2000
|
+
batchOpts?.compression
|
|
2001
|
+
);
|
|
2002
|
+
await tx.send(payload);
|
|
2003
|
+
ctx.metrics.notifyAfterSend(payload.topic, payload.messages.length);
|
|
2004
|
+
}
|
|
2005
|
+
};
|
|
2006
|
+
await fn(txCtx);
|
|
2007
|
+
await tx.commit();
|
|
2008
|
+
} catch (error) {
|
|
2009
|
+
try {
|
|
2010
|
+
await tx.abort();
|
|
2011
|
+
} catch (abortError) {
|
|
2012
|
+
ctx.logger.error(
|
|
2013
|
+
"Failed to abort transaction:",
|
|
2014
|
+
(abortError instanceof Error ? abortError : new Error(String(abortError))).message
|
|
2015
|
+
);
|
|
2016
|
+
}
|
|
2017
|
+
throw error;
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
async function waitIfThrottled(ctx) {
|
|
2021
|
+
if (!ctx._lagThrottled) return;
|
|
2022
|
+
const maxWait = ctx.lagThrottleOpts?.maxWaitMs ?? 3e4;
|
|
2023
|
+
const start = Date.now();
|
|
2024
|
+
while (ctx._lagThrottled) {
|
|
2025
|
+
if (Date.now() - start >= maxWait) {
|
|
2026
|
+
ctx.logger.warn(
|
|
2027
|
+
`lagThrottle: maxWaitMs (${maxWait} ms) exceeded \u2014 sending anyway`
|
|
2028
|
+
);
|
|
2029
|
+
return;
|
|
2030
|
+
}
|
|
2031
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
2032
|
+
}
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
// src/client/kafka.client/consumer/retry-topic.ts
|
|
2036
|
+
async function waitForPartitionAssignment(consumer, topics, logger, timeoutMs = 1e4) {
|
|
2037
|
+
const topicSet = new Set(topics);
|
|
2038
|
+
const deadline = Date.now() + timeoutMs;
|
|
2039
|
+
while (Date.now() < deadline) {
|
|
2040
|
+
try {
|
|
2041
|
+
const assigned = consumer.assignment();
|
|
2042
|
+
if (assigned.some((a) => topicSet.has(a.topic))) return;
|
|
2043
|
+
} catch {
|
|
2044
|
+
}
|
|
2045
|
+
await sleep(200);
|
|
2046
|
+
}
|
|
2047
|
+
logger.warn(
|
|
2048
|
+
`Retry consumer did not receive partition assignments for [${topics.join(", ")}] within ${timeoutMs}ms`
|
|
2049
|
+
);
|
|
2050
|
+
}
|
|
2051
|
+
async function startLevelConsumer(level, levelTopics, levelGroupId, originalTopics, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2052
|
+
const {
|
|
2053
|
+
logger,
|
|
2054
|
+
producer,
|
|
2055
|
+
instrumentation,
|
|
2056
|
+
onMessageLost,
|
|
2057
|
+
onRetry,
|
|
2058
|
+
onDlq,
|
|
2059
|
+
onMessage,
|
|
2060
|
+
ensureTopic: ensureTopic2,
|
|
2061
|
+
getOrCreateConsumer: getOrCreateConsumer2,
|
|
2062
|
+
runningConsumers,
|
|
2063
|
+
createRetryTxProducer: createRetryTxProducer2
|
|
2064
|
+
} = deps;
|
|
2065
|
+
const backoffMs = retry.backoffMs ?? 1e3;
|
|
2066
|
+
const maxBackoffMs = retry.maxBackoffMs ?? 3e4;
|
|
2067
|
+
const pipelineDeps = { logger, producer, instrumentation, onMessageLost };
|
|
2068
|
+
for (const lt of levelTopics) {
|
|
2069
|
+
await ensureTopic2(lt);
|
|
2070
|
+
}
|
|
2071
|
+
const levelTxProducer = await createRetryTxProducer2(`${levelGroupId}-tx`);
|
|
2072
|
+
const consumer = getOrCreateConsumer2(levelGroupId, false, false);
|
|
2073
|
+
await consumer.connect();
|
|
2074
|
+
await subscribeWithRetry(consumer, levelTopics, logger);
|
|
2075
|
+
await consumer.run({
|
|
2076
|
+
eachMessage: async ({ topic: levelTopic, partition, message }) => {
|
|
2077
|
+
const nextOffset = {
|
|
2078
|
+
topic: levelTopic,
|
|
2079
|
+
partition,
|
|
2080
|
+
offset: (parseInt(message.offset, 10) + 1).toString()
|
|
2081
|
+
};
|
|
2082
|
+
if (!message.value) {
|
|
2083
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2084
|
+
return;
|
|
2085
|
+
}
|
|
2086
|
+
const headers = decodeHeaders(message.headers);
|
|
2087
|
+
const retryAfter = parseInt(
|
|
2088
|
+
headers[RETRY_HEADER_AFTER] ?? "0",
|
|
2089
|
+
10
|
|
2090
|
+
);
|
|
2091
|
+
const remaining = retryAfter - Date.now();
|
|
2092
|
+
if (remaining > 0) {
|
|
2093
|
+
consumer.pause([{ topic: levelTopic, partitions: [partition] }]);
|
|
2094
|
+
await sleep(remaining);
|
|
2095
|
+
consumer.resume([{ topic: levelTopic, partitions: [partition] }]);
|
|
2096
|
+
}
|
|
2097
|
+
const raw = message.value.toString();
|
|
2098
|
+
const parsed = parseJsonMessage(raw, levelTopic, logger);
|
|
2099
|
+
if (parsed === null) {
|
|
2100
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2101
|
+
return;
|
|
2102
|
+
}
|
|
2103
|
+
const currentMaxRetries = parseInt(
|
|
2104
|
+
headers[RETRY_HEADER_MAX_RETRIES] ?? String(retry.maxRetries),
|
|
2105
|
+
10
|
|
2106
|
+
);
|
|
2107
|
+
const originalTopic = headers[RETRY_HEADER_ORIGINAL_TOPIC] ?? levelTopic.replace(/\.retry\.\d+$/, "");
|
|
2108
|
+
const validated = await validateWithSchema(
|
|
2109
|
+
parsed,
|
|
2110
|
+
raw,
|
|
2111
|
+
originalTopic,
|
|
2112
|
+
schemaMap,
|
|
2113
|
+
interceptors,
|
|
2114
|
+
dlq,
|
|
2115
|
+
{ ...pipelineDeps, originalHeaders: headers }
|
|
2116
|
+
);
|
|
2117
|
+
if (validated === null) {
|
|
2118
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2119
|
+
return;
|
|
2120
|
+
}
|
|
2121
|
+
const envelope = extractEnvelope(
|
|
2122
|
+
validated,
|
|
2123
|
+
headers,
|
|
2124
|
+
originalTopic,
|
|
2125
|
+
partition,
|
|
2126
|
+
message.offset
|
|
2127
|
+
);
|
|
2128
|
+
const error = await runHandlerWithPipeline(
|
|
2129
|
+
() => runWithEnvelopeContext(
|
|
2130
|
+
{
|
|
2131
|
+
correlationId: envelope.correlationId,
|
|
2132
|
+
traceparent: envelope.traceparent
|
|
2133
|
+
},
|
|
2134
|
+
() => handleMessage(envelope)
|
|
2135
|
+
),
|
|
2136
|
+
[envelope],
|
|
2137
|
+
interceptors,
|
|
2138
|
+
instrumentation
|
|
2139
|
+
);
|
|
2140
|
+
if (!error) {
|
|
2141
|
+
onMessage?.(envelope);
|
|
2142
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2143
|
+
return;
|
|
2144
|
+
}
|
|
2145
|
+
deps.onFailure?.(envelope);
|
|
2146
|
+
const exhausted = level >= currentMaxRetries;
|
|
2147
|
+
const reportedError = exhausted && currentMaxRetries > 1 ? new KafkaRetryExhaustedError(
|
|
2148
|
+
originalTopic,
|
|
2149
|
+
[envelope.payload],
|
|
2150
|
+
currentMaxRetries,
|
|
2151
|
+
{ cause: error }
|
|
2152
|
+
) : error;
|
|
2153
|
+
await notifyInterceptorsOnError([envelope], interceptors, reportedError);
|
|
2154
|
+
logger.error(
|
|
2155
|
+
`Retry consumer error for ${originalTopic} (level ${level}/${currentMaxRetries}):`,
|
|
2156
|
+
error.stack
|
|
2157
|
+
);
|
|
2158
|
+
if (!exhausted) {
|
|
2159
|
+
const nextLevel = level + 1;
|
|
2160
|
+
const cap = Math.min(backoffMs * 2 ** level, maxBackoffMs);
|
|
2161
|
+
const delay = Math.floor(Math.random() * cap);
|
|
2162
|
+
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
2163
|
+
originalTopic,
|
|
2164
|
+
[raw],
|
|
2165
|
+
nextLevel,
|
|
2166
|
+
currentMaxRetries,
|
|
2167
|
+
delay,
|
|
2168
|
+
headers
|
|
2169
|
+
);
|
|
2170
|
+
const tx = await levelTxProducer.transaction();
|
|
2171
|
+
try {
|
|
2172
|
+
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
2173
|
+
await tx.sendOffsets({
|
|
2174
|
+
consumer,
|
|
2175
|
+
topics: [
|
|
2176
|
+
{
|
|
2177
|
+
topic: nextOffset.topic,
|
|
2178
|
+
partitions: [
|
|
2179
|
+
{
|
|
2180
|
+
partition: nextOffset.partition,
|
|
2181
|
+
offset: nextOffset.offset
|
|
2182
|
+
}
|
|
2183
|
+
]
|
|
2184
|
+
}
|
|
2185
|
+
]
|
|
2186
|
+
});
|
|
2187
|
+
await tx.commit();
|
|
2188
|
+
logger.warn(
|
|
2189
|
+
`Message routed to ${rtTopic} (EOS, level ${nextLevel}/${currentMaxRetries})`
|
|
2190
|
+
);
|
|
2191
|
+
onRetry?.(envelope, nextLevel, currentMaxRetries);
|
|
2192
|
+
} catch (txErr) {
|
|
2193
|
+
try {
|
|
2194
|
+
await tx.abort();
|
|
2195
|
+
} catch {
|
|
2196
|
+
}
|
|
2197
|
+
logger.error(
|
|
2198
|
+
`EOS routing to ${rtTopic} failed \u2014 message will be redelivered:`,
|
|
2199
|
+
toError(txErr).stack
|
|
2200
|
+
);
|
|
2201
|
+
return;
|
|
2202
|
+
}
|
|
2203
|
+
} else if (dlq) {
|
|
2204
|
+
const { topic: dTopic, messages: dMsgs } = buildDlqPayload(
|
|
2205
|
+
originalTopic,
|
|
2206
|
+
raw,
|
|
2207
|
+
{
|
|
2208
|
+
error,
|
|
2209
|
+
// +1 to account for the main consumer's initial attempt before routing.
|
|
2210
|
+
attempt: level + 1,
|
|
2211
|
+
originalHeaders: headers
|
|
2212
|
+
}
|
|
2213
|
+
);
|
|
2214
|
+
const tx = await levelTxProducer.transaction();
|
|
2215
|
+
try {
|
|
2216
|
+
await tx.send({ topic: dTopic, messages: dMsgs });
|
|
2217
|
+
await tx.sendOffsets({
|
|
2218
|
+
consumer,
|
|
2219
|
+
topics: [
|
|
2220
|
+
{
|
|
2221
|
+
topic: nextOffset.topic,
|
|
2222
|
+
partitions: [
|
|
2223
|
+
{
|
|
2224
|
+
partition: nextOffset.partition,
|
|
2225
|
+
offset: nextOffset.offset
|
|
2226
|
+
}
|
|
2227
|
+
]
|
|
2228
|
+
}
|
|
2229
|
+
]
|
|
2230
|
+
});
|
|
2231
|
+
await tx.commit();
|
|
2232
|
+
logger.warn(`Message sent to DLQ: ${dTopic} (EOS)`);
|
|
2233
|
+
onDlq?.(envelope, "handler-error");
|
|
2234
|
+
} catch (txErr) {
|
|
2235
|
+
try {
|
|
2236
|
+
await tx.abort();
|
|
2237
|
+
} catch {
|
|
2238
|
+
}
|
|
2239
|
+
logger.error(
|
|
2240
|
+
`EOS DLQ routing to ${dTopic} failed \u2014 message will be redelivered:`,
|
|
2241
|
+
toError(txErr).stack
|
|
2242
|
+
);
|
|
2243
|
+
return;
|
|
2244
|
+
}
|
|
2245
|
+
} else {
|
|
2246
|
+
await onMessageLost?.({
|
|
2247
|
+
topic: originalTopic,
|
|
2248
|
+
error,
|
|
2249
|
+
attempt: level,
|
|
2250
|
+
headers
|
|
2251
|
+
});
|
|
2252
|
+
await consumer.commitOffsets([nextOffset]);
|
|
2253
|
+
}
|
|
2254
|
+
}
|
|
2255
|
+
});
|
|
2256
|
+
runningConsumers.set(levelGroupId, "eachMessage");
|
|
2257
|
+
await waitForPartitionAssignment(
|
|
2258
|
+
consumer,
|
|
2259
|
+
levelTopics,
|
|
2260
|
+
logger,
|
|
2261
|
+
assignmentTimeoutMs
|
|
2262
|
+
);
|
|
2263
|
+
deps.onLevelStarted?.(levelGroupId);
|
|
2264
|
+
logger.log(
|
|
2265
|
+
`Retry level ${level}/${retry.maxRetries} consumer started for: ${originalTopics.join(", ")} (group: ${levelGroupId})`
|
|
2266
|
+
);
|
|
2267
|
+
}
|
|
2268
|
+
async function startRetryTopicConsumers(originalTopics, originalGroupId, handleMessage, retry, dlq, interceptors, schemaMap, deps, assignmentTimeoutMs) {
|
|
2269
|
+
const levelGroupIds = new Array(retry.maxRetries);
|
|
2270
|
+
await Promise.all(
|
|
2271
|
+
Array.from({ length: retry.maxRetries }, async (_, i) => {
|
|
2272
|
+
const level = i + 1;
|
|
2273
|
+
const levelTopics = originalTopics.map((t) => `${t}.retry.${level}`);
|
|
2274
|
+
const levelGroupId = `${originalGroupId}-retry.${level}`;
|
|
2275
|
+
await startLevelConsumer(
|
|
2276
|
+
level,
|
|
2277
|
+
levelTopics,
|
|
2278
|
+
levelGroupId,
|
|
2279
|
+
originalTopics,
|
|
2280
|
+
handleMessage,
|
|
2281
|
+
retry,
|
|
2282
|
+
dlq,
|
|
2283
|
+
interceptors,
|
|
2284
|
+
schemaMap,
|
|
2285
|
+
deps,
|
|
2286
|
+
assignmentTimeoutMs
|
|
2287
|
+
);
|
|
2288
|
+
levelGroupIds[i] = levelGroupId;
|
|
2289
|
+
})
|
|
2290
|
+
);
|
|
2291
|
+
return levelGroupIds;
|
|
2292
|
+
}
|
|
2293
|
+
|
|
2294
|
+
// src/client/kafka.client/consumer/setup.ts
|
|
2295
|
+
function validateTopicConsumerOpts(topics, options) {
|
|
2296
|
+
if (options.retryTopics && !options.retry) {
|
|
2297
|
+
throw new Error(
|
|
2298
|
+
"retryTopics requires retry to be configured \u2014 set retry.maxRetries to enable the retry topic chain"
|
|
2299
|
+
);
|
|
2300
|
+
}
|
|
2301
|
+
if (options.retryTopics && topics.some((t) => t instanceof RegExp)) {
|
|
2302
|
+
throw new Error(
|
|
2303
|
+
"retryTopics is incompatible with regex topic patterns \u2014 retry topics require a fixed topic name to build the retry chain."
|
|
2304
|
+
);
|
|
2305
|
+
}
|
|
2306
|
+
}
|
|
2307
|
+
async function ensureConsumerTopics(ctx, topicNames, dlq, deduplication) {
|
|
2308
|
+
for (const t of topicNames) await ensureTopic(ctx, t);
|
|
2309
|
+
if (dlq) {
|
|
2310
|
+
for (const t of topicNames) await ensureTopic(ctx, `${t}.dlq`);
|
|
2311
|
+
if (!ctx.autoCreateTopicsEnabled && topicNames.length > 0) {
|
|
2312
|
+
await ctx.adminOps.validateDlqTopicsExist(topicNames);
|
|
2313
|
+
}
|
|
2314
|
+
}
|
|
2315
|
+
if (deduplication?.strategy === "topic") {
|
|
2316
|
+
const dest = deduplication.duplicatesTopic;
|
|
2317
|
+
if (ctx.autoCreateTopicsEnabled) {
|
|
2318
|
+
for (const t of topicNames)
|
|
2319
|
+
await ensureTopic(ctx, dest ?? `${t}.duplicates`);
|
|
2320
|
+
} else if (topicNames.length > 0) {
|
|
2321
|
+
await ctx.adminOps.validateDuplicatesTopicsExist(topicNames, dest);
|
|
2322
|
+
}
|
|
2323
|
+
}
|
|
2324
|
+
}
|
|
2325
|
+
async function setupConsumer(ctx, topics, mode, options) {
|
|
2326
|
+
const {
|
|
2327
|
+
groupId: optGroupId,
|
|
2328
|
+
fromBeginning = false,
|
|
2329
|
+
retry,
|
|
2330
|
+
dlq = false,
|
|
2331
|
+
interceptors = [],
|
|
2332
|
+
schemas: optionSchemas
|
|
2333
|
+
} = options;
|
|
2334
|
+
const stringTopics = topics.filter((t) => !(t instanceof RegExp));
|
|
2335
|
+
const regexTopics = topics.filter((t) => t instanceof RegExp);
|
|
2336
|
+
const hasRegex = regexTopics.length > 0;
|
|
2337
|
+
const gid = optGroupId || ctx.defaultGroupId;
|
|
2338
|
+
const existingMode = ctx.runningConsumers.get(gid);
|
|
2339
|
+
const oppositeMode = mode === "eachMessage" ? "eachBatch" : "eachMessage";
|
|
2340
|
+
if (existingMode === oppositeMode) {
|
|
2341
|
+
throw new Error(
|
|
2342
|
+
`Cannot use ${mode} on consumer group "${gid}" \u2014 it is already running with ${oppositeMode}. Use a different groupId for this consumer.`
|
|
2343
|
+
);
|
|
2344
|
+
}
|
|
2345
|
+
if (existingMode === mode) {
|
|
2346
|
+
const callerName = mode === "eachMessage" ? "startConsumer" : "startBatchConsumer";
|
|
2347
|
+
throw new Error(
|
|
2348
|
+
`${callerName}("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
|
|
2349
|
+
);
|
|
2350
|
+
}
|
|
2351
|
+
let resolveReady;
|
|
2352
|
+
const readyPromise = new Promise((resolve) => {
|
|
2353
|
+
resolveReady = resolve;
|
|
2354
|
+
});
|
|
2355
|
+
const consumer = getOrCreateConsumer(
|
|
2356
|
+
gid,
|
|
2357
|
+
fromBeginning,
|
|
2358
|
+
options.autoCommit ?? true,
|
|
2359
|
+
ctx.consumerOpsDeps,
|
|
2360
|
+
options.partitionAssigner,
|
|
2361
|
+
resolveReady,
|
|
2362
|
+
options.groupInstanceId
|
|
2363
|
+
);
|
|
2364
|
+
const schemaMap = buildSchemaMap(
|
|
2365
|
+
stringTopics,
|
|
2366
|
+
ctx.schemaRegistry,
|
|
2367
|
+
optionSchemas,
|
|
2368
|
+
ctx.logger
|
|
2369
|
+
);
|
|
2370
|
+
const topicNames = stringTopics.map((t) => resolveTopicName(t));
|
|
2371
|
+
const subscribeTopics = [...topicNames, ...regexTopics];
|
|
2372
|
+
await ensureConsumerTopics(ctx, topicNames, dlq, options.deduplication);
|
|
2373
|
+
await consumer.connect();
|
|
2374
|
+
if (dlq || options.retryTopics || options.deduplication) {
|
|
2375
|
+
await ctx.producer.connect();
|
|
2376
|
+
}
|
|
2377
|
+
await subscribeWithRetry(
|
|
2378
|
+
consumer,
|
|
2379
|
+
subscribeTopics,
|
|
2380
|
+
ctx.logger,
|
|
2381
|
+
options.subscribeRetry
|
|
2382
|
+
);
|
|
2383
|
+
const displayTopics = subscribeTopics.map((t) => t instanceof RegExp ? t.toString() : t).join(", ");
|
|
2384
|
+
ctx.logger.log(
|
|
2385
|
+
`${mode === "eachBatch" ? "Batch consumer" : "Consumer"} subscribed to topics: ${displayTopics}`
|
|
2386
|
+
);
|
|
2387
|
+
return { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, hasRegex, readyPromise };
|
|
2388
|
+
}
|
|
2389
|
+
function resolveDeduplicationContext(ctx, groupId, options) {
|
|
2390
|
+
if (!options) return void 0;
|
|
2391
|
+
const store = options.store ?? new InMemoryDedupStore(ctx.dedupStates);
|
|
2392
|
+
return { options, store, groupId };
|
|
2393
|
+
}
|
|
2394
|
+
function messageDepsFor(ctx, gid, options) {
|
|
2395
|
+
const notifyRetry = ctx.metrics.notifyRetry.bind(ctx.metrics);
|
|
2396
|
+
return {
|
|
2397
|
+
logger: ctx.logger,
|
|
2398
|
+
producer: ctx.producer,
|
|
2399
|
+
instrumentation: ctx.instrumentation,
|
|
2400
|
+
onMessageLost: options?.onMessageLost ?? ctx.onMessageLost,
|
|
2401
|
+
onTtlExpired: ctx.onTtlExpired,
|
|
2402
|
+
onRetry: options?.onRetry ? (envelope, attempt, max) => {
|
|
2403
|
+
notifyRetry(envelope, attempt, max);
|
|
2404
|
+
return options.onRetry(envelope, attempt, max);
|
|
2405
|
+
} : notifyRetry,
|
|
2406
|
+
onDlq: (envelope, reason) => ctx.metrics.notifyDlq(envelope, reason),
|
|
2407
|
+
onDuplicate: ctx.metrics.notifyDuplicate.bind(ctx.metrics),
|
|
2408
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2409
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid)
|
|
2410
|
+
};
|
|
2411
|
+
}
|
|
2412
|
+
function buildRetryTopicDeps(ctx) {
|
|
2413
|
+
return {
|
|
2414
|
+
logger: ctx.logger,
|
|
2415
|
+
producer: ctx.producer,
|
|
2416
|
+
instrumentation: ctx.instrumentation,
|
|
2417
|
+
onMessageLost: ctx.onMessageLost,
|
|
2418
|
+
onRetry: ctx.metrics.notifyRetry.bind(ctx.metrics),
|
|
2419
|
+
onDlq: ctx.metrics.notifyDlq.bind(ctx.metrics),
|
|
2420
|
+
onMessage: ctx.metrics.notifyMessage.bind(ctx.metrics),
|
|
2421
|
+
ensureTopic: (t) => ensureTopic(ctx, t),
|
|
2422
|
+
getOrCreateConsumer: (gid, fb, ac) => getOrCreateConsumer(gid, fb, ac, ctx.consumerOpsDeps),
|
|
2423
|
+
runningConsumers: ctx.runningConsumers,
|
|
2424
|
+
createRetryTxProducer: (txId) => createRetryTxProducer(ctx, txId)
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
2427
|
+
async function makeEosMainContext(ctx, gid, consumer, options) {
|
|
2428
|
+
if (!options.retryTopics || !options.retry) return void 0;
|
|
2429
|
+
const txProducer = await createRetryTxProducer(ctx, `${gid}-main-tx`);
|
|
2430
|
+
return { txProducer, consumer };
|
|
2431
|
+
}
|
|
2432
|
+
async function launchRetryChain(ctx, gid, topicNames, handleMessage, opts) {
|
|
2433
|
+
const { retry, dlq, interceptors, schemaMap, assignmentTimeoutMs } = opts;
|
|
2434
|
+
if (!ctx.autoCreateTopicsEnabled) {
|
|
2435
|
+
await ctx.adminOps.validateRetryTopicsExist(topicNames, retry.maxRetries);
|
|
2436
|
+
}
|
|
2437
|
+
ctx.companionGroupIds.set(gid, []);
|
|
2438
|
+
await startRetryTopicConsumers(
|
|
2439
|
+
topicNames,
|
|
2440
|
+
gid,
|
|
2441
|
+
handleMessage,
|
|
2442
|
+
retry,
|
|
2443
|
+
dlq,
|
|
2444
|
+
interceptors,
|
|
2445
|
+
schemaMap,
|
|
2446
|
+
{
|
|
2447
|
+
...ctx.retryTopicDeps,
|
|
2448
|
+
// Bind circuit breaker events to the MAIN consumer group so failures and
|
|
2449
|
+
// successes inside the retry chain drive the same breaker as the main
|
|
2450
|
+
// consumer (the retry chain has no breaker config of its own).
|
|
2451
|
+
onFailure: (envelope) => ctx.metrics.notifyFailure(envelope, gid),
|
|
2452
|
+
onMessage: (envelope) => ctx.metrics.notifyMessage(envelope, gid),
|
|
2453
|
+
onLevelStarted: (levelGroupId) => {
|
|
2454
|
+
ctx.companionGroupIds.get(gid).push(levelGroupId);
|
|
2455
|
+
}
|
|
2456
|
+
},
|
|
2457
|
+
assignmentTimeoutMs
|
|
2458
|
+
);
|
|
2459
|
+
}
|
|
2460
|
+
|
|
2461
|
+
// src/client/kafka.client/consumer/handler.ts
|
|
2462
|
+
async function applyDeduplication(envelope, raw, dedup, dlq, deps) {
|
|
2463
|
+
const clockRaw = envelope.headers[HEADER_LAMPORT_CLOCK];
|
|
2464
|
+
if (clockRaw === void 0) return false;
|
|
2465
|
+
const incomingClock = Number(clockRaw);
|
|
2466
|
+
if (Number.isNaN(incomingClock)) return false;
|
|
2467
|
+
const stateKey = `${envelope.topic}:${envelope.partition}`;
|
|
2468
|
+
let lastProcessedClock;
|
|
2469
|
+
try {
|
|
2470
|
+
lastProcessedClock = await dedup.store.getLastClock(dedup.groupId, stateKey) ?? -1;
|
|
2471
|
+
} catch (err) {
|
|
2472
|
+
deps.logger.error(
|
|
2473
|
+
`Dedup store getLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 treating message as not a duplicate (fail-open): ${err.message}`
|
|
2474
|
+
);
|
|
2475
|
+
return false;
|
|
2476
|
+
}
|
|
2477
|
+
if (incomingClock <= lastProcessedClock) {
|
|
2478
|
+
const meta = {
|
|
2479
|
+
incomingClock,
|
|
2480
|
+
lastProcessedClock,
|
|
2481
|
+
originalHeaders: envelope.headers
|
|
2482
|
+
};
|
|
2483
|
+
const strategy = dedup.options.strategy ?? "drop";
|
|
2484
|
+
deps.logger.warn(
|
|
2485
|
+
`Duplicate message on ${envelope.topic}[${envelope.partition}]: clock=${incomingClock} <= last=${lastProcessedClock} \u2014 strategy=${strategy}`
|
|
2486
|
+
);
|
|
2487
|
+
deps.onDuplicate?.(envelope, strategy);
|
|
2488
|
+
if (strategy === "dlq" && dlq) {
|
|
2489
|
+
const augmentedHeaders = {
|
|
2490
|
+
...envelope.headers,
|
|
2491
|
+
"x-dlq-reason": "lamport-clock-duplicate",
|
|
2492
|
+
"x-dlq-duplicate-incoming-clock": String(incomingClock),
|
|
2493
|
+
"x-dlq-duplicate-last-processed-clock": String(lastProcessedClock)
|
|
2494
|
+
};
|
|
2495
|
+
await sendToDlq(envelope.topic, raw, deps, {
|
|
2496
|
+
error: new Error("Lamport Clock duplicate detected"),
|
|
2497
|
+
attempt: 0,
|
|
2498
|
+
originalHeaders: augmentedHeaders
|
|
2499
|
+
});
|
|
2500
|
+
} else if (strategy === "topic") {
|
|
2501
|
+
const destination = dedup.options.duplicatesTopic ?? `${envelope.topic}.duplicates`;
|
|
2502
|
+
await sendToDuplicatesTopic(envelope.topic, raw, destination, deps, meta);
|
|
2503
|
+
}
|
|
2504
|
+
return true;
|
|
2505
|
+
}
|
|
2506
|
+
try {
|
|
2507
|
+
await dedup.store.setLastClock(dedup.groupId, stateKey, incomingClock);
|
|
2508
|
+
} catch (err) {
|
|
2509
|
+
deps.logger.error(
|
|
2510
|
+
`Dedup store setLastClock failed on ${envelope.topic}[${envelope.partition}] \u2014 processing message anyway (fail-open): ${err.message}`
|
|
2511
|
+
);
|
|
2512
|
+
}
|
|
2513
|
+
return false;
|
|
2514
|
+
}
|
|
2515
|
+
async function parseSingleMessage(message, topic2, partition, schemaMap, interceptors, dlq, deps) {
|
|
2516
|
+
if (!message.value) {
|
|
2517
|
+
deps.logger.warn(`Received empty message from topic ${topic2}`);
|
|
2518
|
+
return null;
|
|
2519
|
+
}
|
|
2520
|
+
const raw = message.value.toString();
|
|
2521
|
+
const parsed = parseJsonMessage(raw, topic2, deps.logger);
|
|
2522
|
+
if (parsed === null) return null;
|
|
2523
|
+
const headers = decodeHeaders(message.headers);
|
|
2524
|
+
const validated = await validateWithSchema(
|
|
2525
|
+
parsed,
|
|
2526
|
+
raw,
|
|
2527
|
+
topic2,
|
|
2528
|
+
schemaMap,
|
|
2529
|
+
interceptors,
|
|
2530
|
+
dlq,
|
|
2531
|
+
{ ...deps, originalHeaders: headers }
|
|
2532
|
+
);
|
|
2533
|
+
if (validated === null) return null;
|
|
2534
|
+
return extractEnvelope(validated, headers, topic2, partition, message.offset);
|
|
2535
|
+
}
|
|
2536
|
+
async function handleEachMessage(payload, opts, deps) {
|
|
2537
|
+
const { topic: topic2, partition, message } = payload;
|
|
2538
|
+
const {
|
|
2539
|
+
schemaMap,
|
|
2540
|
+
handleMessage,
|
|
2541
|
+
interceptors,
|
|
2542
|
+
dlq,
|
|
2543
|
+
retry,
|
|
2544
|
+
retryTopics,
|
|
2545
|
+
timeoutMs,
|
|
2546
|
+
wrapWithTimeout
|
|
2547
|
+
} = opts;
|
|
2548
|
+
const eos = opts.eosMainContext;
|
|
2549
|
+
const nextOffsetStr = (parseInt(message.offset, 10) + 1).toString();
|
|
2550
|
+
const commitOffset = eos ? async () => {
|
|
2551
|
+
await eos.consumer.commitOffsets([
|
|
2552
|
+
{ topic: topic2, partition, offset: nextOffsetStr }
|
|
2553
|
+
]);
|
|
2554
|
+
} : void 0;
|
|
2555
|
+
const eosRouteToRetry = eos && retry ? async (rawMsgs, envelopes, delay) => {
|
|
2556
|
+
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
2557
|
+
topic2,
|
|
2558
|
+
rawMsgs,
|
|
2559
|
+
1,
|
|
2560
|
+
retry.maxRetries,
|
|
2561
|
+
delay,
|
|
2562
|
+
envelopes[0]?.headers ?? {}
|
|
2563
|
+
);
|
|
2564
|
+
const tx = await eos.txProducer.transaction();
|
|
2565
|
+
try {
|
|
2566
|
+
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
2567
|
+
await tx.sendOffsets({
|
|
2568
|
+
consumer: eos.consumer,
|
|
2569
|
+
topics: [
|
|
2570
|
+
{
|
|
2571
|
+
topic: topic2,
|
|
2572
|
+
partitions: [{ partition, offset: nextOffsetStr }]
|
|
2573
|
+
}
|
|
2574
|
+
]
|
|
2575
|
+
});
|
|
2576
|
+
await tx.commit();
|
|
2577
|
+
} catch (txErr) {
|
|
2578
|
+
try {
|
|
2579
|
+
await tx.abort();
|
|
2580
|
+
} catch {
|
|
2581
|
+
}
|
|
2582
|
+
throw txErr;
|
|
2583
|
+
}
|
|
2584
|
+
} : void 0;
|
|
2585
|
+
const envelope = await parseSingleMessage(
|
|
2586
|
+
message,
|
|
2587
|
+
topic2,
|
|
2588
|
+
partition,
|
|
2589
|
+
schemaMap,
|
|
2590
|
+
interceptors,
|
|
2591
|
+
dlq,
|
|
2592
|
+
deps
|
|
2593
|
+
);
|
|
2594
|
+
if (envelope === null) {
|
|
2595
|
+
await commitOffset?.();
|
|
2596
|
+
return;
|
|
2597
|
+
}
|
|
2598
|
+
if (opts.deduplication) {
|
|
2599
|
+
const isDuplicate = await applyDeduplication(
|
|
2600
|
+
envelope,
|
|
2601
|
+
message.value.toString(),
|
|
2602
|
+
opts.deduplication,
|
|
2603
|
+
dlq,
|
|
2604
|
+
deps
|
|
2605
|
+
);
|
|
2606
|
+
if (isDuplicate) {
|
|
2607
|
+
await commitOffset?.();
|
|
2608
|
+
return;
|
|
2609
|
+
}
|
|
2610
|
+
}
|
|
2611
|
+
if (opts.messageTtlMs !== void 0) {
|
|
2612
|
+
const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
|
|
2613
|
+
if (ageMs > opts.messageTtlMs) {
|
|
2614
|
+
deps.logger.warn(
|
|
2615
|
+
`[KafkaClient] TTL expired on ${topic2}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2616
|
+
);
|
|
2617
|
+
if (dlq) {
|
|
2618
|
+
await sendToDlq(topic2, message.value.toString(), deps, {
|
|
2619
|
+
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2620
|
+
attempt: 0,
|
|
2621
|
+
originalHeaders: envelope.headers
|
|
2622
|
+
});
|
|
2623
|
+
deps.onDlq?.(envelope, "ttl-expired");
|
|
2624
|
+
} else {
|
|
2625
|
+
const ttlHandler = opts.onTtlExpired ?? deps.onTtlExpired;
|
|
2626
|
+
await ttlHandler?.({
|
|
2627
|
+
topic: topic2,
|
|
2628
|
+
ageMs,
|
|
2629
|
+
messageTtlMs: opts.messageTtlMs,
|
|
2630
|
+
headers: envelope.headers
|
|
2631
|
+
});
|
|
2632
|
+
}
|
|
2633
|
+
await commitOffset?.();
|
|
2634
|
+
return;
|
|
2635
|
+
}
|
|
2636
|
+
}
|
|
2637
|
+
await executeWithRetry(
|
|
2638
|
+
() => {
|
|
2639
|
+
const fn = () => runWithEnvelopeContext(
|
|
2640
|
+
{
|
|
2641
|
+
correlationId: envelope.correlationId,
|
|
2642
|
+
traceparent: envelope.traceparent
|
|
2643
|
+
},
|
|
2644
|
+
() => handleMessage(envelope)
|
|
2645
|
+
);
|
|
2646
|
+
return timeoutMs ? wrapWithTimeout(fn, timeoutMs, topic2) : fn();
|
|
2647
|
+
},
|
|
2648
|
+
{
|
|
2649
|
+
envelope,
|
|
2650
|
+
rawMessages: [message.value.toString()],
|
|
2651
|
+
interceptors,
|
|
2652
|
+
dlq,
|
|
2653
|
+
retry,
|
|
2654
|
+
retryTopics
|
|
2655
|
+
},
|
|
2656
|
+
{ ...deps, eosRouteToRetry, eosCommitOnSuccess: commitOffset }
|
|
2657
|
+
);
|
|
2658
|
+
}
|
|
2659
|
+
async function handleEachBatch(payload, opts, deps) {
|
|
2660
|
+
const { batch, heartbeat, resolveOffset, commitOffsetsIfNecessary } = payload;
|
|
2661
|
+
const {
|
|
2662
|
+
schemaMap,
|
|
2663
|
+
handleBatch,
|
|
2664
|
+
interceptors,
|
|
2665
|
+
dlq,
|
|
2666
|
+
retry,
|
|
2667
|
+
retryTopics,
|
|
2668
|
+
timeoutMs,
|
|
2669
|
+
wrapWithTimeout
|
|
2670
|
+
} = opts;
|
|
2671
|
+
const eos = opts.eosMainContext;
|
|
2672
|
+
const lastRawOffset = batch.messages.length > 0 ? batch.messages[batch.messages.length - 1].offset : void 0;
|
|
2673
|
+
const batchNextOffsetStr = lastRawOffset ? (parseInt(lastRawOffset, 10) + 1).toString() : void 0;
|
|
2674
|
+
const commitBatchOffset = eos && batchNextOffsetStr ? async () => {
|
|
2675
|
+
await eos.consumer.commitOffsets([
|
|
2676
|
+
{
|
|
2677
|
+
topic: batch.topic,
|
|
2678
|
+
partition: batch.partition,
|
|
2679
|
+
offset: batchNextOffsetStr
|
|
2680
|
+
}
|
|
2681
|
+
]);
|
|
2682
|
+
} : void 0;
|
|
2683
|
+
const eosRouteToRetry = eos && retry && batchNextOffsetStr ? async (rawMsgs, envelopes2, delay) => {
|
|
2684
|
+
const { topic: rtTopic, messages: rtMsgs } = buildRetryTopicPayload(
|
|
2685
|
+
batch.topic,
|
|
2686
|
+
rawMsgs,
|
|
2687
|
+
1,
|
|
2688
|
+
retry.maxRetries,
|
|
2689
|
+
delay,
|
|
2690
|
+
envelopes2.map((e) => e.headers)
|
|
2691
|
+
);
|
|
2692
|
+
const tx = await eos.txProducer.transaction();
|
|
2693
|
+
try {
|
|
2694
|
+
await tx.send({ topic: rtTopic, messages: rtMsgs });
|
|
2695
|
+
await tx.sendOffsets({
|
|
2696
|
+
consumer: eos.consumer,
|
|
2697
|
+
topics: [
|
|
2698
|
+
{
|
|
2699
|
+
topic: batch.topic,
|
|
2700
|
+
partitions: [
|
|
2701
|
+
{ partition: batch.partition, offset: batchNextOffsetStr }
|
|
2702
|
+
]
|
|
2703
|
+
}
|
|
2704
|
+
]
|
|
2705
|
+
});
|
|
2706
|
+
await tx.commit();
|
|
2707
|
+
} catch (txErr) {
|
|
2708
|
+
try {
|
|
2709
|
+
await tx.abort();
|
|
2710
|
+
} catch {
|
|
2711
|
+
}
|
|
2712
|
+
throw txErr;
|
|
2713
|
+
}
|
|
2714
|
+
} : void 0;
|
|
2715
|
+
const envelopes = [];
|
|
2716
|
+
const rawMessages = [];
|
|
2717
|
+
for (const message of batch.messages) {
|
|
2718
|
+
const envelope = await parseSingleMessage(
|
|
2719
|
+
message,
|
|
2720
|
+
batch.topic,
|
|
2721
|
+
batch.partition,
|
|
2722
|
+
schemaMap,
|
|
2723
|
+
interceptors,
|
|
2724
|
+
dlq,
|
|
2725
|
+
deps
|
|
2726
|
+
);
|
|
2727
|
+
if (envelope === null) continue;
|
|
2728
|
+
if (opts.deduplication) {
|
|
2729
|
+
const raw = message.value.toString();
|
|
2730
|
+
const isDuplicate = await applyDeduplication(
|
|
2731
|
+
envelope,
|
|
2732
|
+
raw,
|
|
2733
|
+
opts.deduplication,
|
|
2734
|
+
dlq,
|
|
2735
|
+
deps
|
|
2736
|
+
);
|
|
2737
|
+
if (isDuplicate) continue;
|
|
2738
|
+
}
|
|
2739
|
+
if (opts.messageTtlMs !== void 0) {
|
|
2740
|
+
const ageMs = Date.now() - new Date(envelope.timestamp).getTime();
|
|
2741
|
+
if (ageMs > opts.messageTtlMs) {
|
|
2742
|
+
deps.logger.warn(
|
|
2743
|
+
`[KafkaClient] TTL expired on ${batch.topic}: age ${ageMs}ms > ${opts.messageTtlMs}ms`
|
|
2744
|
+
);
|
|
2745
|
+
if (dlq) {
|
|
2746
|
+
await sendToDlq(batch.topic, message.value.toString(), deps, {
|
|
2747
|
+
error: new Error(`Message TTL expired: age ${ageMs}ms`),
|
|
2748
|
+
attempt: 0,
|
|
2749
|
+
originalHeaders: envelope.headers
|
|
2750
|
+
});
|
|
2751
|
+
deps.onDlq?.(envelope, "ttl-expired");
|
|
2752
|
+
} else {
|
|
2753
|
+
const ttlHandler = opts.onTtlExpired ?? deps.onTtlExpired;
|
|
2754
|
+
await ttlHandler?.({
|
|
2755
|
+
topic: batch.topic,
|
|
2756
|
+
ageMs,
|
|
2757
|
+
messageTtlMs: opts.messageTtlMs,
|
|
2758
|
+
headers: envelope.headers
|
|
2759
|
+
});
|
|
2760
|
+
}
|
|
2761
|
+
continue;
|
|
2762
|
+
}
|
|
2763
|
+
}
|
|
2764
|
+
envelopes.push(envelope);
|
|
2765
|
+
rawMessages.push(message.value.toString());
|
|
2766
|
+
}
|
|
2767
|
+
if (envelopes.length === 0) {
|
|
2768
|
+
await commitBatchOffset?.();
|
|
2769
|
+
return;
|
|
2770
|
+
}
|
|
2771
|
+
const meta = {
|
|
2772
|
+
partition: batch.partition,
|
|
2773
|
+
highWatermark: batch.highWatermark,
|
|
2774
|
+
heartbeat,
|
|
2775
|
+
resolveOffset,
|
|
2776
|
+
commitOffsetsIfNecessary
|
|
2777
|
+
};
|
|
2778
|
+
await executeWithRetry(
|
|
2779
|
+
() => {
|
|
2780
|
+
const fn = () => handleBatch(envelopes, meta);
|
|
2781
|
+
return timeoutMs ? wrapWithTimeout(fn, timeoutMs, batch.topic) : fn();
|
|
2782
|
+
},
|
|
2783
|
+
{
|
|
2784
|
+
envelope: envelopes,
|
|
2785
|
+
rawMessages,
|
|
2786
|
+
interceptors,
|
|
2787
|
+
dlq,
|
|
2788
|
+
retry,
|
|
2789
|
+
isBatch: true,
|
|
2790
|
+
retryTopics
|
|
2791
|
+
},
|
|
2792
|
+
{ ...deps, eosRouteToRetry, eosCommitOnSuccess: commitBatchOffset }
|
|
2793
|
+
);
|
|
2794
|
+
}
|
|
2795
|
+
|
|
2796
|
+
// src/client/kafka.client/consumer/stop.ts
|
|
2797
|
+
async function stopConsumerImpl(ctx, groupId) {
|
|
2798
|
+
if (groupId === void 0) {
|
|
2799
|
+
const tasks = [
|
|
2800
|
+
...Array.from(ctx.consumers.values()).map(
|
|
2801
|
+
(c) => c.disconnect().catch(() => {
|
|
2802
|
+
})
|
|
2803
|
+
),
|
|
2804
|
+
...Array.from(ctx.retryTxProducers.values()).map(
|
|
2805
|
+
(p) => p.disconnect().catch(() => {
|
|
2806
|
+
})
|
|
2807
|
+
)
|
|
2808
|
+
];
|
|
2809
|
+
await Promise.allSettled(tasks);
|
|
2810
|
+
ctx.consumers.clear();
|
|
2811
|
+
ctx.runningConsumers.clear();
|
|
2812
|
+
ctx.consumerCreationOptions.clear();
|
|
2813
|
+
ctx.companionGroupIds.clear();
|
|
2814
|
+
ctx.retryTxProducers.clear();
|
|
2815
|
+
ctx.dedupStates.clear();
|
|
2816
|
+
ctx.circuitBreaker.clear();
|
|
2817
|
+
ctx.logger.log("All consumers disconnected");
|
|
2818
|
+
return;
|
|
2819
|
+
}
|
|
2820
|
+
const consumer = ctx.consumers.get(groupId);
|
|
2821
|
+
if (!consumer) {
|
|
2822
|
+
ctx.logger.warn(
|
|
2823
|
+
`stopConsumer: no active consumer for group "${groupId}"`
|
|
2824
|
+
);
|
|
2825
|
+
return;
|
|
2826
|
+
}
|
|
2827
|
+
await consumer.disconnect().catch(
|
|
2828
|
+
(e) => ctx.logger.warn(
|
|
2829
|
+
`Error disconnecting consumer "${groupId}":`,
|
|
2830
|
+
toError(e).message
|
|
2831
|
+
)
|
|
2832
|
+
);
|
|
2833
|
+
ctx.consumers.delete(groupId);
|
|
2834
|
+
ctx.runningConsumers.delete(groupId);
|
|
2835
|
+
ctx.consumerCreationOptions.delete(groupId);
|
|
2836
|
+
ctx.dedupStates.delete(groupId);
|
|
2837
|
+
ctx.circuitBreaker.removeGroup(groupId);
|
|
2838
|
+
ctx.logger.log(`Consumer disconnected: group "${groupId}"`);
|
|
2839
|
+
const mainTxId = `${groupId}-main-tx`;
|
|
2840
|
+
const mainTxProducer = ctx.retryTxProducers.get(mainTxId);
|
|
2841
|
+
if (mainTxProducer) {
|
|
2842
|
+
await mainTxProducer.disconnect().catch(
|
|
2843
|
+
(e) => ctx.logger.warn(
|
|
2844
|
+
`Error disconnecting main tx producer "${mainTxId}":`,
|
|
2845
|
+
toError(e).message
|
|
2846
|
+
)
|
|
2847
|
+
);
|
|
2848
|
+
_activeTransactionalIds.delete(mainTxId);
|
|
2849
|
+
ctx.retryTxProducers.delete(mainTxId);
|
|
2850
|
+
}
|
|
2851
|
+
const companions = ctx.companionGroupIds.get(groupId) ?? [];
|
|
2852
|
+
for (const cGroupId of companions) {
|
|
2853
|
+
const cConsumer = ctx.consumers.get(cGroupId);
|
|
2854
|
+
if (cConsumer) {
|
|
2855
|
+
await cConsumer.disconnect().catch(
|
|
2856
|
+
(e) => ctx.logger.warn(
|
|
2857
|
+
`Error disconnecting retry consumer "${cGroupId}":`,
|
|
2858
|
+
toError(e).message
|
|
2859
|
+
)
|
|
2860
|
+
);
|
|
2861
|
+
ctx.consumers.delete(cGroupId);
|
|
2862
|
+
ctx.runningConsumers.delete(cGroupId);
|
|
2863
|
+
ctx.consumerCreationOptions.delete(cGroupId);
|
|
2864
|
+
ctx.logger.log(`Retry consumer disconnected: group "${cGroupId}"`);
|
|
2865
|
+
}
|
|
2866
|
+
const txId = `${cGroupId}-tx`;
|
|
2867
|
+
const txProducer = ctx.retryTxProducers.get(txId);
|
|
2868
|
+
if (txProducer) {
|
|
2869
|
+
await txProducer.disconnect().catch(
|
|
2870
|
+
(e) => ctx.logger.warn(
|
|
2871
|
+
`Error disconnecting retry tx producer "${txId}":`,
|
|
2872
|
+
toError(e).message
|
|
2873
|
+
)
|
|
2874
|
+
);
|
|
2875
|
+
_activeTransactionalIds.delete(txId);
|
|
2876
|
+
ctx.retryTxProducers.delete(txId);
|
|
2877
|
+
}
|
|
2878
|
+
}
|
|
2879
|
+
ctx.companionGroupIds.delete(groupId);
|
|
2880
|
+
}
|
|
2881
|
+
function pauseConsumerImpl(ctx, groupId, assignments) {
|
|
2882
|
+
const gid = groupId ?? ctx.defaultGroupId;
|
|
2883
|
+
const consumer = ctx.consumers.get(gid);
|
|
2884
|
+
if (!consumer) {
|
|
2885
|
+
ctx.logger.warn(`pauseConsumer: no active consumer for group "${gid}"`);
|
|
2886
|
+
return;
|
|
2887
|
+
}
|
|
2888
|
+
consumer.pause(
|
|
2889
|
+
assignments.flatMap(
|
|
2890
|
+
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
2891
|
+
)
|
|
2892
|
+
);
|
|
2893
|
+
}
|
|
2894
|
+
function resumeConsumerImpl(ctx, groupId, assignments) {
|
|
2895
|
+
const gid = groupId ?? ctx.defaultGroupId;
|
|
2896
|
+
const consumer = ctx.consumers.get(gid);
|
|
2897
|
+
if (!consumer) {
|
|
2898
|
+
ctx.logger.warn(`resumeConsumer: no active consumer for group "${gid}"`);
|
|
2899
|
+
return;
|
|
2900
|
+
}
|
|
2901
|
+
consumer.resume(
|
|
2902
|
+
assignments.flatMap(
|
|
2903
|
+
({ topic: topic2, partitions }) => partitions.map((p) => ({ topic: topic2, partitions: [p] }))
|
|
2904
|
+
)
|
|
2905
|
+
);
|
|
2906
|
+
}
|
|
2907
|
+
function pauseTopicAllPartitions(ctx, gid, topic2) {
|
|
2908
|
+
const consumer = ctx.consumers.get(gid);
|
|
2909
|
+
if (!consumer) return;
|
|
2910
|
+
const assignment = consumer.assignment();
|
|
2911
|
+
const partitions = assignment.filter((a) => a.topic === topic2).map((a) => a.partition);
|
|
2912
|
+
if (partitions.length > 0)
|
|
2913
|
+
consumer.pause(partitions.map((p) => ({ topic: topic2, partitions: [p] })));
|
|
2914
|
+
}
|
|
2915
|
+
function resumeTopicAllPartitions(ctx, gid, topic2) {
|
|
2916
|
+
const consumer = ctx.consumers.get(gid);
|
|
2917
|
+
if (!consumer) return;
|
|
2918
|
+
const assignment = consumer.assignment();
|
|
2919
|
+
const partitions = assignment.filter((a) => a.topic === topic2).map((a) => a.partition);
|
|
2920
|
+
if (partitions.length > 0)
|
|
2921
|
+
consumer.resume(partitions.map((p) => ({ topic: topic2, partitions: [p] })));
|
|
2922
|
+
}
|
|
2923
|
+
|
|
2924
|
+
// src/client/kafka.client/consumer/start.ts
|
|
2925
|
+
async function startConsumerImpl(ctx, topics, handleMessage, options = {}) {
|
|
2926
|
+
validateTopicConsumerOpts(topics, options);
|
|
2927
|
+
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
2928
|
+
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachMessage", setupOptions);
|
|
2929
|
+
if (options.circuitBreaker)
|
|
2930
|
+
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
2931
|
+
const deps = messageDepsFor(ctx, gid, options);
|
|
2932
|
+
const eosMainContext = await makeEosMainContext(ctx, gid, consumer, options);
|
|
2933
|
+
await consumer.run({
|
|
2934
|
+
eachMessage: (payload) => ctx.inFlight.track(
|
|
2935
|
+
() => handleEachMessage(
|
|
2936
|
+
payload,
|
|
2937
|
+
{
|
|
2938
|
+
schemaMap,
|
|
2939
|
+
handleMessage,
|
|
2940
|
+
interceptors,
|
|
2941
|
+
dlq,
|
|
2942
|
+
retry,
|
|
2943
|
+
retryTopics: options.retryTopics,
|
|
2944
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
2945
|
+
wrapWithTimeout: (fn, ms, topic2) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic2),
|
|
2946
|
+
deduplication: resolveDeduplicationContext(
|
|
2947
|
+
ctx,
|
|
2948
|
+
gid,
|
|
2949
|
+
options.deduplication
|
|
2950
|
+
),
|
|
2951
|
+
messageTtlMs: options.messageTtlMs,
|
|
2952
|
+
onTtlExpired: options.onTtlExpired,
|
|
2953
|
+
eosMainContext
|
|
2954
|
+
},
|
|
2955
|
+
deps
|
|
2956
|
+
)
|
|
2957
|
+
)
|
|
2958
|
+
});
|
|
2959
|
+
ctx.runningConsumers.set(gid, "eachMessage");
|
|
2960
|
+
if (options.retryTopics && retry) {
|
|
2961
|
+
await launchRetryChain(ctx, gid, topicNames, handleMessage, {
|
|
2962
|
+
retry,
|
|
2963
|
+
dlq,
|
|
2964
|
+
interceptors,
|
|
2965
|
+
schemaMap,
|
|
2966
|
+
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
2967
|
+
});
|
|
2968
|
+
}
|
|
2969
|
+
return { groupId: gid, stop: () => stopConsumerByGid(ctx, gid), ready: () => readyPromise };
|
|
2970
|
+
}
|
|
2971
|
+
async function startBatchConsumerImpl(ctx, topics, handleBatch, options = {}) {
|
|
2972
|
+
validateTopicConsumerOpts(topics, options);
|
|
2973
|
+
if (!options.retryTopics && options.autoCommit !== false) {
|
|
2974
|
+
ctx.logger.debug?.(
|
|
2975
|
+
`startBatchConsumer: autoCommit is enabled (default true). If your handler calls resolveOffset() or commitOffsetsIfNecessary(), set autoCommit: false to avoid offset conflicts.`
|
|
2976
|
+
);
|
|
2977
|
+
}
|
|
2978
|
+
const setupOptions = options.retryTopics ? { ...options, autoCommit: false } : options;
|
|
2979
|
+
const { consumer, schemaMap, topicNames, gid, dlq, interceptors, retry, readyPromise } = await setupConsumer(ctx, topics, "eachBatch", setupOptions);
|
|
2980
|
+
if (options.circuitBreaker)
|
|
2981
|
+
ctx.circuitBreaker.setConfig(gid, options.circuitBreaker);
|
|
2982
|
+
const deps = messageDepsFor(ctx, gid, options);
|
|
2983
|
+
const eosMainContext = await makeEosMainContext(ctx, gid, consumer, options);
|
|
2984
|
+
await consumer.run({
|
|
2985
|
+
eachBatch: (payload) => ctx.inFlight.track(
|
|
2986
|
+
() => handleEachBatch(
|
|
2987
|
+
payload,
|
|
2988
|
+
{
|
|
2989
|
+
schemaMap,
|
|
2990
|
+
handleBatch,
|
|
2991
|
+
interceptors,
|
|
2992
|
+
dlq,
|
|
2993
|
+
retry,
|
|
2994
|
+
retryTopics: options.retryTopics,
|
|
2995
|
+
timeoutMs: options.handlerTimeoutMs,
|
|
2996
|
+
wrapWithTimeout: (fn, ms, topic2) => wrapWithTimeoutWarning(ctx.logger, fn, ms, topic2),
|
|
2997
|
+
deduplication: resolveDeduplicationContext(
|
|
2998
|
+
ctx,
|
|
2999
|
+
gid,
|
|
3000
|
+
options.deduplication
|
|
3001
|
+
),
|
|
3002
|
+
messageTtlMs: options.messageTtlMs,
|
|
3003
|
+
onTtlExpired: options.onTtlExpired,
|
|
3004
|
+
eosMainContext
|
|
3005
|
+
},
|
|
3006
|
+
deps
|
|
3007
|
+
)
|
|
3008
|
+
)
|
|
3009
|
+
});
|
|
3010
|
+
ctx.runningConsumers.set(gid, "eachBatch");
|
|
3011
|
+
if (options.retryTopics && retry) {
|
|
3012
|
+
const handleMessageForRetry = (env) => handleBatch([env], {
|
|
3013
|
+
partition: env.partition,
|
|
3014
|
+
highWatermark: null,
|
|
3015
|
+
heartbeat: async () => {
|
|
3016
|
+
},
|
|
3017
|
+
resolveOffset: () => {
|
|
3018
|
+
},
|
|
3019
|
+
commitOffsetsIfNecessary: async () => {
|
|
3020
|
+
}
|
|
3021
|
+
});
|
|
3022
|
+
await launchRetryChain(ctx, gid, topicNames, handleMessageForRetry, {
|
|
3023
|
+
retry,
|
|
3024
|
+
dlq,
|
|
3025
|
+
interceptors,
|
|
3026
|
+
schemaMap,
|
|
3027
|
+
assignmentTimeoutMs: options.retryTopicAssignmentTimeoutMs
|
|
3028
|
+
});
|
|
3029
|
+
}
|
|
3030
|
+
return { groupId: gid, stop: () => stopConsumerByGid(ctx, gid), ready: () => readyPromise };
|
|
3031
|
+
}
|
|
3032
|
+
async function startTransactionalConsumerImpl(ctx, topics, handler, options = {}) {
|
|
3033
|
+
if (options.retryTopics) {
|
|
3034
|
+
throw new Error(
|
|
3035
|
+
"startTransactionalConsumer: retryTopics is not supported. EOS is already guaranteed by the transaction \u2014 redelivery on failure is handled automatically."
|
|
3036
|
+
);
|
|
3037
|
+
}
|
|
3038
|
+
const setupOptions = { ...options, autoCommit: false };
|
|
3039
|
+
const { consumer, schemaMap, gid, readyPromise } = await setupConsumer(
|
|
3040
|
+
ctx,
|
|
3041
|
+
topics,
|
|
3042
|
+
"eachMessage",
|
|
3043
|
+
setupOptions
|
|
3044
|
+
);
|
|
3045
|
+
const txProducer = await createRetryTxProducer(ctx, `${gid}-txc`);
|
|
3046
|
+
const deps = messageDepsFor(ctx, gid);
|
|
3047
|
+
await consumer.run({
|
|
3048
|
+
eachMessage: ({ topic: topic2, partition, message }) => ctx.inFlight.track(async () => {
|
|
3049
|
+
const envelope = await parseSingleMessage(
|
|
3050
|
+
message,
|
|
3051
|
+
topic2,
|
|
3052
|
+
partition,
|
|
3053
|
+
schemaMap,
|
|
3054
|
+
options.interceptors ?? [],
|
|
3055
|
+
false,
|
|
3056
|
+
deps
|
|
3057
|
+
);
|
|
3058
|
+
const nextOffset = String(Number.parseInt(message.offset, 10) + 1);
|
|
3059
|
+
if (envelope === null) {
|
|
3060
|
+
await consumer.commitOffsets([
|
|
3061
|
+
{ topic: topic2, partition, offset: nextOffset }
|
|
3062
|
+
]);
|
|
3063
|
+
return;
|
|
3064
|
+
}
|
|
3065
|
+
const tx = await txProducer.transaction();
|
|
3066
|
+
const txCtx = {
|
|
3067
|
+
send: async (t, msg, sendOpts) => {
|
|
3068
|
+
const payload = await preparePayload(ctx, t, [
|
|
3069
|
+
{
|
|
3070
|
+
value: msg,
|
|
3071
|
+
key: sendOpts?.key,
|
|
3072
|
+
headers: sendOpts?.headers,
|
|
3073
|
+
correlationId: sendOpts?.correlationId,
|
|
3074
|
+
schemaVersion: sendOpts?.schemaVersion,
|
|
3075
|
+
eventId: sendOpts?.eventId
|
|
3076
|
+
}
|
|
3077
|
+
]);
|
|
3078
|
+
await tx.send(payload);
|
|
3079
|
+
},
|
|
3080
|
+
sendBatch: async (t, msgs, batchOpts) => {
|
|
3081
|
+
const payload = await preparePayload(
|
|
3082
|
+
ctx,
|
|
3083
|
+
t,
|
|
3084
|
+
msgs,
|
|
3085
|
+
batchOpts?.compression
|
|
3086
|
+
);
|
|
3087
|
+
await tx.send(payload);
|
|
3088
|
+
}
|
|
3089
|
+
};
|
|
3090
|
+
try {
|
|
3091
|
+
await runWithEnvelopeContext(
|
|
3092
|
+
{
|
|
3093
|
+
correlationId: envelope.correlationId,
|
|
3094
|
+
traceparent: envelope.traceparent
|
|
3095
|
+
},
|
|
3096
|
+
() => handler(envelope, txCtx)
|
|
3097
|
+
);
|
|
3098
|
+
await tx.sendOffsets({
|
|
3099
|
+
consumer,
|
|
3100
|
+
topics: [
|
|
3101
|
+
{ topic: topic2, partitions: [{ partition, offset: nextOffset }] }
|
|
3102
|
+
]
|
|
3103
|
+
});
|
|
3104
|
+
await tx.commit();
|
|
3105
|
+
deps.onMessage?.(envelope);
|
|
3106
|
+
} catch (err) {
|
|
3107
|
+
try {
|
|
3108
|
+
await tx.abort();
|
|
3109
|
+
} catch {
|
|
3110
|
+
}
|
|
3111
|
+
ctx.logger.warn(
|
|
3112
|
+
`startTransactionalConsumer: handler failed on ${topic2}[${partition}]@${message.offset} \u2014 tx aborted, message will be redelivered (${toError(err).message})`
|
|
3113
|
+
);
|
|
3114
|
+
throw err;
|
|
3115
|
+
}
|
|
3116
|
+
})
|
|
3117
|
+
});
|
|
3118
|
+
ctx.runningConsumers.set(gid, "eachMessage");
|
|
3119
|
+
return { groupId: gid, stop: () => stopConsumerByGid(ctx, gid), ready: () => readyPromise };
|
|
3120
|
+
}
|
|
3121
|
+
function stopConsumerByGid(ctx, gid) {
|
|
3122
|
+
return stopConsumerImpl(ctx, gid);
|
|
3123
|
+
}
|
|
3124
|
+
|
|
3125
|
+
// src/client/kafka.client/consumer/features/window.ts
|
|
3126
|
+
async function startWindowConsumerImpl(ctx, topic2, handler, options) {
|
|
3127
|
+
const { maxMessages, maxMs, ...consumerOptions } = options;
|
|
3128
|
+
if (maxMessages <= 0)
|
|
3129
|
+
throw new Error("startWindowConsumer: maxMessages must be > 0");
|
|
3130
|
+
if (maxMs <= 0) throw new Error("startWindowConsumer: maxMs must be > 0");
|
|
3131
|
+
if (consumerOptions.retryTopics) {
|
|
3132
|
+
throw new Error(
|
|
3133
|
+
"startWindowConsumer() does not support retryTopics. Use startConsumer() with retryTopics: true for guaranteed retry delivery."
|
|
3134
|
+
);
|
|
3135
|
+
}
|
|
3136
|
+
const buffer = [];
|
|
3137
|
+
let flushTimer = null;
|
|
3138
|
+
let windowStart = 0;
|
|
3139
|
+
const onLost = consumerOptions.onMessageLost ?? ctx.onMessageLost;
|
|
3140
|
+
const flush = async (trigger) => {
|
|
3141
|
+
if (flushTimer !== null) {
|
|
3142
|
+
clearTimeout(flushTimer);
|
|
3143
|
+
flushTimer = null;
|
|
3144
|
+
}
|
|
3145
|
+
if (buffer.length === 0) return;
|
|
3146
|
+
const envelopes = buffer.splice(0);
|
|
3147
|
+
try {
|
|
3148
|
+
await handler(envelopes, { trigger, windowStart, windowEnd: Date.now() });
|
|
3149
|
+
} catch (err) {
|
|
3150
|
+
const error = toError(err);
|
|
3151
|
+
ctx.logger.error(
|
|
3152
|
+
`startWindowConsumer: ${trigger}-triggered flush failed \u2014 window of ${envelopes.length} message(s) lost:`,
|
|
3153
|
+
error.stack
|
|
3154
|
+
);
|
|
3155
|
+
for (const envelope of envelopes) {
|
|
3156
|
+
await Promise.resolve(
|
|
3157
|
+
onLost?.({
|
|
3158
|
+
topic: envelope.topic,
|
|
3159
|
+
error,
|
|
3160
|
+
attempt: 0,
|
|
3161
|
+
headers: envelope.headers
|
|
3162
|
+
})
|
|
3163
|
+
).catch(() => {
|
|
3164
|
+
});
|
|
3165
|
+
}
|
|
3166
|
+
}
|
|
3167
|
+
};
|
|
3168
|
+
const scheduleFlush = () => {
|
|
3169
|
+
if (flushTimer !== null) return;
|
|
3170
|
+
flushTimer = setTimeout(() => {
|
|
3171
|
+
flushTimer = null;
|
|
3172
|
+
void flush("time");
|
|
3173
|
+
}, maxMs);
|
|
3174
|
+
};
|
|
3175
|
+
const handle = await startConsumerImpl(
|
|
3176
|
+
ctx,
|
|
3177
|
+
[topic2],
|
|
3178
|
+
async (envelope) => {
|
|
3179
|
+
if (buffer.length === 0) windowStart = Date.now();
|
|
3180
|
+
buffer.push(envelope);
|
|
3181
|
+
scheduleFlush();
|
|
3182
|
+
if (buffer.length >= maxMessages) await flush("size");
|
|
3183
|
+
},
|
|
3184
|
+
consumerOptions
|
|
3185
|
+
);
|
|
3186
|
+
const originalStop = handle.stop.bind(handle);
|
|
3187
|
+
handle.stop = async () => {
|
|
3188
|
+
await flush("time");
|
|
3189
|
+
return originalStop();
|
|
3190
|
+
};
|
|
3191
|
+
return handle;
|
|
3192
|
+
}
|
|
3193
|
+
|
|
3194
|
+
// src/client/kafka.client/consumer/features/routed.ts
|
|
3195
|
+
async function startRoutedConsumerImpl(ctx, topics, routing, options) {
|
|
3196
|
+
const { header, routes, fallback } = routing;
|
|
3197
|
+
const handleMessage = async (envelope) => {
|
|
3198
|
+
const headerValue = envelope.headers[header];
|
|
3199
|
+
const routeHandler = headerValue === void 0 ? void 0 : routes[headerValue];
|
|
3200
|
+
if (routeHandler) {
|
|
3201
|
+
await routeHandler(envelope);
|
|
3202
|
+
} else {
|
|
3203
|
+
await fallback?.(envelope);
|
|
3204
|
+
}
|
|
3205
|
+
};
|
|
3206
|
+
return startConsumerImpl(ctx, topics, handleMessage, options);
|
|
3207
|
+
}
|
|
3208
|
+
|
|
3209
|
+
// src/client/kafka.client/consumer/features/delayed.ts
|
|
3210
|
+
function delayedTopicName(topic2) {
|
|
3211
|
+
return `${topic2}.delayed`;
|
|
3212
|
+
}
|
|
3213
|
+
async function startDelayedRelayImpl(ctx, topics, options) {
|
|
3214
|
+
if (topics.length === 0) {
|
|
3215
|
+
throw new Error("startDelayedRelay: at least one topic is required");
|
|
3216
|
+
}
|
|
3217
|
+
const gid = options?.groupId ?? `${ctx.defaultGroupId}-delayed-relay`;
|
|
3218
|
+
if (ctx.runningConsumers.has(gid)) {
|
|
3219
|
+
throw new Error(
|
|
3220
|
+
`startDelayedRelay("${gid}") called twice \u2014 this group is already consuming. Call stopConsumer("${gid}") first or pass a different groupId.`
|
|
3221
|
+
);
|
|
3222
|
+
}
|
|
3223
|
+
const delayedTopics = topics.map(delayedTopicName);
|
|
3224
|
+
for (const t of delayedTopics) await ensureTopic(ctx, t);
|
|
3225
|
+
const txProducer = await createRetryTxProducer(ctx, `${gid}-tx`);
|
|
3226
|
+
let resolveReady;
|
|
3227
|
+
const readyPromise = new Promise((resolve) => {
|
|
3228
|
+
resolveReady = resolve;
|
|
3229
|
+
});
|
|
3230
|
+
const consumer = getOrCreateConsumer(
|
|
3231
|
+
gid,
|
|
3232
|
+
false,
|
|
3233
|
+
false,
|
|
3234
|
+
ctx.consumerOpsDeps,
|
|
3235
|
+
void 0,
|
|
3236
|
+
resolveReady
|
|
3237
|
+
);
|
|
3238
|
+
await consumer.connect();
|
|
3239
|
+
await subscribeWithRetry(consumer, delayedTopics, ctx.logger);
|
|
3240
|
+
await consumer.run({
|
|
3241
|
+
eachMessage: async ({ topic: stagingTopic, partition, message }) => {
|
|
3242
|
+
const nextOffset = {
|
|
3243
|
+
topic: stagingTopic,
|
|
3244
|
+
partition,
|
|
3245
|
+
offset: (parseInt(message.offset, 10) + 1).toString()
|
|
3246
|
+
};
|
|
3247
|
+
if (!message.value) {
|
|
3248
|
+
await consumer.commitOffsets([nextOffset]);
|
|
3249
|
+
return;
|
|
3250
|
+
}
|
|
3251
|
+
const headers = decodeHeaders(message.headers);
|
|
3252
|
+
const target = headers[HEADER_DELAYED_TARGET] ?? stagingTopic.replace(/\.delayed$/, "");
|
|
3253
|
+
const until = parseInt(
|
|
3254
|
+
headers[HEADER_DELAYED_UNTIL] ?? "0",
|
|
3255
|
+
10
|
|
3256
|
+
);
|
|
3257
|
+
const remaining = until - Date.now();
|
|
3258
|
+
if (remaining > 0) {
|
|
3259
|
+
consumer.pause([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3260
|
+
await sleep(remaining);
|
|
3261
|
+
consumer.resume([{ topic: stagingTopic, partitions: [partition] }]);
|
|
3262
|
+
}
|
|
3263
|
+
const forwardHeaders = Object.fromEntries(
|
|
3264
|
+
Object.entries(headers).filter(
|
|
3265
|
+
([k]) => k !== HEADER_DELAYED_UNTIL && k !== HEADER_DELAYED_TARGET
|
|
3266
|
+
)
|
|
3267
|
+
);
|
|
3268
|
+
const tx = await txProducer.transaction();
|
|
3269
|
+
try {
|
|
3270
|
+
await tx.send({
|
|
3271
|
+
topic: target,
|
|
3272
|
+
messages: [
|
|
3273
|
+
{
|
|
3274
|
+
value: message.value.toString(),
|
|
3275
|
+
key: message.key ? message.key.toString() : null,
|
|
3276
|
+
headers: forwardHeaders
|
|
3277
|
+
}
|
|
3278
|
+
]
|
|
3279
|
+
});
|
|
3280
|
+
await tx.sendOffsets({
|
|
3281
|
+
consumer,
|
|
3282
|
+
topics: [
|
|
3283
|
+
{
|
|
3284
|
+
topic: nextOffset.topic,
|
|
3285
|
+
partitions: [
|
|
3286
|
+
{ partition: nextOffset.partition, offset: nextOffset.offset }
|
|
3287
|
+
]
|
|
3288
|
+
}
|
|
3289
|
+
]
|
|
3290
|
+
});
|
|
3291
|
+
await tx.commit();
|
|
3292
|
+
ctx.logger.debug?.(
|
|
3293
|
+
`Delayed message relayed to "${target}" (deadline ${new Date(until).toISOString()})`
|
|
3294
|
+
);
|
|
3295
|
+
} catch (txErr) {
|
|
3296
|
+
try {
|
|
3297
|
+
await tx.abort();
|
|
3298
|
+
} catch {
|
|
3299
|
+
}
|
|
3300
|
+
ctx.logger.error(
|
|
3301
|
+
`Delayed relay to "${target}" failed \u2014 message will be redelivered:`,
|
|
3302
|
+
toError(txErr).stack
|
|
3303
|
+
);
|
|
3304
|
+
}
|
|
3305
|
+
}
|
|
3306
|
+
});
|
|
3307
|
+
ctx.runningConsumers.set(gid, "eachMessage");
|
|
3308
|
+
ctx.logger.log(
|
|
3309
|
+
`Delayed relay started for: ${delayedTopics.join(", ")} (group: ${gid})`
|
|
3310
|
+
);
|
|
3311
|
+
return {
|
|
3312
|
+
groupId: gid,
|
|
3313
|
+
ready: () => readyPromise,
|
|
3314
|
+
stop: async () => {
|
|
3315
|
+
await stopConsumerImpl(ctx, gid);
|
|
3316
|
+
}
|
|
3317
|
+
};
|
|
3318
|
+
}
|
|
3319
|
+
|
|
3320
|
+
// src/client/kafka.client/consumer/features/snapshot.ts
|
|
3321
|
+
async function readSnapshotImpl(ctx, topic2, options = {}) {
|
|
3322
|
+
await ctx.adminOps.ensureConnected();
|
|
3323
|
+
let offsets;
|
|
3324
|
+
try {
|
|
3325
|
+
offsets = await ctx.adminOps.admin.fetchTopicOffsets(topic2);
|
|
3326
|
+
} catch {
|
|
3327
|
+
ctx.logger.warn(
|
|
3328
|
+
`readSnapshot: could not fetch offsets for "${String(topic2)}", returning empty snapshot`
|
|
3329
|
+
);
|
|
3330
|
+
return /* @__PURE__ */ new Map();
|
|
3331
|
+
}
|
|
3332
|
+
const targets = /* @__PURE__ */ new Map();
|
|
3333
|
+
for (const { partition, high, low } of offsets) {
|
|
3334
|
+
const highN = Number.parseInt(high, 10);
|
|
3335
|
+
const lowN = Number.parseInt(low, 10);
|
|
3336
|
+
if (highN > lowN) targets.set(partition, highN - 1);
|
|
3337
|
+
}
|
|
3338
|
+
if (targets.size === 0) {
|
|
3339
|
+
ctx.logger.debug?.(
|
|
3340
|
+
`readSnapshot: topic "${String(topic2)}" is empty \u2014 returning empty snapshot`
|
|
3341
|
+
);
|
|
3342
|
+
return /* @__PURE__ */ new Map();
|
|
3343
|
+
}
|
|
3344
|
+
const snapshot = /* @__PURE__ */ new Map();
|
|
3345
|
+
const remaining = new Set(targets.keys());
|
|
3346
|
+
const snapshotGroupId = `${ctx.clientId}-snapshot-${Date.now()}`;
|
|
3347
|
+
await new Promise((resolve, reject) => {
|
|
3348
|
+
const consumer = ctx.transport.consumer({
|
|
3349
|
+
groupId: snapshotGroupId,
|
|
3350
|
+
fromBeginning: true
|
|
3351
|
+
});
|
|
3352
|
+
const cleanup = () => {
|
|
3353
|
+
consumer.disconnect().catch(() => {
|
|
3354
|
+
}).finally(() => {
|
|
3355
|
+
ctx.adminOps.deleteGroups([snapshotGroupId]).catch(() => {
|
|
3356
|
+
});
|
|
3357
|
+
});
|
|
3358
|
+
};
|
|
3359
|
+
consumer.connect().then(() => consumer.subscribe({ topics: [topic2] })).then(
|
|
3360
|
+
() => consumer.run({
|
|
3361
|
+
eachMessage: async ({ topic: t, partition, message }) => {
|
|
3362
|
+
if (!remaining.has(partition)) return;
|
|
3363
|
+
const msgOffsetN = Number.parseInt(message.offset, 10);
|
|
3364
|
+
applySnapshotMessage(snapshot, options, ctx, t, partition, message);
|
|
3365
|
+
if (msgOffsetN >= targets.get(partition)) {
|
|
3366
|
+
remaining.delete(partition);
|
|
3367
|
+
if (remaining.size === 0) {
|
|
3368
|
+
cleanup();
|
|
3369
|
+
resolve();
|
|
3370
|
+
}
|
|
3371
|
+
}
|
|
3372
|
+
}
|
|
3373
|
+
})
|
|
3374
|
+
).catch((err) => {
|
|
3375
|
+
cleanup();
|
|
3376
|
+
reject(err);
|
|
3377
|
+
});
|
|
3378
|
+
});
|
|
3379
|
+
ctx.logger.log(
|
|
3380
|
+
`readSnapshot: ${snapshot.size} key(s) from "${String(topic2)}"`
|
|
3381
|
+
);
|
|
3382
|
+
return snapshot;
|
|
3383
|
+
}
|
|
3384
|
+
function applySnapshotMessage(snapshot, options, ctx, t, partition, message) {
|
|
3385
|
+
let key = null;
|
|
3386
|
+
if (message.key) {
|
|
3387
|
+
key = Buffer.isBuffer(message.key) ? message.key.toString() : String(message.key);
|
|
3388
|
+
}
|
|
3389
|
+
if (message.value === null || message.value === void 0) {
|
|
3390
|
+
if (key !== null) {
|
|
3391
|
+
snapshot.delete(key);
|
|
3392
|
+
options.onTombstone?.(key);
|
|
3393
|
+
}
|
|
3394
|
+
return;
|
|
3395
|
+
}
|
|
3396
|
+
if (key === null) return;
|
|
3397
|
+
const rawValue = Buffer.isBuffer(message.value) ? message.value.toString() : String(message.value);
|
|
3398
|
+
try {
|
|
3399
|
+
const jsonValue = JSON.parse(rawValue);
|
|
3400
|
+
const headers = decodeHeaders(message.headers);
|
|
3401
|
+
const parsed = options.schema ? options.schema.parse(jsonValue) : jsonValue;
|
|
3402
|
+
snapshot.set(
|
|
3403
|
+
key,
|
|
3404
|
+
extractEnvelope(parsed, headers, t, partition, message.offset)
|
|
3405
|
+
);
|
|
3406
|
+
} catch (err) {
|
|
3407
|
+
ctx.logger.warn(
|
|
3408
|
+
`readSnapshot: skipping ${t}:${partition}@${message.offset} \u2014 ${toError(err).message}`
|
|
3409
|
+
);
|
|
3410
|
+
}
|
|
3411
|
+
}
|
|
3412
|
+
async function checkpointOffsetsImpl(ctx, groupId, checkpointTopic) {
|
|
3413
|
+
const gid = groupId ?? ctx.defaultGroupId;
|
|
3414
|
+
await ctx.adminOps.ensureConnected();
|
|
3415
|
+
const committed = await ctx.adminOps.admin.fetchOffsets({ groupId: gid });
|
|
3416
|
+
const offsets = [];
|
|
3417
|
+
for (const { topic: topic2, partitions } of committed) {
|
|
3418
|
+
for (const { partition, offset } of partitions) {
|
|
3419
|
+
offsets.push({ topic: topic2, partition, offset });
|
|
3420
|
+
}
|
|
3421
|
+
}
|
|
3422
|
+
const savedAt = Date.now();
|
|
3423
|
+
const payload = JSON.stringify({ groupId: gid, offsets, savedAt });
|
|
3424
|
+
await ctx.producer.send({
|
|
3425
|
+
topic: checkpointTopic,
|
|
3426
|
+
messages: [
|
|
3427
|
+
{
|
|
3428
|
+
key: gid,
|
|
3429
|
+
value: payload,
|
|
3430
|
+
headers: {
|
|
3431
|
+
"x-checkpoint-group-id": [gid],
|
|
3432
|
+
"x-checkpoint-timestamp": [String(savedAt)]
|
|
3433
|
+
}
|
|
3434
|
+
}
|
|
3435
|
+
]
|
|
3436
|
+
});
|
|
3437
|
+
const topics = [...new Set(offsets.map((e) => e.topic))];
|
|
3438
|
+
ctx.logger.log(
|
|
3439
|
+
`checkpointOffsets: saved ${offsets.length} partition(s) for group "${gid}" \u2192 "${checkpointTopic}"`
|
|
3440
|
+
);
|
|
3441
|
+
return { groupId: gid, topics, partitionCount: offsets.length, savedAt };
|
|
3442
|
+
}
|
|
3443
|
+
async function restoreFromCheckpointImpl(ctx, groupId, checkpointTopic, options = {}) {
|
|
3444
|
+
const gid = groupId ?? ctx.defaultGroupId;
|
|
3445
|
+
if (ctx.runningConsumers.has(gid)) {
|
|
3446
|
+
throw new Error(
|
|
3447
|
+
`restoreFromCheckpoint: consumer group "${gid}" is still running. Call stopConsumer("${gid}") before restoring offsets.`
|
|
3448
|
+
);
|
|
3449
|
+
}
|
|
3450
|
+
await ctx.adminOps.ensureConnected();
|
|
3451
|
+
const checkpoints = [];
|
|
3452
|
+
let hwmOffsets;
|
|
3453
|
+
try {
|
|
3454
|
+
hwmOffsets = await ctx.adminOps.admin.fetchTopicOffsets(checkpointTopic);
|
|
3455
|
+
} catch {
|
|
3456
|
+
throw new Error(
|
|
3457
|
+
`restoreFromCheckpoint: could not fetch offsets for "${checkpointTopic}" \u2014 does the topic exist?`
|
|
3458
|
+
);
|
|
3459
|
+
}
|
|
3460
|
+
const targets = /* @__PURE__ */ new Map();
|
|
3461
|
+
for (const { partition, high, low } of hwmOffsets) {
|
|
3462
|
+
const highN = Number.parseInt(high, 10);
|
|
3463
|
+
if (highN > Number.parseInt(low, 10)) targets.set(partition, highN - 1);
|
|
3464
|
+
}
|
|
3465
|
+
if (targets.size > 0) {
|
|
3466
|
+
const checkpointGroupId = `${ctx.clientId}-checkpoint-restore-${Date.now()}`;
|
|
3467
|
+
await new Promise((resolve, reject) => {
|
|
3468
|
+
const consumer = ctx.transport.consumer({
|
|
3469
|
+
groupId: checkpointGroupId,
|
|
3470
|
+
fromBeginning: true
|
|
3471
|
+
});
|
|
3472
|
+
const cleanup = () => {
|
|
3473
|
+
consumer.disconnect().catch(() => {
|
|
3474
|
+
}).finally(() => {
|
|
3475
|
+
ctx.adminOps.deleteGroups([checkpointGroupId]).catch(() => {
|
|
3476
|
+
});
|
|
3477
|
+
});
|
|
3478
|
+
};
|
|
3479
|
+
const remaining = new Set(targets.keys());
|
|
3480
|
+
consumer.connect().then(() => consumer.subscribe({ topics: [checkpointTopic] })).then(
|
|
3481
|
+
() => consumer.run({
|
|
3482
|
+
eachMessage: async ({ partition, message }) => {
|
|
3483
|
+
if (!remaining.has(partition)) return;
|
|
3484
|
+
let msgKey = null;
|
|
3485
|
+
if (message.key) {
|
|
3486
|
+
msgKey = Buffer.isBuffer(message.key) ? message.key.toString() : String(message.key);
|
|
3487
|
+
}
|
|
3488
|
+
if (msgKey === gid && message.value) {
|
|
3489
|
+
try {
|
|
3490
|
+
const raw = Buffer.isBuffer(message.value) ? message.value.toString() : String(message.value);
|
|
3491
|
+
const parsed = JSON.parse(raw);
|
|
3492
|
+
checkpoints.push({
|
|
3493
|
+
savedAt: parsed.savedAt,
|
|
3494
|
+
offsets: parsed.offsets
|
|
3495
|
+
});
|
|
3496
|
+
} catch {
|
|
3497
|
+
ctx.logger.warn(
|
|
3498
|
+
`restoreFromCheckpoint: skipping malformed checkpoint at partition ${partition}@${message.offset}`
|
|
3499
|
+
);
|
|
3500
|
+
}
|
|
3501
|
+
}
|
|
3502
|
+
if (Number.parseInt(message.offset, 10) >= targets.get(partition)) {
|
|
3503
|
+
remaining.delete(partition);
|
|
3504
|
+
if (remaining.size === 0) {
|
|
3505
|
+
cleanup();
|
|
3506
|
+
resolve();
|
|
3507
|
+
}
|
|
3508
|
+
}
|
|
3509
|
+
}
|
|
3510
|
+
})
|
|
3511
|
+
).catch((err) => {
|
|
3512
|
+
cleanup();
|
|
3513
|
+
reject(err);
|
|
3514
|
+
});
|
|
3515
|
+
});
|
|
3516
|
+
}
|
|
3517
|
+
if (checkpoints.length === 0) {
|
|
3518
|
+
throw new Error(
|
|
3519
|
+
`restoreFromCheckpoint: no checkpoints found for group "${gid}" in "${checkpointTopic}"`
|
|
3520
|
+
);
|
|
3521
|
+
}
|
|
3522
|
+
const target = options.timestamp;
|
|
3523
|
+
let best;
|
|
3524
|
+
if (target === void 0) {
|
|
3525
|
+
best = checkpoints.reduce(
|
|
3526
|
+
(acc, c) => c.savedAt > acc.savedAt ? c : acc,
|
|
3527
|
+
checkpoints[0]
|
|
3528
|
+
);
|
|
3529
|
+
} else {
|
|
3530
|
+
const candidates = checkpoints.filter((c) => c.savedAt <= target);
|
|
3531
|
+
if (candidates.length > 0) {
|
|
3532
|
+
best = candidates.reduce(
|
|
3533
|
+
(acc, c) => c.savedAt > acc.savedAt ? c : acc,
|
|
3534
|
+
candidates[0]
|
|
3535
|
+
);
|
|
3536
|
+
} else {
|
|
3537
|
+
best = checkpoints.reduce(
|
|
3538
|
+
(acc, c) => c.savedAt < acc.savedAt ? c : acc,
|
|
3539
|
+
checkpoints[0]
|
|
3540
|
+
);
|
|
3541
|
+
ctx.logger.warn(
|
|
3542
|
+
`restoreFromCheckpoint: no checkpoint at or before ${new Date(target).toISOString()} \u2014 using oldest available (${new Date(best.savedAt).toISOString()})`
|
|
3543
|
+
);
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
await ctx.adminOps.seekToOffset(gid, best.offsets);
|
|
3547
|
+
const checkpointAge = Date.now() - best.savedAt;
|
|
3548
|
+
ctx.logger.log(
|
|
3549
|
+
`restoreFromCheckpoint: restored ${best.offsets.length} partition(s) for group "${gid}" from checkpoint at ${new Date(best.savedAt).toISOString()} (age: ${checkpointAge}ms)`
|
|
3550
|
+
);
|
|
3551
|
+
return {
|
|
3552
|
+
groupId: gid,
|
|
3553
|
+
offsets: best.offsets,
|
|
3554
|
+
restoredAt: best.savedAt,
|
|
3555
|
+
checkpointAge
|
|
3556
|
+
};
|
|
3557
|
+
}
|
|
3558
|
+
|
|
3559
|
+
// src/client/kafka.client/index.ts
|
|
3560
|
+
var DLQ_HEADER_KEYS = /* @__PURE__ */ new Set([
|
|
3561
|
+
"x-dlq-original-topic",
|
|
3562
|
+
"x-dlq-failed-at",
|
|
3563
|
+
"x-dlq-error-message",
|
|
3564
|
+
"x-dlq-error-stack",
|
|
3565
|
+
"x-dlq-attempt-count"
|
|
3566
|
+
]);
|
|
3567
|
+
var KafkaClient = class {
|
|
3568
|
+
clientId;
|
|
3569
|
+
ctx;
|
|
3570
|
+
/**
|
|
3571
|
+
* Create a new KafkaClient.
|
|
3572
|
+
* @param clientId Unique client identifier (used in Kafka metadata and logs).
|
|
3573
|
+
* @param groupId Default consumer group ID for this client.
|
|
3574
|
+
* @param brokers Array of broker addresses, e.g. `['localhost:9092']`.
|
|
3575
|
+
* @param options Optional client-wide configuration.
|
|
3576
|
+
* @example
|
|
3577
|
+
* ```ts
|
|
3578
|
+
* const kafka = new KafkaClient('my-service', 'my-service-group', ['localhost:9092'], {
|
|
3579
|
+
* lagThrottle: { maxLag: 10_000 },
|
|
3580
|
+
* onMessageLost: (ctx) => logger.error('Message lost', ctx),
|
|
3581
|
+
* });
|
|
3582
|
+
* await kafka.connectProducer();
|
|
3583
|
+
* ```
|
|
3584
|
+
*/
|
|
3585
|
+
constructor(clientId, groupId, brokers, options) {
|
|
3586
|
+
validateClientOptions(clientId, groupId, brokers, options);
|
|
3587
|
+
this.clientId = clientId;
|
|
3588
|
+
const logger = options?.logger ?? {
|
|
3589
|
+
log: (msg) => console.log(`[KafkaClient:${clientId}] ${msg}`),
|
|
3590
|
+
warn: (msg, ...args) => console.warn(`[KafkaClient:${clientId}] ${msg}`, ...args),
|
|
3591
|
+
error: (msg, ...args) => console.error(`[KafkaClient:${clientId}] ${msg}`, ...args),
|
|
3592
|
+
debug: (msg, ...args) => console.debug(`[KafkaClient:${clientId}] ${msg}`, ...args)
|
|
3593
|
+
};
|
|
3594
|
+
const security = resolveSecurityOptions(options?.security, brokers, logger);
|
|
3595
|
+
const transport = options?.transport ?? new ConfluentTransport(clientId, brokers, security);
|
|
3596
|
+
const producer = transport.producer();
|
|
3597
|
+
const runningConsumers = /* @__PURE__ */ new Map();
|
|
3598
|
+
const consumers = /* @__PURE__ */ new Map();
|
|
3599
|
+
const consumerCreationOptions = /* @__PURE__ */ new Map();
|
|
3600
|
+
const schemaRegistry = /* @__PURE__ */ new Map();
|
|
3601
|
+
const adminOps = new AdminOps({
|
|
3602
|
+
admin: transport.admin(),
|
|
3603
|
+
logger,
|
|
3604
|
+
runningConsumers,
|
|
3605
|
+
defaultGroupId: groupId,
|
|
3606
|
+
clientId
|
|
3607
|
+
});
|
|
3608
|
+
const circuitBreaker = new CircuitBreakerManager({
|
|
3609
|
+
pauseConsumer: (gid, assignments) => pauseConsumerImpl(this.ctx, gid, assignments),
|
|
3610
|
+
resumeConsumer: (gid, assignments) => resumeConsumerImpl(this.ctx, gid, assignments),
|
|
3611
|
+
logger,
|
|
3612
|
+
instrumentation: options?.instrumentation ?? []
|
|
3613
|
+
});
|
|
3614
|
+
const metrics = new MetricsManager({
|
|
3615
|
+
instrumentation: options?.instrumentation ?? [],
|
|
3616
|
+
onCircuitFailure: (envelope, gid) => circuitBreaker.onFailure(envelope, gid),
|
|
3617
|
+
onCircuitSuccess: (envelope, gid) => circuitBreaker.onSuccess(envelope, gid)
|
|
3618
|
+
});
|
|
3619
|
+
const inFlight = new InFlightTracker((msg) => logger.warn(msg));
|
|
3620
|
+
const ctx = {
|
|
3621
|
+
clientId,
|
|
3622
|
+
defaultGroupId: groupId,
|
|
3623
|
+
logger,
|
|
3624
|
+
autoCreateTopicsEnabled: options?.autoCreateTopics ?? false,
|
|
3625
|
+
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3626
|
+
numPartitions: options?.numPartitions ?? 1,
|
|
3627
|
+
txId: options?.transactionalId ?? `${clientId}-tx`,
|
|
3628
|
+
clockRecoveryTopics: options?.clockRecovery?.topics ?? [],
|
|
3629
|
+
clockRecoveryTimeoutMs: options?.clockRecovery?.timeoutMs ?? 3e4,
|
|
3630
|
+
lagThrottleOpts: options?.lagThrottle,
|
|
3631
|
+
instrumentation: options?.instrumentation ?? [],
|
|
3632
|
+
onMessageLost: options?.onMessageLost,
|
|
3633
|
+
onTtlExpired: options?.onTtlExpired,
|
|
3634
|
+
onRebalance: options?.onRebalance,
|
|
3635
|
+
transport,
|
|
3636
|
+
producer,
|
|
3637
|
+
txProducer: void 0,
|
|
3638
|
+
txProducerInitPromise: void 0,
|
|
3639
|
+
_txChain: Promise.resolve(),
|
|
3640
|
+
retryTxProducers: /* @__PURE__ */ new Map(),
|
|
3641
|
+
consumers,
|
|
3642
|
+
runningConsumers,
|
|
3643
|
+
consumerCreationOptions,
|
|
3644
|
+
companionGroupIds: /* @__PURE__ */ new Map(),
|
|
3645
|
+
dedupStates: /* @__PURE__ */ new Map(),
|
|
3646
|
+
ensuredTopics: /* @__PURE__ */ new Set(),
|
|
3647
|
+
ensureTopicPromises: /* @__PURE__ */ new Map(),
|
|
3648
|
+
schemaRegistry,
|
|
3649
|
+
_lagThrottled: false,
|
|
3650
|
+
_lagThrottleTimer: void 0,
|
|
3651
|
+
_lamportClock: 0,
|
|
3652
|
+
circuitBreaker,
|
|
3653
|
+
adminOps,
|
|
3654
|
+
metrics,
|
|
3655
|
+
inFlight,
|
|
3656
|
+
producerOpsDeps: {
|
|
3657
|
+
schemaRegistry,
|
|
3658
|
+
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3659
|
+
instrumentation: options?.instrumentation ?? [],
|
|
3660
|
+
logger,
|
|
3661
|
+
nextLamportClock: () => 0
|
|
3662
|
+
// patched below
|
|
3663
|
+
},
|
|
3664
|
+
consumerOpsDeps: {
|
|
3665
|
+
consumers,
|
|
3666
|
+
consumerCreationOptions,
|
|
3667
|
+
transport,
|
|
3668
|
+
onRebalance: options?.onRebalance,
|
|
3669
|
+
logger
|
|
3670
|
+
},
|
|
3671
|
+
retryTopicDeps: {}
|
|
3672
|
+
// patched below
|
|
3673
|
+
};
|
|
3674
|
+
ctx.producerOpsDeps = {
|
|
3675
|
+
schemaRegistry,
|
|
3676
|
+
strictSchemasEnabled: options?.strictSchemas ?? true,
|
|
3677
|
+
instrumentation: options?.instrumentation ?? [],
|
|
3678
|
+
logger,
|
|
3679
|
+
nextLamportClock: () => ++ctx._lamportClock
|
|
3680
|
+
};
|
|
3681
|
+
ctx.retryTopicDeps = buildRetryTopicDeps(ctx);
|
|
3682
|
+
this.ctx = ctx;
|
|
3683
|
+
}
|
|
3684
|
+
async sendMessage(topicOrDesc, message, options = {}) {
|
|
3685
|
+
return sendMessageImpl(this.ctx, topicOrDesc, message, options);
|
|
3686
|
+
}
|
|
3687
|
+
/** @inheritDoc */
|
|
3688
|
+
async sendTombstone(topic2, key, headers) {
|
|
3689
|
+
return sendTombstoneImpl(this.ctx, topic2, key, headers);
|
|
3690
|
+
}
|
|
3691
|
+
async sendBatch(topicOrDesc, messages, options) {
|
|
3692
|
+
return sendBatchImpl(this.ctx, topicOrDesc, messages, options);
|
|
3693
|
+
}
|
|
3694
|
+
/** @inheritDoc */
|
|
3695
|
+
async transaction(fn) {
|
|
3696
|
+
return transactionImpl(this.ctx, fn);
|
|
3697
|
+
}
|
|
3698
|
+
// ── Producer lifecycle ────────────────────────────────────────────
|
|
3699
|
+
/** @inheritDoc */
|
|
3700
|
+
async connectProducer() {
|
|
3701
|
+
return connectProducerImpl(this.ctx);
|
|
3702
|
+
}
|
|
3703
|
+
/** @internal */
|
|
3704
|
+
async disconnectProducer() {
|
|
3705
|
+
if (this.ctx._lagThrottleTimer) {
|
|
3706
|
+
clearInterval(this.ctx._lagThrottleTimer);
|
|
3707
|
+
this.ctx._lagThrottleTimer = void 0;
|
|
3708
|
+
}
|
|
3709
|
+
await this.ctx.producer.disconnect();
|
|
3710
|
+
this.ctx.logger.log("Producer disconnected");
|
|
3711
|
+
}
|
|
3712
|
+
async startConsumer(topics, handleMessage, options = {}) {
|
|
3713
|
+
return startConsumerImpl(this.ctx, topics, handleMessage, options);
|
|
3714
|
+
}
|
|
3715
|
+
async startBatchConsumer(topics, handleBatch, options = {}) {
|
|
3716
|
+
return startBatchConsumerImpl(this.ctx, topics, handleBatch, options);
|
|
3717
|
+
}
|
|
3718
|
+
// ── Consumer: AsyncIterableIterator ──────────────────────────────
|
|
3719
|
+
/** @inheritDoc */
|
|
3720
|
+
consume(topic2, options) {
|
|
3721
|
+
if (options?.retryTopics) {
|
|
3722
|
+
throw new Error(
|
|
3723
|
+
"consume() does not support retryTopics (EOS retry chains). Use startConsumer() with retryTopics: true for guaranteed retry delivery."
|
|
3724
|
+
);
|
|
3725
|
+
}
|
|
3726
|
+
const gid = options?.groupId ?? this.ctx.defaultGroupId;
|
|
3727
|
+
const queue = new AsyncQueue(
|
|
3728
|
+
options?.queueHighWaterMark,
|
|
3729
|
+
() => pauseTopicAllPartitions(this.ctx, gid, topic2),
|
|
3730
|
+
() => resumeTopicAllPartitions(this.ctx, gid, topic2)
|
|
3731
|
+
);
|
|
3732
|
+
const handlePromise = this.startConsumer(
|
|
3733
|
+
[topic2],
|
|
3734
|
+
async (envelope) => {
|
|
3735
|
+
queue.push(envelope);
|
|
3736
|
+
},
|
|
3737
|
+
options
|
|
3738
|
+
);
|
|
3739
|
+
handlePromise.catch((err) => queue.fail(err));
|
|
3740
|
+
return {
|
|
3741
|
+
[Symbol.asyncIterator]() {
|
|
3742
|
+
return this;
|
|
3743
|
+
},
|
|
3744
|
+
next: () => queue.next(),
|
|
3745
|
+
return: async () => {
|
|
3746
|
+
queue.close();
|
|
3747
|
+
const handle = await handlePromise;
|
|
3748
|
+
await handle.stop();
|
|
3749
|
+
return { value: void 0, done: true };
|
|
3750
|
+
}
|
|
3751
|
+
};
|
|
3752
|
+
}
|
|
3753
|
+
// ── Consumer: windowed ────────────────────────────────────────────
|
|
3754
|
+
/** @inheritDoc */
|
|
3755
|
+
startWindowConsumer(topic2, handler, options) {
|
|
3756
|
+
return startWindowConsumerImpl(this.ctx, topic2, handler, options);
|
|
3757
|
+
}
|
|
3758
|
+
// ── Consumer: header routing ──────────────────────────────────────
|
|
3759
|
+
/** @inheritDoc */
|
|
3760
|
+
startRoutedConsumer(topics, routing, options) {
|
|
3761
|
+
return startRoutedConsumerImpl(this.ctx, topics, routing, options);
|
|
3762
|
+
}
|
|
3763
|
+
// ── Consumer: delayed delivery relay ──────────────────────────────
|
|
3764
|
+
/**
|
|
3765
|
+
* Start a relay that delivers messages produced with
|
|
3766
|
+
* `SendOptions.deliverAfterMs` from `<topic>.delayed` to their target topic
|
|
3767
|
+
* once their deadline passes.
|
|
3768
|
+
*
|
|
3769
|
+
* Forwarding is transactional (produce + source-offset commit are atomic),
|
|
3770
|
+
* so no duplicates are relayed even if the relay crashes mid-forward.
|
|
3771
|
+
* Delivery time is a lower bound — the relay must be running for delayed
|
|
3772
|
+
* messages to be delivered at all.
|
|
3773
|
+
*
|
|
3774
|
+
* @param topics Target topic name(s) whose `<topic>.delayed` staging topics to relay.
|
|
3775
|
+
* @param options Optional `groupId` override (default: `<defaultGroupId>-delayed-relay`).
|
|
3776
|
+
*
|
|
3777
|
+
* @example
|
|
3778
|
+
* ```ts
|
|
3779
|
+
* await kafka.startDelayedRelay(['orders.reminder']);
|
|
3780
|
+
* await kafka.sendMessage('orders.reminder', payload, { deliverAfterMs: 60_000 });
|
|
3781
|
+
* // → delivered to orders.reminder ~60 s later
|
|
3782
|
+
* ```
|
|
3783
|
+
*/
|
|
3784
|
+
async startDelayedRelay(topics, options) {
|
|
3785
|
+
const list = Array.isArray(topics) ? topics : [topics];
|
|
3786
|
+
return startDelayedRelayImpl(this.ctx, list, options);
|
|
3787
|
+
}
|
|
3788
|
+
// ── Consumer: transactional EOS ───────────────────────────────────
|
|
3789
|
+
/** @inheritDoc */
|
|
3790
|
+
async startTransactionalConsumer(topics, handler, options = {}) {
|
|
3791
|
+
return startTransactionalConsumerImpl(this.ctx, topics, handler, options);
|
|
3792
|
+
}
|
|
3793
|
+
// ── Consumer lifecycle ────────────────────────────────────────────
|
|
3794
|
+
/** @inheritDoc */
|
|
3795
|
+
async stopConsumer(groupId) {
|
|
3796
|
+
return stopConsumerImpl(this.ctx, groupId);
|
|
3797
|
+
}
|
|
3798
|
+
/** @inheritDoc */
|
|
3799
|
+
pauseConsumer(groupId, assignments) {
|
|
3800
|
+
return pauseConsumerImpl(this.ctx, groupId, assignments);
|
|
3801
|
+
}
|
|
3802
|
+
/** @inheritDoc */
|
|
3803
|
+
resumeConsumer(groupId, assignments) {
|
|
3804
|
+
return resumeConsumerImpl(this.ctx, groupId, assignments);
|
|
3805
|
+
}
|
|
3806
|
+
// ── DLQ replay ────────────────────────────────────────────────────
|
|
3807
|
+
/** @inheritDoc */
|
|
3808
|
+
async replayDlq(topic2, options = {}) {
|
|
3809
|
+
await this.ctx.adminOps.ensureConnected();
|
|
3810
|
+
return replayDlqTopic(
|
|
3811
|
+
topic2,
|
|
3812
|
+
{
|
|
3813
|
+
logger: this.ctx.logger,
|
|
3814
|
+
fetchTopicOffsets: (t) => this.ctx.adminOps.admin.fetchTopicOffsets(t),
|
|
3815
|
+
send: async (t, messages) => {
|
|
3816
|
+
await this.ctx.producer.send({ topic: t, messages });
|
|
3817
|
+
},
|
|
3818
|
+
createConsumer: (gid, fromBeginning) => getOrCreateConsumer(gid, fromBeginning, true, this.ctx.consumerOpsDeps),
|
|
3819
|
+
cleanupConsumer: (consumer, gid, deleteGroup) => {
|
|
3820
|
+
consumer.disconnect().catch(() => {
|
|
3821
|
+
}).finally(() => {
|
|
3822
|
+
this.ctx.consumers.delete(gid);
|
|
3823
|
+
this.ctx.runningConsumers.delete(gid);
|
|
3824
|
+
this.ctx.consumerCreationOptions.delete(gid);
|
|
3825
|
+
if (deleteGroup) {
|
|
3826
|
+
this.ctx.adminOps.deleteGroups([gid]).catch(() => {
|
|
3827
|
+
});
|
|
3828
|
+
}
|
|
3829
|
+
});
|
|
3830
|
+
},
|
|
3831
|
+
dlqHeaderKeys: DLQ_HEADER_KEYS
|
|
3832
|
+
},
|
|
3833
|
+
options
|
|
3834
|
+
);
|
|
3835
|
+
}
|
|
3836
|
+
// ── Snapshot & checkpoint ─────────────────────────────────────────
|
|
3837
|
+
/** @inheritDoc */
|
|
3838
|
+
async readSnapshot(topic2, options = {}) {
|
|
3839
|
+
return readSnapshotImpl(this.ctx, topic2, options);
|
|
3840
|
+
}
|
|
3841
|
+
/** @inheritDoc */
|
|
3842
|
+
async checkpointOffsets(groupId, checkpointTopic) {
|
|
3843
|
+
return checkpointOffsetsImpl(this.ctx, groupId, checkpointTopic);
|
|
3844
|
+
}
|
|
3845
|
+
/** @inheritDoc */
|
|
3846
|
+
async restoreFromCheckpoint(groupId, checkpointTopic, options = {}) {
|
|
3847
|
+
return restoreFromCheckpointImpl(this.ctx, groupId, checkpointTopic, options);
|
|
3848
|
+
}
|
|
3849
|
+
// ── Admin ─────────────────────────────────────────────────────────
|
|
3850
|
+
/** @inheritDoc */
|
|
3851
|
+
async resetOffsets(groupId, topic2, position) {
|
|
3852
|
+
return this.ctx.adminOps.resetOffsets(groupId, topic2, position);
|
|
3853
|
+
}
|
|
3854
|
+
/** @inheritDoc */
|
|
3855
|
+
async seekToOffset(groupId, assignments) {
|
|
3856
|
+
return this.ctx.adminOps.seekToOffset(groupId, assignments);
|
|
3857
|
+
}
|
|
3858
|
+
/** @inheritDoc */
|
|
3859
|
+
async seekToTimestamp(groupId, assignments) {
|
|
3860
|
+
return this.ctx.adminOps.seekToTimestamp(groupId, assignments);
|
|
3861
|
+
}
|
|
3862
|
+
/** @inheritDoc */
|
|
3863
|
+
async getConsumerLag(groupId) {
|
|
3864
|
+
return this.ctx.adminOps.getConsumerLag(groupId);
|
|
3865
|
+
}
|
|
3866
|
+
/** @inheritDoc */
|
|
3867
|
+
async checkStatus() {
|
|
3868
|
+
return this.ctx.adminOps.checkStatus();
|
|
3869
|
+
}
|
|
3870
|
+
/** @inheritDoc */
|
|
3871
|
+
async listConsumerGroups() {
|
|
3872
|
+
return this.ctx.adminOps.listConsumerGroups();
|
|
3873
|
+
}
|
|
3874
|
+
/** @inheritDoc */
|
|
3875
|
+
async describeTopics(topics) {
|
|
3876
|
+
return this.ctx.adminOps.describeTopics(topics);
|
|
3877
|
+
}
|
|
3878
|
+
/** @inheritDoc */
|
|
3879
|
+
async deleteRecords(topic2, partitions) {
|
|
3880
|
+
return this.ctx.adminOps.deleteRecords(topic2, partitions);
|
|
3881
|
+
}
|
|
3882
|
+
// ── Circuit breaker ───────────────────────────────────────────────
|
|
3883
|
+
/** @inheritDoc */
|
|
3884
|
+
getCircuitState(topic2, partition, groupId) {
|
|
3885
|
+
return this.ctx.circuitBreaker.getState(
|
|
3886
|
+
topic2,
|
|
3887
|
+
partition,
|
|
3888
|
+
groupId ?? this.ctx.defaultGroupId
|
|
3889
|
+
);
|
|
3890
|
+
}
|
|
3891
|
+
// ── Metrics ───────────────────────────────────────────────────────
|
|
3892
|
+
/** @inheritDoc */
|
|
3893
|
+
getMetrics(topic2) {
|
|
3894
|
+
return this.ctx.metrics.getMetrics(topic2);
|
|
3895
|
+
}
|
|
3896
|
+
/** @inheritDoc */
|
|
3897
|
+
resetMetrics(topic2) {
|
|
3898
|
+
this.ctx.metrics.resetMetrics(topic2);
|
|
3899
|
+
}
|
|
3900
|
+
getClientId() {
|
|
3901
|
+
return this.clientId;
|
|
3902
|
+
}
|
|
3903
|
+
// ── Lifecycle ─────────────────────────────────────────────────────
|
|
3904
|
+
/** @inheritDoc */
|
|
3905
|
+
async disconnect(drainTimeoutMs = 3e4) {
|
|
3906
|
+
return disconnectImpl(this.ctx, drainTimeoutMs);
|
|
3907
|
+
}
|
|
3908
|
+
/** NestJS lifecycle hook — called automatically on module teardown. */
|
|
3909
|
+
async onModuleDestroy() {
|
|
3910
|
+
await this.disconnect();
|
|
3911
|
+
}
|
|
3912
|
+
/** @inheritDoc */
|
|
3913
|
+
enableGracefulShutdown(signals = ["SIGTERM", "SIGINT"], drainTimeoutMs = 3e4) {
|
|
3914
|
+
const handler = () => {
|
|
3915
|
+
this.ctx.logger.log(
|
|
3916
|
+
"Shutdown signal received \u2014 draining in-flight handlers..."
|
|
3917
|
+
);
|
|
3918
|
+
this.disconnect(drainTimeoutMs).catch(
|
|
3919
|
+
(err) => this.ctx.logger.error(
|
|
3920
|
+
"Error during graceful shutdown:",
|
|
3921
|
+
toError(err).message
|
|
3922
|
+
)
|
|
3923
|
+
).finally(() => process.exit(0));
|
|
3924
|
+
};
|
|
3925
|
+
for (const signal of signals) process.once(signal, handler);
|
|
3926
|
+
}
|
|
3927
|
+
};
|
|
3928
|
+
|
|
3929
|
+
// src/client/message/topic.ts
|
|
3930
|
+
function topic(name) {
|
|
3931
|
+
return {
|
|
3932
|
+
/** Provide an explicit message type without a runtime schema. */
|
|
3933
|
+
type: () => keyable({
|
|
3934
|
+
__topic: name,
|
|
3935
|
+
__type: void 0
|
|
3936
|
+
}),
|
|
3937
|
+
schema: (schema) => keyable({
|
|
3938
|
+
__topic: name,
|
|
3939
|
+
__type: void 0,
|
|
3940
|
+
__schema: schema
|
|
3941
|
+
})
|
|
3942
|
+
};
|
|
3943
|
+
}
|
|
3944
|
+
function keyable(desc) {
|
|
3945
|
+
return {
|
|
3946
|
+
...desc,
|
|
3947
|
+
key: (extractor) => ({
|
|
3948
|
+
...desc,
|
|
3949
|
+
__key: extractor
|
|
3950
|
+
})
|
|
3951
|
+
};
|
|
3952
|
+
}
|
|
3953
|
+
|
|
3954
|
+
// src/client/message/versioned-schema.ts
|
|
3955
|
+
function versionedSchema(versions, options) {
|
|
3956
|
+
const registered = Object.keys(versions).map(Number).filter((v) => Number.isInteger(v) && v > 0).sort((a, b) => a - b);
|
|
3957
|
+
if (registered.length === 0) {
|
|
3958
|
+
throw new Error(
|
|
3959
|
+
"versionedSchema: at least one schema version must be registered (keys must be positive integers)"
|
|
3960
|
+
);
|
|
3961
|
+
}
|
|
3962
|
+
const latestVersion = registered[registered.length - 1];
|
|
3963
|
+
return {
|
|
3964
|
+
async parse(data, ctx) {
|
|
3965
|
+
const version = ctx?.version ?? latestVersion;
|
|
3966
|
+
const schema = versions[version];
|
|
3967
|
+
if (!schema) {
|
|
3968
|
+
throw new Error(
|
|
3969
|
+
`versionedSchema: no schema registered for version ${version}${ctx?.topic ? ` (topic "${ctx.topic}")` : ""} \u2014 registered versions: ${registered.join(", ")}`
|
|
3970
|
+
);
|
|
3971
|
+
}
|
|
3972
|
+
const parsed = await schema.parse(data, ctx);
|
|
3973
|
+
if (version < latestVersion && options?.migrate) {
|
|
3974
|
+
return options.migrate(parsed, version, latestVersion);
|
|
3975
|
+
}
|
|
3976
|
+
return parsed;
|
|
3977
|
+
}
|
|
3978
|
+
};
|
|
3979
|
+
}
|
|
3980
|
+
|
|
3981
|
+
// src/client/message/schema-registry.ts
|
|
3982
|
+
var SchemaRegistryClient = class {
|
|
3983
|
+
constructor(options) {
|
|
3984
|
+
this.options = options;
|
|
3985
|
+
if (!options.baseUrl) {
|
|
3986
|
+
throw new Error("SchemaRegistryClient: baseUrl is required");
|
|
3987
|
+
}
|
|
3988
|
+
this.fetchFn = options.fetchFn ?? fetch;
|
|
3989
|
+
this.cacheTtlMs = options.cacheTtlMs ?? 3e5;
|
|
3990
|
+
}
|
|
3991
|
+
options;
|
|
3992
|
+
fetchFn;
|
|
3993
|
+
cacheTtlMs;
|
|
3994
|
+
latestCache = /* @__PURE__ */ new Map();
|
|
3995
|
+
headers() {
|
|
3996
|
+
const h = {
|
|
3997
|
+
"Content-Type": "application/vnd.schemaregistry.v1+json"
|
|
3998
|
+
};
|
|
3999
|
+
if (this.options.auth) {
|
|
4000
|
+
const { username, password } = this.options.auth;
|
|
4001
|
+
h["Authorization"] = "Basic " + Buffer.from(`${username}:${password}`).toString("base64");
|
|
4002
|
+
}
|
|
4003
|
+
return h;
|
|
4004
|
+
}
|
|
4005
|
+
async request(method, path, body) {
|
|
4006
|
+
const url = `${this.options.baseUrl.replace(/\/$/, "")}${path}`;
|
|
4007
|
+
const res = await this.fetchFn(url, {
|
|
4008
|
+
method,
|
|
4009
|
+
headers: this.headers(),
|
|
4010
|
+
...body !== void 0 && { body: JSON.stringify(body) }
|
|
4011
|
+
});
|
|
4012
|
+
if (!res.ok) {
|
|
4013
|
+
const text = await res.text().catch(() => "");
|
|
4014
|
+
throw new Error(
|
|
4015
|
+
`SchemaRegistry ${method} ${path} failed: ${res.status} ${res.statusText}${text ? ` \u2014 ${text}` : ""}`
|
|
4016
|
+
);
|
|
4017
|
+
}
|
|
4018
|
+
return await res.json();
|
|
4019
|
+
}
|
|
4020
|
+
/** Fetch the latest schema registered under `subject`. Cached for `cacheTtlMs`. */
|
|
4021
|
+
async getLatestSchema(subject) {
|
|
4022
|
+
const cached = this.latestCache.get(subject);
|
|
4023
|
+
if (cached && cached.expiresAt > Date.now()) return cached.value;
|
|
4024
|
+
const raw = await this.request("GET", `/subjects/${encodeURIComponent(subject)}/versions/latest`);
|
|
4025
|
+
const value = {
|
|
4026
|
+
id: raw.id,
|
|
4027
|
+
version: raw.version,
|
|
4028
|
+
schema: raw.schema
|
|
4029
|
+
};
|
|
4030
|
+
this.latestCache.set(subject, {
|
|
4031
|
+
value,
|
|
4032
|
+
expiresAt: Date.now() + this.cacheTtlMs
|
|
4033
|
+
});
|
|
4034
|
+
return value;
|
|
4035
|
+
}
|
|
4036
|
+
/** Fetch a specific schema version of a subject. */
|
|
4037
|
+
async getSchemaVersion(subject, version) {
|
|
4038
|
+
const raw = await this.request(
|
|
4039
|
+
"GET",
|
|
4040
|
+
`/subjects/${encodeURIComponent(subject)}/versions/${version}`
|
|
4041
|
+
);
|
|
4042
|
+
return { id: raw.id, version: raw.version, schema: raw.schema };
|
|
4043
|
+
}
|
|
4044
|
+
/**
|
|
4045
|
+
* Register a schema under `subject` (idempotent — re-registering the same
|
|
4046
|
+
* schema returns the existing id). Returns the registry-assigned schema id.
|
|
4047
|
+
*/
|
|
4048
|
+
async registerSchema(subject, schema, schemaType = "JSON") {
|
|
4049
|
+
this.latestCache.delete(subject);
|
|
4050
|
+
return this.request(
|
|
4051
|
+
"POST",
|
|
4052
|
+
`/subjects/${encodeURIComponent(subject)}/versions`,
|
|
4053
|
+
{ schema, schemaType }
|
|
4054
|
+
);
|
|
4055
|
+
}
|
|
4056
|
+
/**
|
|
4057
|
+
* Test `schema` against the subject's compatibility policy without registering.
|
|
4058
|
+
* Returns `true` when the registry reports the schema as compatible.
|
|
4059
|
+
*/
|
|
4060
|
+
async checkCompatibility(subject, schema, schemaType = "JSON") {
|
|
4061
|
+
const res = await this.request(
|
|
4062
|
+
"POST",
|
|
4063
|
+
`/compatibility/subjects/${encodeURIComponent(subject)}/versions/latest`,
|
|
4064
|
+
{ schema, schemaType }
|
|
4065
|
+
);
|
|
4066
|
+
return res.is_compatible;
|
|
4067
|
+
}
|
|
4068
|
+
};
|
|
4069
|
+
function registrySchema(client, subject, options) {
|
|
4070
|
+
const enforceVersion = options?.enforceVersion ?? true;
|
|
4071
|
+
return {
|
|
4072
|
+
async parse(data, ctx) {
|
|
4073
|
+
const latest = await client.getLatestSchema(subject);
|
|
4074
|
+
if (enforceVersion && ctx?.version !== void 0 && ctx.version > latest.version) {
|
|
4075
|
+
throw new Error(
|
|
4076
|
+
`registrySchema: message version ${ctx.version} for subject "${subject}" is newer than the latest registered version ${latest.version} \u2014 register the schema before producing with it`
|
|
4077
|
+
);
|
|
4078
|
+
}
|
|
4079
|
+
if (options?.validator) {
|
|
4080
|
+
return options.validator.parse(data, ctx);
|
|
4081
|
+
}
|
|
4082
|
+
return data;
|
|
4083
|
+
}
|
|
4084
|
+
};
|
|
4085
|
+
}
|
|
4086
|
+
|
|
4087
|
+
// src/client/outbox/outbox.store.ts
|
|
4088
|
+
var InMemoryOutboxStore = class {
|
|
4089
|
+
/** Insertion-ordered rows. `published` flips to true after `markPublished`. */
|
|
4090
|
+
rows = [];
|
|
4091
|
+
/**
|
|
4092
|
+
* Append a message to the outbox. In a real store this INSERT would run inside
|
|
4093
|
+
* the same DB transaction as the corresponding business write.
|
|
4094
|
+
*/
|
|
4095
|
+
add(message) {
|
|
4096
|
+
this.rows.push({ message, published: false });
|
|
4097
|
+
}
|
|
4098
|
+
async fetchUnpublished(limit) {
|
|
4099
|
+
const out = [];
|
|
4100
|
+
for (const row of this.rows) {
|
|
4101
|
+
if (row.published) continue;
|
|
4102
|
+
out.push(row.message);
|
|
4103
|
+
if (out.length >= limit) break;
|
|
4104
|
+
}
|
|
4105
|
+
return out;
|
|
4106
|
+
}
|
|
4107
|
+
async markPublished(ids) {
|
|
4108
|
+
const idSet = new Set(ids);
|
|
4109
|
+
for (const row of this.rows) {
|
|
4110
|
+
if (idSet.has(row.message.id)) row.published = true;
|
|
4111
|
+
}
|
|
4112
|
+
}
|
|
4113
|
+
/** Test helper: count of rows not yet marked published. */
|
|
4114
|
+
get pendingCount() {
|
|
4115
|
+
return this.rows.filter((r) => !r.published).length;
|
|
4116
|
+
}
|
|
4117
|
+
/** Test helper: count of rows marked published. */
|
|
4118
|
+
get publishedCount() {
|
|
4119
|
+
return this.rows.filter((r) => r.published).length;
|
|
4120
|
+
}
|
|
4121
|
+
};
|
|
4122
|
+
|
|
4123
|
+
// src/client/outbox/outbox.relay.ts
|
|
4124
|
+
function toError2(e) {
|
|
4125
|
+
return e instanceof Error ? e : new Error(String(e));
|
|
4126
|
+
}
|
|
4127
|
+
function startOutboxRelay(kafka, store, options = {}) {
|
|
4128
|
+
const pollIntervalMs = options.pollIntervalMs ?? 1e3;
|
|
4129
|
+
const batchSize = options.batchSize ?? 100;
|
|
4130
|
+
const onError = options.onError ?? ((error, batch) => {
|
|
4131
|
+
console.error(
|
|
4132
|
+
`[outbox] batch of ${batch.length} message(s) failed \u2014 will retry:`,
|
|
4133
|
+
error
|
|
4134
|
+
);
|
|
4135
|
+
});
|
|
4136
|
+
const onPublished = options.onPublished;
|
|
4137
|
+
let stopped = false;
|
|
4138
|
+
let running = false;
|
|
4139
|
+
let inFlight = Promise.resolve();
|
|
4140
|
+
const iterate = async () => {
|
|
4141
|
+
let batch = [];
|
|
4142
|
+
try {
|
|
4143
|
+
batch = await store.fetchUnpublished(batchSize);
|
|
4144
|
+
if (batch.length === 0) return;
|
|
4145
|
+
await kafka.transaction(async (tx) => {
|
|
4146
|
+
for (const msg of batch) {
|
|
4147
|
+
await tx.send(msg.topic, msg.payload, {
|
|
4148
|
+
key: msg.key,
|
|
4149
|
+
headers: msg.headers,
|
|
4150
|
+
correlationId: msg.correlationId,
|
|
4151
|
+
eventId: msg.eventId
|
|
4152
|
+
});
|
|
4153
|
+
}
|
|
4154
|
+
});
|
|
4155
|
+
await store.markPublished(batch.map((m) => m.id));
|
|
4156
|
+
onPublished?.(batch.length);
|
|
4157
|
+
} catch (err) {
|
|
4158
|
+
onError(toError2(err), batch);
|
|
4159
|
+
}
|
|
4160
|
+
};
|
|
4161
|
+
const tick = () => {
|
|
4162
|
+
if (stopped || running) return;
|
|
4163
|
+
running = true;
|
|
4164
|
+
inFlight = iterate().finally(() => {
|
|
4165
|
+
running = false;
|
|
4166
|
+
});
|
|
4167
|
+
};
|
|
4168
|
+
const timer = setInterval(tick, pollIntervalMs);
|
|
4169
|
+
timer.unref?.();
|
|
4170
|
+
return {
|
|
4171
|
+
stop: async () => {
|
|
4172
|
+
stopped = true;
|
|
4173
|
+
clearInterval(timer);
|
|
4174
|
+
await inFlight;
|
|
4175
|
+
}
|
|
4176
|
+
};
|
|
4177
|
+
}
|
|
4178
|
+
|
|
4179
|
+
// src/client/security/providers.ts
|
|
4180
|
+
var defaultImport = (specifier) => import(specifier);
|
|
4181
|
+
function awsMskIamProvider(options) {
|
|
4182
|
+
const importFn = options.importFn ?? defaultImport;
|
|
4183
|
+
return async () => {
|
|
4184
|
+
let signer;
|
|
4185
|
+
try {
|
|
4186
|
+
signer = await importFn("aws-msk-iam-sasl-signer-js");
|
|
4187
|
+
} catch {
|
|
4188
|
+
throw new Error(
|
|
4189
|
+
"awsMskIamProvider: package 'aws-msk-iam-sasl-signer-js' is not installed. Run `npm install aws-msk-iam-sasl-signer-js` to enable MSK IAM authentication."
|
|
4190
|
+
);
|
|
4191
|
+
}
|
|
4192
|
+
const { token, expiryTime } = await signer.generateAuthToken({
|
|
4193
|
+
region: options.region
|
|
4194
|
+
});
|
|
4195
|
+
return {
|
|
4196
|
+
value: token,
|
|
4197
|
+
principal: "msk-iam",
|
|
4198
|
+
// expiryTime is epoch ms per the signer's contract
|
|
4199
|
+
lifetimeMs: expiryTime
|
|
4200
|
+
};
|
|
4201
|
+
};
|
|
4202
|
+
}
|
|
4203
|
+
function gcpAccessTokenProvider(options = {}) {
|
|
4204
|
+
const importFn = options.importFn ?? defaultImport;
|
|
4205
|
+
const ttlMs = options.tokenTtlMs ?? 50 * 6e4;
|
|
4206
|
+
return async () => {
|
|
4207
|
+
let lib;
|
|
4208
|
+
try {
|
|
4209
|
+
lib = await importFn("google-auth-library");
|
|
4210
|
+
} catch {
|
|
4211
|
+
throw new Error(
|
|
4212
|
+
"gcpAccessTokenProvider: package 'google-auth-library' is not installed. Run `npm install google-auth-library` to enable GCP authentication."
|
|
4213
|
+
);
|
|
4214
|
+
}
|
|
4215
|
+
const auth = new lib.GoogleAuth({
|
|
4216
|
+
scopes: options.scopes ?? ["https://www.googleapis.com/auth/cloud-platform"]
|
|
4217
|
+
});
|
|
4218
|
+
const token = await auth.getAccessToken();
|
|
4219
|
+
if (!token) {
|
|
4220
|
+
throw new Error(
|
|
4221
|
+
"gcpAccessTokenProvider: google-auth-library returned no access token \u2014 check Application Default Credentials."
|
|
4222
|
+
);
|
|
4223
|
+
}
|
|
4224
|
+
return {
|
|
4225
|
+
value: token,
|
|
4226
|
+
principal: options.principal ?? "gcp",
|
|
4227
|
+
lifetimeMs: Date.now() + ttlMs
|
|
4228
|
+
};
|
|
4229
|
+
};
|
|
4230
|
+
}
|
|
4231
|
+
|
|
4232
|
+
// src/client/security/acl.ts
|
|
4233
|
+
function addResource(out, r) {
|
|
4234
|
+
const key = `${r.resourceType}:${r.patternType}:${r.name}`;
|
|
4235
|
+
const existing = out.get(key);
|
|
4236
|
+
if (existing) {
|
|
4237
|
+
for (const op of r.operations)
|
|
4238
|
+
if (!existing.operations.includes(op)) existing.operations.push(op);
|
|
4239
|
+
if (!existing.reason.includes(r.reason))
|
|
4240
|
+
existing.reason += `; ${r.reason}`;
|
|
4241
|
+
} else {
|
|
4242
|
+
out.set(key, { ...r, operations: [...r.operations] });
|
|
4243
|
+
}
|
|
4244
|
+
}
|
|
4245
|
+
function describeRequiredAcls(input) {
|
|
4246
|
+
const out = /* @__PURE__ */ new Map();
|
|
4247
|
+
const f = input.features ?? {};
|
|
4248
|
+
const produce = input.produceTopics ?? [];
|
|
4249
|
+
const consume = input.consumeTopics ?? [];
|
|
4250
|
+
const groups = input.groupIds ?? [];
|
|
4251
|
+
for (const t of produce) {
|
|
4252
|
+
addResource(out, {
|
|
4253
|
+
resourceType: "topic",
|
|
4254
|
+
patternType: "literal",
|
|
4255
|
+
name: t,
|
|
4256
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4257
|
+
reason: "sendMessage/sendBatch"
|
|
4258
|
+
});
|
|
4259
|
+
}
|
|
4260
|
+
for (const t of consume) {
|
|
4261
|
+
addResource(out, {
|
|
4262
|
+
resourceType: "topic",
|
|
4263
|
+
patternType: "literal",
|
|
4264
|
+
name: t,
|
|
4265
|
+
operations: ["READ", "DESCRIBE"],
|
|
4266
|
+
reason: "startConsumer"
|
|
4267
|
+
});
|
|
4268
|
+
}
|
|
4269
|
+
for (const g of groups) {
|
|
4270
|
+
addResource(out, {
|
|
4271
|
+
resourceType: "group",
|
|
4272
|
+
patternType: "literal",
|
|
4273
|
+
name: g,
|
|
4274
|
+
operations: ["READ", "DESCRIBE"],
|
|
4275
|
+
reason: "consumer group membership + offset commits"
|
|
4276
|
+
});
|
|
4277
|
+
}
|
|
4278
|
+
if (f.dlq) {
|
|
4279
|
+
for (const t of consume) {
|
|
4280
|
+
addResource(out, {
|
|
4281
|
+
resourceType: "topic",
|
|
4282
|
+
patternType: "literal",
|
|
4283
|
+
name: `${t}.dlq`,
|
|
4284
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4285
|
+
reason: "dlq: true \u2014 failed messages routed to DLQ"
|
|
4286
|
+
});
|
|
4287
|
+
}
|
|
4288
|
+
}
|
|
4289
|
+
if (f.retryTopics) {
|
|
4290
|
+
for (const t of consume) {
|
|
4291
|
+
for (let level = 1; level <= f.retryTopics.maxRetries; level++) {
|
|
4292
|
+
addResource(out, {
|
|
4293
|
+
resourceType: "topic",
|
|
4294
|
+
patternType: "literal",
|
|
4295
|
+
name: `${t}.retry.${level}`,
|
|
4296
|
+
operations: ["READ", "WRITE", "DESCRIBE"],
|
|
4297
|
+
reason: "retryTopics \u2014 retry chain produce + companion consume"
|
|
4298
|
+
});
|
|
4299
|
+
}
|
|
4300
|
+
}
|
|
4301
|
+
for (const g of groups) {
|
|
4302
|
+
addResource(out, {
|
|
4303
|
+
resourceType: "group",
|
|
4304
|
+
patternType: "prefixed",
|
|
4305
|
+
name: `${g}-retry.`,
|
|
4306
|
+
operations: ["READ", "DESCRIBE"],
|
|
4307
|
+
reason: "retryTopics \u2014 companion retry-level consumer groups"
|
|
4308
|
+
});
|
|
4309
|
+
addResource(out, {
|
|
4310
|
+
resourceType: "transactional-id",
|
|
4311
|
+
patternType: "prefixed",
|
|
4312
|
+
name: `${g}-`,
|
|
4313
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4314
|
+
reason: "retryTopics \u2014 EOS routing transactions per retry level"
|
|
4315
|
+
});
|
|
4316
|
+
}
|
|
4317
|
+
}
|
|
4318
|
+
if (f.delayedDelivery) {
|
|
4319
|
+
for (const t of [.../* @__PURE__ */ new Set([...produce, ...consume])]) {
|
|
4320
|
+
addResource(out, {
|
|
4321
|
+
resourceType: "topic",
|
|
4322
|
+
patternType: "literal",
|
|
4323
|
+
name: `${t}.delayed`,
|
|
4324
|
+
operations: ["READ", "WRITE", "DESCRIBE"],
|
|
4325
|
+
reason: "deliverAfterMs staging + startDelayedRelay consume"
|
|
4326
|
+
});
|
|
4327
|
+
}
|
|
4328
|
+
for (const g of groups) {
|
|
4329
|
+
addResource(out, {
|
|
4330
|
+
resourceType: "group",
|
|
4331
|
+
patternType: "literal",
|
|
4332
|
+
name: `${g}-delayed-relay`,
|
|
4333
|
+
operations: ["READ", "DESCRIBE"],
|
|
4334
|
+
reason: "startDelayedRelay consumer group"
|
|
4335
|
+
});
|
|
4336
|
+
addResource(out, {
|
|
4337
|
+
resourceType: "transactional-id",
|
|
4338
|
+
patternType: "literal",
|
|
4339
|
+
name: `${g}-delayed-relay-tx`,
|
|
4340
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4341
|
+
reason: "startDelayedRelay transactional forwarding"
|
|
4342
|
+
});
|
|
4343
|
+
}
|
|
4344
|
+
}
|
|
4345
|
+
if (f.duplicatesTopic) {
|
|
4346
|
+
if (typeof f.duplicatesTopic === "string") {
|
|
4347
|
+
addResource(out, {
|
|
4348
|
+
resourceType: "topic",
|
|
4349
|
+
patternType: "literal",
|
|
4350
|
+
name: f.duplicatesTopic,
|
|
4351
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4352
|
+
reason: "deduplication.strategy 'topic' \u2014 custom duplicates topic"
|
|
4353
|
+
});
|
|
4354
|
+
} else {
|
|
4355
|
+
for (const t of consume) {
|
|
4356
|
+
addResource(out, {
|
|
4357
|
+
resourceType: "topic",
|
|
4358
|
+
patternType: "literal",
|
|
4359
|
+
name: `${t}.duplicates`,
|
|
4360
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4361
|
+
reason: "deduplication.strategy 'topic'"
|
|
4362
|
+
});
|
|
4363
|
+
}
|
|
4364
|
+
}
|
|
4365
|
+
}
|
|
4366
|
+
if (f.dlqReplay) {
|
|
4367
|
+
for (const t of consume) {
|
|
4368
|
+
addResource(out, {
|
|
4369
|
+
resourceType: "group",
|
|
4370
|
+
patternType: "prefixed",
|
|
4371
|
+
name: `${t}.dlq-replay`,
|
|
4372
|
+
operations: ["READ", "DESCRIBE", "DELETE"],
|
|
4373
|
+
reason: "replayDlq \u2014 ephemeral/stable replay groups (deleted after use)"
|
|
4374
|
+
});
|
|
4375
|
+
addResource(out, {
|
|
4376
|
+
resourceType: "topic",
|
|
4377
|
+
patternType: "literal",
|
|
4378
|
+
name: `${t}.dlq`,
|
|
4379
|
+
operations: ["READ", "DESCRIBE"],
|
|
4380
|
+
reason: "replayDlq \u2014 reads the DLQ"
|
|
4381
|
+
});
|
|
4382
|
+
}
|
|
4383
|
+
}
|
|
4384
|
+
if (f.snapshots) {
|
|
4385
|
+
addResource(out, {
|
|
4386
|
+
resourceType: "group",
|
|
4387
|
+
patternType: "prefixed",
|
|
4388
|
+
name: `${input.clientId}-snapshot-`,
|
|
4389
|
+
operations: ["READ", "DESCRIBE", "DELETE"],
|
|
4390
|
+
reason: "readSnapshot \u2014 timestamped ephemeral groups (deleted after use)"
|
|
4391
|
+
});
|
|
4392
|
+
}
|
|
4393
|
+
if (f.clockRecovery) {
|
|
4394
|
+
addResource(out, {
|
|
4395
|
+
resourceType: "group",
|
|
4396
|
+
patternType: "prefixed",
|
|
4397
|
+
name: `${input.clientId}-clock-recovery-`,
|
|
4398
|
+
operations: ["READ", "DESCRIBE", "DELETE"],
|
|
4399
|
+
reason: "clockRecovery \u2014 timestamped ephemeral groups (deleted after use)"
|
|
4400
|
+
});
|
|
4401
|
+
}
|
|
4402
|
+
if (f.transactions) {
|
|
4403
|
+
addResource(out, {
|
|
4404
|
+
resourceType: "transactional-id",
|
|
4405
|
+
patternType: "literal",
|
|
4406
|
+
name: `${input.clientId}-tx`,
|
|
4407
|
+
operations: ["WRITE", "DESCRIBE"],
|
|
4408
|
+
reason: "transaction() \u2014 default transactionalId (override-aware: adjust if you set one)"
|
|
4409
|
+
});
|
|
4410
|
+
}
|
|
4411
|
+
if (f.autoCreateTopics) {
|
|
4412
|
+
addResource(out, {
|
|
4413
|
+
resourceType: "cluster",
|
|
4414
|
+
patternType: "literal",
|
|
4415
|
+
name: "kafka-cluster",
|
|
4416
|
+
operations: ["CREATE"],
|
|
4417
|
+
reason: "autoCreateTopics: true \u2014 not recommended in production"
|
|
4418
|
+
});
|
|
4419
|
+
}
|
|
4420
|
+
return [...out.values()];
|
|
4421
|
+
}
|
|
4422
|
+
function toKafkaAclCommands(resources, principal, bootstrapServer = "<bootstrap-server>") {
|
|
4423
|
+
return resources.map((r) => {
|
|
4424
|
+
const ops = r.operations.map((o) => `--operation ${o}`).join(" ");
|
|
4425
|
+
const resourceFlag = r.resourceType === "topic" ? `--topic '${r.name}'` : r.resourceType === "group" ? `--group '${r.name}'` : r.resourceType === "transactional-id" ? `--transactional-id '${r.name}'` : "--cluster";
|
|
4426
|
+
const pattern = r.patternType === "prefixed" ? " --resource-pattern-type prefixed" : "";
|
|
4427
|
+
return `kafka-acls.sh --bootstrap-server ${bootstrapServer} --add --allow-principal '${principal}' ${ops} ${resourceFlag}${pattern} # ${r.reason}`;
|
|
4428
|
+
});
|
|
4429
|
+
}
|
|
4430
|
+
var MSK_TOPIC_ACTIONS = {
|
|
4431
|
+
READ: ["kafka-cluster:ReadData", "kafka-cluster:DescribeTopic"],
|
|
4432
|
+
WRITE: ["kafka-cluster:WriteData", "kafka-cluster:DescribeTopic"],
|
|
4433
|
+
DESCRIBE: ["kafka-cluster:DescribeTopic"],
|
|
4434
|
+
CREATE: ["kafka-cluster:CreateTopic"],
|
|
4435
|
+
DELETE: ["kafka-cluster:DeleteTopic"]
|
|
4436
|
+
};
|
|
4437
|
+
var MSK_GROUP_ACTIONS = {
|
|
4438
|
+
READ: ["kafka-cluster:AlterGroup", "kafka-cluster:DescribeGroup"],
|
|
4439
|
+
DESCRIBE: ["kafka-cluster:DescribeGroup"],
|
|
4440
|
+
DELETE: ["kafka-cluster:DeleteGroup"]
|
|
4441
|
+
};
|
|
4442
|
+
var MSK_TX_ACTIONS = {
|
|
4443
|
+
WRITE: [
|
|
4444
|
+
"kafka-cluster:AlterTransactionalId",
|
|
4445
|
+
"kafka-cluster:DescribeTransactionalId"
|
|
4446
|
+
],
|
|
4447
|
+
DESCRIBE: ["kafka-cluster:DescribeTransactionalId"]
|
|
4448
|
+
};
|
|
4449
|
+
function toMskIamPolicy(resources, cluster) {
|
|
4450
|
+
const { region, accountId, clusterName, clusterUuid } = cluster;
|
|
4451
|
+
const arn = (type, name) => `arn:aws:kafka:${region}:${accountId}:${type}/${clusterName}/${clusterUuid}/${name}`;
|
|
4452
|
+
const statements = [
|
|
4453
|
+
{
|
|
4454
|
+
Sid: "Connect",
|
|
4455
|
+
Effect: "Allow",
|
|
4456
|
+
Action: ["kafka-cluster:Connect"],
|
|
4457
|
+
Resource: [
|
|
4458
|
+
`arn:aws:kafka:${region}:${accountId}:cluster/${clusterName}/${clusterUuid}`
|
|
4459
|
+
]
|
|
4460
|
+
}
|
|
4461
|
+
];
|
|
4462
|
+
let sid = 0;
|
|
4463
|
+
for (const r of resources) {
|
|
4464
|
+
const suffix = r.patternType === "prefixed" ? `${r.name}*` : r.name;
|
|
4465
|
+
let actions = [];
|
|
4466
|
+
let resource;
|
|
4467
|
+
if (r.resourceType === "topic") {
|
|
4468
|
+
actions = [...new Set(r.operations.flatMap((o) => MSK_TOPIC_ACTIONS[o] ?? []))];
|
|
4469
|
+
resource = arn("topic", suffix);
|
|
4470
|
+
} else if (r.resourceType === "group") {
|
|
4471
|
+
actions = [...new Set(r.operations.flatMap((o) => MSK_GROUP_ACTIONS[o] ?? []))];
|
|
4472
|
+
resource = arn("group", suffix);
|
|
4473
|
+
} else if (r.resourceType === "transactional-id") {
|
|
4474
|
+
actions = [...new Set(r.operations.flatMap((o) => MSK_TX_ACTIONS[o] ?? []))];
|
|
4475
|
+
resource = arn("transactional-id", suffix);
|
|
4476
|
+
} else {
|
|
4477
|
+
actions = ["kafka-cluster:CreateTopic"];
|
|
4478
|
+
resource = `arn:aws:kafka:${region}:${accountId}:topic/${clusterName}/${clusterUuid}/*`;
|
|
4479
|
+
}
|
|
4480
|
+
if (actions.length === 0 || !resource) continue;
|
|
4481
|
+
statements.push({
|
|
4482
|
+
Sid: `Acl${sid++}`,
|
|
4483
|
+
Effect: "Allow",
|
|
4484
|
+
Action: actions,
|
|
4485
|
+
Resource: [resource]
|
|
4486
|
+
});
|
|
4487
|
+
}
|
|
4488
|
+
return { Version: "2012-10-17", Statement: statements };
|
|
4489
|
+
}
|
|
4490
|
+
|
|
4491
|
+
// src/client/config/from-env.ts
|
|
4492
|
+
var TRUE_VALUES = /* @__PURE__ */ new Set(["true", "1", "yes"]);
|
|
4493
|
+
var FALSE_VALUES = /* @__PURE__ */ new Set(["false", "0", "no"]);
|
|
4494
|
+
function parseBool(name, raw) {
|
|
4495
|
+
const normalized = raw.trim().toLowerCase();
|
|
4496
|
+
if (TRUE_VALUES.has(normalized)) return true;
|
|
4497
|
+
if (FALSE_VALUES.has(normalized)) return false;
|
|
4498
|
+
throw new Error(
|
|
4499
|
+
`Invalid boolean for ${name}: "${raw}". Use one of true/false, 1/0, yes/no (case-insensitive).`
|
|
4500
|
+
);
|
|
4501
|
+
}
|
|
4502
|
+
function parseNum(name, raw) {
|
|
4503
|
+
const value = Number(raw.trim());
|
|
4504
|
+
if (Number.isNaN(value)) {
|
|
4505
|
+
throw new Error(`Invalid number for ${name}: "${raw}".`);
|
|
4506
|
+
}
|
|
4507
|
+
return value;
|
|
4508
|
+
}
|
|
4509
|
+
function parseList(raw) {
|
|
4510
|
+
return raw.split(",").map((part) => part.trim()).filter((part) => part.length > 0);
|
|
4511
|
+
}
|
|
4512
|
+
function parseEnum(name, raw, allowed) {
|
|
4513
|
+
const value = raw.trim();
|
|
4514
|
+
if (!allowed.includes(value)) {
|
|
4515
|
+
throw new Error(
|
|
4516
|
+
`Invalid value for ${name}: "${raw}". Allowed: ${allowed.join(", ")}.`
|
|
4517
|
+
);
|
|
4518
|
+
}
|
|
4519
|
+
return value;
|
|
4520
|
+
}
|
|
4521
|
+
function readVar(env, key, apply) {
|
|
4522
|
+
const raw = env[key];
|
|
4523
|
+
if (raw === void 0 || raw.trim() === "") return;
|
|
4524
|
+
apply(raw);
|
|
4525
|
+
}
|
|
4526
|
+
function kafkaClientConfigFromEnv(env = process.env, prefix = "KAFKA_") {
|
|
4527
|
+
const options = {};
|
|
4528
|
+
const result = { options };
|
|
4529
|
+
readVar(env, `${prefix}CLIENT_ID`, (raw) => {
|
|
4530
|
+
result.clientId = raw.trim();
|
|
4531
|
+
});
|
|
4532
|
+
readVar(env, `${prefix}GROUP_ID`, (raw) => {
|
|
4533
|
+
result.groupId = raw.trim();
|
|
4534
|
+
});
|
|
4535
|
+
readVar(env, `${prefix}BROKERS`, (raw) => {
|
|
4536
|
+
result.brokers = parseList(raw);
|
|
4537
|
+
});
|
|
4538
|
+
readVar(env, `${prefix}AUTO_CREATE_TOPICS`, (raw) => {
|
|
4539
|
+
options.autoCreateTopics = parseBool(`${prefix}AUTO_CREATE_TOPICS`, raw);
|
|
4540
|
+
});
|
|
4541
|
+
readVar(env, `${prefix}STRICT_SCHEMAS`, (raw) => {
|
|
4542
|
+
options.strictSchemas = parseBool(`${prefix}STRICT_SCHEMAS`, raw);
|
|
4543
|
+
});
|
|
4544
|
+
readVar(env, `${prefix}NUM_PARTITIONS`, (raw) => {
|
|
4545
|
+
options.numPartitions = parseNum(`${prefix}NUM_PARTITIONS`, raw);
|
|
4546
|
+
});
|
|
4547
|
+
readVar(env, `${prefix}TRANSACTIONAL_ID`, (raw) => {
|
|
4548
|
+
options.transactionalId = raw.trim();
|
|
4549
|
+
});
|
|
4550
|
+
readVar(env, `${prefix}CLOCK_RECOVERY_TOPICS`, (raw) => {
|
|
4551
|
+
const topics = parseList(raw);
|
|
4552
|
+
if (topics.length === 0) return;
|
|
4553
|
+
options.clockRecovery = { topics };
|
|
4554
|
+
});
|
|
4555
|
+
readVar(env, `${prefix}CLOCK_RECOVERY_TIMEOUT_MS`, (raw) => {
|
|
4556
|
+
const timeoutMs = parseNum(`${prefix}CLOCK_RECOVERY_TIMEOUT_MS`, raw);
|
|
4557
|
+
if (options.clockRecovery) {
|
|
4558
|
+
options.clockRecovery.timeoutMs = timeoutMs;
|
|
4559
|
+
}
|
|
4560
|
+
});
|
|
4561
|
+
readVar(env, `${prefix}LAG_THROTTLE_MAX_LAG`, (raw) => {
|
|
4562
|
+
options.lagThrottle = {
|
|
4563
|
+
maxLag: parseNum(`${prefix}LAG_THROTTLE_MAX_LAG`, raw)
|
|
4564
|
+
};
|
|
4565
|
+
});
|
|
4566
|
+
readVar(env, `${prefix}LAG_THROTTLE_GROUP_ID`, (raw) => {
|
|
4567
|
+
if (options.lagThrottle) options.lagThrottle.groupId = raw.trim();
|
|
4568
|
+
});
|
|
4569
|
+
readVar(env, `${prefix}LAG_THROTTLE_POLL_INTERVAL_MS`, (raw) => {
|
|
4570
|
+
if (options.lagThrottle) {
|
|
4571
|
+
options.lagThrottle.pollIntervalMs = parseNum(
|
|
4572
|
+
`${prefix}LAG_THROTTLE_POLL_INTERVAL_MS`,
|
|
4573
|
+
raw
|
|
4574
|
+
);
|
|
4575
|
+
}
|
|
4576
|
+
});
|
|
4577
|
+
readVar(env, `${prefix}LAG_THROTTLE_MAX_WAIT_MS`, (raw) => {
|
|
4578
|
+
if (options.lagThrottle) {
|
|
4579
|
+
options.lagThrottle.maxWaitMs = parseNum(
|
|
4580
|
+
`${prefix}LAG_THROTTLE_MAX_WAIT_MS`,
|
|
4581
|
+
raw
|
|
4582
|
+
);
|
|
4583
|
+
}
|
|
4584
|
+
});
|
|
4585
|
+
const security = securityFromEnv(env, prefix);
|
|
4586
|
+
if (security) options.security = security;
|
|
4587
|
+
return result;
|
|
4588
|
+
}
|
|
4589
|
+
function securityFromEnv(env, prefix) {
|
|
4590
|
+
let ssl;
|
|
4591
|
+
let allowInsecure;
|
|
4592
|
+
let mechanism;
|
|
4593
|
+
let username;
|
|
4594
|
+
let password;
|
|
4595
|
+
readVar(env, `${prefix}SSL`, (raw) => {
|
|
4596
|
+
ssl = parseBool(`${prefix}SSL`, raw);
|
|
4597
|
+
});
|
|
4598
|
+
readVar(env, `${prefix}ALLOW_INSECURE`, (raw) => {
|
|
4599
|
+
allowInsecure = parseBool(`${prefix}ALLOW_INSECURE`, raw);
|
|
4600
|
+
});
|
|
4601
|
+
readVar(env, `${prefix}SASL_MECHANISM`, (raw) => {
|
|
4602
|
+
mechanism = parseEnum(`${prefix}SASL_MECHANISM`, raw, [
|
|
4603
|
+
"plain",
|
|
4604
|
+
"scram-sha-256",
|
|
4605
|
+
"scram-sha-512"
|
|
4606
|
+
]);
|
|
4607
|
+
});
|
|
4608
|
+
readVar(env, `${prefix}SASL_USERNAME`, (raw) => {
|
|
4609
|
+
username = raw.trim();
|
|
4610
|
+
});
|
|
4611
|
+
readVar(env, `${prefix}SASL_PASSWORD`, (raw) => {
|
|
4612
|
+
password = raw;
|
|
4613
|
+
});
|
|
4614
|
+
if (ssl === void 0 && allowInsecure === void 0 && mechanism === void 0 && username === void 0 && password === void 0) {
|
|
4615
|
+
return void 0;
|
|
4616
|
+
}
|
|
4617
|
+
const security = {};
|
|
4618
|
+
if (ssl !== void 0) security.ssl = ssl;
|
|
4619
|
+
if (allowInsecure !== void 0) security.allowInsecure = allowInsecure;
|
|
4620
|
+
if (mechanism !== void 0 || username !== void 0 || password !== void 0) {
|
|
4621
|
+
if (mechanism === void 0 || username === void 0 || password === void 0) {
|
|
4622
|
+
throw new Error(
|
|
4623
|
+
`Incomplete SASL configuration: ${prefix}SASL_MECHANISM, ${prefix}SASL_USERNAME, and ${prefix}SASL_PASSWORD must all be set together (oauthbearer must be configured in code).`
|
|
4624
|
+
);
|
|
4625
|
+
}
|
|
4626
|
+
const sasl = { mechanism, username, password };
|
|
4627
|
+
security.sasl = sasl;
|
|
4628
|
+
}
|
|
4629
|
+
return security;
|
|
4630
|
+
}
|
|
4631
|
+
function consumerOptionsFromEnv(env = process.env, prefix = "KAFKA_CONSUMER_") {
|
|
4632
|
+
const options = {};
|
|
4633
|
+
readVar(env, `${prefix}GROUP_ID`, (raw) => {
|
|
4634
|
+
options.groupId = raw.trim();
|
|
4635
|
+
});
|
|
4636
|
+
readVar(env, `${prefix}FROM_BEGINNING`, (raw) => {
|
|
4637
|
+
options.fromBeginning = parseBool(`${prefix}FROM_BEGINNING`, raw);
|
|
4638
|
+
});
|
|
4639
|
+
readVar(env, `${prefix}AUTO_COMMIT`, (raw) => {
|
|
4640
|
+
options.autoCommit = parseBool(`${prefix}AUTO_COMMIT`, raw);
|
|
4641
|
+
});
|
|
4642
|
+
readVar(env, `${prefix}DLQ`, (raw) => {
|
|
4643
|
+
options.dlq = parseBool(`${prefix}DLQ`, raw);
|
|
4644
|
+
});
|
|
4645
|
+
readVar(env, `${prefix}RETRY_MAX_RETRIES`, (raw) => {
|
|
4646
|
+
const retry = {
|
|
4647
|
+
maxRetries: parseNum(`${prefix}RETRY_MAX_RETRIES`, raw)
|
|
4648
|
+
};
|
|
4649
|
+
options.retry = retry;
|
|
4650
|
+
});
|
|
4651
|
+
readVar(env, `${prefix}RETRY_BACKOFF_MS`, (raw) => {
|
|
4652
|
+
if (options.retry) {
|
|
4653
|
+
options.retry.backoffMs = parseNum(`${prefix}RETRY_BACKOFF_MS`, raw);
|
|
4654
|
+
}
|
|
4655
|
+
});
|
|
4656
|
+
readVar(env, `${prefix}RETRY_MAX_BACKOFF_MS`, (raw) => {
|
|
4657
|
+
if (options.retry) {
|
|
4658
|
+
options.retry.maxBackoffMs = parseNum(`${prefix}RETRY_MAX_BACKOFF_MS`, raw);
|
|
4659
|
+
}
|
|
4660
|
+
});
|
|
4661
|
+
readVar(env, `${prefix}RETRY_TOPICS`, (raw) => {
|
|
4662
|
+
options.retryTopics = parseBool(`${prefix}RETRY_TOPICS`, raw);
|
|
4663
|
+
});
|
|
4664
|
+
readVar(env, `${prefix}RETRY_TOPIC_ASSIGNMENT_TIMEOUT_MS`, (raw) => {
|
|
4665
|
+
options.retryTopicAssignmentTimeoutMs = parseNum(
|
|
4666
|
+
`${prefix}RETRY_TOPIC_ASSIGNMENT_TIMEOUT_MS`,
|
|
4667
|
+
raw
|
|
4668
|
+
);
|
|
4669
|
+
});
|
|
4670
|
+
readVar(env, `${prefix}HANDLER_TIMEOUT_MS`, (raw) => {
|
|
4671
|
+
options.handlerTimeoutMs = parseNum(`${prefix}HANDLER_TIMEOUT_MS`, raw);
|
|
4672
|
+
});
|
|
4673
|
+
readVar(env, `${prefix}MESSAGE_TTL_MS`, (raw) => {
|
|
4674
|
+
options.messageTtlMs = parseNum(`${prefix}MESSAGE_TTL_MS`, raw);
|
|
4675
|
+
});
|
|
4676
|
+
readVar(env, `${prefix}DEDUPLICATION_STRATEGY`, (raw) => {
|
|
4677
|
+
const strategy = parseEnum(`${prefix}DEDUPLICATION_STRATEGY`, raw, [
|
|
4678
|
+
"drop",
|
|
4679
|
+
"dlq",
|
|
4680
|
+
"topic"
|
|
4681
|
+
]);
|
|
4682
|
+
const dedup = { strategy };
|
|
4683
|
+
options.deduplication = dedup;
|
|
4684
|
+
});
|
|
4685
|
+
readVar(env, `${prefix}DEDUPLICATION_TOPIC`, (raw) => {
|
|
4686
|
+
if (options.deduplication) {
|
|
4687
|
+
options.deduplication.duplicatesTopic = raw.trim();
|
|
4688
|
+
}
|
|
4689
|
+
});
|
|
4690
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_THRESHOLD`, (raw) => {
|
|
4691
|
+
const cb = {
|
|
4692
|
+
threshold: parseNum(`${prefix}CIRCUIT_BREAKER_THRESHOLD`, raw)
|
|
4693
|
+
};
|
|
4694
|
+
options.circuitBreaker = cb;
|
|
4695
|
+
});
|
|
4696
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_RECOVERY_MS`, (raw) => {
|
|
4697
|
+
if (options.circuitBreaker) {
|
|
4698
|
+
options.circuitBreaker.recoveryMs = parseNum(
|
|
4699
|
+
`${prefix}CIRCUIT_BREAKER_RECOVERY_MS`,
|
|
4700
|
+
raw
|
|
4701
|
+
);
|
|
4702
|
+
}
|
|
4703
|
+
});
|
|
4704
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_WINDOW_SIZE`, (raw) => {
|
|
4705
|
+
if (options.circuitBreaker) {
|
|
4706
|
+
options.circuitBreaker.windowSize = parseNum(
|
|
4707
|
+
`${prefix}CIRCUIT_BREAKER_WINDOW_SIZE`,
|
|
4708
|
+
raw
|
|
4709
|
+
);
|
|
4710
|
+
}
|
|
4711
|
+
});
|
|
4712
|
+
readVar(env, `${prefix}CIRCUIT_BREAKER_HALF_OPEN_SUCCESSES`, (raw) => {
|
|
4713
|
+
if (options.circuitBreaker) {
|
|
4714
|
+
options.circuitBreaker.halfOpenSuccesses = parseNum(
|
|
4715
|
+
`${prefix}CIRCUIT_BREAKER_HALF_OPEN_SUCCESSES`,
|
|
4716
|
+
raw
|
|
4717
|
+
);
|
|
4718
|
+
}
|
|
4719
|
+
});
|
|
4720
|
+
readVar(env, `${prefix}QUEUE_HIGH_WATER_MARK`, (raw) => {
|
|
4721
|
+
options.queueHighWaterMark = parseNum(`${prefix}QUEUE_HIGH_WATER_MARK`, raw);
|
|
4722
|
+
});
|
|
4723
|
+
readVar(env, `${prefix}PARTITION_ASSIGNER`, (raw) => {
|
|
4724
|
+
options.partitionAssigner = parseEnum(`${prefix}PARTITION_ASSIGNER`, raw, [
|
|
4725
|
+
"roundrobin",
|
|
4726
|
+
"range",
|
|
4727
|
+
"cooperative-sticky"
|
|
4728
|
+
]);
|
|
4729
|
+
});
|
|
4730
|
+
readVar(env, `${prefix}GROUP_INSTANCE_ID`, (raw) => {
|
|
4731
|
+
options.groupInstanceId = raw.trim();
|
|
4732
|
+
});
|
|
4733
|
+
readVar(env, `${prefix}SUBSCRIBE_RETRY_RETRIES`, (raw) => {
|
|
4734
|
+
const subscribeRetry = {
|
|
4735
|
+
retries: parseNum(`${prefix}SUBSCRIBE_RETRY_RETRIES`, raw)
|
|
4736
|
+
};
|
|
4737
|
+
options.subscribeRetry = subscribeRetry;
|
|
4738
|
+
});
|
|
4739
|
+
readVar(env, `${prefix}SUBSCRIBE_RETRY_DELAY_MS`, (raw) => {
|
|
4740
|
+
if (options.subscribeRetry) {
|
|
4741
|
+
options.subscribeRetry.backoffMs = parseNum(
|
|
4742
|
+
`${prefix}SUBSCRIBE_RETRY_DELAY_MS`,
|
|
4743
|
+
raw
|
|
4744
|
+
);
|
|
4745
|
+
}
|
|
4746
|
+
});
|
|
4747
|
+
return options;
|
|
4748
|
+
}
|
|
4749
|
+
var NESTED_CONSUMER_KEYS = [
|
|
4750
|
+
"retry",
|
|
4751
|
+
"deduplication",
|
|
4752
|
+
"circuitBreaker",
|
|
4753
|
+
"subscribeRetry"
|
|
4754
|
+
];
|
|
4755
|
+
function isPlainObject(value) {
|
|
4756
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
4757
|
+
}
|
|
4758
|
+
function mergeConsumerOptions(...layers) {
|
|
4759
|
+
const result = {};
|
|
4760
|
+
for (const layer of layers) {
|
|
4761
|
+
if (!layer) continue;
|
|
4762
|
+
for (const [key, value] of Object.entries(layer)) {
|
|
4763
|
+
if (value === void 0) continue;
|
|
4764
|
+
if (NESTED_CONSUMER_KEYS.includes(key) && isPlainObject(value) && isPlainObject(result[key])) {
|
|
4765
|
+
result[key] = {
|
|
4766
|
+
...result[key],
|
|
4767
|
+
...value
|
|
4768
|
+
};
|
|
4769
|
+
} else {
|
|
4770
|
+
result[key] = value;
|
|
4771
|
+
}
|
|
4772
|
+
}
|
|
4773
|
+
}
|
|
4774
|
+
return result;
|
|
4775
|
+
}
|
|
4776
|
+
|
|
4777
|
+
export {
|
|
4778
|
+
ConfluentTransport,
|
|
4779
|
+
InMemoryDedupStore,
|
|
4780
|
+
HEADER_EVENT_ID,
|
|
4781
|
+
HEADER_CORRELATION_ID,
|
|
4782
|
+
HEADER_TIMESTAMP,
|
|
4783
|
+
HEADER_SCHEMA_VERSION,
|
|
4784
|
+
HEADER_TRACEPARENT,
|
|
4785
|
+
HEADER_LAMPORT_CLOCK,
|
|
4786
|
+
HEADER_DELAYED_UNTIL,
|
|
4787
|
+
HEADER_DELAYED_TARGET,
|
|
4788
|
+
getEnvelopeContext,
|
|
4789
|
+
runWithEnvelopeContext,
|
|
4790
|
+
buildEnvelopeHeaders,
|
|
4791
|
+
decodeHeaders,
|
|
4792
|
+
extractEnvelope,
|
|
4793
|
+
toError,
|
|
4794
|
+
KafkaProcessingError,
|
|
4795
|
+
KafkaValidationError,
|
|
4796
|
+
KafkaRetryExhaustedError,
|
|
4797
|
+
resolveSecurityOptions,
|
|
4798
|
+
KafkaClient,
|
|
4799
|
+
topic,
|
|
4800
|
+
versionedSchema,
|
|
4801
|
+
SchemaRegistryClient,
|
|
4802
|
+
registrySchema,
|
|
4803
|
+
InMemoryOutboxStore,
|
|
4804
|
+
startOutboxRelay,
|
|
4805
|
+
awsMskIamProvider,
|
|
4806
|
+
gcpAccessTokenProvider,
|
|
4807
|
+
describeRequiredAcls,
|
|
4808
|
+
toKafkaAclCommands,
|
|
4809
|
+
toMskIamPolicy,
|
|
4810
|
+
kafkaClientConfigFromEnv,
|
|
4811
|
+
consumerOptionsFromEnv,
|
|
4812
|
+
mergeConsumerOptions
|
|
4813
|
+
};
|
|
4814
|
+
//# sourceMappingURL=chunk-CMO7SMVK.mjs.map
|