@drarzter/kafka-client 0.9.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +625 -8
- package/dist/chunk-CMO7SMVK.mjs +4814 -0
- package/dist/chunk-CMO7SMVK.mjs.map +1 -0
- package/dist/cli/dlq.d.ts +119 -0
- package/dist/cli/dlq.d.ts.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/{chunk-TPIP5VV7.mjs → cli/index.js} +965 -265
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/index.mjs +355 -0
- package/dist/cli/index.mjs.map +1 -0
- package/dist/client/config/from-env.d.ts +188 -0
- package/dist/client/config/from-env.d.ts.map +1 -0
- package/dist/client/config/index.d.ts +2 -0
- package/dist/client/config/index.d.ts.map +1 -0
- package/dist/client/errors.d.ts +67 -0
- package/dist/client/errors.d.ts.map +1 -0
- package/dist/client/kafka.client/admin/ops.d.ts +114 -0
- package/dist/client/kafka.client/admin/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts +24 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +52 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts +4 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts +10 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts +5 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/handler.d.ts +149 -0
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/ops.d.ts +51 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts +167 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/queue.d.ts +37 -0
- package/dist/client/kafka.client/consumer/queue.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +65 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/setup.d.ts +63 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/start.d.ts +7 -0
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/stop.d.ts +19 -0
- package/dist/client/kafka.client/consumer/stop.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts +4 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts.map +1 -0
- package/dist/client/kafka.client/context.d.ts +72 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -0
- package/dist/client/kafka.client/index.d.ts +155 -0
- package/dist/client/kafka.client/index.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts +61 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts +28 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts +22 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts +67 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts +41 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/ops.d.ts +70 -0
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/send.d.ts +21 -0
- package/dist/client/kafka.client/producer/send.d.ts.map +1 -0
- package/dist/client/kafka.client/validate-options.d.ts +11 -0
- package/dist/client/kafka.client/validate-options.d.ts.map +1 -0
- package/dist/client/message/envelope.d.ts +105 -0
- package/dist/client/message/envelope.d.ts.map +1 -0
- package/dist/client/message/schema-registry.d.ts +105 -0
- package/dist/client/message/schema-registry.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +138 -0
- package/dist/client/message/topic.d.ts.map +1 -0
- package/dist/client/message/versioned-schema.d.ts +53 -0
- package/dist/client/message/versioned-schema.d.ts.map +1 -0
- package/dist/client/outbox/index.d.ts +4 -0
- package/dist/client/outbox/index.d.ts.map +1 -0
- package/dist/client/outbox/outbox.relay.d.ts +90 -0
- package/dist/client/outbox/outbox.relay.d.ts.map +1 -0
- package/dist/client/outbox/outbox.store.d.ts +42 -0
- package/dist/client/outbox/outbox.store.d.ts.map +1 -0
- package/dist/client/outbox/outbox.types.d.ts +144 -0
- package/dist/client/outbox/outbox.types.d.ts.map +1 -0
- package/dist/client/security/acl.d.ts +108 -0
- package/dist/client/security/acl.d.ts.map +1 -0
- package/dist/client/security/index.d.ts +5 -0
- package/dist/client/security/index.d.ts.map +1 -0
- package/dist/client/security/providers.d.ts +88 -0
- package/dist/client/security/providers.d.ts.map +1 -0
- package/dist/client/security/resolve-security.d.ts +19 -0
- package/dist/client/security/resolve-security.d.ts.map +1 -0
- package/dist/client/security/security.types.d.ts +76 -0
- package/dist/client/security/security.types.d.ts.map +1 -0
- package/dist/client/transport/confluent.transport.d.ts +32 -0
- package/dist/client/transport/confluent.transport.d.ts.map +1 -0
- package/dist/client/transport/transport.interface.d.ts +216 -0
- package/dist/client/transport/transport.interface.d.ts.map +1 -0
- package/dist/client/types/admin.interface.d.ts +174 -0
- package/dist/client/types/admin.interface.d.ts.map +1 -0
- package/dist/client/types/admin.types.d.ts +140 -0
- package/dist/client/types/admin.types.d.ts.map +1 -0
- package/dist/client/types/client.d.ts +21 -0
- package/dist/client/types/client.d.ts.map +1 -0
- package/dist/client/types/common.d.ts +84 -0
- package/dist/client/types/common.d.ts.map +1 -0
- package/dist/client/types/config.types.d.ts +150 -0
- package/dist/client/types/config.types.d.ts.map +1 -0
- package/dist/client/types/consumer.interface.d.ts +115 -0
- package/dist/client/types/consumer.interface.d.ts.map +1 -0
- package/dist/{consumer.types-fFCag3VJ.d.mts → client/types/consumer.types.d.ts} +62 -383
- package/dist/client/types/consumer.types.d.ts.map +1 -0
- package/dist/client/types/dedup.types.d.ts +50 -0
- package/dist/client/types/dedup.types.d.ts.map +1 -0
- package/dist/client/types/lifecycle.interface.d.ts +72 -0
- package/dist/client/types/lifecycle.interface.d.ts.map +1 -0
- package/dist/client/types/producer.interface.d.ts +52 -0
- package/dist/client/types/producer.interface.d.ts.map +1 -0
- package/dist/client/types/producer.types.d.ts +90 -0
- package/dist/client/types/producer.types.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/core.d.ts +10 -314
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +1326 -74
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +39 -3
- package/dist/index.d.ts +7 -128
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1343 -74
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +56 -3
- package/dist/index.mjs.map +1 -1
- package/dist/nest/kafka.constants.d.ts +5 -0
- package/dist/nest/kafka.constants.d.ts.map +1 -0
- package/dist/nest/kafka.decorator.d.ts +49 -0
- package/dist/nest/kafka.decorator.d.ts.map +1 -0
- package/dist/nest/kafka.explorer.d.ts +17 -0
- package/dist/nest/kafka.explorer.d.ts.map +1 -0
- package/dist/nest/kafka.health.d.ts +7 -0
- package/dist/nest/kafka.health.d.ts.map +1 -0
- package/dist/nest/kafka.module.d.ts +61 -0
- package/dist/nest/kafka.module.d.ts.map +1 -0
- package/dist/otel.d.ts +83 -5
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +100 -6
- package/dist/otel.js.map +1 -1
- package/dist/otel.mjs +98 -5
- package/dist/otel.mjs.map +1 -1
- package/dist/testing/client.mock.d.ts +47 -0
- package/dist/testing/client.mock.d.ts.map +1 -0
- package/dist/testing/index.d.ts +4 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/test.container.d.ts +63 -0
- package/dist/testing/test.container.d.ts.map +1 -0
- package/dist/{testing.d.mts → testing/transport.fake.d.ts} +7 -111
- package/dist/testing/transport.fake.d.ts.map +1 -0
- package/dist/testing.d.ts +2 -318
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +28 -2
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +28 -2
- package/dist/testing.mjs.map +1 -1
- package/package.json +22 -9
- package/dist/chunk-TPIP5VV7.mjs.map +0 -1
- package/dist/client-CBBUDDtu.d.ts +0 -751
- package/dist/client-D-SxYV2b.d.mts +0 -751
- package/dist/consumer.types-fFCag3VJ.d.ts +0 -958
- package/dist/core.d.mts +0 -314
- package/dist/index.d.mts +0 -128
- package/dist/otel.d.mts +0 -27
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
ConfluentTransport,
|
|
4
|
+
KafkaClient
|
|
5
|
+
} from "../chunk-CMO7SMVK.mjs";
|
|
6
|
+
import "../chunk-EQQGB2QZ.mjs";
|
|
7
|
+
|
|
8
|
+
// src/cli/dlq.ts
|
|
9
|
+
var DLQ_SUFFIX = ".dlq";
|
|
10
|
+
var DlqUsageError = class extends Error {
|
|
11
|
+
constructor(message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = "DlqUsageError";
|
|
14
|
+
}
|
|
15
|
+
};
|
|
16
|
+
var USAGE = `kafka-client-dlq \u2014 dead-letter queue operations
|
|
17
|
+
|
|
18
|
+
Usage:
|
|
19
|
+
kafka-client-dlq ls --brokers <b1,b2> [--prefix <name>]
|
|
20
|
+
kafka-client-dlq peek --brokers <b1,b2> --topic <name> [--limit <n>]
|
|
21
|
+
kafka-client-dlq replay --brokers <b1,b2> --topic <name> [--target <t>] [--dry-run] [--from-beginning | --incremental]
|
|
22
|
+
|
|
23
|
+
Commands:
|
|
24
|
+
ls List DLQ topics (ending in .dlq) with per-topic message counts.
|
|
25
|
+
peek Print up to N messages from <topic>.dlq (offset, x-dlq-* headers, value).
|
|
26
|
+
replay Re-publish <topic>.dlq messages to their original topic (or --target).
|
|
27
|
+
|
|
28
|
+
Options:
|
|
29
|
+
--brokers <list> Comma-separated broker addresses (required). e.g. localhost:9092
|
|
30
|
+
--prefix <name> ls: only show DLQ topics whose base name starts with <name>.
|
|
31
|
+
--topic <name> peek/replay: base topic name (the CLI uses <name>.dlq).
|
|
32
|
+
--limit <n> peek: max messages to print (default 10).
|
|
33
|
+
--target <t> replay: override destination topic.
|
|
34
|
+
--dry-run replay: log without publishing.
|
|
35
|
+
--from-beginning replay: full replay every call (default).
|
|
36
|
+
--incremental replay: only messages added since the previous replay.
|
|
37
|
+
-h, --help Show this help.
|
|
38
|
+
|
|
39
|
+
Examples:
|
|
40
|
+
kafka-client-dlq ls --brokers localhost:9092
|
|
41
|
+
kafka-client-dlq ls --brokers localhost:9092 --prefix orders
|
|
42
|
+
kafka-client-dlq peek --brokers localhost:9092 --topic orders.created --limit 5
|
|
43
|
+
kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --dry-run
|
|
44
|
+
kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --target orders.manual --incremental
|
|
45
|
+
`;
|
|
46
|
+
var VALUE_FLAGS = /* @__PURE__ */ new Set([
|
|
47
|
+
"--brokers",
|
|
48
|
+
"--prefix",
|
|
49
|
+
"--topic",
|
|
50
|
+
"--limit",
|
|
51
|
+
"--target"
|
|
52
|
+
]);
|
|
53
|
+
var BOOL_FLAGS = /* @__PURE__ */ new Set(["--dry-run", "--from-beginning", "--incremental"]);
|
|
54
|
+
function parseFlags(args) {
|
|
55
|
+
const values = {};
|
|
56
|
+
const bools = /* @__PURE__ */ new Set();
|
|
57
|
+
for (let i = 0; i < args.length; i++) {
|
|
58
|
+
const arg = args[i];
|
|
59
|
+
if (!arg.startsWith("--")) {
|
|
60
|
+
throw new DlqUsageError(`Unexpected argument: "${arg}"`);
|
|
61
|
+
}
|
|
62
|
+
if (VALUE_FLAGS.has(arg)) {
|
|
63
|
+
const value = args[i + 1];
|
|
64
|
+
if (value === void 0 || value.startsWith("--")) {
|
|
65
|
+
throw new DlqUsageError(`Flag "${arg}" requires a value.`);
|
|
66
|
+
}
|
|
67
|
+
values[arg] = value;
|
|
68
|
+
i++;
|
|
69
|
+
} else if (BOOL_FLAGS.has(arg)) {
|
|
70
|
+
bools.add(arg);
|
|
71
|
+
} else {
|
|
72
|
+
throw new DlqUsageError(`Unknown flag: "${arg}"`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return { values, bools };
|
|
76
|
+
}
|
|
77
|
+
function requireBrokers(flags) {
|
|
78
|
+
const raw = flags.values["--brokers"];
|
|
79
|
+
if (raw === void 0) {
|
|
80
|
+
throw new DlqUsageError("Missing required flag: --brokers");
|
|
81
|
+
}
|
|
82
|
+
const brokers = raw.split(",").map((b) => b.trim()).filter((b) => b.length > 0);
|
|
83
|
+
if (brokers.length === 0) {
|
|
84
|
+
throw new DlqUsageError("--brokers must list at least one broker address.");
|
|
85
|
+
}
|
|
86
|
+
return brokers;
|
|
87
|
+
}
|
|
88
|
+
function requireTopic(flags) {
|
|
89
|
+
const topic = flags.values["--topic"];
|
|
90
|
+
if (topic === void 0 || topic.length === 0) {
|
|
91
|
+
throw new DlqUsageError("Missing required flag: --topic");
|
|
92
|
+
}
|
|
93
|
+
return topic;
|
|
94
|
+
}
|
|
95
|
+
function parseArgs(argv) {
|
|
96
|
+
const [command, ...rest] = argv;
|
|
97
|
+
if (command === void 0 || command === "-h" || command === "--help" || command === "help") {
|
|
98
|
+
return { command: "help" };
|
|
99
|
+
}
|
|
100
|
+
switch (command) {
|
|
101
|
+
case "ls": {
|
|
102
|
+
const flags = parseFlags(rest);
|
|
103
|
+
const brokers = requireBrokers(flags);
|
|
104
|
+
const prefix = flags.values["--prefix"];
|
|
105
|
+
return { command: "ls", brokers, prefix };
|
|
106
|
+
}
|
|
107
|
+
case "peek": {
|
|
108
|
+
const flags = parseFlags(rest);
|
|
109
|
+
const brokers = requireBrokers(flags);
|
|
110
|
+
const topic = requireTopic(flags);
|
|
111
|
+
const limit = parseLimit(flags.values["--limit"]);
|
|
112
|
+
return { command: "peek", brokers, topic, limit };
|
|
113
|
+
}
|
|
114
|
+
case "replay": {
|
|
115
|
+
const flags = parseFlags(rest);
|
|
116
|
+
const brokers = requireBrokers(flags);
|
|
117
|
+
const topic = requireTopic(flags);
|
|
118
|
+
const target = flags.values["--target"];
|
|
119
|
+
const dryRun = flags.bools.has("--dry-run");
|
|
120
|
+
if (flags.bools.has("--from-beginning") && flags.bools.has("--incremental")) {
|
|
121
|
+
throw new DlqUsageError(
|
|
122
|
+
"--from-beginning and --incremental are mutually exclusive."
|
|
123
|
+
);
|
|
124
|
+
}
|
|
125
|
+
const fromBeginning = !flags.bools.has("--incremental");
|
|
126
|
+
return { command: "replay", brokers, topic, target, dryRun, fromBeginning };
|
|
127
|
+
}
|
|
128
|
+
default:
|
|
129
|
+
throw new DlqUsageError(`Unknown command: "${command}"`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
function parseLimit(raw) {
|
|
133
|
+
if (raw === void 0) return 10;
|
|
134
|
+
const n = Number(raw);
|
|
135
|
+
if (!Number.isInteger(n) || n <= 0) {
|
|
136
|
+
throw new DlqUsageError(`--limit must be a positive integer, got "${raw}".`);
|
|
137
|
+
}
|
|
138
|
+
return n;
|
|
139
|
+
}
|
|
140
|
+
function countFromWatermarks(watermarks) {
|
|
141
|
+
let total = 0;
|
|
142
|
+
for (const { low, high } of watermarks) {
|
|
143
|
+
const width = Number(high) - Number(low);
|
|
144
|
+
total += width > 0 ? width : 0;
|
|
145
|
+
}
|
|
146
|
+
return total;
|
|
147
|
+
}
|
|
148
|
+
function truncate(value, max = 200) {
|
|
149
|
+
if (value.length <= max) return value;
|
|
150
|
+
return `${value.slice(0, max)}\u2026 (${value.length} chars)`;
|
|
151
|
+
}
|
|
152
|
+
async function runDlqCommand(cmd, deps) {
|
|
153
|
+
if (cmd.command === "help") {
|
|
154
|
+
deps.out(USAGE);
|
|
155
|
+
return { command: "help" };
|
|
156
|
+
}
|
|
157
|
+
const client = await deps.createClient(cmd.brokers);
|
|
158
|
+
try {
|
|
159
|
+
switch (cmd.command) {
|
|
160
|
+
case "ls":
|
|
161
|
+
return await runLs(cmd, client, deps);
|
|
162
|
+
case "peek":
|
|
163
|
+
return await runPeek(cmd, client, deps);
|
|
164
|
+
case "replay":
|
|
165
|
+
return await runReplay(cmd, client, deps);
|
|
166
|
+
}
|
|
167
|
+
} finally {
|
|
168
|
+
await client.close();
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
async function runLs(cmd, client, deps) {
|
|
172
|
+
const allTopics = await client.listTopics();
|
|
173
|
+
let dlqTopics = allTopics.filter((t) => t.endsWith(DLQ_SUFFIX));
|
|
174
|
+
if (cmd.prefix) {
|
|
175
|
+
const prefix = cmd.prefix;
|
|
176
|
+
dlqTopics = dlqTopics.filter(
|
|
177
|
+
(t) => t.slice(0, -DLQ_SUFFIX.length).startsWith(prefix)
|
|
178
|
+
);
|
|
179
|
+
}
|
|
180
|
+
dlqTopics.sort();
|
|
181
|
+
const counts = [];
|
|
182
|
+
for (const dlqTopic of dlqTopics) {
|
|
183
|
+
const watermarks = await client.fetchTopicOffsets(dlqTopic);
|
|
184
|
+
counts.push({
|
|
185
|
+
dlqTopic,
|
|
186
|
+
baseTopic: dlqTopic.slice(0, -DLQ_SUFFIX.length),
|
|
187
|
+
count: countFromWatermarks(watermarks)
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
if (counts.length === 0) {
|
|
191
|
+
deps.out(
|
|
192
|
+
cmd.prefix ? `No DLQ topics found matching prefix "${cmd.prefix}".` : "No DLQ topics found."
|
|
193
|
+
);
|
|
194
|
+
} else {
|
|
195
|
+
const width = Math.max(...counts.map((c) => c.dlqTopic.length));
|
|
196
|
+
deps.out(`${"TOPIC".padEnd(width)} MESSAGES`);
|
|
197
|
+
for (const c of counts) {
|
|
198
|
+
deps.out(`${c.dlqTopic.padEnd(width)} ${c.count}`);
|
|
199
|
+
}
|
|
200
|
+
const total = counts.reduce((s, c) => s + c.count, 0);
|
|
201
|
+
deps.out(`${counts.length} DLQ topic(s), ${total} message(s) total.`);
|
|
202
|
+
}
|
|
203
|
+
return { command: "ls", topics: counts };
|
|
204
|
+
}
|
|
205
|
+
async function runPeek(cmd, client, deps) {
|
|
206
|
+
const dlqTopic = `${cmd.topic}${DLQ_SUFFIX}`;
|
|
207
|
+
const messages = await client.peekMessages(dlqTopic, cmd.limit);
|
|
208
|
+
if (messages.length === 0) {
|
|
209
|
+
deps.out(`No messages in ${dlqTopic}.`);
|
|
210
|
+
return { command: "peek", printed: 0 };
|
|
211
|
+
}
|
|
212
|
+
deps.out(`Peeking up to ${cmd.limit} message(s) from ${dlqTopic}:`);
|
|
213
|
+
let printed = 0;
|
|
214
|
+
for (const env of messages) {
|
|
215
|
+
if (printed >= cmd.limit) break;
|
|
216
|
+
deps.out("");
|
|
217
|
+
deps.out(
|
|
218
|
+
`\u2500 offset ${env.offset} \xB7 partition ${env.partition} \xB7 ${env.timestamp}`
|
|
219
|
+
);
|
|
220
|
+
const dlqHeaders = Object.entries(env.headers).filter(([k]) => k.startsWith("x-dlq-")).sort(([a], [b]) => a.localeCompare(b));
|
|
221
|
+
for (const [k, v] of dlqHeaders) {
|
|
222
|
+
deps.out(` ${k}: ${truncate(String(v), 500)}`);
|
|
223
|
+
}
|
|
224
|
+
deps.out(` value: ${truncate(JSON.stringify(env.payload))}`);
|
|
225
|
+
printed++;
|
|
226
|
+
}
|
|
227
|
+
deps.out("");
|
|
228
|
+
deps.out(`Printed ${printed} message(s).`);
|
|
229
|
+
return { command: "peek", printed };
|
|
230
|
+
}
|
|
231
|
+
async function runReplay(cmd, client, deps) {
|
|
232
|
+
const options = {
|
|
233
|
+
dryRun: cmd.dryRun,
|
|
234
|
+
fromBeginning: cmd.fromBeginning
|
|
235
|
+
};
|
|
236
|
+
if (cmd.target !== void 0) options.targetTopic = cmd.target;
|
|
237
|
+
const mode = cmd.fromBeginning ? "full" : "incremental";
|
|
238
|
+
const targetDesc = cmd.target ? ` \u2192 ${cmd.target}` : " \u2192 original topic";
|
|
239
|
+
deps.out(
|
|
240
|
+
`Replaying ${cmd.topic}${DLQ_SUFFIX}${targetDesc} (${mode}${cmd.dryRun ? ", dry-run" : ""})\u2026`
|
|
241
|
+
);
|
|
242
|
+
const { replayed, skipped } = await client.replayDlq(cmd.topic, options);
|
|
243
|
+
deps.out(
|
|
244
|
+
cmd.dryRun ? `Dry-run: ${replayed} message(s) would be replayed, ${skipped} skipped.` : `Replayed ${replayed} message(s), ${skipped} skipped.`
|
|
245
|
+
);
|
|
246
|
+
return { command: "replay", replayed, skipped, dryRun: cmd.dryRun };
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// src/cli/index.ts
|
|
250
|
+
var CLIENT_ID = `dlq-cli-${process.pid}`;
|
|
251
|
+
var GROUP_ID = `${CLIENT_ID}-peek`;
|
|
252
|
+
function createRealClient(brokers) {
|
|
253
|
+
const transport = new ConfluentTransport(CLIENT_ID, brokers);
|
|
254
|
+
const kafka = new KafkaClient(
|
|
255
|
+
CLIENT_ID,
|
|
256
|
+
GROUP_ID,
|
|
257
|
+
brokers,
|
|
258
|
+
{ transport, autoCreateTopics: false, strictSchemas: false }
|
|
259
|
+
);
|
|
260
|
+
const admin = transport.admin();
|
|
261
|
+
let adminConnected = false;
|
|
262
|
+
async function ensureAdmin() {
|
|
263
|
+
if (!adminConnected) {
|
|
264
|
+
await admin.connect();
|
|
265
|
+
adminConnected = true;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
return {
|
|
269
|
+
async listTopics() {
|
|
270
|
+
const status = await kafka.checkStatus();
|
|
271
|
+
if (status.status === "down") {
|
|
272
|
+
throw new Error(`Broker unreachable: ${status.error}`);
|
|
273
|
+
}
|
|
274
|
+
return status.topics;
|
|
275
|
+
},
|
|
276
|
+
async fetchTopicOffsets(topic) {
|
|
277
|
+
await ensureAdmin();
|
|
278
|
+
return admin.fetchTopicOffsets(topic);
|
|
279
|
+
},
|
|
280
|
+
async peekMessages(dlqTopic, limit) {
|
|
281
|
+
const collected = [];
|
|
282
|
+
const iterator = kafka.consume(dlqTopic, {
|
|
283
|
+
groupId: `${dlqTopic}.dlq-peek-${Date.now()}`,
|
|
284
|
+
fromBeginning: true
|
|
285
|
+
});
|
|
286
|
+
await ensureAdmin();
|
|
287
|
+
const watermarks = await admin.fetchTopicOffsets(dlqTopic);
|
|
288
|
+
const available = watermarks.reduce(
|
|
289
|
+
(sum, w) => sum + Math.max(0, Number(w.high) - Number(w.low)),
|
|
290
|
+
0
|
|
291
|
+
);
|
|
292
|
+
const target = Math.min(limit, available);
|
|
293
|
+
if (target === 0) {
|
|
294
|
+
await iterator.return?.();
|
|
295
|
+
return collected;
|
|
296
|
+
}
|
|
297
|
+
try {
|
|
298
|
+
for await (const env of iterator) {
|
|
299
|
+
collected.push(env);
|
|
300
|
+
if (collected.length >= target) break;
|
|
301
|
+
}
|
|
302
|
+
} finally {
|
|
303
|
+
await iterator.return?.();
|
|
304
|
+
}
|
|
305
|
+
return collected;
|
|
306
|
+
},
|
|
307
|
+
replayDlq(topic, options) {
|
|
308
|
+
return kafka.replayDlq(topic, options);
|
|
309
|
+
},
|
|
310
|
+
async close() {
|
|
311
|
+
if (adminConnected) {
|
|
312
|
+
await admin.disconnect().catch(() => {
|
|
313
|
+
});
|
|
314
|
+
}
|
|
315
|
+
await kafka.disconnect().catch(() => {
|
|
316
|
+
});
|
|
317
|
+
}
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
async function main() {
|
|
321
|
+
let cmd;
|
|
322
|
+
try {
|
|
323
|
+
cmd = parseArgs(process.argv.slice(2));
|
|
324
|
+
} catch (err) {
|
|
325
|
+
if (err instanceof DlqUsageError) {
|
|
326
|
+
process.stderr.write(`Error: ${err.message}
|
|
327
|
+
|
|
328
|
+
`);
|
|
329
|
+
process.stderr.write(USAGE);
|
|
330
|
+
return 2;
|
|
331
|
+
}
|
|
332
|
+
throw err;
|
|
333
|
+
}
|
|
334
|
+
try {
|
|
335
|
+
await runDlqCommand(cmd, {
|
|
336
|
+
createClient: createRealClient,
|
|
337
|
+
out: (line) => process.stdout.write(`${line}
|
|
338
|
+
`)
|
|
339
|
+
});
|
|
340
|
+
return 0;
|
|
341
|
+
} catch (err) {
|
|
342
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
343
|
+
process.stderr.write(`Error: ${message}
|
|
344
|
+
`);
|
|
345
|
+
return 1;
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
main().then((code) => {
|
|
349
|
+
process.exitCode = code;
|
|
350
|
+
}).catch((err) => {
|
|
351
|
+
process.stderr.write(`Fatal: ${err?.stack ?? err}
|
|
352
|
+
`);
|
|
353
|
+
process.exitCode = 1;
|
|
354
|
+
});
|
|
355
|
+
//# sourceMappingURL=index.mjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/cli/dlq.ts","../../src/cli/index.ts"],"sourcesContent":["import type { EventEnvelope } from \"../core\";\nimport type { DlqReplayOptions } from \"../core\";\n\n// ── Parsed command shapes ─────────────────────────────────────────────────────\n\n/** DLQ topic suffix — the CLI operates on `<topic>.dlq`. */\nexport const DLQ_SUFFIX = \".dlq\";\n\n/** `ls` — list DLQ topics with message counts. */\nexport interface LsCommand {\n command: \"ls\";\n brokers: string[];\n /** Optional topic-name prefix filter (matched against the base topic). */\n prefix?: string;\n}\n\n/** `peek` — print the first N messages of `<topic>.dlq`. */\nexport interface PeekCommand {\n command: \"peek\";\n brokers: string[];\n /** Base topic name — the CLI reads from `<topic>.dlq`. */\n topic: string;\n /** Maximum number of messages to print. Default: 10. */\n limit: number;\n}\n\n/** `replay` — re-publish `<topic>.dlq` messages via `KafkaClient.replayDlq`. */\nexport interface ReplayCommand {\n command: \"replay\";\n brokers: string[];\n /** Base topic name — the CLI replays `<topic>.dlq`. */\n topic: string;\n /** Override destination topic (default: read from `x-dlq-original-topic`). */\n target?: string;\n /** Log what would be replayed without publishing. */\n dryRun: boolean;\n /**\n * `true` → full replay of all DLQ messages on every call (ephemeral group).\n * `false` → incremental — only messages since the previous replay (stable group).\n */\n fromBeginning: boolean;\n}\n\n/** `help` — print usage. */\nexport interface HelpCommand {\n command: \"help\";\n}\n\n/** Discriminated union of every parsed CLI invocation. */\nexport type ParsedCommand = LsCommand | PeekCommand | ReplayCommand | HelpCommand;\n\n/** Thrown by `parseArgs` when the argv is invalid. Carries the usage text. */\nexport class DlqUsageError extends Error {\n constructor(message: string) {\n super(message);\n this.name = \"DlqUsageError\";\n }\n}\n\n// ── Usage text ────────────────────────────────────────────────────────────────\n\nexport const USAGE = `kafka-client-dlq — dead-letter queue operations\n\nUsage:\n kafka-client-dlq ls --brokers <b1,b2> [--prefix <name>]\n kafka-client-dlq peek --brokers <b1,b2> --topic <name> [--limit <n>]\n kafka-client-dlq replay --brokers <b1,b2> --topic <name> [--target <t>] [--dry-run] [--from-beginning | --incremental]\n\nCommands:\n ls List DLQ topics (ending in .dlq) with per-topic message counts.\n peek Print up to N messages from <topic>.dlq (offset, x-dlq-* headers, value).\n replay Re-publish <topic>.dlq messages to their original topic (or --target).\n\nOptions:\n --brokers <list> Comma-separated broker addresses (required). e.g. localhost:9092\n --prefix <name> ls: only show DLQ topics whose base name starts with <name>.\n --topic <name> peek/replay: base topic name (the CLI uses <name>.dlq).\n --limit <n> peek: max messages to print (default 10).\n --target <t> replay: override destination topic.\n --dry-run replay: log without publishing.\n --from-beginning replay: full replay every call (default).\n --incremental replay: only messages added since the previous replay.\n -h, --help Show this help.\n\nExamples:\n kafka-client-dlq ls --brokers localhost:9092\n kafka-client-dlq ls --brokers localhost:9092 --prefix orders\n kafka-client-dlq peek --brokers localhost:9092 --topic orders.created --limit 5\n kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --dry-run\n kafka-client-dlq replay --brokers localhost:9092 --topic orders.created --target orders.manual --incremental\n`;\n\n// ── Argument parsing ────────────────────────────────────────────────────────────\n\n/** Flags that take a value (`--flag value`). */\nconst VALUE_FLAGS = new Set([\n \"--brokers\",\n \"--prefix\",\n \"--topic\",\n \"--limit\",\n \"--target\",\n]);\n/** Boolean flags (no value). */\nconst BOOL_FLAGS = new Set([\"--dry-run\", \"--from-beginning\", \"--incremental\"]);\n\ninterface RawFlags {\n values: Record<string, string>;\n bools: Set<string>;\n}\n\n/** Split the flag portion of argv into value flags and boolean flags. */\nfunction parseFlags(args: string[]): RawFlags {\n const values: Record<string, string> = {};\n const bools = new Set<string>();\n\n for (let i = 0; i < args.length; i++) {\n const arg = args[i];\n if (!arg.startsWith(\"--\")) {\n throw new DlqUsageError(`Unexpected argument: \"${arg}\"`);\n }\n if (VALUE_FLAGS.has(arg)) {\n const value = args[i + 1];\n if (value === undefined || value.startsWith(\"--\")) {\n throw new DlqUsageError(`Flag \"${arg}\" requires a value.`);\n }\n values[arg] = value;\n i++; // consume the value\n } else if (BOOL_FLAGS.has(arg)) {\n bools.add(arg);\n } else {\n throw new DlqUsageError(`Unknown flag: \"${arg}\"`);\n }\n }\n\n return { values, bools };\n}\n\n/** Parse comma-separated brokers, requiring at least one non-empty entry. */\nfunction requireBrokers(flags: RawFlags): string[] {\n const raw = flags.values[\"--brokers\"];\n if (raw === undefined) {\n throw new DlqUsageError(\"Missing required flag: --brokers\");\n }\n const brokers = raw\n .split(\",\")\n .map((b) => b.trim())\n .filter((b) => b.length > 0);\n if (brokers.length === 0) {\n throw new DlqUsageError(\"--brokers must list at least one broker address.\");\n }\n return brokers;\n}\n\nfunction requireTopic(flags: RawFlags): string {\n const topic = flags.values[\"--topic\"];\n if (topic === undefined || topic.length === 0) {\n throw new DlqUsageError(\"Missing required flag: --topic\");\n }\n return topic;\n}\n\n/**\n * Parse the process argv (already sliced past `node <script>`) into a\n * `ParsedCommand`. Pure and synchronous — no I/O — so it is fully unit-testable.\n *\n * @throws {DlqUsageError} on unknown commands, missing/invalid flags.\n */\nexport function parseArgs(argv: string[]): ParsedCommand {\n const [command, ...rest] = argv;\n\n if (\n command === undefined ||\n command === \"-h\" ||\n command === \"--help\" ||\n command === \"help\"\n ) {\n return { command: \"help\" };\n }\n\n switch (command) {\n case \"ls\": {\n const flags = parseFlags(rest);\n const brokers = requireBrokers(flags);\n const prefix = flags.values[\"--prefix\"];\n return { command: \"ls\", brokers, prefix };\n }\n case \"peek\": {\n const flags = parseFlags(rest);\n const brokers = requireBrokers(flags);\n const topic = requireTopic(flags);\n const limit = parseLimit(flags.values[\"--limit\"]);\n return { command: \"peek\", brokers, topic, limit };\n }\n case \"replay\": {\n const flags = parseFlags(rest);\n const brokers = requireBrokers(flags);\n const topic = requireTopic(flags);\n const target = flags.values[\"--target\"];\n const dryRun = flags.bools.has(\"--dry-run\");\n if (\n flags.bools.has(\"--from-beginning\") &&\n flags.bools.has(\"--incremental\")\n ) {\n throw new DlqUsageError(\n \"--from-beginning and --incremental are mutually exclusive.\",\n );\n }\n // Default is full replay (fromBeginning: true); --incremental opts out.\n const fromBeginning = !flags.bools.has(\"--incremental\");\n return { command: \"replay\", brokers, topic, target, dryRun, fromBeginning };\n }\n default:\n throw new DlqUsageError(`Unknown command: \"${command}\"`);\n }\n}\n\nfunction parseLimit(raw: string | undefined): number {\n if (raw === undefined) return 10;\n const n = Number(raw);\n if (!Number.isInteger(n) || n <= 0) {\n throw new DlqUsageError(`--limit must be a positive integer, got \"${raw}\".`);\n }\n return n;\n}\n\n// ── Runtime deps (injected — tests pass a fake) ──────────────────────────────────\n\n/** Per-partition low/high watermarks for a topic. */\nexport interface PartitionWatermarks {\n partition: number;\n low: string;\n high: string;\n}\n\n/**\n * Minimal client surface the CLI needs. The production factory backs this with\n * a real `KafkaClient` plus its transport admin; tests inject a fake.\n */\nexport interface DlqCliClient {\n /** List all topics visible to the broker (via `checkStatus`). */\n listTopics(): Promise<string[]>;\n /** Per-partition low/high watermarks for a single topic. */\n fetchTopicOffsets(topic: string): Promise<PartitionWatermarks[]>;\n /** Read up to `limit` messages from `dlqTopic`, from the earliest offset. */\n peekMessages(\n dlqTopic: string,\n limit: number,\n ): Promise<Array<EventEnvelope<unknown>>>;\n /** Delegate to `KafkaClient.replayDlq`. */\n replayDlq(\n topic: string,\n options: DlqReplayOptions,\n ): Promise<{ replayed: number; skipped: number }>;\n /** Release all connections. */\n close(): Promise<void>;\n}\n\n/** Injected dependencies for `runDlqCommand`. */\nexport interface RunDeps {\n /** Build a connected client for the given brokers. */\n createClient(brokers: string[]): DlqCliClient | Promise<DlqCliClient>;\n /** Sink for human-readable output. Defaults are wired by the bin entrypoint. */\n out: (line: string) => void;\n}\n\n// ── Result shapes (returned so the entrypoint / tests can assert) ────────────────\n\nexport interface DlqTopicCount {\n /** The `<name>.dlq` topic. */\n dlqTopic: string;\n /** The base topic (`dlqTopic` without the `.dlq` suffix). */\n baseTopic: string;\n /** Sum over partitions of (high − low). */\n count: number;\n}\n\nexport type RunResult =\n | { command: \"ls\"; topics: DlqTopicCount[] }\n | { command: \"peek\"; printed: number }\n | { command: \"replay\"; replayed: number; skipped: number; dryRun: boolean }\n | { command: \"help\" };\n\n// ── Command runner ───────────────────────────────────────────────────────────\n\n/** Sum (high − low) across a topic's partitions. Negative widths clamp to 0. */\nexport function countFromWatermarks(watermarks: PartitionWatermarks[]): number {\n let total = 0;\n for (const { low, high } of watermarks) {\n const width = Number(high) - Number(low);\n total += width > 0 ? width : 0;\n }\n return total;\n}\n\n/** Truncate a value for display, appending an ellipsis marker when cut. */\nexport function truncate(value: string, max = 200): string {\n if (value.length <= max) return value;\n return `${value.slice(0, max)}… (${value.length} chars)`;\n}\n\n/**\n * Execute a parsed command against injected deps.\n * Returns a structured result; all human-readable output goes through `deps.out`.\n * The client is always closed in a `finally` block.\n */\nexport async function runDlqCommand(\n cmd: ParsedCommand,\n deps: RunDeps,\n): Promise<RunResult> {\n if (cmd.command === \"help\") {\n deps.out(USAGE);\n return { command: \"help\" };\n }\n\n const client = await deps.createClient(cmd.brokers);\n try {\n switch (cmd.command) {\n case \"ls\":\n return await runLs(cmd, client, deps);\n case \"peek\":\n return await runPeek(cmd, client, deps);\n case \"replay\":\n return await runReplay(cmd, client, deps);\n }\n } finally {\n await client.close();\n }\n}\n\nasync function runLs(\n cmd: LsCommand,\n client: DlqCliClient,\n deps: RunDeps,\n): Promise<RunResult> {\n const allTopics = await client.listTopics();\n let dlqTopics = allTopics.filter((t) => t.endsWith(DLQ_SUFFIX));\n if (cmd.prefix) {\n const prefix = cmd.prefix;\n dlqTopics = dlqTopics.filter((t) =>\n t.slice(0, -DLQ_SUFFIX.length).startsWith(prefix),\n );\n }\n dlqTopics.sort();\n\n const counts: DlqTopicCount[] = [];\n for (const dlqTopic of dlqTopics) {\n const watermarks = await client.fetchTopicOffsets(dlqTopic);\n counts.push({\n dlqTopic,\n baseTopic: dlqTopic.slice(0, -DLQ_SUFFIX.length),\n count: countFromWatermarks(watermarks),\n });\n }\n\n if (counts.length === 0) {\n deps.out(\n cmd.prefix\n ? `No DLQ topics found matching prefix \"${cmd.prefix}\".`\n : \"No DLQ topics found.\",\n );\n } else {\n const width = Math.max(...counts.map((c) => c.dlqTopic.length));\n deps.out(`${\"TOPIC\".padEnd(width)} MESSAGES`);\n for (const c of counts) {\n deps.out(`${c.dlqTopic.padEnd(width)} ${c.count}`);\n }\n const total = counts.reduce((s, c) => s + c.count, 0);\n deps.out(`${counts.length} DLQ topic(s), ${total} message(s) total.`);\n }\n\n return { command: \"ls\", topics: counts };\n}\n\nasync function runPeek(\n cmd: PeekCommand,\n client: DlqCliClient,\n deps: RunDeps,\n): Promise<RunResult> {\n const dlqTopic = `${cmd.topic}${DLQ_SUFFIX}`;\n const messages = await client.peekMessages(dlqTopic, cmd.limit);\n\n if (messages.length === 0) {\n deps.out(`No messages in ${dlqTopic}.`);\n return { command: \"peek\", printed: 0 };\n }\n\n deps.out(`Peeking up to ${cmd.limit} message(s) from ${dlqTopic}:`);\n let printed = 0;\n for (const env of messages) {\n if (printed >= cmd.limit) break;\n deps.out(\"\");\n deps.out(\n `─ offset ${env.offset} · partition ${env.partition} · ${env.timestamp}`,\n );\n const dlqHeaders = Object.entries(env.headers)\n .filter(([k]) => k.startsWith(\"x-dlq-\"))\n .sort(([a], [b]) => a.localeCompare(b));\n for (const [k, v] of dlqHeaders) {\n deps.out(` ${k}: ${truncate(String(v), 500)}`);\n }\n deps.out(` value: ${truncate(JSON.stringify(env.payload))}`);\n printed++;\n }\n\n deps.out(\"\");\n deps.out(`Printed ${printed} message(s).`);\n return { command: \"peek\", printed };\n}\n\nasync function runReplay(\n cmd: ReplayCommand,\n client: DlqCliClient,\n deps: RunDeps,\n): Promise<RunResult> {\n const options: DlqReplayOptions = {\n dryRun: cmd.dryRun,\n fromBeginning: cmd.fromBeginning,\n };\n if (cmd.target !== undefined) options.targetTopic = cmd.target;\n\n const mode = cmd.fromBeginning ? \"full\" : \"incremental\";\n const targetDesc = cmd.target ? ` → ${cmd.target}` : \" → original topic\";\n deps.out(\n `Replaying ${cmd.topic}${DLQ_SUFFIX}${targetDesc} (${mode}${\n cmd.dryRun ? \", dry-run\" : \"\"\n })…`,\n );\n\n const { replayed, skipped } = await client.replayDlq(cmd.topic, options);\n\n deps.out(\n cmd.dryRun\n ? `Dry-run: ${replayed} message(s) would be replayed, ${skipped} skipped.`\n : `Replayed ${replayed} message(s), ${skipped} skipped.`,\n );\n return { command: \"replay\", replayed, skipped, dryRun: cmd.dryRun };\n}\n","#!/usr/bin/env node\nimport { KafkaClient } from \"../core\";\nimport type { EventEnvelope } from \"../core\";\nimport { ConfluentTransport } from \"../client/transport/confluent.transport\";\nimport type { KafkaTransport } from \"../client/transport/transport.interface\";\nimport {\n DlqUsageError,\n USAGE,\n parseArgs,\n runDlqCommand,\n type DlqCliClient,\n type PartitionWatermarks,\n} from \"./dlq\";\n\n/** Unique-per-process client id so concurrent invocations don't collide. */\nconst CLIENT_ID = `dlq-cli-${process.pid}`;\nconst GROUP_ID = `${CLIENT_ID}-peek`;\n\n/**\n * Production `DlqCliClient` factory. Shares a single `ConfluentTransport`\n * between the `KafkaClient` (for `replayDlq` / `consume`) and a standalone\n * admin (for topic listing and per-partition watermarks used by `ls`).\n */\nfunction createRealClient(brokers: string[]): DlqCliClient {\n const transport: KafkaTransport = new ConfluentTransport(CLIENT_ID, brokers);\n const kafka = new KafkaClient<Record<string, Record<string, unknown>>>(\n CLIENT_ID,\n GROUP_ID,\n brokers,\n { transport, autoCreateTopics: false, strictSchemas: false },\n );\n const admin = transport.admin();\n let adminConnected = false;\n\n async function ensureAdmin() {\n if (!adminConnected) {\n await admin.connect();\n adminConnected = true;\n }\n }\n\n return {\n async listTopics(): Promise<string[]> {\n const status = await kafka.checkStatus();\n if (status.status === \"down\") {\n throw new Error(`Broker unreachable: ${status.error}`);\n }\n return status.topics;\n },\n\n async fetchTopicOffsets(topic: string): Promise<PartitionWatermarks[]> {\n await ensureAdmin();\n return admin.fetchTopicOffsets(topic);\n },\n\n async peekMessages(\n dlqTopic: string,\n limit: number,\n ): Promise<Array<EventEnvelope<unknown>>> {\n const collected: Array<EventEnvelope<unknown>> = [];\n // Ephemeral group so peeking never disturbs a real consumer's offsets.\n const iterator = kafka.consume(dlqTopic as never, {\n groupId: `${dlqTopic}.dlq-peek-${Date.now()}`,\n fromBeginning: true,\n });\n // Guard against an empty topic hanging the iterator forever: if the topic\n // has no messages, stop once the high watermark says there's nothing to read.\n await ensureAdmin();\n const watermarks = await admin.fetchTopicOffsets(dlqTopic);\n const available = watermarks.reduce(\n (sum, w) => sum + Math.max(0, Number(w.high) - Number(w.low)),\n 0,\n );\n const target = Math.min(limit, available);\n if (target === 0) {\n await iterator.return?.();\n return collected;\n }\n try {\n for await (const env of iterator) {\n collected.push(env as EventEnvelope<unknown>);\n if (collected.length >= target) break;\n }\n } finally {\n await iterator.return?.();\n }\n return collected;\n },\n\n replayDlq(topic, options) {\n return kafka.replayDlq(topic, options);\n },\n\n async close(): Promise<void> {\n if (adminConnected) {\n await admin.disconnect().catch(() => {});\n }\n await kafka.disconnect().catch(() => {});\n },\n };\n}\n\nasync function main(): Promise<number> {\n let cmd;\n try {\n cmd = parseArgs(process.argv.slice(2));\n } catch (err) {\n if (err instanceof DlqUsageError) {\n process.stderr.write(`Error: ${err.message}\\n\\n`);\n process.stderr.write(USAGE);\n return 2;\n }\n throw err;\n }\n\n try {\n await runDlqCommand(cmd, {\n createClient: createRealClient,\n out: (line) => process.stdout.write(`${line}\\n`),\n });\n return 0;\n } catch (err) {\n const message = err instanceof Error ? err.message : String(err);\n process.stderr.write(`Error: ${message}\\n`);\n return 1;\n }\n}\n\nmain()\n .then((code) => {\n process.exitCode = code;\n })\n .catch((err) => {\n process.stderr.write(`Fatal: ${err?.stack ?? err}\\n`);\n process.exitCode = 1;\n });\n"],"mappings":";;;;;;;;AAMO,IAAM,aAAa;AA8CnB,IAAM,gBAAN,cAA4B,MAAM;AAAA,EACvC,YAAY,SAAiB;AAC3B,UAAM,OAAO;AACb,SAAK,OAAO;AAAA,EACd;AACF;AAIO,IAAM,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAkCrB,IAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,aAAa,oBAAI,IAAI,CAAC,aAAa,oBAAoB,eAAe,CAAC;AAQ7E,SAAS,WAAW,MAA0B;AAC5C,QAAM,SAAiC,CAAC;AACxC,QAAM,QAAQ,oBAAI,IAAY;AAE9B,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,QAAI,CAAC,IAAI,WAAW,IAAI,GAAG;AACzB,YAAM,IAAI,cAAc,yBAAyB,GAAG,GAAG;AAAA,IACzD;AACA,QAAI,YAAY,IAAI,GAAG,GAAG;AACxB,YAAM,QAAQ,KAAK,IAAI,CAAC;AACxB,UAAI,UAAU,UAAa,MAAM,WAAW,IAAI,GAAG;AACjD,cAAM,IAAI,cAAc,SAAS,GAAG,qBAAqB;AAAA,MAC3D;AACA,aAAO,GAAG,IAAI;AACd;AAAA,IACF,WAAW,WAAW,IAAI,GAAG,GAAG;AAC9B,YAAM,IAAI,GAAG;AAAA,IACf,OAAO;AACL,YAAM,IAAI,cAAc,kBAAkB,GAAG,GAAG;AAAA,IAClD;AAAA,EACF;AAEA,SAAO,EAAE,QAAQ,MAAM;AACzB;AAGA,SAAS,eAAe,OAA2B;AACjD,QAAM,MAAM,MAAM,OAAO,WAAW;AACpC,MAAI,QAAQ,QAAW;AACrB,UAAM,IAAI,cAAc,kCAAkC;AAAA,EAC5D;AACA,QAAM,UAAU,IACb,MAAM,GAAG,EACT,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EACnB,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC;AAC7B,MAAI,QAAQ,WAAW,GAAG;AACxB,UAAM,IAAI,cAAc,kDAAkD;AAAA,EAC5E;AACA,SAAO;AACT;AAEA,SAAS,aAAa,OAAyB;AAC7C,QAAM,QAAQ,MAAM,OAAO,SAAS;AACpC,MAAI,UAAU,UAAa,MAAM,WAAW,GAAG;AAC7C,UAAM,IAAI,cAAc,gCAAgC;AAAA,EAC1D;AACA,SAAO;AACT;AAQO,SAAS,UAAU,MAA+B;AACvD,QAAM,CAAC,SAAS,GAAG,IAAI,IAAI;AAE3B,MACE,YAAY,UACZ,YAAY,QACZ,YAAY,YACZ,YAAY,QACZ;AACA,WAAO,EAAE,SAAS,OAAO;AAAA,EAC3B;AAEA,UAAQ,SAAS;AAAA,IACf,KAAK,MAAM;AACT,YAAM,QAAQ,WAAW,IAAI;AAC7B,YAAM,UAAU,eAAe,KAAK;AACpC,YAAM,SAAS,MAAM,OAAO,UAAU;AACtC,aAAO,EAAE,SAAS,MAAM,SAAS,OAAO;AAAA,IAC1C;AAAA,IACA,KAAK,QAAQ;AACX,YAAM,QAAQ,WAAW,IAAI;AAC7B,YAAM,UAAU,eAAe,KAAK;AACpC,YAAM,QAAQ,aAAa,KAAK;AAChC,YAAM,QAAQ,WAAW,MAAM,OAAO,SAAS,CAAC;AAChD,aAAO,EAAE,SAAS,QAAQ,SAAS,OAAO,MAAM;AAAA,IAClD;AAAA,IACA,KAAK,UAAU;AACb,YAAM,QAAQ,WAAW,IAAI;AAC7B,YAAM,UAAU,eAAe,KAAK;AACpC,YAAM,QAAQ,aAAa,KAAK;AAChC,YAAM,SAAS,MAAM,OAAO,UAAU;AACtC,YAAM,SAAS,MAAM,MAAM,IAAI,WAAW;AAC1C,UACE,MAAM,MAAM,IAAI,kBAAkB,KAClC,MAAM,MAAM,IAAI,eAAe,GAC/B;AACA,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AAEA,YAAM,gBAAgB,CAAC,MAAM,MAAM,IAAI,eAAe;AACtD,aAAO,EAAE,SAAS,UAAU,SAAS,OAAO,QAAQ,QAAQ,cAAc;AAAA,IAC5E;AAAA,IACA;AACE,YAAM,IAAI,cAAc,qBAAqB,OAAO,GAAG;AAAA,EAC3D;AACF;AAEA,SAAS,WAAW,KAAiC;AACnD,MAAI,QAAQ,OAAW,QAAO;AAC9B,QAAM,IAAI,OAAO,GAAG;AACpB,MAAI,CAAC,OAAO,UAAU,CAAC,KAAK,KAAK,GAAG;AAClC,UAAM,IAAI,cAAc,4CAA4C,GAAG,IAAI;AAAA,EAC7E;AACA,SAAO;AACT;AA8DO,SAAS,oBAAoB,YAA2C;AAC7E,MAAI,QAAQ;AACZ,aAAW,EAAE,KAAK,KAAK,KAAK,YAAY;AACtC,UAAM,QAAQ,OAAO,IAAI,IAAI,OAAO,GAAG;AACvC,aAAS,QAAQ,IAAI,QAAQ;AAAA,EAC/B;AACA,SAAO;AACT;AAGO,SAAS,SAAS,OAAe,MAAM,KAAa;AACzD,MAAI,MAAM,UAAU,IAAK,QAAO;AAChC,SAAO,GAAG,MAAM,MAAM,GAAG,GAAG,CAAC,WAAM,MAAM,MAAM;AACjD;AAOA,eAAsB,cACpB,KACA,MACoB;AACpB,MAAI,IAAI,YAAY,QAAQ;AAC1B,SAAK,IAAI,KAAK;AACd,WAAO,EAAE,SAAS,OAAO;AAAA,EAC3B;AAEA,QAAM,SAAS,MAAM,KAAK,aAAa,IAAI,OAAO;AAClD,MAAI;AACF,YAAQ,IAAI,SAAS;AAAA,MACnB,KAAK;AACH,eAAO,MAAM,MAAM,KAAK,QAAQ,IAAI;AAAA,MACtC,KAAK;AACH,eAAO,MAAM,QAAQ,KAAK,QAAQ,IAAI;AAAA,MACxC,KAAK;AACH,eAAO,MAAM,UAAU,KAAK,QAAQ,IAAI;AAAA,IAC5C;AAAA,EACF,UAAE;AACA,UAAM,OAAO,MAAM;AAAA,EACrB;AACF;AAEA,eAAe,MACb,KACA,QACA,MACoB;AACpB,QAAM,YAAY,MAAM,OAAO,WAAW;AAC1C,MAAI,YAAY,UAAU,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC;AAC9D,MAAI,IAAI,QAAQ;AACd,UAAM,SAAS,IAAI;AACnB,gBAAY,UAAU;AAAA,MAAO,CAAC,MAC5B,EAAE,MAAM,GAAG,CAAC,WAAW,MAAM,EAAE,WAAW,MAAM;AAAA,IAClD;AAAA,EACF;AACA,YAAU,KAAK;AAEf,QAAM,SAA0B,CAAC;AACjC,aAAW,YAAY,WAAW;AAChC,UAAM,aAAa,MAAM,OAAO,kBAAkB,QAAQ;AAC1D,WAAO,KAAK;AAAA,MACV;AAAA,MACA,WAAW,SAAS,MAAM,GAAG,CAAC,WAAW,MAAM;AAAA,MAC/C,OAAO,oBAAoB,UAAU;AAAA,IACvC,CAAC;AAAA,EACH;AAEA,MAAI,OAAO,WAAW,GAAG;AACvB,SAAK;AAAA,MACH,IAAI,SACA,wCAAwC,IAAI,MAAM,OAClD;AAAA,IACN;AAAA,EACF,OAAO;AACL,UAAM,QAAQ,KAAK,IAAI,GAAG,OAAO,IAAI,CAAC,MAAM,EAAE,SAAS,MAAM,CAAC;AAC9D,SAAK,IAAI,GAAG,QAAQ,OAAO,KAAK,CAAC,YAAY;AAC7C,eAAW,KAAK,QAAQ;AACtB,WAAK,IAAI,GAAG,EAAE,SAAS,OAAO,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE;AAAA,IACpD;AACA,UAAM,QAAQ,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC;AACpD,SAAK,IAAI,GAAG,OAAO,MAAM,kBAAkB,KAAK,oBAAoB;AAAA,EACtE;AAEA,SAAO,EAAE,SAAS,MAAM,QAAQ,OAAO;AACzC;AAEA,eAAe,QACb,KACA,QACA,MACoB;AACpB,QAAM,WAAW,GAAG,IAAI,KAAK,GAAG,UAAU;AAC1C,QAAM,WAAW,MAAM,OAAO,aAAa,UAAU,IAAI,KAAK;AAE9D,MAAI,SAAS,WAAW,GAAG;AACzB,SAAK,IAAI,kBAAkB,QAAQ,GAAG;AACtC,WAAO,EAAE,SAAS,QAAQ,SAAS,EAAE;AAAA,EACvC;AAEA,OAAK,IAAI,iBAAiB,IAAI,KAAK,oBAAoB,QAAQ,GAAG;AAClE,MAAI,UAAU;AACd,aAAW,OAAO,UAAU;AAC1B,QAAI,WAAW,IAAI,MAAO;AAC1B,SAAK,IAAI,EAAE;AACX,SAAK;AAAA,MACH,iBAAY,IAAI,MAAM,mBAAgB,IAAI,SAAS,SAAM,IAAI,SAAS;AAAA,IACxE;AACA,UAAM,aAAa,OAAO,QAAQ,IAAI,OAAO,EAC1C,OAAO,CAAC,CAAC,CAAC,MAAM,EAAE,WAAW,QAAQ,CAAC,EACtC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;AACxC,eAAW,CAAC,GAAG,CAAC,KAAK,YAAY;AAC/B,WAAK,IAAI,OAAO,CAAC,KAAK,SAAS,OAAO,CAAC,GAAG,GAAG,CAAC,EAAE;AAAA,IAClD;AACA,SAAK,IAAI,cAAc,SAAS,KAAK,UAAU,IAAI,OAAO,CAAC,CAAC,EAAE;AAC9D;AAAA,EACF;AAEA,OAAK,IAAI,EAAE;AACX,OAAK,IAAI,WAAW,OAAO,cAAc;AACzC,SAAO,EAAE,SAAS,QAAQ,QAAQ;AACpC;AAEA,eAAe,UACb,KACA,QACA,MACoB;AACpB,QAAM,UAA4B;AAAA,IAChC,QAAQ,IAAI;AAAA,IACZ,eAAe,IAAI;AAAA,EACrB;AACA,MAAI,IAAI,WAAW,OAAW,SAAQ,cAAc,IAAI;AAExD,QAAM,OAAO,IAAI,gBAAgB,SAAS;AAC1C,QAAM,aAAa,IAAI,SAAS,WAAM,IAAI,MAAM,KAAK;AACrD,OAAK;AAAA,IACH,aAAa,IAAI,KAAK,GAAG,UAAU,GAAG,UAAU,KAAK,IAAI,GACvD,IAAI,SAAS,cAAc,EAC7B;AAAA,EACF;AAEA,QAAM,EAAE,UAAU,QAAQ,IAAI,MAAM,OAAO,UAAU,IAAI,OAAO,OAAO;AAEvE,OAAK;AAAA,IACH,IAAI,SACA,YAAY,QAAQ,kCAAkC,OAAO,cAC7D,YAAY,QAAQ,gBAAgB,OAAO;AAAA,EACjD;AACA,SAAO,EAAE,SAAS,UAAU,UAAU,SAAS,QAAQ,IAAI,OAAO;AACpE;;;ACraA,IAAM,YAAY,WAAW,QAAQ,GAAG;AACxC,IAAM,WAAW,GAAG,SAAS;AAO7B,SAAS,iBAAiB,SAAiC;AACzD,QAAM,YAA4B,IAAI,mBAAmB,WAAW,OAAO;AAC3E,QAAM,QAAQ,IAAI;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA,EAAE,WAAW,kBAAkB,OAAO,eAAe,MAAM;AAAA,EAC7D;AACA,QAAM,QAAQ,UAAU,MAAM;AAC9B,MAAI,iBAAiB;AAErB,iBAAe,cAAc;AAC3B,QAAI,CAAC,gBAAgB;AACnB,YAAM,MAAM,QAAQ;AACpB,uBAAiB;AAAA,IACnB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,MAAM,aAAgC;AACpC,YAAM,SAAS,MAAM,MAAM,YAAY;AACvC,UAAI,OAAO,WAAW,QAAQ;AAC5B,cAAM,IAAI,MAAM,uBAAuB,OAAO,KAAK,EAAE;AAAA,MACvD;AACA,aAAO,OAAO;AAAA,IAChB;AAAA,IAEA,MAAM,kBAAkB,OAA+C;AACrE,YAAM,YAAY;AAClB,aAAO,MAAM,kBAAkB,KAAK;AAAA,IACtC;AAAA,IAEA,MAAM,aACJ,UACA,OACwC;AACxC,YAAM,YAA2C,CAAC;AAElD,YAAM,WAAW,MAAM,QAAQ,UAAmB;AAAA,QAChD,SAAS,GAAG,QAAQ,aAAa,KAAK,IAAI,CAAC;AAAA,QAC3C,eAAe;AAAA,MACjB,CAAC;AAGD,YAAM,YAAY;AAClB,YAAM,aAAa,MAAM,MAAM,kBAAkB,QAAQ;AACzD,YAAM,YAAY,WAAW;AAAA,QAC3B,CAAC,KAAK,MAAM,MAAM,KAAK,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,OAAO,EAAE,GAAG,CAAC;AAAA,QAC5D;AAAA,MACF;AACA,YAAM,SAAS,KAAK,IAAI,OAAO,SAAS;AACxC,UAAI,WAAW,GAAG;AAChB,cAAM,SAAS,SAAS;AACxB,eAAO;AAAA,MACT;AACA,UAAI;AACF,yBAAiB,OAAO,UAAU;AAChC,oBAAU,KAAK,GAA6B;AAC5C,cAAI,UAAU,UAAU,OAAQ;AAAA,QAClC;AAAA,MACF,UAAE;AACA,cAAM,SAAS,SAAS;AAAA,MAC1B;AACA,aAAO;AAAA,IACT;AAAA,IAEA,UAAU,OAAO,SAAS;AACxB,aAAO,MAAM,UAAU,OAAO,OAAO;AAAA,IACvC;AAAA,IAEA,MAAM,QAAuB;AAC3B,UAAI,gBAAgB;AAClB,cAAM,MAAM,WAAW,EAAE,MAAM,MAAM;AAAA,QAAC,CAAC;AAAA,MACzC;AACA,YAAM,MAAM,WAAW,EAAE,MAAM,MAAM;AAAA,MAAC,CAAC;AAAA,IACzC;AAAA,EACF;AACF;AAEA,eAAe,OAAwB;AACrC,MAAI;AACJ,MAAI;AACF,UAAM,UAAU,QAAQ,KAAK,MAAM,CAAC,CAAC;AAAA,EACvC,SAAS,KAAK;AACZ,QAAI,eAAe,eAAe;AAChC,cAAQ,OAAO,MAAM,UAAU,IAAI,OAAO;AAAA;AAAA,CAAM;AAChD,cAAQ,OAAO,MAAM,KAAK;AAC1B,aAAO;AAAA,IACT;AACA,UAAM;AAAA,EACR;AAEA,MAAI;AACF,UAAM,cAAc,KAAK;AAAA,MACvB,cAAc;AAAA,MACd,KAAK,CAAC,SAAS,QAAQ,OAAO,MAAM,GAAG,IAAI;AAAA,CAAI;AAAA,IACjD,CAAC;AACD,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,UAAM,UAAU,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC/D,YAAQ,OAAO,MAAM,UAAU,OAAO;AAAA,CAAI;AAC1C,WAAO;AAAA,EACT;AACF;AAEA,KAAK,EACF,KAAK,CAAC,SAAS;AACd,UAAQ,WAAW;AACrB,CAAC,EACA,MAAM,CAAC,QAAQ;AACd,UAAQ,OAAO,MAAM,UAAU,KAAK,SAAS,GAAG;AAAA,CAAI;AACpD,UAAQ,WAAW;AACrB,CAAC;","names":[]}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import type { TTopicMessageMap } from "../types/common";
|
|
2
|
+
import type { KafkaClientOptions } from "../types/config.types";
|
|
3
|
+
import type { ConsumerOptions } from "../types/consumer.types";
|
|
4
|
+
/**
|
|
5
|
+
* Structured result of {@link kafkaClientConfigFromEnv}.
|
|
6
|
+
*
|
|
7
|
+
* The three positional `KafkaClient` constructor arguments (`clientId`,
|
|
8
|
+
* `groupId`, `brokers`) are surfaced separately from {@link options} so callers
|
|
9
|
+
* can supply their own fallbacks with `??`:
|
|
10
|
+
*
|
|
11
|
+
* ```ts
|
|
12
|
+
* const { clientId, groupId, brokers, options } = kafkaClientConfigFromEnv();
|
|
13
|
+
* const kafka = new KafkaClient(
|
|
14
|
+
* clientId ?? 'my-svc',
|
|
15
|
+
* groupId ?? 'my-grp',
|
|
16
|
+
* brokers ?? ['localhost:9092'],
|
|
17
|
+
* options,
|
|
18
|
+
* );
|
|
19
|
+
* ```
|
|
20
|
+
*
|
|
21
|
+
* Any of `clientId`, `groupId`, or `brokers` is `undefined` when its variable is
|
|
22
|
+
* unset, so the caller's fallback (not an empty string / empty array) is used.
|
|
23
|
+
* {@link options} only carries the keys whose variables were present — unset
|
|
24
|
+
* variables never appear, so library defaults and any code-level values you spread
|
|
25
|
+
* afterwards remain untouched.
|
|
26
|
+
*/
|
|
27
|
+
export interface EnvClientConfig {
|
|
28
|
+
/** `KAFKA_CLIENT_ID` — first `KafkaClient` constructor argument. */
|
|
29
|
+
clientId?: string;
|
|
30
|
+
/** `KAFKA_GROUP_ID` — second `KafkaClient` constructor argument. */
|
|
31
|
+
groupId?: string;
|
|
32
|
+
/** `KAFKA_BROKERS` (comma-separated) — third `KafkaClient` constructor argument. */
|
|
33
|
+
brokers?: string[];
|
|
34
|
+
/** Partial {@link KafkaClientOptions} — only keys whose env vars were present. */
|
|
35
|
+
options: KafkaClientOptions;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Build partial `KafkaClient` constructor arguments from environment variables.
|
|
39
|
+
*
|
|
40
|
+
* **Precedence — read this carefully.** These helpers only *feed* the constructor
|
|
41
|
+
* arguments; they never override anything. The effective precedence is:
|
|
42
|
+
*
|
|
43
|
+
* > **explicit code options > env vars > built-in library defaults**
|
|
44
|
+
*
|
|
45
|
+
* Env vars fill in what the developer did not hard-code; anything you spread
|
|
46
|
+
* *after* `...options` wins over the env value, and any key absent from the
|
|
47
|
+
* environment leaves the library default in place. The library never reads a
|
|
48
|
+
* `.env` file itself — load one with `node --env-file=.env` (Node 20.6+) or the
|
|
49
|
+
* `dotenv` package **before** constructing the client.
|
|
50
|
+
*
|
|
51
|
+
* ### Supported variables (default prefix `KAFKA_`)
|
|
52
|
+
*
|
|
53
|
+
* | Variable | Type | Maps to |
|
|
54
|
+
* |---|---|---|
|
|
55
|
+
* | `CLIENT_ID` | string | `clientId` (arg) |
|
|
56
|
+
* | `GROUP_ID` | string | `groupId` (arg) |
|
|
57
|
+
* | `BROKERS` | list | `brokers` (arg) |
|
|
58
|
+
* | `AUTO_CREATE_TOPICS` | boolean | `autoCreateTopics` |
|
|
59
|
+
* | `STRICT_SCHEMAS` | boolean | `strictSchemas` |
|
|
60
|
+
* | `NUM_PARTITIONS` | number | `numPartitions` |
|
|
61
|
+
* | `TRANSACTIONAL_ID` | string | `transactionalId` |
|
|
62
|
+
* | `CLOCK_RECOVERY_TOPICS` | list | `clockRecovery.topics` |
|
|
63
|
+
* | `CLOCK_RECOVERY_TIMEOUT_MS` | number | `clockRecovery.timeoutMs` |
|
|
64
|
+
* | `LAG_THROTTLE_MAX_LAG` | number | `lagThrottle.maxLag` (enables the block) |
|
|
65
|
+
* | `LAG_THROTTLE_GROUP_ID` | string | `lagThrottle.groupId` |
|
|
66
|
+
* | `LAG_THROTTLE_POLL_INTERVAL_MS` | number | `lagThrottle.pollIntervalMs` |
|
|
67
|
+
* | `LAG_THROTTLE_MAX_WAIT_MS` | number | `lagThrottle.maxWaitMs` |
|
|
68
|
+
* | `SSL` | boolean | `security.ssl` |
|
|
69
|
+
* | `SASL_MECHANISM` | `plain`\|`scram-sha-256`\|`scram-sha-512` | `security.sasl.mechanism` |
|
|
70
|
+
* | `SASL_USERNAME` | string | `security.sasl.username` |
|
|
71
|
+
* | `SASL_PASSWORD` | string | `security.sasl.password` |
|
|
72
|
+
* | `ALLOW_INSECURE` | boolean | `security.allowInsecure` |
|
|
73
|
+
*
|
|
74
|
+
* `lagThrottle` is only present when `LAG_THROTTLE_MAX_LAG` is set (it is the
|
|
75
|
+
* only required field). `clockRecovery` is only present when
|
|
76
|
+
* `CLOCK_RECOVERY_TOPICS` is set. `security` is only present when at least one of
|
|
77
|
+
* its variables is set.
|
|
78
|
+
*
|
|
79
|
+
* **`oauthbearer` cannot come from env.** Token providers
|
|
80
|
+
* (`awsMskIamProvider`, `gcpAccessTokenProvider`, custom async factories) are
|
|
81
|
+
* functions — configure them in code, not via environment variables. Only the
|
|
82
|
+
* username/password SASL mechanisms are env-configurable.
|
|
83
|
+
*
|
|
84
|
+
* @param env Environment source. Defaults to `process.env`. Pass an explicit
|
|
85
|
+
* object in tests to avoid mutating the real environment.
|
|
86
|
+
* @param prefix Variable prefix. Defaults to `"KAFKA_"`.
|
|
87
|
+
*
|
|
88
|
+
* @throws {Error} when a boolean/number/enum value is malformed (the message
|
|
89
|
+
* names the offending variable).
|
|
90
|
+
*
|
|
91
|
+
* @example Basic usage with fallbacks — code always wins / extends
|
|
92
|
+
* ```ts
|
|
93
|
+
* import { KafkaClient } from '@drarzter/kafka-client/core';
|
|
94
|
+
* import { kafkaClientConfigFromEnv } from '@drarzter/kafka-client/core';
|
|
95
|
+
*
|
|
96
|
+
* const { clientId, groupId, brokers, options } = kafkaClientConfigFromEnv();
|
|
97
|
+
* const kafka = new KafkaClient(
|
|
98
|
+
* clientId ?? 'my-svc',
|
|
99
|
+
* groupId ?? 'my-grp',
|
|
100
|
+
* brokers ?? ['localhost:9092'],
|
|
101
|
+
* {
|
|
102
|
+
* ...options,
|
|
103
|
+
* onMessageLost: alerting, // code-level value: not env-configurable, always applied
|
|
104
|
+
* },
|
|
105
|
+
* );
|
|
106
|
+
* ```
|
|
107
|
+
*/
|
|
108
|
+
export declare function kafkaClientConfigFromEnv(env?: NodeJS.ProcessEnv, prefix?: string): EnvClientConfig;
|
|
109
|
+
/**
|
|
110
|
+
* Build a partial {@link ConsumerOptions} from environment variables.
|
|
111
|
+
*
|
|
112
|
+
* Intended as a base layer of defaults that you merge with code-level per-consumer
|
|
113
|
+
* options via {@link mergeConsumerOptions}. As with {@link kafkaClientConfigFromEnv},
|
|
114
|
+
* unset variables emit no key, and env values never override code — merge order
|
|
115
|
+
* decides precedence.
|
|
116
|
+
*
|
|
117
|
+
* ### Supported variables (default prefix `KAFKA_CONSUMER_`)
|
|
118
|
+
*
|
|
119
|
+
* | Variable | Type | Maps to |
|
|
120
|
+
* |---|---|---|
|
|
121
|
+
* | `GROUP_ID` | string | `groupId` |
|
|
122
|
+
* | `FROM_BEGINNING` | boolean | `fromBeginning` |
|
|
123
|
+
* | `AUTO_COMMIT` | boolean | `autoCommit` |
|
|
124
|
+
* | `DLQ` | boolean | `dlq` |
|
|
125
|
+
* | `RETRY_MAX_RETRIES` | number | `retry.maxRetries` (enables `retry`) |
|
|
126
|
+
* | `RETRY_BACKOFF_MS` | number | `retry.backoffMs` |
|
|
127
|
+
* | `RETRY_MAX_BACKOFF_MS` | number | `retry.maxBackoffMs` |
|
|
128
|
+
* | `RETRY_TOPICS` | boolean | `retryTopics` |
|
|
129
|
+
* | `RETRY_TOPIC_ASSIGNMENT_TIMEOUT_MS` | number | `retryTopicAssignmentTimeoutMs` |
|
|
130
|
+
* | `HANDLER_TIMEOUT_MS` | number | `handlerTimeoutMs` |
|
|
131
|
+
* | `MESSAGE_TTL_MS` | number | `messageTtlMs` |
|
|
132
|
+
* | `DEDUPLICATION_STRATEGY` | `drop`\|`dlq`\|`topic` | `deduplication.strategy` (enables `deduplication`) |
|
|
133
|
+
* | `DEDUPLICATION_TOPIC` | string | `deduplication.duplicatesTopic` |
|
|
134
|
+
* | `CIRCUIT_BREAKER_THRESHOLD` | number | `circuitBreaker.threshold` (enables `circuitBreaker`) |
|
|
135
|
+
* | `CIRCUIT_BREAKER_RECOVERY_MS` | number | `circuitBreaker.recoveryMs` |
|
|
136
|
+
* | `CIRCUIT_BREAKER_WINDOW_SIZE` | number | `circuitBreaker.windowSize` |
|
|
137
|
+
* | `CIRCUIT_BREAKER_HALF_OPEN_SUCCESSES` | number | `circuitBreaker.halfOpenSuccesses` |
|
|
138
|
+
* | `QUEUE_HIGH_WATER_MARK` | number | `queueHighWaterMark` |
|
|
139
|
+
* | `PARTITION_ASSIGNER` | `roundrobin`\|`range`\|`cooperative-sticky` | `partitionAssigner` |
|
|
140
|
+
* | `GROUP_INSTANCE_ID` | string | `groupInstanceId` |
|
|
141
|
+
* | `SUBSCRIBE_RETRY_RETRIES` | number | `subscribeRetry.retries` (enables `subscribeRetry`) |
|
|
142
|
+
* | `SUBSCRIBE_RETRY_DELAY_MS` | number | `subscribeRetry.backoffMs` |
|
|
143
|
+
*
|
|
144
|
+
* `retry` is only present when `RETRY_MAX_RETRIES` is set (it is the only required
|
|
145
|
+
* field). `circuitBreaker` is only present when `CIRCUIT_BREAKER_THRESHOLD` is set.
|
|
146
|
+
* `deduplication` is only present when `DEDUPLICATION_STRATEGY` is set.
|
|
147
|
+
* `subscribeRetry` is only present when `SUBSCRIBE_RETRY_RETRIES` is set.
|
|
148
|
+
*
|
|
149
|
+
* @param env Environment source. Defaults to `process.env`.
|
|
150
|
+
* @param prefix Variable prefix. Defaults to `"KAFKA_CONSUMER_"`.
|
|
151
|
+
*
|
|
152
|
+
* @throws {Error} when a boolean/number/enum value is malformed.
|
|
153
|
+
*
|
|
154
|
+
* @example
|
|
155
|
+
* ```ts
|
|
156
|
+
* const envDefaults = consumerOptionsFromEnv();
|
|
157
|
+
* await kafka.startConsumer(
|
|
158
|
+
* ['orders'],
|
|
159
|
+
* handler,
|
|
160
|
+
* mergeConsumerOptions(envDefaults, { dlq: true }), // code layer wins
|
|
161
|
+
* );
|
|
162
|
+
* ```
|
|
163
|
+
*/
|
|
164
|
+
export declare function consumerOptionsFromEnv<T extends TTopicMessageMap = TTopicMessageMap>(env?: NodeJS.ProcessEnv, prefix?: string): Partial<ConsumerOptions<T>>;
|
|
165
|
+
/**
|
|
166
|
+
* Merge several partial {@link ConsumerOptions} layers into one, **later layers
|
|
167
|
+
* winning** on conflict. This is the mechanism that implements the
|
|
168
|
+
* *code > env > defaults* precedence rule — pass the env layer first and the
|
|
169
|
+
* code layer last.
|
|
170
|
+
*
|
|
171
|
+
* Top-level keys are replaced by the last defined value. The nested option
|
|
172
|
+
* objects — `retry`, `deduplication`, `circuitBreaker`, and `subscribeRetry` —
|
|
173
|
+
* are **deep-merged** so a later layer can override a single nested field (e.g.
|
|
174
|
+
* bump `retry.maxRetries`) without discarding the other fields set by an earlier
|
|
175
|
+
* layer. `undefined` values never overwrite a previously set value, and
|
|
176
|
+
* `undefined` layers are skipped entirely.
|
|
177
|
+
*
|
|
178
|
+
* @example Env base overridden by code
|
|
179
|
+
* ```ts
|
|
180
|
+
* const env = { retry: { maxRetries: 3, backoffMs: 500 }, dlq: false };
|
|
181
|
+
* const code = { retry: { maxRetries: 5 }, dlq: true };
|
|
182
|
+
* mergeConsumerOptions(env, code);
|
|
183
|
+
* // → { retry: { maxRetries: 5, backoffMs: 500 }, dlq: true }
|
|
184
|
+
* // maxRetries overridden, backoffMs preserved, dlq overridden
|
|
185
|
+
* ```
|
|
186
|
+
*/
|
|
187
|
+
export declare function mergeConsumerOptions<T extends TTopicMessageMap = TTopicMessageMap>(...layers: Array<Partial<ConsumerOptions<T>> | undefined>): ConsumerOptions<T>;
|
|
188
|
+
//# sourceMappingURL=from-env.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"from-env.d.ts","sourceRoot":"","sources":["../../../src/client/config/from-env.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,KAAK,EACV,eAAe,EAKhB,MAAM,yBAAyB,CAAC;AAMjC;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,WAAW,eAAe;IAC9B,oEAAoE;IACpE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oEAAoE;IACpE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oFAAoF;IACpF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,kFAAkF;IAClF,OAAO,EAAE,kBAAkB,CAAC;CAC7B;AAmFD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsEG;AACH,wBAAgB,wBAAwB,CACtC,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,MAAM,SAAW,GAChB,eAAe,CAsEjB;AA+ED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsDG;AACH,wBAAgB,sBAAsB,CACpC,CAAC,SAAS,gBAAgB,GAAG,gBAAgB,EAE7C,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,MAAM,SAAoB,GACzB,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAiI7B;AAoBD;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,oBAAoB,CAClC,CAAC,SAAS,gBAAgB,GAAG,gBAAgB,EAE7C,GAAG,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,GACxD,eAAe,CAAC,CAAC,CAAC,CAwBpB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/client/config/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC"}
|