@drarzter/kafka-client 0.9.4 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +625 -8
- package/dist/chunk-CMO7SMVK.mjs +4814 -0
- package/dist/chunk-CMO7SMVK.mjs.map +1 -0
- package/dist/cli/dlq.d.ts +119 -0
- package/dist/cli/dlq.d.ts.map +1 -0
- package/dist/cli/index.d.ts +3 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/{chunk-SM4FZKAZ.mjs → cli/index.js} +964 -264
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/index.mjs +355 -0
- package/dist/cli/index.mjs.map +1 -0
- package/dist/client/config/from-env.d.ts +188 -0
- package/dist/client/config/from-env.d.ts.map +1 -0
- package/dist/client/config/index.d.ts +2 -0
- package/dist/client/config/index.d.ts.map +1 -0
- package/dist/client/errors.d.ts +67 -0
- package/dist/client/errors.d.ts.map +1 -0
- package/dist/client/kafka.client/admin/ops.d.ts +114 -0
- package/dist/client/kafka.client/admin/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts +24 -0
- package/dist/client/kafka.client/consumer/features/delayed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts +52 -0
- package/dist/client/kafka.client/consumer/features/dlq-replay.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts +4 -0
- package/dist/client/kafka.client/consumer/features/routed.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts +10 -0
- package/dist/client/kafka.client/consumer/features/snapshot.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts +5 -0
- package/dist/client/kafka.client/consumer/features/window.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/handler.d.ts +149 -0
- package/dist/client/kafka.client/consumer/handler.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/ops.d.ts +51 -0
- package/dist/client/kafka.client/consumer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts +167 -0
- package/dist/client/kafka.client/consumer/pipeline.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/queue.d.ts +37 -0
- package/dist/client/kafka.client/consumer/queue.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts +65 -0
- package/dist/client/kafka.client/consumer/retry-topic.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/setup.d.ts +63 -0
- package/dist/client/kafka.client/consumer/setup.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/start.d.ts +7 -0
- package/dist/client/kafka.client/consumer/start.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/stop.d.ts +19 -0
- package/dist/client/kafka.client/consumer/stop.d.ts.map +1 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts +4 -0
- package/dist/client/kafka.client/consumer/subscribe-retry.d.ts.map +1 -0
- package/dist/client/kafka.client/context.d.ts +72 -0
- package/dist/client/kafka.client/context.d.ts.map +1 -0
- package/dist/client/kafka.client/index.d.ts +155 -0
- package/dist/client/kafka.client/index.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts +61 -0
- package/dist/client/kafka.client/infra/circuit-breaker.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts +28 -0
- package/dist/client/kafka.client/infra/dedup.store.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts +22 -0
- package/dist/client/kafka.client/infra/inflight.tracker.d.ts.map +1 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts +67 -0
- package/dist/client/kafka.client/infra/metrics.manager.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts +41 -0
- package/dist/client/kafka.client/producer/lifecycle.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/ops.d.ts +70 -0
- package/dist/client/kafka.client/producer/ops.d.ts.map +1 -0
- package/dist/client/kafka.client/producer/send.d.ts +21 -0
- package/dist/client/kafka.client/producer/send.d.ts.map +1 -0
- package/dist/client/kafka.client/validate-options.d.ts +11 -0
- package/dist/client/kafka.client/validate-options.d.ts.map +1 -0
- package/dist/client/message/envelope.d.ts +105 -0
- package/dist/client/message/envelope.d.ts.map +1 -0
- package/dist/client/message/schema-registry.d.ts +105 -0
- package/dist/client/message/schema-registry.d.ts.map +1 -0
- package/dist/client/message/topic.d.ts +138 -0
- package/dist/client/message/topic.d.ts.map +1 -0
- package/dist/client/message/versioned-schema.d.ts +53 -0
- package/dist/client/message/versioned-schema.d.ts.map +1 -0
- package/dist/client/outbox/index.d.ts +4 -0
- package/dist/client/outbox/index.d.ts.map +1 -0
- package/dist/client/outbox/outbox.relay.d.ts +90 -0
- package/dist/client/outbox/outbox.relay.d.ts.map +1 -0
- package/dist/client/outbox/outbox.store.d.ts +42 -0
- package/dist/client/outbox/outbox.store.d.ts.map +1 -0
- package/dist/client/outbox/outbox.types.d.ts +144 -0
- package/dist/client/outbox/outbox.types.d.ts.map +1 -0
- package/dist/client/security/acl.d.ts +108 -0
- package/dist/client/security/acl.d.ts.map +1 -0
- package/dist/client/security/index.d.ts +5 -0
- package/dist/client/security/index.d.ts.map +1 -0
- package/dist/client/security/providers.d.ts +88 -0
- package/dist/client/security/providers.d.ts.map +1 -0
- package/dist/client/security/resolve-security.d.ts +19 -0
- package/dist/client/security/resolve-security.d.ts.map +1 -0
- package/dist/client/security/security.types.d.ts +76 -0
- package/dist/client/security/security.types.d.ts.map +1 -0
- package/dist/client/transport/confluent.transport.d.ts +32 -0
- package/dist/client/transport/confluent.transport.d.ts.map +1 -0
- package/dist/client/transport/transport.interface.d.ts +216 -0
- package/dist/client/transport/transport.interface.d.ts.map +1 -0
- package/dist/client/types/admin.interface.d.ts +174 -0
- package/dist/client/types/admin.interface.d.ts.map +1 -0
- package/dist/client/types/admin.types.d.ts +140 -0
- package/dist/client/types/admin.types.d.ts.map +1 -0
- package/dist/client/types/client.d.ts +21 -0
- package/dist/client/types/client.d.ts.map +1 -0
- package/dist/client/types/common.d.ts +84 -0
- package/dist/client/types/common.d.ts.map +1 -0
- package/dist/client/types/config.types.d.ts +150 -0
- package/dist/client/types/config.types.d.ts.map +1 -0
- package/dist/client/types/consumer.interface.d.ts +115 -0
- package/dist/client/types/consumer.interface.d.ts.map +1 -0
- package/dist/{consumer.types-fFCag3VJ.d.mts → client/types/consumer.types.d.ts} +62 -383
- package/dist/client/types/consumer.types.d.ts.map +1 -0
- package/dist/client/types/dedup.types.d.ts +50 -0
- package/dist/client/types/dedup.types.d.ts.map +1 -0
- package/dist/client/types/lifecycle.interface.d.ts +72 -0
- package/dist/client/types/lifecycle.interface.d.ts.map +1 -0
- package/dist/client/types/producer.interface.d.ts +52 -0
- package/dist/client/types/producer.interface.d.ts.map +1 -0
- package/dist/client/types/producer.types.d.ts +90 -0
- package/dist/client/types/producer.types.d.ts.map +1 -0
- package/dist/client/types.d.ts +8 -0
- package/dist/client/types.d.ts.map +1 -0
- package/dist/core.d.ts +10 -314
- package/dist/core.d.ts.map +1 -0
- package/dist/core.js +1325 -73
- package/dist/core.js.map +1 -1
- package/dist/core.mjs +39 -3
- package/dist/index.d.ts +7 -128
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1342 -73
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +56 -3
- package/dist/index.mjs.map +1 -1
- package/dist/nest/kafka.constants.d.ts +5 -0
- package/dist/nest/kafka.constants.d.ts.map +1 -0
- package/dist/nest/kafka.decorator.d.ts +49 -0
- package/dist/nest/kafka.decorator.d.ts.map +1 -0
- package/dist/nest/kafka.explorer.d.ts +17 -0
- package/dist/nest/kafka.explorer.d.ts.map +1 -0
- package/dist/nest/kafka.health.d.ts +7 -0
- package/dist/nest/kafka.health.d.ts.map +1 -0
- package/dist/nest/kafka.module.d.ts +61 -0
- package/dist/nest/kafka.module.d.ts.map +1 -0
- package/dist/otel.d.ts +83 -5
- package/dist/otel.d.ts.map +1 -0
- package/dist/otel.js +100 -6
- package/dist/otel.js.map +1 -1
- package/dist/otel.mjs +98 -5
- package/dist/otel.mjs.map +1 -1
- package/dist/testing/client.mock.d.ts +47 -0
- package/dist/testing/client.mock.d.ts.map +1 -0
- package/dist/testing/index.d.ts +4 -0
- package/dist/testing/index.d.ts.map +1 -0
- package/dist/testing/test.container.d.ts +63 -0
- package/dist/testing/test.container.d.ts.map +1 -0
- package/dist/{testing.d.mts → testing/transport.fake.d.ts} +7 -111
- package/dist/testing/transport.fake.d.ts.map +1 -0
- package/dist/testing.d.ts +2 -318
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +26 -0
- package/dist/testing.js.map +1 -1
- package/dist/testing.mjs +26 -0
- package/dist/testing.mjs.map +1 -1
- package/package.json +21 -8
- package/dist/chunk-SM4FZKAZ.mjs.map +0 -1
- package/dist/client-1irhGEu0.d.mts +0 -751
- package/dist/client-BpFjkHhr.d.ts +0 -751
- package/dist/consumer.types-fFCag3VJ.d.ts +0 -958
- package/dist/core.d.mts +0 -314
- package/dist/index.d.mts +0 -128
- package/dist/otel.d.mts +0 -27
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Coerce an unknown thrown value to an `Error` instance.
|
|
3
|
+
* Returns the value as-is if it is already an `Error`; otherwise wraps it with `String(error)`.
|
|
4
|
+
* @param error The value caught in a `catch` clause.
|
|
5
|
+
* @returns A guaranteed `Error` instance.
|
|
6
|
+
*/
|
|
7
|
+
export declare function toError(error: unknown): Error;
|
|
8
|
+
/**
|
|
9
|
+
* Error thrown when a consumer message handler fails.
|
|
10
|
+
* @example
|
|
11
|
+
* ```ts
|
|
12
|
+
* await kafka.startConsumer(['orders'], async (envelope) => {
|
|
13
|
+
* try { await process(envelope); }
|
|
14
|
+
* catch (err) {
|
|
15
|
+
* if (err instanceof KafkaProcessingError) {
|
|
16
|
+
* console.error(err.topic, err.originalMessage);
|
|
17
|
+
* }
|
|
18
|
+
* }
|
|
19
|
+
* });
|
|
20
|
+
* ```
|
|
21
|
+
*/
|
|
22
|
+
export declare class KafkaProcessingError extends Error {
|
|
23
|
+
readonly topic: string;
|
|
24
|
+
readonly originalMessage: unknown;
|
|
25
|
+
readonly cause?: Error;
|
|
26
|
+
constructor(message: string, topic: string, originalMessage: unknown, options?: {
|
|
27
|
+
cause?: Error;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Error thrown when schema validation fails on send or consume.
|
|
32
|
+
* @example
|
|
33
|
+
* ```ts
|
|
34
|
+
* try { await kafka.sendMessage('orders.created', invalidPayload); }
|
|
35
|
+
* catch (err) {
|
|
36
|
+
* if (err instanceof KafkaValidationError) {
|
|
37
|
+
* console.error('Validation failed for topic:', err.topic);
|
|
38
|
+
* }
|
|
39
|
+
* }
|
|
40
|
+
* ```
|
|
41
|
+
*/
|
|
42
|
+
export declare class KafkaValidationError extends Error {
|
|
43
|
+
readonly topic: string;
|
|
44
|
+
readonly originalMessage: unknown;
|
|
45
|
+
readonly cause?: Error;
|
|
46
|
+
constructor(topic: string, originalMessage: unknown, options?: {
|
|
47
|
+
cause?: Error;
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Error thrown when all retry attempts are exhausted for a message.
|
|
52
|
+
* @example
|
|
53
|
+
* ```ts
|
|
54
|
+
* const kafka = new KafkaClient(config, groupId, { onMessageLost: (ctx) => {
|
|
55
|
+
* if (ctx.error instanceof KafkaRetryExhaustedError) {
|
|
56
|
+
* console.error(`Exhausted after ${ctx.error.attempts} attempts on ${ctx.error.topic}`);
|
|
57
|
+
* }
|
|
58
|
+
* }});
|
|
59
|
+
* ```
|
|
60
|
+
*/
|
|
61
|
+
export declare class KafkaRetryExhaustedError extends KafkaProcessingError {
|
|
62
|
+
readonly attempts: number;
|
|
63
|
+
constructor(topic: string, originalMessage: unknown, attempts: number, options?: {
|
|
64
|
+
cause?: Error;
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/client/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAE7C;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAK3B,KAAK,EAAE,MAAM;aACb,eAAe,EAAE,OAAO;IAL1C,SAAiB,KAAK,CAAC,EAAE,KAAK,CAAC;gBAG7B,OAAO,EAAE,MAAM,EACC,KAAK,EAAE,MAAM,EACb,eAAe,EAAE,OAAO,EACxC,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,CAAA;KAAE;CAM9B;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAI3B,KAAK,EAAE,MAAM;aACb,eAAe,EAAE,OAAO;IAJ1C,SAAiB,KAAK,CAAC,EAAE,KAAK,CAAC;gBAGb,KAAK,EAAE,MAAM,EACb,eAAe,EAAE,OAAO,EACxC,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,CAAA;KAAE;CAM9B;AAED;;;;;;;;;;GAUG;AACH,qBAAa,wBAAyB,SAAQ,oBAAoB;aAI9C,QAAQ,EAAE,MAAM;gBAFhC,KAAK,EAAE,MAAM,EACb,eAAe,EAAE,OAAO,EACR,QAAQ,EAAE,MAAM,EAChC,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,CAAA;KAAE;CAU9B"}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
import type { IAdmin } from "../../transport/transport.interface";
|
|
2
|
+
type Admin = IAdmin;
|
|
3
|
+
import type { ClientId, ConsumerGroupSummary, KafkaHealthResult, KafkaLogger, TopicDescription } from "../../types";
|
|
4
|
+
export type AdminOpsDeps = {
|
|
5
|
+
admin: Admin;
|
|
6
|
+
logger: KafkaLogger;
|
|
7
|
+
runningConsumers: Map<string, "eachMessage" | "eachBatch">;
|
|
8
|
+
defaultGroupId: string;
|
|
9
|
+
clientId: ClientId;
|
|
10
|
+
};
|
|
11
|
+
export declare class AdminOps {
|
|
12
|
+
private readonly deps;
|
|
13
|
+
private isConnected;
|
|
14
|
+
constructor(deps: AdminOpsDeps);
|
|
15
|
+
/** Underlying admin client — used by index.ts for topic validation. */
|
|
16
|
+
get admin(): Admin;
|
|
17
|
+
/** Whether the admin client is currently connected. */
|
|
18
|
+
get connected(): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Connect the admin client if not already connected.
|
|
21
|
+
* The flag is only set to `true` after a successful connect — if `admin.connect()`
|
|
22
|
+
* throws the flag remains `false` so the next call will retry the connection.
|
|
23
|
+
*/
|
|
24
|
+
ensureConnected(): Promise<void>;
|
|
25
|
+
/** Disconnect admin if connected. Resets the connected flag. */
|
|
26
|
+
disconnect(): Promise<void>;
|
|
27
|
+
resetOffsets(groupId: string | undefined, topic: string, position: "earliest" | "latest"): Promise<void>;
|
|
28
|
+
/**
|
|
29
|
+
* Seek specific topic-partition pairs to explicit offsets for a stopped consumer group.
|
|
30
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
31
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
32
|
+
*/
|
|
33
|
+
seekToOffset(groupId: string | undefined, assignments: Array<{
|
|
34
|
+
topic: string;
|
|
35
|
+
partition: number;
|
|
36
|
+
offset: string;
|
|
37
|
+
}>): Promise<void>;
|
|
38
|
+
/**
|
|
39
|
+
* Seek specific topic-partition pairs to the offset nearest to a given timestamp
|
|
40
|
+
* (in milliseconds) for a stopped consumer group.
|
|
41
|
+
* Throws if the group is still running — call `stopConsumer(groupId)` first.
|
|
42
|
+
* Assignments are grouped by topic and committed via `admin.setOffsets`.
|
|
43
|
+
* If no offset exists at the requested timestamp (e.g. empty partition or
|
|
44
|
+
* future timestamp), the partition falls back to `-1` (end of topic — new messages only).
|
|
45
|
+
*/
|
|
46
|
+
seekToTimestamp(groupId: string | undefined, assignments: Array<{
|
|
47
|
+
topic: string;
|
|
48
|
+
partition: number;
|
|
49
|
+
timestamp: number;
|
|
50
|
+
}>): Promise<void>;
|
|
51
|
+
/**
|
|
52
|
+
* Query consumer group lag per partition.
|
|
53
|
+
* Lag = broker high-watermark − last committed offset.
|
|
54
|
+
* A committed offset of -1 (nothing committed yet) counts as full lag.
|
|
55
|
+
*
|
|
56
|
+
* Returns an empty array when the consumer group has never committed any
|
|
57
|
+
* offsets (freshly created group, `autoCommit: false` with no manual commits,
|
|
58
|
+
* or group not yet assigned). This is a Kafka protocol limitation:
|
|
59
|
+
* `fetchOffsets` only returns data for topic-partitions that have at least one
|
|
60
|
+
* committed offset. Use `checkStatus()` to verify broker connectivity in that case.
|
|
61
|
+
*/
|
|
62
|
+
getConsumerLag(groupId?: string): Promise<Array<{
|
|
63
|
+
topic: string;
|
|
64
|
+
partition: number;
|
|
65
|
+
lag: number;
|
|
66
|
+
}>>;
|
|
67
|
+
/** Check broker connectivity. Never throws — returns a discriminated union. */
|
|
68
|
+
checkStatus(): Promise<KafkaHealthResult>;
|
|
69
|
+
/**
|
|
70
|
+
* List all consumer groups known to the broker.
|
|
71
|
+
* Useful for monitoring which groups are active and their current state.
|
|
72
|
+
*/
|
|
73
|
+
listConsumerGroups(): Promise<ConsumerGroupSummary[]>;
|
|
74
|
+
/**
|
|
75
|
+
* Describe topics — returns partition layout, leader, replicas, and ISR.
|
|
76
|
+
* @param topics Topic names to describe. Omit to describe all topics.
|
|
77
|
+
*/
|
|
78
|
+
describeTopics(topics?: string[]): Promise<TopicDescription[]>;
|
|
79
|
+
/**
|
|
80
|
+
* Delete consumer groups from the broker.
|
|
81
|
+
* Groups must be empty (no active members) before deletion.
|
|
82
|
+
* Silently skips unknown group IDs.
|
|
83
|
+
* @param groupIds Consumer group IDs to delete.
|
|
84
|
+
*/
|
|
85
|
+
deleteGroups(groupIds: string[]): Promise<void>;
|
|
86
|
+
/**
|
|
87
|
+
* Delete records from a topic up to (but not including) the given offsets.
|
|
88
|
+
* All messages with offsets **before** the given offset are deleted.
|
|
89
|
+
*/
|
|
90
|
+
deleteRecords(topic: string, partitions: Array<{
|
|
91
|
+
partition: number;
|
|
92
|
+
offset: string;
|
|
93
|
+
}>): Promise<void>;
|
|
94
|
+
/**
|
|
95
|
+
* When `retryTopics: true` and `autoCreateTopics: false`, verify that every
|
|
96
|
+
* `<topic>.retry.<level>` topic already exists. Throws a clear error at startup
|
|
97
|
+
* rather than silently discovering missing topics on the first handler failure.
|
|
98
|
+
*/
|
|
99
|
+
validateRetryTopicsExist(topicNames: string[], maxRetries: number): Promise<void>;
|
|
100
|
+
/**
|
|
101
|
+
* When `autoCreateTopics` is disabled, verify that `<topic>.dlq` exists for every
|
|
102
|
+
* consumed topic. Throws a clear error at startup rather than silently discovering
|
|
103
|
+
* missing DLQ topics on the first handler failure.
|
|
104
|
+
*/
|
|
105
|
+
validateDlqTopicsExist(topicNames: string[]): Promise<void>;
|
|
106
|
+
/**
|
|
107
|
+
* When `deduplication.strategy: 'topic'` and `autoCreateTopics: false`, verify
|
|
108
|
+
* that every `<topic>.duplicates` destination topic already exists. Throws a
|
|
109
|
+
* clear error at startup rather than silently dropping duplicates on first hit.
|
|
110
|
+
*/
|
|
111
|
+
validateDuplicatesTopicsExist(topicNames: string[], customDestination: string | undefined): Promise<void>;
|
|
112
|
+
}
|
|
113
|
+
export {};
|
|
114
|
+
//# sourceMappingURL=ops.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ops.d.ts","sourceRoot":"","sources":["../../../../src/client/kafka.client/admin/ops.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAClE,KAAK,KAAK,GAAG,MAAM,CAAC;AACpB,OAAO,KAAK,EACV,QAAQ,EACR,oBAAoB,EACpB,iBAAiB,EACjB,WAAW,EACX,gBAAgB,EACjB,MAAM,aAAa,CAAC;AAErB,MAAM,MAAM,YAAY,GAAG;IACzB,KAAK,EAAE,KAAK,CAAC;IACb,MAAM,EAAE,WAAW,CAAC;IACpB,gBAAgB,EAAE,GAAG,CAAC,MAAM,EAAE,aAAa,GAAG,WAAW,CAAC,CAAC;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,QAAQ,CAAC;CACpB,CAAC;AAEF,qBAAa,QAAQ;IAGP,OAAO,CAAC,QAAQ,CAAC,IAAI;IAFjC,OAAO,CAAC,WAAW,CAAS;gBAEC,IAAI,EAAE,YAAY;IAE/C,uEAAuE;IACvE,IAAI,KAAK,IAAI,KAAK,CAEjB;IAED,uDAAuD;IACvD,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED;;;;OAIG;IACG,eAAe,IAAI,OAAO,CAAC,IAAI,CAAC;IAWtC,gEAAgE;IAC1D,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAMpB,YAAY,CACvB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,UAAU,GAAG,QAAQ,GAC9B,OAAO,CAAC,IAAI,CAAC;IAoBhB;;;;OAIG;IACU,YAAY,CACvB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,WAAW,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,GACvE,OAAO,CAAC,IAAI,CAAC;IAuBhB;;;;;;;OAOG;IACU,eAAe,CAC1B,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,WAAW,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,GAC1E,OAAO,CAAC,IAAI,CAAC;IA2ChB;;;;;;;;;;OAUG;IACU,cAAc,CACzB,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IA+BpE,+EAA+E;IAClE,WAAW,IAAI,OAAO,CAAC,iBAAiB,CAAC;IActD;;;OAGG;IACU,kBAAkB,IAAI,OAAO,CAAC,oBAAoB,EAAE,CAAC;IASlE;;;OAGG;IACU,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;IAqB3E;;;;;OAKG;IACU,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM5D;;;OAGG;IACU,aAAa,CACxB,KAAK,EAAE,MAAM,EACb,UAAU,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,GACvD,OAAO,CAAC,IAAI,CAAC;IAKhB;;;;OAIG;IACU,wBAAwB,CACnC,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IAkBhB;;;;OAIG;IACU,sBAAsB,CAAC,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAcxE;;;;OAIG;IACU,6BAA6B,CACxC,UAAU,EAAE,MAAM,EAAE,EACpB,iBAAiB,EAAE,MAAM,GAAG,SAAS,GACpC,OAAO,CAAC,IAAI,CAAC;CAcjB"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { KafkaClientContext } from "../../context";
|
|
2
|
+
import type { TopicMapConstraint, ConsumerHandle } from "../../../types";
|
|
3
|
+
/** Staging topic name for delayed messages of `topic`. */
|
|
4
|
+
export declare function delayedTopicName(topic: string): string;
|
|
5
|
+
/**
|
|
6
|
+
* Start a relay consumer that delivers messages produced with
|
|
7
|
+
* `SendOptions.deliverAfterMs` from `<topic>.delayed` to their target topic
|
|
8
|
+
* once their `x-delayed-until` deadline passes.
|
|
9
|
+
*
|
|
10
|
+
* Semantics mirror the retry-topic chain:
|
|
11
|
+
* - The partition is paused while waiting for the deadline; the offset is not
|
|
12
|
+
* committed during the wait, so a crash redelivers the message (at-least-once).
|
|
13
|
+
* - Forwarding uses a Kafka transaction (produce to target + commit source
|
|
14
|
+
* offset atomically), so no duplicates are relayed even on crash.
|
|
15
|
+
* - Delivery time is a lower bound — head-of-line waiting applies per partition.
|
|
16
|
+
*
|
|
17
|
+
* The relay preserves the original key, value, and headers (minus the
|
|
18
|
+
* `x-delayed-*` control headers), so envelope metadata (`x-event-id`,
|
|
19
|
+
* `x-correlation-id`, `x-lamport-clock`, `traceparent`) survives the hop.
|
|
20
|
+
*/
|
|
21
|
+
export declare function startDelayedRelayImpl<T extends TopicMapConstraint<T>>(ctx: KafkaClientContext<T>, topics: string[], options?: {
|
|
22
|
+
groupId?: string;
|
|
23
|
+
}): Promise<ConsumerHandle>;
|
|
24
|
+
//# sourceMappingURL=delayed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"delayed.d.ts","sourceRoot":"","sources":["../../../../../src/client/kafka.client/consumer/features/delayed.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,KAAK,EAAE,kBAAkB,EAAE,cAAc,EAAkB,MAAM,gBAAgB,CAAC;AAYzF,0DAA0D;AAC1D,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEtD;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,qBAAqB,CAAC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EACzE,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAC1B,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAA;CAAE,GAC7B,OAAO,CAAC,cAAc,CAAC,CA6HzB"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import type { IConsumer } from "../../../transport/transport.interface";
|
|
2
|
+
type Consumer = IConsumer;
|
|
3
|
+
import type { KafkaLogger, DlqReplayOptions } from "../../../types";
|
|
4
|
+
/**
|
|
5
|
+
* Dependencies injected into `replayDlqTopic` by `KafkaClient`.
|
|
6
|
+
* Abstracts broker access (offset fetching, producing, consumer creation) so the
|
|
7
|
+
* replay logic can be unit-tested without a real Kafka connection.
|
|
8
|
+
*/
|
|
9
|
+
export type DlqReplayDeps = {
|
|
10
|
+
logger: KafkaLogger;
|
|
11
|
+
fetchTopicOffsets: (topic: string) => Promise<Array<{
|
|
12
|
+
partition: number;
|
|
13
|
+
low: string;
|
|
14
|
+
high: string;
|
|
15
|
+
}>>;
|
|
16
|
+
send: (topic: string, messages: Array<{
|
|
17
|
+
value: string;
|
|
18
|
+
headers: Record<string, string>;
|
|
19
|
+
}>) => Promise<void>;
|
|
20
|
+
/**
|
|
21
|
+
* Create a consumer for the given group.
|
|
22
|
+
* @param groupId Consumer group ID.
|
|
23
|
+
* @param fromBeginning When `true`, `auto.offset.reset=earliest` (no committed offsets on a temp group).
|
|
24
|
+
*/
|
|
25
|
+
createConsumer: (groupId: string, fromBeginning: boolean) => Consumer;
|
|
26
|
+
/**
|
|
27
|
+
* Disconnect the consumer and, when `deleteGroup` is `true`, delete the group
|
|
28
|
+
* from the broker (used for ephemeral temp groups that should not accumulate).
|
|
29
|
+
*/
|
|
30
|
+
cleanupConsumer: (consumer: Consumer, groupId: string, deleteGroup: boolean) => void;
|
|
31
|
+
dlqHeaderKeys: Set<string>;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Re-publish messages from a dead letter queue back to the original topic.
|
|
35
|
+
*
|
|
36
|
+
* Messages are consumed from `<topic>.dlq` and re-published to `<topic>`.
|
|
37
|
+
* The original topic is determined by the `x-dlq-original-topic` header.
|
|
38
|
+
* The `x-dlq-*` headers are stripped before re-publishing.
|
|
39
|
+
*
|
|
40
|
+
* ### Group ID strategy (driven by `options.fromBeginning`):
|
|
41
|
+
* - `fromBeginning: true` (default) — a new ephemeral group `<topic>.dlq-replay-<ts>` is used
|
|
42
|
+
* on every call so there are no committed offsets; reads all messages from the beginning
|
|
43
|
+
* every time. The group is deleted from the broker after the replay finishes.
|
|
44
|
+
* - `fromBeginning: false` — a stable group `<topic>.dlq-replay` is used; committed offsets
|
|
45
|
+
* persist between calls so only messages added since the previous call are replayed.
|
|
46
|
+
*/
|
|
47
|
+
export declare function replayDlqTopic(topic: string, deps: DlqReplayDeps, options?: DlqReplayOptions): Promise<{
|
|
48
|
+
replayed: number;
|
|
49
|
+
skipped: number;
|
|
50
|
+
}>;
|
|
51
|
+
export {};
|
|
52
|
+
//# sourceMappingURL=dlq-replay.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"dlq-replay.d.ts","sourceRoot":"","sources":["../../../../../src/client/kafka.client/consumer/features/dlq-replay.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wCAAwC,CAAC;AACxE,KAAK,QAAQ,GAAG,SAAS,CAAC;AAC1B,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAIpE;;;;GAIG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,MAAM,EAAE,WAAW,CAAC;IACpB,iBAAiB,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;IACvG,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5G;;;;OAIG;IACH,cAAc,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,OAAO,KAAK,QAAQ,CAAC;IACtE;;;OAGG;IACH,eAAe,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,KAAK,IAAI,CAAC;IACrF,aAAa,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;CAC5B,CAAC;AAEF;;;;;;;;;;;;;GAaG;AACH,wBAAsB,cAAc,CAClC,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,aAAa,EACnB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAC,CAwFhD"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { KafkaClientContext } from "../../context";
|
|
2
|
+
import type { TopicMapConstraint, ConsumerOptions, ConsumerHandle, RoutingOptions } from "../../../types";
|
|
3
|
+
export declare function startRoutedConsumerImpl<T extends TopicMapConstraint<T>, K extends Array<keyof T>>(ctx: KafkaClientContext<T>, topics: K, routing: RoutingOptions<T[K[number]]>, options?: ConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
4
|
+
//# sourceMappingURL=routed.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"routed.d.ts","sourceRoot":"","sources":["../../../../../src/client/kafka.client/consumer/features/routed.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,KAAK,EACV,kBAAkB,EAClB,eAAe,EACf,cAAc,EACd,cAAc,EACf,MAAM,gBAAgB,CAAC;AAMxB,wBAAsB,uBAAuB,CAC3C,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAC/B,CAAC,SAAS,KAAK,CAAC,MAAM,CAAC,CAAC,EAExB,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAC1B,MAAM,EAAE,CAAC,EACT,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EACrC,OAAO,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,GAC3B,OAAO,CAAC,cAAc,CAAC,CAezB"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { KafkaClientContext } from "../../context";
|
|
2
|
+
import type { TopicMapConstraint, ReadSnapshotOptions, CheckpointResult, CheckpointRestoreResult, RestoreCheckpointOptions } from "../../../types";
|
|
3
|
+
import type { EventEnvelope } from "../../../message/envelope";
|
|
4
|
+
/** Minimal context surface required by this module. */
|
|
5
|
+
type SnapshotCtx<T extends TopicMapConstraint<T>> = Pick<KafkaClientContext<T>, "adminOps" | "clientId" | "defaultGroupId" | "transport" | "logger" | "producer" | "runningConsumers">;
|
|
6
|
+
export declare function readSnapshotImpl<T extends TopicMapConstraint<T>, K extends keyof T & string>(ctx: SnapshotCtx<T>, topic: K, options?: ReadSnapshotOptions): Promise<Map<string, EventEnvelope<T[K]>>>;
|
|
7
|
+
export declare function checkpointOffsetsImpl<T extends TopicMapConstraint<T>>(ctx: SnapshotCtx<T>, groupId: string | undefined, checkpointTopic: string): Promise<CheckpointResult>;
|
|
8
|
+
export declare function restoreFromCheckpointImpl<T extends TopicMapConstraint<T>>(ctx: SnapshotCtx<T>, groupId: string | undefined, checkpointTopic: string, options?: RestoreCheckpointOptions): Promise<CheckpointRestoreResult>;
|
|
9
|
+
export {};
|
|
10
|
+
//# sourceMappingURL=snapshot.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"snapshot.d.ts","sourceRoot":"","sources":["../../../../../src/client/kafka.client/consumer/features/snapshot.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,KAAK,EACV,kBAAkB,EAClB,mBAAmB,EACnB,gBAAgB,EAChB,uBAAuB,EACvB,wBAAwB,EAEzB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAI/D,uDAAuD;AACvD,KAAK,WAAW,CAAC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,IAAI,IAAI,CACtD,kBAAkB,CAAC,CAAC,CAAC,EACnB,UAAU,GACV,UAAU,GACV,gBAAgB,GAChB,WAAW,GACX,QAAQ,GACR,UAAU,GACV,kBAAkB,CACrB,CAAC;AAIF,wBAAsB,gBAAgB,CACpC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAC/B,CAAC,SAAS,MAAM,CAAC,GAAG,MAAM,EAE1B,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,EACnB,KAAK,EAAE,CAAC,EACR,OAAO,GAAE,mBAAwB,GAChC,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CA4E3C;AAkDD,wBAAsB,qBAAqB,CAAC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EACzE,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,EACnB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,eAAe,EAAE,MAAM,GACtB,OAAO,CAAC,gBAAgB,CAAC,CAmC3B;AAID,wBAAsB,yBAAyB,CAAC,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAC7E,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC,EACnB,OAAO,EAAE,MAAM,GAAG,SAAS,EAC3B,eAAe,EAAE,MAAM,EACvB,OAAO,GAAE,wBAA6B,GACrC,OAAO,CAAC,uBAAuB,CAAC,CAmJlC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { KafkaClientContext } from "../../context";
|
|
2
|
+
import type { TopicMapConstraint, WindowConsumerOptions, WindowMeta, ConsumerHandle } from "../../../types";
|
|
3
|
+
import type { EventEnvelope } from "../../../message/envelope";
|
|
4
|
+
export declare function startWindowConsumerImpl<T extends TopicMapConstraint<T>, K extends keyof T & string>(ctx: KafkaClientContext<T>, topic: K, handler: (envelopes: EventEnvelope<T[K]>[], meta: WindowMeta) => Promise<void>, options: WindowConsumerOptions<T>): Promise<ConsumerHandle>;
|
|
5
|
+
//# sourceMappingURL=window.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"window.d.ts","sourceRoot":"","sources":["../../../../../src/client/kafka.client/consumer/features/window.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,KAAK,EACV,kBAAkB,EAClB,qBAAqB,EACrB,UAAU,EACV,cAAc,EACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAM/D,wBAAsB,uBAAuB,CAC3C,CAAC,SAAS,kBAAkB,CAAC,CAAC,CAAC,EAC/B,CAAC,SAAS,MAAM,CAAC,GAAG,MAAM,EAE1B,GAAG,EAAE,kBAAkB,CAAC,CAAC,CAAC,EAC1B,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,EAAE,UAAU,KAAK,OAAO,CAAC,IAAI,CAAC,EAC9E,OAAO,EAAE,qBAAqB,CAAC,CAAC,CAAC,GAChC,OAAO,CAAC,cAAc,CAAC,CAgFzB"}
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import type { IProducer, IConsumer } from "../../transport/transport.interface";
|
|
2
|
+
type Producer = IProducer;
|
|
3
|
+
type Consumer = IConsumer;
|
|
4
|
+
import type { EventEnvelope } from "../../message/envelope";
|
|
5
|
+
import type { SchemaLike } from "../../message/topic";
|
|
6
|
+
import type { BatchMeta, ConsumerInterceptor, DedupStore, DeduplicationOptions, DlqReason, KafkaClientOptions, KafkaInstrumentation, KafkaLogger, RetryOptions } from "../../types";
|
|
7
|
+
/**
|
|
8
|
+
* Runtime dependencies injected into message handler functions by `KafkaClient`.
|
|
9
|
+
* Holds shared infrastructure (producer, logger, instrumentation) and per-consumer
|
|
10
|
+
* lifecycle callbacks (onMessageLost, onRetry, onDlq, etc.).
|
|
11
|
+
*/
|
|
12
|
+
export type MessageHandlerDeps = {
|
|
13
|
+
logger: KafkaLogger;
|
|
14
|
+
producer: Producer;
|
|
15
|
+
instrumentation: KafkaInstrumentation[];
|
|
16
|
+
onMessageLost: KafkaClientOptions["onMessageLost"];
|
|
17
|
+
onTtlExpired?: KafkaClientOptions["onTtlExpired"];
|
|
18
|
+
onRetry?: (envelope: EventEnvelope<any>, attempt: number, maxRetries: number) => void;
|
|
19
|
+
onDlq?: (envelope: EventEnvelope<any>, reason: DlqReason) => void;
|
|
20
|
+
onDuplicate?: (envelope: EventEnvelope<any>, strategy: "drop" | "dlq" | "topic") => void;
|
|
21
|
+
onMessage?: (envelope: EventEnvelope<any>) => void;
|
|
22
|
+
/** Fired on every failed handler attempt — drives the circuit breaker. */
|
|
23
|
+
onFailure?: (envelope: EventEnvelope<any>) => void;
|
|
24
|
+
};
|
|
25
|
+
/** Active deduplication context passed from KafkaClient to the message handler. */
|
|
26
|
+
export type DeduplicationContext = {
|
|
27
|
+
options: DeduplicationOptions;
|
|
28
|
+
/** Backing store for the last processed Lamport clock (in-memory by default). */
|
|
29
|
+
store: DedupStore;
|
|
30
|
+
/** Consumer group ID this context belongs to — used as the store key. */
|
|
31
|
+
groupId: string;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Per-consumer configuration forwarded from `startConsumer` into `handleEachMessage`.
|
|
35
|
+
* Carries schema map, handler function, interceptors, retry/DLQ policy, timeout,
|
|
36
|
+
* deduplication context, TTL options, and the optional EOS routing context.
|
|
37
|
+
*/
|
|
38
|
+
export type EachMessageOpts = {
|
|
39
|
+
schemaMap: Map<string, SchemaLike>;
|
|
40
|
+
handleMessage: (envelope: EventEnvelope<any>) => Promise<void>;
|
|
41
|
+
interceptors: ConsumerInterceptor<any>[];
|
|
42
|
+
dlq: boolean;
|
|
43
|
+
retry: RetryOptions | undefined;
|
|
44
|
+
retryTopics: boolean | undefined;
|
|
45
|
+
timeoutMs: number | undefined;
|
|
46
|
+
wrapWithTimeout: <R>(fn: () => Promise<R>, ms: number, topic: string) => Promise<R>;
|
|
47
|
+
deduplication?: DeduplicationContext;
|
|
48
|
+
/** Drop messages older than this threshold (ms). See `ConsumerOptions.messageTtlMs`. */
|
|
49
|
+
messageTtlMs?: number;
|
|
50
|
+
/**
|
|
51
|
+
* Per-consumer TTL expiry callback. Takes precedence over the client-level
|
|
52
|
+
* `KafkaClientOptions.onTtlExpired` stored in `deps.onTtlExpired`.
|
|
53
|
+
*/
|
|
54
|
+
onTtlExpired?: KafkaClientOptions["onTtlExpired"];
|
|
55
|
+
/**
|
|
56
|
+
* EOS context for main consumer → retry.1 routing.
|
|
57
|
+
* When set, the main consumer runs with `autoCommit: false`. On handler failure,
|
|
58
|
+
* routing to the retry topic and the source offset commit are wrapped in a single
|
|
59
|
+
* Kafka transaction — a crash at any point rolls back the transaction, ensuring
|
|
60
|
+
* the message is not duplicated between the main topic and retry.1.
|
|
61
|
+
* On success, the offset is committed manually (no transaction needed).
|
|
62
|
+
*/
|
|
63
|
+
eosMainContext?: {
|
|
64
|
+
txProducer: Producer;
|
|
65
|
+
consumer: Consumer;
|
|
66
|
+
};
|
|
67
|
+
};
|
|
68
|
+
/** Parse, validate and extract an envelope from a single raw Kafka message. Returns null to skip. */
|
|
69
|
+
export declare function parseSingleMessage(message: {
|
|
70
|
+
value: Buffer | null;
|
|
71
|
+
headers?: Record<string, any>;
|
|
72
|
+
offset: string;
|
|
73
|
+
}, topic: string, partition: number, schemaMap: Map<string, SchemaLike>, interceptors: ConsumerInterceptor<any>[], dlq: boolean, deps: MessageHandlerDeps): Promise<EventEnvelope<any> | null>;
|
|
74
|
+
/**
|
|
75
|
+
* Core single-message processing pipeline invoked by the kafkajs `eachMessage` callback.
|
|
76
|
+
* Parses, validates, deduplicates, checks TTL, runs the handler with retry/DLQ support,
|
|
77
|
+
* and manages EOS offset commits when the main consumer runs with `autoCommit: false`.
|
|
78
|
+
* @param payload Raw kafkajs `eachMessage` payload (topic, partition, message).
|
|
79
|
+
* @param opts Consumer-level options including handler, schema map, retry policy, and EOS context.
|
|
80
|
+
* @param deps Shared infrastructure dependencies.
|
|
81
|
+
*/
|
|
82
|
+
export declare function handleEachMessage(payload: {
|
|
83
|
+
topic: string;
|
|
84
|
+
partition: number;
|
|
85
|
+
message: {
|
|
86
|
+
value: Buffer | null;
|
|
87
|
+
headers?: Record<string, any>;
|
|
88
|
+
offset: string;
|
|
89
|
+
};
|
|
90
|
+
}, opts: EachMessageOpts, deps: MessageHandlerDeps): Promise<void>;
|
|
91
|
+
/**
|
|
92
|
+
* Per-consumer configuration forwarded from `startBatchConsumer` into `handleEachBatch`.
|
|
93
|
+
* Mirrors `EachMessageOpts` but carries a batch handler and batch-level EOS context.
|
|
94
|
+
*/
|
|
95
|
+
export type EachBatchOpts = {
|
|
96
|
+
schemaMap: Map<string, SchemaLike>;
|
|
97
|
+
handleBatch: (envelopes: EventEnvelope<any>[], meta: BatchMeta) => Promise<void>;
|
|
98
|
+
interceptors: ConsumerInterceptor<any>[];
|
|
99
|
+
dlq: boolean;
|
|
100
|
+
retry: RetryOptions | undefined;
|
|
101
|
+
retryTopics: boolean | undefined;
|
|
102
|
+
timeoutMs: number | undefined;
|
|
103
|
+
wrapWithTimeout: <R>(fn: () => Promise<R>, ms: number, topic: string) => Promise<R>;
|
|
104
|
+
deduplication?: DeduplicationContext;
|
|
105
|
+
/** Drop messages older than this threshold (ms). See `ConsumerOptions.messageTtlMs`. */
|
|
106
|
+
messageTtlMs?: number;
|
|
107
|
+
/**
|
|
108
|
+
* Per-consumer TTL expiry callback. Takes precedence over the client-level
|
|
109
|
+
* `KafkaClientOptions.onTtlExpired` stored in `deps.onTtlExpired`.
|
|
110
|
+
*/
|
|
111
|
+
onTtlExpired?: KafkaClientOptions["onTtlExpired"];
|
|
112
|
+
/**
|
|
113
|
+
* EOS context for batch consumer → retry.1 routing.
|
|
114
|
+
* When set, the batch consumer runs with `autoCommit: false`.
|
|
115
|
+
* On handler failure, all messages are routed to retry.1 and the partition
|
|
116
|
+
* offset is committed atomically in a single Kafka transaction.
|
|
117
|
+
* On success, the offset is committed manually.
|
|
118
|
+
*/
|
|
119
|
+
eosMainContext?: {
|
|
120
|
+
txProducer: Producer;
|
|
121
|
+
consumer: Consumer;
|
|
122
|
+
};
|
|
123
|
+
};
|
|
124
|
+
/**
|
|
125
|
+
* Core batch processing pipeline invoked by the kafkajs `eachBatch` callback.
|
|
126
|
+
* Iterates messages, parses, validates, deduplicates, and checks TTL for each one,
|
|
127
|
+
* then calls the batch handler with the surviving envelopes and retry/DLQ support.
|
|
128
|
+
* Manages EOS offset commits when the consumer runs with `autoCommit: false`.
|
|
129
|
+
* @param payload Raw kafkajs `eachBatch` payload (batch, heartbeat, resolveOffset, commitOffsetsIfNecessary).
|
|
130
|
+
* @param opts Consumer-level options including batch handler, schema map, retry policy, and EOS context.
|
|
131
|
+
* @param deps Shared infrastructure dependencies.
|
|
132
|
+
*/
|
|
133
|
+
export declare function handleEachBatch(payload: {
|
|
134
|
+
batch: {
|
|
135
|
+
topic: string;
|
|
136
|
+
partition: number;
|
|
137
|
+
highWatermark: string;
|
|
138
|
+
messages: Array<{
|
|
139
|
+
value: Buffer | null;
|
|
140
|
+
headers?: Record<string, any>;
|
|
141
|
+
offset: string;
|
|
142
|
+
}>;
|
|
143
|
+
};
|
|
144
|
+
heartbeat(): Promise<void>;
|
|
145
|
+
resolveOffset(offset: string): void;
|
|
146
|
+
commitOffsetsIfNecessary(): Promise<void>;
|
|
147
|
+
}, opts: EachBatchOpts, deps: MessageHandlerDeps): Promise<void>;
|
|
148
|
+
export {};
|
|
149
|
+
//# sourceMappingURL=handler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"handler.d.ts","sourceRoot":"","sources":["../../../../src/client/kafka.client/consumer/handler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,qCAAqC,CAAC;AAChF,KAAK,QAAQ,GAAG,SAAS,CAAC;AAC1B,KAAK,QAAQ,GAAG,SAAS,CAAC;AAM1B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AAW5D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EACV,SAAS,EACT,mBAAmB,EACnB,UAAU,EACV,oBAAoB,EACpB,SAAS,EACT,kBAAkB,EAClB,oBAAoB,EACpB,WAAW,EACX,YAAY,EACb,MAAM,aAAa,CAAC;AAErB;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,MAAM,EAAE,WAAW,CAAC;IACpB,QAAQ,EAAE,QAAQ,CAAC;IACnB,eAAe,EAAE,oBAAoB,EAAE,CAAC;IACxC,aAAa,EAAE,kBAAkB,CAAC,eAAe,CAAC,CAAC;IACnD,YAAY,CAAC,EAAE,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAClD,OAAO,CAAC,EAAE,CACR,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,EAC5B,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,KACf,IAAI,CAAC;IACV,KAAK,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,SAAS,KAAK,IAAI,CAAC;IAClE,WAAW,CAAC,EAAE,CACZ,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,EAC5B,QAAQ,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,KAC/B,IAAI,CAAC;IACV,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;IACnD,0EAA0E;IAC1E,SAAS,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC;CACpD,CAAC;AAEF,mFAAmF;AACnF,MAAM,MAAM,oBAAoB,GAAG;IACjC,OAAO,EAAE,oBAAoB,CAAC;IAC9B,iFAAiF;IACjF,KAAK,EAAE,UAAU,CAAC;IAClB,yEAAyE;IACzE,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACnC,aAAa,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,GAAG,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/D,YAAY,EAAE,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC;IACzC,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,YAAY,GAAG,SAAS,CAAC;IAChC,WAAW,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,eAAe,EAAE,CAAC,CAAC,EACjB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,MAAM,KACV,OAAO,CAAC,CAAC,CAAC,CAAC;IAChB,aAAa,CAAC,EAAE,oBAAoB,CAAC;IACrC,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,YAAY,CAAC,EAAE,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAClD;;;;;;;OAOG;IACH,cAAc,CAAC,EAAE;QACf,UAAU,EAAE,QAAQ,CAAC;QACrB,QAAQ,EAAE,QAAQ,CAAC;KACpB,CAAC;CACH,CAAC;AAmFF,qGAAqG;AACrG,wBAAsB,kBAAkB,CACtC,OAAO,EAAE;IACP,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC9B,MAAM,EAAE,MAAM,CAAC;CAChB,EACD,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EACjB,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EAClC,YAAY,EAAE,mBAAmB,CAAC,GAAG,CAAC,EAAE,EACxC,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAuBpC;AAED;;;;;;;GAOG;AACH,wBAAsB,iBAAiB,CACrC,OAAO,EAAE;IACP,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC9B,MAAM,EAAE,MAAM,CAAC;KAChB,CAAC;CACH,EACD,IAAI,EAAE,eAAe,EACrB,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,IAAI,CAAC,CA8If;AAED;;;GAGG;AACH,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACnC,WAAW,EAAE,CACX,SAAS,EAAE,aAAa,CAAC,GAAG,CAAC,EAAE,EAC/B,IAAI,EAAE,SAAS,KACZ,OAAO,CAAC,IAAI,CAAC,CAAC;IACnB,YAAY,EAAE,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC;IACzC,GAAG,EAAE,OAAO,CAAC;IACb,KAAK,EAAE,YAAY,GAAG,SAAS,CAAC;IAChC,WAAW,EAAE,OAAO,GAAG,SAAS,CAAC;IACjC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,eAAe,EAAE,CAAC,CAAC,EACjB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,MAAM,KACV,OAAO,CAAC,CAAC,CAAC,CAAC;IAChB,aAAa,CAAC,EAAE,oBAAoB,CAAC;IACrC,wFAAwF;IACxF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;OAGG;IACH,YAAY,CAAC,EAAE,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAClD;;;;;;OAMG;IACH,cAAc,CAAC,EAAE;QACf,UAAU,EAAE,QAAQ,CAAC;QACrB,QAAQ,EAAE,QAAQ,CAAC;KACpB,CAAC;CACH,CAAC;AAEF;;;;;;;;GAQG;AACH,wBAAsB,eAAe,CACnC,OAAO,EAAE;IACP,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,MAAM,CAAC;QACtB,QAAQ,EAAE,KAAK,CAAC;YACd,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;YACrB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC9B,MAAM,EAAE,MAAM,CAAC;SAChB,CAAC,CAAC;KACJ,CAAC;IACF,SAAS,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3C,EACD,IAAI,EAAE,aAAa,EACnB,IAAI,EAAE,kBAAkB,GACvB,OAAO,CAAC,IAAI,CAAC,CAoKf"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { IConsumer, KafkaTransport } from "../../transport/transport.interface";
|
|
2
|
+
import type { SchemaLike } from "../../message/topic";
|
|
3
|
+
import type { KafkaClientOptions, KafkaLogger } from "../../types";
|
|
4
|
+
export type ConsumerOpsDeps = {
|
|
5
|
+
consumers: Map<string, IConsumer>;
|
|
6
|
+
consumerCreationOptions: Map<string, {
|
|
7
|
+
fromBeginning: boolean;
|
|
8
|
+
autoCommit: boolean;
|
|
9
|
+
}>;
|
|
10
|
+
transport: KafkaTransport;
|
|
11
|
+
onRebalance: KafkaClientOptions["onRebalance"];
|
|
12
|
+
logger: KafkaLogger;
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Return an existing consumer for `groupId`, or create and register a new one.
|
|
16
|
+
*
|
|
17
|
+
* If the group already exists with different `fromBeginning` / `autoCommit` options the
|
|
18
|
+
* existing consumer is returned unchanged and a warning is logged — use a distinct
|
|
19
|
+
* `groupId` if different options are required.
|
|
20
|
+
*
|
|
21
|
+
* Partition assignment strategy defaults to `cooperative-sticky`, which minimises
|
|
22
|
+
* partition movement during rebalances and is the safest choice for horizontally
|
|
23
|
+
* scaled deployments.
|
|
24
|
+
*
|
|
25
|
+
* @param groupId Kafka consumer group ID.
|
|
26
|
+
* @param fromBeginning When `true`, the group starts from the earliest available offset.
|
|
27
|
+
* @param autoCommit When `true`, offsets are committed automatically. Set to `false` for manual EOS commits.
|
|
28
|
+
* @param deps Shared client dependencies (consumer map, transport, logger, …).
|
|
29
|
+
* @param partitionAssigner Assignment strategy — `'cooperative-sticky'` (default), `'roundrobin'`, or `'range'`.
|
|
30
|
+
* @returns The consumer instance for the group (existing or newly created).
|
|
31
|
+
*/
|
|
32
|
+
export declare function getOrCreateConsumer(groupId: string, fromBeginning: boolean, autoCommit: boolean, deps: ConsumerOpsDeps, partitionAssigner?: "roundrobin" | "range" | "cooperative-sticky", onFirstAssignment?: () => void, groupInstanceId?: string): IConsumer;
|
|
33
|
+
/**
|
|
34
|
+
* Build a local schema map for the topics being subscribed to.
|
|
35
|
+
*
|
|
36
|
+
* Schemas are collected from two sources in order:
|
|
37
|
+
* 1. Inline schemas on `TopicDescriptor` objects in the `topics` array.
|
|
38
|
+
* 2. Explicit overrides passed in `optionSchemas` (e.g. from `ConsumerOptions.schemas`).
|
|
39
|
+
*
|
|
40
|
+
* Both sources are also registered in the shared `schemaRegistry` so that the same
|
|
41
|
+
* schema is used consistently across producers and consumers. A warning is logged when
|
|
42
|
+
* a topic already has a different schema registered.
|
|
43
|
+
*
|
|
44
|
+
* @param topics Array of topic names or `TopicDescriptor` objects.
|
|
45
|
+
* @param schemaRegistry Shared client-wide schema map (mutated in place).
|
|
46
|
+
* @param optionSchemas Additional topic → schema overrides from consumer options.
|
|
47
|
+
* @param logger Optional logger for schema-conflict warnings.
|
|
48
|
+
* @returns A topic → schema map scoped to the current subscription.
|
|
49
|
+
*/
|
|
50
|
+
export declare function buildSchemaMap(topics: any[], schemaRegistry: Map<string, SchemaLike>, optionSchemas?: Map<string, SchemaLike>, logger?: KafkaLogger): Map<string, SchemaLike>;
|
|
51
|
+
//# sourceMappingURL=ops.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ops.d.ts","sourceRoot":"","sources":["../../../../src/client/kafka.client/consumer/ops.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAC;AACrF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAGnE,MAAM,MAAM,eAAe,GAAG;IAC5B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAClC,uBAAuB,EAAE,GAAG,CAC1B,MAAM,EACN;QAAE,aAAa,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,OAAO,CAAA;KAAE,CAChD,CAAC;IACF,SAAS,EAAE,cAAc,CAAC;IAC1B,WAAW,EAAE,kBAAkB,CAAC,aAAa,CAAC,CAAC;IAC/C,MAAM,EAAE,WAAW,CAAC;CACrB,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,aAAa,EAAE,OAAO,EACtB,UAAU,EAAE,OAAO,EACnB,IAAI,EAAE,eAAe,EACrB,iBAAiB,CAAC,EAAE,YAAY,GAAG,OAAO,GAAG,oBAAoB,EACjE,iBAAiB,CAAC,EAAE,MAAM,IAAI,EAC9B,eAAe,CAAC,EAAE,MAAM,GACvB,SAAS,CAyFX;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,GAAG,EAAE,EACb,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EACvC,aAAa,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,EACvC,MAAM,CAAC,EAAE,WAAW,GACnB,GAAG,CAAC,MAAM,EAAE,UAAU,CAAC,CA0BzB"}
|