kafka-ts 0.0.3-beta → 0.0.4

package/dist/consumer/fetch-manager.js

@@ -0,0 +1,140 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.FetchManager = void 0;
+const error_1 = require("../utils/error");
+const tracer_1 = require("../utils/tracer");
+const fetcher_1 = require("./fetcher");
+const processor_1 = require("./processor");
+const trace = (0, tracer_1.createTracer)('FetchManager');
+class FetchManager {
+    options;
+    queue = [];
+    isRunning = false;
+    fetchers;
+    processors;
+    pollQueue = [];
+    fetcherCallbacks = {};
+    constructor(options) {
+        this.options = options;
+        const { fetch, process, consumerGroup, nodeAssignments, concurrency } = this.options;
+        this.fetchers = nodeAssignments.map(({ nodeId, assignment }, index) => new fetcher_1.Fetcher(index, {
+            nodeId,
+            assignment,
+            consumerGroup,
+            fetch,
+            onResponse: this.onResponse.bind(this),
+        }));
+        this.processors = Array.from({ length: concurrency }).map(() => new processor_1.Processor({ process, poll: this.poll.bind(this) }));
+    }
+    async start() {
+        this.queue = [];
+        this.isRunning = true;
+        try {
+            await Promise.all([
+                ...this.fetchers.map((fetcher) => fetcher.loop()),
+                ...this.processors.map((processor) => processor.loop()),
+            ]);
+        }
+        finally {
+            await this.stop();
+        }
+    }
+    async stop() {
+        this.isRunning = false;
+        const stopPromise = Promise.all([
+            ...this.fetchers.map((fetcher) => fetcher.stop()),
+            ...this.processors.map((processor) => processor.stop()),
+        ]);
+        this.pollQueue.forEach((resolve) => resolve());
+        this.pollQueue = [];
+        Object.values(this.fetcherCallbacks).forEach((callback) => callback());
+        this.fetcherCallbacks = {};
+        await stopPromise;
+    }
+    async poll() {
+        if (!this.isRunning) {
+            return [];
+        }
+        const batch = this.queue.shift();
+        if (!batch) {
+            // wait until new data is available or fetch manager is requested to stop
+            await new Promise((resolve) => {
+                this.pollQueue.push(resolve);
+            });
+            return this.poll();
+        }
+        if ('kind' in batch && batch.kind === 'checkpoint') {
+            this.fetcherCallbacks[batch.fetcherId]?.();
+            return this.poll();
+        }
+        this.pollQueue?.shift()?.();
+        return batch;
+    }
+    async onResponse(fetcherId, response) {
+        const { metadata, batchGranularity } = this.options;
+        const batches = fetchResponseToBatches(response, batchGranularity, metadata);
+        if (!batches.length) {
+            return;
+        }
+        // wait until all broker batches have been processed or fetch manager is requested to stop
+        await new Promise((resolve) => {
+            this.fetcherCallbacks[fetcherId] = resolve;
+            this.queue.push(...batches, { kind: 'checkpoint', fetcherId });
+            this.pollQueue?.shift()?.();
+        });
+    }
+}
+exports.FetchManager = FetchManager;
+__decorate([
+    trace(() => ({ root: true })),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], FetchManager.prototype, "start", null);
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], FetchManager.prototype, "poll", null);
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Number, Object]),
+    __metadata("design:returntype", Promise)
+], FetchManager.prototype, "onResponse", null);
+const fetchResponseToBatches = (batch, batchGranularity, metadata) => {
+    const brokerTopics = batch.responses.map(({ topicId, partitions }) => partitions.map(({ partitionIndex, records }) => records.flatMap(({ baseTimestamp, baseOffset, records }) => records.map((message) => ({
+        topic: metadata.getTopicNameById(topicId),
+        partition: partitionIndex,
+        key: message.key ?? null,
+        value: message.value ?? null,
+        headers: Object.fromEntries(message.headers.map(({ key, value }) => [key, value])),
+        timestamp: baseTimestamp + BigInt(message.timestampDelta),
+        offset: baseOffset + BigInt(message.offsetDelta),
+    })))));
+    switch (batchGranularity) {
+        case 'broker':
+            const messages = brokerTopics.flatMap((topicPartition) => topicPartition.flatMap((partitionMessages) => partitionMessages));
+            return messages.length ? [messages] : [];
+        case 'topic':
+            return brokerTopics
+                .map((topicPartition) => topicPartition.flatMap((partitionMessages) => partitionMessages))
+                .filter((messages) => messages.length);
+        case 'partition':
+            return brokerTopics
+                .flatMap((topicPartition) => topicPartition.map((partitionMessages) => partitionMessages))
+                .filter((messages) => messages.length);
+        default:
+            throw new error_1.KafkaTSError(`Unhandled batch granularity: ${batchGranularity}`);
+    }
+};

package/dist/consumer/fetcher.d.ts

@@ -0,0 +1,25 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { EventEmitter } from 'stream';
+import { FetchResponse } from '../api/fetch';
+import { Assignment } from '../api/sync-group';
+import { ConsumerGroup } from './consumer-group';
+type FetcherOptions = {
+    nodeId: number;
+    assignment: Assignment;
+    consumerGroup?: ConsumerGroup;
+    fetch: (nodeId: number, assignment: Assignment) => Promise<FetchResponse>;
+    onResponse: (fetcherId: number, response: FetchResponse) => Promise<void>;
+};
+export declare class Fetcher extends EventEmitter<{
+    stopped: [];
+}> {
+    private fetcherId;
+    private options;
+    private isRunning;
+    constructor(fetcherId: number, options: FetcherOptions);
+    loop(): Promise<void>;
+    private step;
+    stop(): Promise<void>;
+}
+export {};

package/dist/consumer/fetcher.js

@@ -0,0 +1,64 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Fetcher = void 0;
+const stream_1 = require("stream");
+const tracer_1 = require("../utils/tracer");
+const trace = (0, tracer_1.createTracer)('Fetcher');
+class Fetcher extends stream_1.EventEmitter {
+    fetcherId;
+    options;
+    isRunning = false;
+    constructor(fetcherId, options) {
+        super();
+        this.fetcherId = fetcherId;
+        this.options = options;
+    }
+    async loop() {
+        this.isRunning = true;
+        try {
+            while (this.isRunning) {
+                await this.step();
+            }
+        }
+        finally {
+            this.isRunning = false;
+            this.emit('stopped');
+        }
+    }
+    async step() {
+        const { nodeId, assignment, consumerGroup, fetch, onResponse } = this.options;
+        const response = await fetch(nodeId, assignment);
+        if (!this.isRunning) {
+            return;
+        }
+        consumerGroup?.handleLastHeartbeat();
+        await onResponse(this.fetcherId, response);
+        consumerGroup?.handleLastHeartbeat();
+    }
+    async stop() {
+        if (!this.isRunning) {
+            return;
+        }
+        const stopPromise = new Promise((resolve) => {
+            this.once('stopped', resolve);
+        });
+        this.isRunning = false;
+        return stopPromise;
+    }
+}
+exports.Fetcher = Fetcher;
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], Fetcher.prototype, "step", null);

package/dist/consumer/offset-manager.d.ts

@@ -0,0 +1,22 @@
+import { IsolationLevel } from '../api/fetch';
+import { Cluster } from '../cluster';
+import { ConsumerMetadata } from './consumer-metadata';
+type OffsetManagerOptions = {
+    cluster: Cluster;
+    metadata: ConsumerMetadata;
+    isolationLevel: IsolationLevel;
+};
+export declare class OffsetManager {
+    private options;
+    private currentOffsets;
+    pendingOffsets: Record<string, Record<number, bigint>>;
+    constructor(options: OffsetManagerOptions);
+    getCurrentOffset(topic: string, partition: number): bigint;
+    resolve(topic: string, partition: number, offset: bigint): void;
+    flush(topicPartitions: Record<string, Set<number>>): void;
+    fetchOffsets(options: {
+        fromBeginning: boolean;
+    }): Promise<void>;
+    private listOffsets;
+}
+export {};

package/dist/consumer/offset-manager.js

@@ -0,0 +1,66 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OffsetManager = void 0;
+const api_1 = require("../api");
+const messages_to_topic_partition_leaders_1 = require("../distributors/messages-to-topic-partition-leaders");
+const tracer_1 = require("../utils/tracer");
+const trace = (0, tracer_1.createTracer)('OffsetManager');
+class OffsetManager {
+    options;
+    currentOffsets = {};
+    pendingOffsets = {};
+    constructor(options) {
+        this.options = options;
+    }
+    getCurrentOffset(topic, partition) {
+        return this.currentOffsets[topic]?.[partition] ?? 0n;
+    }
+    resolve(topic, partition, offset) {
+        this.pendingOffsets[topic] ??= {};
+        this.pendingOffsets[topic][partition] = offset;
+    }
+    flush(topicPartitions) {
+        Object.entries(topicPartitions).forEach(([topic, partitions]) => {
+            this.currentOffsets[topic] ??= {};
+            partitions.forEach((partition) => {
+                if (this.pendingOffsets[topic]?.[partition]) {
+                    this.currentOffsets[topic][partition] = this.pendingOffsets[topic][partition];
+                    delete this.pendingOffsets[topic][partition];
+                }
+            });
+        });
+    }
+    async fetchOffsets(options) {
+        const { metadata } = this.options;
+        const topicPartitions = Object.entries(metadata.getAssignment()).flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })));
+        const nodeTopicPartitions = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)(topicPartitions, metadata.getTopicPartitionLeaderIds());
+        await Promise.all(Object.entries(nodeTopicPartitions).map(([nodeId, topicPartitions]) => this.listOffsets({
+            ...options,
+            nodeId: parseInt(nodeId),
+            nodeAssignment: Object.fromEntries(Object.entries(topicPartitions).map(([topicName, partitions]) => [topicName, Object.keys(partitions).map(Number)])),
+        })));
+    }
+    async listOffsets({ nodeId, nodeAssignment, fromBeginning, }) {
+        const { cluster, isolationLevel } = this.options;
+        const offsets = await cluster.sendRequestToNode(nodeId)(api_1.API.LIST_OFFSETS, {
+            replicaId: -1,
+            isolationLevel,
+            topics: Object.entries(nodeAssignment)
+                .flatMap(([topic, partitions]) => partitions.map((partition) => ({ topic, partition })))
+                .map(({ topic, partition }) => ({
+                name: topic,
+                partitions: [{ partitionIndex: partition, currentLeaderEpoch: -1, timestamp: -1n }],
+            })),
+        });
+        const topicPartitions = {};
+        offsets.topics.forEach(({ name, partitions }) => {
+            topicPartitions[name] ??= new Set();
+            partitions.forEach(({ partitionIndex, offset }) => {
+                topicPartitions[name].add(partitionIndex);
+                this.resolve(name, partitionIndex, fromBeginning ? 0n : offset);
+            });
+        });
+        this.flush(topicPartitions);
+    }
+}
+exports.OffsetManager = OffsetManager;

package/dist/consumer/processor.d.ts

@@ -0,0 +1,19 @@
+/// <reference types="node" />
+/// <reference types="node" />
+import { EventEmitter } from 'stream';
+import { Batch } from '../types';
+type ProcessorOptions = {
+    poll: () => Promise<Batch>;
+    process: (batch: Batch) => Promise<void>;
+};
+export declare class Processor extends EventEmitter<{
+    stopped: [];
+}> {
+    private options;
+    private isRunning;
+    constructor(options: ProcessorOptions);
+    loop(): Promise<void>;
+    private step;
+    stop(): Promise<void>;
+}
+export {};

package/dist/consumer/processor.js

@@ -0,0 +1,59 @@
+"use strict";
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Processor = void 0;
+const stream_1 = require("stream");
+const tracer_1 = require("../utils/tracer");
+const trace = (0, tracer_1.createTracer)('Processor');
+class Processor extends stream_1.EventEmitter {
+    options;
+    isRunning = false;
+    constructor(options) {
+        super();
+        this.options = options;
+    }
+    async loop() {
+        this.isRunning = true;
+        try {
+            while (this.isRunning) {
+                await this.step();
+            }
+        }
+        finally {
+            this.isRunning = false;
+            this.emit('stopped');
+        }
+    }
+    async step() {
+        const { poll, process } = this.options;
+        const batch = await poll();
+        if (batch.length) {
+            await process(batch);
+        }
+    }
+    async stop() {
+        if (!this.isRunning) {
+            return;
+        }
+        const stopPromise = new Promise((resolve) => {
+            this.once('stopped', resolve);
+        });
+        this.isRunning = false;
+        return stopPromise;
+    }
+}
+exports.Processor = Processor;
+__decorate([
+    trace(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], Processor.prototype, "step", null);

package/dist/distributors/assignments-to-replicas.d.ts

@@ -0,0 +1,16 @@
+type Assignment = {
+    [topicName: string]: number[];
+};
+type TopicPartitionReplicaIds = {
+    [topicName: string]: {
+        [partition: number]: number[];
+    };
+};
+type NodeAssignment = {
+    [replicaId: number]: Assignment;
+};
+/** From replica ids pick the one with fewest assignments to balance the load across brokers */
+export declare const distributeAssignmentsToNodesBalanced: (assignment: Assignment, topicPartitionReplicaIds: TopicPartitionReplicaIds) => NodeAssignment;
+/** Minimize the total number of replicas in the result to reduce the number of requests to different brokers */
+export declare const distributeAssignmentsToNodesOptimized: (assignment: Assignment, topicPartitionReplicaIds: TopicPartitionReplicaIds) => NodeAssignment;
+export {};

package/{src/distributors/assignments-to-replicas.ts → dist/distributors/assignments-to-replicas.js}

@@ -1,16 +1,10 @@
-type Assignment = { [topicName: string]: number[] };
-type TopicPartitionReplicaIds = { [topicName: string]: { [partition: number]: number[] } };
-export type NodeAssignment = { [replicaId: number]: Assignment };
-
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.distributeAssignmentsToNodesOptimized = exports.distributeAssignmentsToNodesBalanced = void 0;
 /** From replica ids pick the one with fewest assignments to balance the load across brokers */
-export const distributeAssignmentsToNodesBalanced = (
-    assignment: Assignment,
-    topicPartitionReplicaIds: TopicPartitionReplicaIds,
-) => {
+const distributeAssignmentsToNodesBalanced = (assignment, topicPartitionReplicaIds) => {
     const replicaPartitions = getPartitionsByReplica(assignment, topicPartitionReplicaIds);
-
-    const result: NodeAssignment = {};
-
+    const result = {};
     for (const [topicName, partitions] of Object.entries(assignment)) {
         for (const partition of partitions) {
             const replicaIds = topicPartitionReplicaIds[topicName][partition];
@@ -25,51 +19,33 @@ export const distributeAssignmentsToNodesBalanced = (
             result[replicaId][topicName].push(partition);
         }
     }
-
     return result;
 };
-
+exports.distributeAssignmentsToNodesBalanced = distributeAssignmentsToNodesBalanced;
 /** Minimize the total number of replicas in the result to reduce the number of requests to different brokers */
-export const distributeAssignmentsToNodesOptimized = (
-    assignment: Assignment,
-    topicPartitionReplicaIds: TopicPartitionReplicaIds,
-) => {
-    const result: NodeAssignment = {};
-
-    const sortFn = ([, partitionsA]: [string, string[]], [, partitionsB]: [string, string[]]) =>
-        partitionsB.length - partitionsA.length;
-
+const distributeAssignmentsToNodesOptimized = (assignment, topicPartitionReplicaIds) => {
+    const result = {};
+    const sortFn = ([, partitionsA], [, partitionsB]) => partitionsB.length - partitionsA.length;
     let replicaPartitions = getPartitionsByReplica(assignment, topicPartitionReplicaIds);
-
     while (replicaPartitions.length) {
         replicaPartitions.sort(sortFn);
-
-        const [replicaId, partitions] = replicaPartitions.shift()!;
+        const [replicaId, partitions] = replicaPartitions.shift();
         if (!partitions.length) {
             continue;
         }
-
         result[parseInt(replicaId)] = partitions.reduce((acc, partition) => {
             const [topicName, partitionId] = partition.split(':');
             acc[topicName] ??= [];
             acc[topicName].push(parseInt(partitionId));
             return acc;
-        }, {} as Assignment);
-
-        replicaPartitions = replicaPartitions.map(
-            ([replicaId, replicaPartitions]) =>
-                [replicaId, replicaPartitions.filter((partition) => !partitions.includes(partition))] as [
-                    string,
-                    string[],
-                ],
-        );
+        }, {});
+        replicaPartitions = replicaPartitions.map(([replicaId, replicaPartitions]) => [replicaId, replicaPartitions.filter((partition) => !partitions.includes(partition))]);
     }
-
     return result;
 };
-
-const getPartitionsByReplica = (assignment: Assignment, topicPartitionReplicaIds: TopicPartitionReplicaIds) => {
-    const partitionsByReplicaId: { [replicaId: number]: string[] } = {};
+exports.distributeAssignmentsToNodesOptimized = distributeAssignmentsToNodesOptimized;
+const getPartitionsByReplica = (assignment, topicPartitionReplicaIds) => {
+    const partitionsByReplicaId = {};
     for (const [topicName, partitions] of Object.entries(assignment)) {
         for (const partition of partitions) {
             const replicaIds = topicPartitionReplicaIds[topicName][partition];
@@ -81,5 +57,3 @@ const getPartitionsByReplica = (assignment: Assignment, topicPartitionReplicaIds
     }
     return Object.entries(partitionsByReplicaId);
 };
-
-export const distributeAssignmentsToNodes = distributeAssignmentsToNodesBalanced;

package/dist/distributors/assignments-to-replicas.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/distributors/assignments-to-replicas.test.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const assignments_to_replicas_1 = require("./assignments-to-replicas");
+(0, vitest_1.describe)('Distribute assignments to replica ids', () => {
+    (0, vitest_1.describe)('distributeAssignmentsToNodesBalanced', () => {
+        (0, vitest_1.it)('smoke', () => {
+            const result = (0, assignments_to_replicas_1.distributeAssignmentsToNodesBalanced)({ topic: [0, 1] }, { topic: { 0: [0, 1], 1: [1, 2] } });
+            (0, vitest_1.expect)(result).toMatchInlineSnapshot(`
+              {
+                "1": {
+                  "topic": [
+                    0,
+                  ],
+                },
+                "2": {
+                  "topic": [
+                    1,
+                  ],
+                },
+              }
+            `);
+        });
+    });
+    (0, vitest_1.describe)('distributeAssignmentsToNodesOptimized', () => {
+        (0, vitest_1.it)('smoke', () => {
+            const result = (0, assignments_to_replicas_1.distributeAssignmentsToNodesOptimized)({ topic: [0, 1] }, { topic: { 0: [0, 1], 1: [1, 2] } });
+            (0, vitest_1.expect)(result).toMatchInlineSnapshot(`
+              {
+                "1": {
+                  "topic": [
+                    0,
+                    1,
+                  ],
+                },
+              }
+            `);
+        });
+    });
+});

package/dist/distributors/messages-to-topic-partition-leaders.d.ts

@@ -0,0 +1,17 @@
+type TopicPartitionLeader = {
+    [topicName: string]: {
+        [partitionId: number]: number;
+    };
+};
+type MessagesByNodeTopicPartition<T> = {
+    [nodeId: number]: {
+        [topicName: string]: {
+            [partitionId: number]: T[];
+        };
+    };
+};
+export declare const distributeMessagesToTopicPartitionLeaders: <T extends {
+    topic: string;
+    partition: number;
+}>(messages: T[], topicPartitionLeader: TopicPartitionLeader) => MessagesByNodeTopicPartition<T>;
+export {};

package/dist/distributors/messages-to-topic-partition-leaders.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.distributeMessagesToTopicPartitionLeaders = void 0;
+const distributeMessagesToTopicPartitionLeaders = (messages, topicPartitionLeader) => {
+    const result = {};
+    messages.forEach((message) => {
+        const leaderId = topicPartitionLeader[message.topic][message.partition];
+        result[leaderId] ??= {};
+        result[leaderId][message.topic] ??= {};
+        result[leaderId][message.topic][message.partition] ??= [];
+        result[leaderId][message.topic][message.partition].push(message);
+    });
+    return result;
+};
+exports.distributeMessagesToTopicPartitionLeaders = distributeMessagesToTopicPartitionLeaders;

package/dist/distributors/messages-to-topic-partition-leaders.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/distributors/messages-to-topic-partition-leaders.test.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const vitest_1 = require("vitest");
+const messages_to_topic_partition_leaders_1 = require("./messages-to-topic-partition-leaders");
+(0, vitest_1.describe)('Distribute messages to partition leader ids', () => {
+    (0, vitest_1.describe)('distributeMessagesToTopicPartitionLeaders', () => {
+        (0, vitest_1.it)('snoke', () => {
+            const result = (0, messages_to_topic_partition_leaders_1.distributeMessagesToTopicPartitionLeaders)([{ topic: 'topic', partition: 0, key: null, value: null, offset: 0n, timestamp: 0n, headers: {} }], { topic: { 0: 1 } });
+            (0, vitest_1.expect)(result).toMatchInlineSnapshot(`
+              {
+                "1": {
+                  "topic": {
+                    "0": [
+                      {
+                        "headers": {},
+                        "key": null,
+                        "offset": 0n,
+                        "partition": 0,
+                        "timestamp": 0n,
+                        "topic": "topic",
+                        "value": null,
+                      },
+                    ],
+                  },
+                },
+              }
+            `);
+        });
+    });
+});

package/dist/distributors/partitioner.d.ts

@@ -0,0 +1,7 @@
+import { Metadata } from '../metadata';
+import { Message } from '../types';
+export type Partition = (message: Message) => number;
+export type Partitioner = (context: {
+    metadata: Metadata;
+}) => Partition;
+export declare const defaultPartitioner: Partitioner;

package/dist/distributors/partitioner.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.defaultPartitioner = void 0;
+const murmur2_1 = require("../utils/murmur2");
+const defaultPartitioner = ({ metadata }) => {
+    const topicCounterMap = {};
+    const getNextValue = (topic) => {
+        topicCounterMap[topic] ??= 0;
+        return topicCounterMap[topic]++;
+    };
+    return ({ topic, partition, key }) => {
+        if (partition !== null && partition !== undefined) {
+            return partition;
+        }
+        const partitions = metadata.getTopicPartitions()[topic];
+        const numPartitions = partitions.length;
+        if (key) {
+            return (0, murmur2_1.toPositive)((0, murmur2_1.murmur2)(key)) % numPartitions;
+        }
+        return (0, murmur2_1.toPositive)(getNextValue(topic)) % numPartitions;
+    };
+};
+exports.defaultPartitioner = defaultPartitioner;

package/dist/index.d.ts

@@ -0,0 +1,9 @@
+export * from './api';
+export * from './auth';
+export { SASLProvider } from './broker';
+export * from './client';
+export * from './distributors/partitioner';
+export * from './types';
+export * from './utils/error';
+export * from './utils/logger';
+export { Tracer, setTracer } from './utils/tracer';