kafka-ts 0.0.1-beta.2 → 0.0.1-beta.4

package/src/consumer/fetch-manager.ts

@@ -14,7 +14,7 @@ const trace = createTracer('FetchManager');
 export type BatchGranularity = 'partition' | 'topic' | 'broker';
 
 type FetchManagerOptions = {
-    fetch: (nodeId: number, assignment: Assignment) => Promise<ReturnType<(typeof API.FETCH)['response']>>;
+    fetch: (nodeId: number, assignment: Assignment) => Promise<Awaited<ReturnType<(typeof API.FETCH)['response']>>>;
     process: (batch: Batch) => Promise<void>;
     metadata: Metadata;
     consumerGroup?: ConsumerGroup;
@@ -86,6 +86,7 @@ export class FetchManager extends EventEmitter<{ data: []; checkpoint: [number];
 
         const batch = this.queue.shift();
         if (!batch) {
+            // wait until new data is available or fetch manager is requested to stop
             await new Promise<void>((resolve) => {
                 const onData = () => {
                     this.removeListener('stop', onStop);
@@ -109,35 +110,40 @@ export class FetchManager extends EventEmitter<{ data: []; checkpoint: [number];
         return batch as Exclude<Entry, Checkpoint>;
     }
 
-    private async onResponse(fetcherId: number, response: ReturnType<(typeof API.FETCH)['response']>) {
+    private async onResponse(fetcherId: number, response: Awaited<ReturnType<(typeof API.FETCH)['response']>>) {
         const { metadata, batchGranularity } = this.options;
 
         const batches = fetchResponseToBatches(response, batchGranularity, metadata);
-        if (batches.length) {
+        if (!batches.length) {
+            return;
+        }
+
+        // wait until all broker batches have been processed or fetch manager is requested to stop
+        await new Promise<void>((resolve) => {
+            const onCheckpoint = (id: number) => {
+                if (id === fetcherId) {
+                    this.removeListener('checkpoint', onCheckpoint);
+                    this.removeListener('stop', onStop);
+                    resolve();
+                }
+            };
+            const onStop = () => {
+                this.removeListener('checkpoint', onCheckpoint);
+                resolve();
+            };
+            this.on('checkpoint', onCheckpoint);
+            this.once('stop', onStop);
+
             this.queue.push(...batches);
             this.queue.push({ kind: 'checkpoint', fetcherId });
 
             this.emit('data');
-            await new Promise<void>((resolve) => {
-                const onCheckpoint = (id: number) => {
-                    if (id === fetcherId) {
-                        this.removeListener('stop', onStop);
-                        resolve();
-                    }
-                };
-                const onStop = () => {
-                    this.removeListener('checkpoint', onCheckpoint);
-                    resolve();
-                };
-                this.once('checkpoint', onCheckpoint);
-                this.once('stop', onStop);
-            });
-        }
+        });
     }
 }
 
 const fetchResponseToBatches = (
-    batch: ReturnType<typeof API.FETCH.response>,
+    batch: Awaited<ReturnType<typeof API.FETCH.response>>,
     batchGranularity: BatchGranularity,
     metadata: Metadata,
 ): Batch[] => {

package/src/consumer/fetcher.ts

@@ -10,8 +10,8 @@ type FetcherOptions = {
     nodeId: number;
     assignment: Assignment;
     consumerGroup?: ConsumerGroup;
-    fetch: (nodeId: number, assignment: Assignment) => Promise<ReturnType<(typeof API.FETCH)['response']>>;
-    onResponse: (fetcherId: number, response: ReturnType<(typeof API.FETCH)['response']>) => Promise<void>;
+    fetch: (nodeId: number, assignment: Assignment) => Promise<Awaited<ReturnType<(typeof API.FETCH)['response']>>>;
+    onResponse: (fetcherId: number, response: Awaited<ReturnType<(typeof API.FETCH)['response']>>) => Promise<void>;
 };
 
 export class Fetcher extends EventEmitter<{ stop: []; stopped: []; data: []; drain: [] }> {
@@ -26,10 +26,10 @@ export class Fetcher extends EventEmitter<{ stop: []; stopped: []; data: []; dra
 
     public async loop() {
         const { nodeId, assignment, consumerGroup, fetch, onResponse } = this.options;
-
+        
         this.isRunning = true;
         this.once('stop', () => (this.isRunning = false));
-
+        
         try {
             while (this.isRunning) {
                 const response = await fetch(nodeId, assignment);

package/src/consumer/processor.ts

@@ -25,7 +25,9 @@ export class Processor extends EventEmitter<{ stop: []; stopped: [] }> {
         try {
             while (this.isRunning) {
                 const batch = await poll();
-                await process(batch);
+                if (batch.length) {
+                    await process(batch);
+                }
             }
         } finally {
             this.isRunning = false;

package/src/index.ts

@@ -5,3 +5,5 @@ export * from './client';
 export * from './distributors/partitioner';
 export * from './types';
 export * from './utils/error';
+export * from './utils/logger';
+

package/src/producer/producer.ts

@@ -34,7 +34,7 @@ export class Producer {
         this.partition = this.options.partitioner({ metadata: this.metadata });
     }
 
-    public async send(messages: Message[]) {
+    public async send(messages: Message[], { acks = -1 }: { acks?: -1 | 1 } = {}) {
         await this.ensureConnected();
 
         const { allowTopicAutoCreation } = this.options;
@@ -44,7 +44,7 @@ export class Producer {
         await this.metadata.fetchMetadataIfNecessary({ topics, allowTopicAutoCreation });
 
         const nodeTopicPartitionMessages = distributeMessagesToTopicPartitionLeaders(
-            messages.map(message => ({ ...message, partition: this.partition(message) })),
+            messages.map((message) => ({ ...message, partition: this.partition(message) })),
             this.metadata.getTopicPartitionLeaderIds(),
         );
 
@@ -52,7 +52,7 @@ export class Producer {
             Object.entries(nodeTopicPartitionMessages).map(async ([nodeId, topicPartitionMessages]) => {
                 await this.cluster.sendRequestToNode(parseInt(nodeId))(API.PRODUCE, {
                     transactionalId: null,
-                    acks: 1,
+                    acks,
                     timeoutMs: 5000,
                     topicData: Object.entries(topicPartitionMessages).map(([topic, partitionMessages]) => ({
                         name: topic,
@@ -85,7 +85,7 @@ export class Producer {
                                     attributes: 0,
                                     timestampDelta: (message.timestamp ?? defaultTimestamp) - (baseTimestamp ?? 0n),
                                     offsetDelta: index,
-                                    key: message.key,
+                                    key: message.key ?? null,
                                     value: message.value,
                                     headers: Object.entries(message.headers ?? {}).map(([key, value]) => ({
                                         key: Buffer.from(key),

package/src/types.ts

@@ -1,9 +1,9 @@
 export type Message = {
     topic: string;
-    partition: number;
+    partition?: number;
     offset?: bigint;
     timestamp?: bigint;
-    key: Buffer | null;
+    key?: Buffer | null;
     value: Buffer | null;
     headers?: Record<string, string>;
 };

package/src/utils/api.ts

@@ -5,7 +5,7 @@ export type Api<Request, Response> = {
     apiKey: number;
     apiVersion: number;
     request: (encoder: Encoder, body: Request) => Encoder;
-    response: (buffer: Decoder) => Response;
+    response: (buffer: Decoder) => Promise<Response> | Response;
 };
 
 export const createApi = <Request, Response>(api: Api<Request, Response>) => api;

package/src/utils/decoder.ts

@@ -7,6 +7,10 @@ export class Decoder {
         return this.offset;
     }
 
+    public getBuffer() {
+        return this.buffer;
+    }
+
     public getBufferLength() {
         return this.buffer.length;
     }
@@ -132,6 +136,12 @@ export class Decoder {
         return results;
     }
 
+    public readVarIntArray<T>(callback: (opts: Decoder) => T): T[] {
+        const length = this.readVarInt();
+        const results = Array.from({ length }).map(() => callback(this));
+        return results;
+    }
+
     public readRecords<T>(callback: (opts: Decoder) => T): T[] {
         const length = this.readInt32();
 
@@ -143,9 +153,9 @@ export class Decoder {
         });
     }
 
-    public read(length: number) {
-        const value = this.buffer.subarray(this.offset, this.offset + length);
-        this.offset += length;
+    public read(length?: number) {
+        const value = this.buffer.subarray(this.offset, length !== undefined ? this.offset + length : undefined);
+        this.offset += Buffer.byteLength(value);
         return value;
     }
 

package/src/utils/logger.ts

@@ -0,0 +1,37 @@
+export interface Logger {
+    debug: (message: string, metadata?: unknown) => void;
+    info: (message: string, metadata?: unknown) => void;
+    warn: (message: string, metadata?: unknown) => void;
+    error: (message: string, metadata?: unknown) => void;
+}
+
+export const jsonSerializer = (_: unknown, v: unknown) => {
+    if (v instanceof Error) {
+        return { name: v.name, message: v.message, stack: v.stack, cause: v.cause };
+    }
+    if (typeof v === 'bigint') {
+        return v.toString();
+    }
+    return v;
+};
+
+class JsonLogger implements Logger {
+    debug(message: string, metadata?: unknown) {
+        console.debug(JSON.stringify({ message, metadata }, jsonSerializer));
+    }
+    info(message: string, metadata?: unknown) {
+        console.info(JSON.stringify({ message, metadata }, jsonSerializer));
+    }
+    warn(message: string, metadata?: unknown) {
+        console.warn(JSON.stringify({ message, metadata }, jsonSerializer));
+    }
+    error(message: string, metadata?: unknown) {
+        console.error(JSON.stringify({ message, metadata }, jsonSerializer));
+    }
+}
+
+export let log: Logger = new JsonLogger();
+
+export const setLogger = (newLogger: Logger) => {
+    log = newLogger;
+};

package/src/utils/mutex.ts

@@ -0,0 +1,31 @@
+export const createMutex = () => {
+    const queue: (() => void)[] = [];
+    let isLocked = false;
+
+    const acquire = () => {
+        return new Promise<void>((resolve) => {
+            if (!isLocked) {
+                isLocked = true;
+                return resolve();
+            }
+            queue.push(resolve);
+        });
+    };
+
+    const release = () => {
+        isLocked = false;
+        const next = queue.shift();
+        next && next();
+    };
+
+    const exclusive = async (fn: () => Promise<void>) => {
+        await acquire();
+        try {
+            await fn();
+        } finally {
+            release();
+        }
+    };
+
+    return { exclusive };
+};

package/src/utils/tracer.ts

@@ -1,4 +1,4 @@
-import { serializer } from './debug';
+import { log } from './logger';
 
 export const createTracer =
     (module: string, attributes?: Record<string, unknown>) =>
@@ -12,9 +12,11 @@ export const createTracer =
             const metadata = fn?.(...args);
 
             const onEnd = <T>(result: T): T => {
-                console.log(
-                    `[${module}.${propertyKey}] +${Date.now() - startTime}ms ${JSON.stringify({ ...attributes, ...metadata, result }, serializer)}`,
-                );
+                log.debug(`[${module}.${propertyKey}] ${metadata?.message ?? ''} +${Date.now() - startTime}ms`, {
+                    ...attributes,
+                    ...metadata,
+                    ...result && { result},
+                });
                 return result;
             };
 

package/src/utils/debug.ts

@@ -1,9 +0,0 @@
-export const serializer = (_: string, value: unknown) => (typeof value === 'bigint' ? value.toString() : value);
-
-export const createDebugger = (module: string) => (func: string, message: string, data?: unknown) => {
-    if (!process.env.DEBUG?.includes('kafka-ts')) return;
-    console.debug(
-        `[${module}] ${func}: ${message}`,
-        data && `(${data instanceof Error ? data : JSON.stringify(data, serializer, 4)})`,
-    );
-};