@platformatic/kafka 0.2.0 → 0.4.0

package/dist/clients/consumer/consumer.js

@@ -16,7 +16,7 @@ import { defaultBaseOptions } from "../base/options.js";
 import { createPromisifiedCallback, kCallbackPromise, runConcurrentCallbacks } from "../callbacks.js";
 import { ensureMetric } from "../metrics.js";
 import { MessagesStream } from "./messages-stream.js";
-import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
+import { commitOptionsValidator, consumeOptionsValidator, consumerOptionsValidator, defaultConsumerOptions, fetchOptionsValidator, groupIdAndOptionsValidator, groupOptionsValidator, listCommitsOptionsValidator, listOffsetsOptionsValidator } from "./options.js";
 import { TopicsMap } from "./topics-map.js";
 export class Consumer extends Base {
     groupId;
@@ -64,7 +64,7 @@ export class Consumer extends Base {
         this.#coordinatorId = null;
         this.#heartbeatInterval = null;
         this.#streams = new Set();
-        this.#validateGroupOptions(this[kOptions]);
+        this.#validateGroupOptions(this[kOptions], groupIdAndOptionsValidator);
         // Initialize connection pool
         this[kFetchConnections] = this[kCreateConnectionPool]();
         if (this[kPrometheus]) {
@@ -688,10 +688,11 @@ export class Consumer extends Base {
             });
         });
     }
-    #validateGroupOptions(options) {
-        const valid = groupOptionsValidator(options);
+    #validateGroupOptions(options, validator) {
+        validator ??= groupOptionsValidator;
+        const valid = validator(options);
         if (!valid) {
-            throw new UserError(this[kFormatValidationErrors](groupOptionsValidator, '/options'));
+            throw new UserError(this[kFormatValidationErrors](validator, '/options'));
         }
     }
     /*

package/dist/clients/consumer/messages-stream.js

@@ -269,12 +269,12 @@ export class MessagesStream extends Readable {
                 const firstTimestamp = records.firstTimestamp;
                 const leaderEpoch = metadata.topics.get(topic).partitions[partition].leaderEpoch;
                 for (const record of records.records) {
-                    const key = keyDeserializer(record.key);
-                    const value = valueDeserializer(record.value);
                     const headers = new Map();
                     for (const [headerKey, headerValue] of record.headers) {
                         headers.set(headerKeyDeserializer(headerKey), headerValueDeserializer(headerValue));
                     }
+                    const key = keyDeserializer(record.key, headers);
+                    const value = valueDeserializer(record.value, headers);
                     this.#metricsConsumedMessages?.inc();
                     canPush = this.push({
                         key,

package/dist/clients/consumer/options.d.ts

@@ -1,4 +1,120 @@
 import { type ConsumerOptions } from './types.ts';
+export declare const groupOptionsProperties: {
+    sessionTimeout: {
+        type: string;
+        minimum: number;
+    };
+    rebalanceTimeout: {
+        type: string;
+        minimum: number;
+    };
+    heartbeatInterval: {
+        type: string;
+        minimum: number;
+    };
+    protocols: {
+        type: string;
+        items: {
+            type: string;
+            properties: {
+                name: {
+                    type: string;
+                    pattern: string;
+                };
+                version: {
+                    type: string;
+                    minimum: number;
+                };
+                topics: {
+                    type: string;
+                    items: {
+                        type: string;
+                    };
+                };
+                metadata: {
+                    oneOf: ({
+                        type: string;
+                        buffer?: undefined;
+                    } | {
+                        buffer: boolean;
+                        type?: undefined;
+                    })[];
+                };
+            };
+        };
+    };
+};
+export declare const groupOptionsAdditionalValidations: {
+    rebalanceTimeout: {
+        properties: {
+            rebalanceTimeout: {
+                type: string;
+                minimum: number;
+                gteProperty: string;
+            };
+        };
+    };
+    heartbeatInterval: {
+        properties: {
+            heartbeatInterval: {
+                type: string;
+                minimum: number;
+                allOf: {
+                    lteProperty: string;
+                }[];
+            };
+        };
+    };
+};
+export declare const consumeOptionsProperties: {
+    autocommit: {
+        oneOf: ({
+            type: string;
+            minimum?: undefined;
+        } | {
+            type: string;
+            minimum: number;
+        })[];
+    };
+    minBytes: {
+        type: string;
+        minimum: number;
+    };
+    maxBytes: {
+        type: string;
+        minimum: number;
+    };
+    maxWaitTime: {
+        type: string;
+        minimum: number;
+    };
+    isolationLevel: {
+        type: string;
+        enum: string[];
+    };
+    deserializers: {
+        type: string;
+        properties: {
+            key: {
+                function: boolean;
+            };
+            value: {
+                function: boolean;
+            };
+            headerKey: {
+                function: boolean;
+            };
+            headerValue: {
+                function: boolean;
+            };
+        };
+        additionalProperties: boolean;
+    };
+    highWaterMark: {
+        type: string;
+        minimum: number;
+    };
+};
 export declare const groupOptionsSchema: {
     type: string;
     properties: {
@@ -500,6 +616,9 @@ export declare const listOffsetsOptionsSchema: {
     additionalProperties: boolean;
 };
 export declare const groupOptionsValidator: import("ajv").ValidateFunction<unknown>;
+export declare const groupIdAndOptionsValidator: import("ajv").ValidateFunction<{
+    groupId: any;
+}>;
 export declare const consumeOptionsValidator: import("ajv").ValidateFunction<{
     [x: string]: {};
 }>;

package/dist/clients/consumer/options.js

@@ -3,7 +3,7 @@ import { ajv } from "../../utils.js";
 import { idProperty, topicWithPartitionAndOffsetProperties } from "../base/options.js";
 import { serdeProperties } from "../serde.js";
 import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
-const groupOptionsProperties = {
+export const groupOptionsProperties = {
     sessionTimeout: { type: 'number', minimum: 0 },
     rebalanceTimeout: { type: 'number', minimum: 0 },
     heartbeatInterval: { type: 'number', minimum: 0 },
@@ -23,7 +23,7 @@ const groupOptionsProperties = {
         }
     }
 };
-const groupOptionsAdditionalValidations = {
+export const groupOptionsAdditionalValidations = {
     rebalanceTimeout: {
         properties: {
             rebalanceTimeout: {
@@ -50,7 +50,7 @@ const groupOptionsAdditionalValidations = {
         }
     }
 };
-const consumeOptionsProperties = {
+export const consumeOptionsProperties = {
     autocommit: { oneOf: [{ type: 'boolean' }, { type: 'number', minimum: 100 }] },
     minBytes: { type: 'number', minimum: 0 },
     maxBytes: { type: 'number', minimum: 0 },
@@ -187,6 +187,16 @@ export const groupOptionsValidator = ajv.compile({
     ...groupOptionsSchema,
     dependentSchemas: groupOptionsAdditionalValidations
 });
+export const groupIdAndOptionsValidator = ajv.compile({
+    type: 'object',
+    properties: {
+        groupId: idProperty,
+        ...groupOptionsProperties
+    },
+    required: ['groupId'],
+    additionalProperties: true,
+    dependentSchemas: groupOptionsAdditionalValidations
+});
 export const consumeOptionsValidator = ajv.compile(consumeOptionsSchema);
 export const consumerOptionsValidator = ajv.compile(consumerOptionsSchema);
 export const fetchOptionsValidator = ajv.compile(fetchOptionsSchema);

package/dist/clients/producer/options.d.ts

@@ -1,3 +1,33 @@
+export declare const produceOptionsProperties: {
+    producerId: {
+        bigint: boolean;
+    };
+    producerEpoch: {
+        type: string;
+    };
+    idempotent: {
+        type: string;
+    };
+    acks: {
+        type: string;
+        enum: (0 | 1 | -1)[];
+        errorMessage: string;
+    };
+    compression: {
+        type: string;
+        enum: string[];
+        errorMessage: string;
+    };
+    partitioner: {
+        function: boolean;
+    };
+    autocreateTopics: {
+        type: string;
+    };
+    repeatOnStaleMetadata: {
+        type: string;
+    };
+};
 export declare const produceOptionsSchema: {
     type: string;
     properties: {

package/dist/clients/producer/options.js

@@ -3,7 +3,7 @@ import { compressionsAlgorithms } from "../../protocol/compression.js";
 import { messageSchema } from "../../protocol/records.js";
 import { ajv, enumErrorMessage } from "../../utils.js";
 import { serdeProperties } from "../serde.js";
-const produceOptionsProperties = {
+export const produceOptionsProperties = {
     producerId: { bigint: true },
     producerEpoch: { type: 'number' },
     idempotent: { type: 'boolean' },

package/dist/clients/producer/producer.js

@@ -159,8 +159,19 @@ export class Producer extends Base {
         const messages = [];
         for (const message of options.messages) {
             const topic = message.topic;
-            const key = this.#keySerializer(message.key);
-            const headers = new Map();
+            let headers = new Map();
+            const serializedHeaders = new Map();
+            if (message.headers) {
+                headers =
+                    message.headers instanceof Map
+                        ? message.headers
+                        : new Map(Object.entries(message.headers));
+                for (const [key, value] of headers) {
+                    serializedHeaders.set(this.#headerKeySerializer(key), this.#headerValueSerializer(value));
+                }
+            }
+            const key = this.#keySerializer(message.key, headers);
+            const value = this.#valueSerializer(message.value, headers);
             let partition = 0;
             if (typeof message.partition !== 'number') {
                 if (partitioner) {
@@ -177,17 +188,11 @@ export class Producer extends Base {
             else {
                 partition = message.partition;
             }
-            if (message.headers) {
-                const entries = message.headers instanceof Map ? message.headers : Object.entries(message.headers);
-                for (const [key, value] of entries) {
-                    headers.set(this.#headerKeySerializer(key), this.#headerValueSerializer(value));
-                }
-            }
             topics.add(topic);
             messages.push({
                 key,
-                value: this.#valueSerializer(message.value),
-                headers,
+                value,
+                headers: serializedHeaders,
                 topic,
                 partition,
                 timestamp: message.timestamp

package/dist/clients/serde.d.ts

@@ -1,14 +1,16 @@
 export type Serializer<InputType = unknown> = (data?: InputType) => Buffer | undefined;
 export type Deserializer<OutputType = unknown> = (data?: Buffer) => OutputType | undefined;
+export type SerializerWithHeaders<InputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: InputType, headers?: Map<HeaderKey, HeaderValue>) => Buffer | undefined;
+export type DeserializerWithHeaders<OutputType = unknown, HeaderKey = unknown, HeaderValue = unknown> = (data?: Buffer, headers?: Map<HeaderKey, HeaderValue>) => OutputType | undefined;
 export interface Serializers<Key, Value, HeaderKey, HeaderValue> {
-    key: Serializer<Key>;
-    value: Serializer<Value>;
+    key: SerializerWithHeaders<Key, HeaderKey, HeaderValue>;
+    value: SerializerWithHeaders<Value, HeaderKey, HeaderValue>;
     headerKey: Serializer<HeaderKey>;
     headerValue: Serializer<HeaderValue>;
 }
 export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
-    key: Deserializer<Key>;
-    value: Deserializer<Value>;
+    key: DeserializerWithHeaders<Key>;
+    value: DeserializerWithHeaders<Value>;
     headerKey: Deserializer<HeaderKey>;
     headerValue: Deserializer<HeaderValue>;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "0.2.0",
+  "version": "0.4.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",