@platformatic/kafka 0.1.0

package/dist/clients/consumer/options.d.ts

@@ -0,0 +1,521 @@
+import { type ConsumerOptions } from './types.ts';
+export declare const groupOptionsSchema: {
+    type: string;
+    properties: {
+        sessionTimeout: {
+            type: string;
+            minimum: number;
+        };
+        rebalanceTimeout: {
+            type: string;
+            minimum: number;
+        };
+        heartbeatInterval: {
+            type: string;
+            minimum: number;
+        };
+        protocols: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    name: {
+                        type: string;
+                        pattern: string;
+                    };
+                    version: {
+                        type: string;
+                        minimum: number;
+                    };
+                    topics: {
+                        type: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                    metadata: {
+                        oneOf: ({
+                            type: string;
+                            buffer?: undefined;
+                        } | {
+                            buffer: boolean;
+                            type?: undefined;
+                        })[];
+                    };
+                };
+            };
+        };
+    };
+    additionalProperties: boolean;
+};
+export declare const consumeOptionsSchema: {
+    type: string;
+    properties: {
+        autocommit: {
+            oneOf: ({
+                type: string;
+                minimum?: undefined;
+            } | {
+                type: string;
+                minimum: number;
+            })[];
+        };
+        minBytes: {
+            type: string;
+            minimum: number;
+        };
+        maxBytes: {
+            type: string;
+            minimum: number;
+        };
+        maxWaitTime: {
+            type: string;
+            minimum: number;
+        };
+        isolationLevel: {
+            type: string;
+            enum: string[];
+        };
+        deserializers: {
+            type: string;
+            properties: {
+                key: {
+                    function: boolean;
+                };
+                value: {
+                    function: boolean;
+                };
+                headerKey: {
+                    function: boolean;
+                };
+                headerValue: {
+                    function: boolean;
+                };
+            };
+            additionalProperties: boolean;
+        };
+        highWaterMark: {
+            type: string;
+            minimum: number;
+        };
+        sessionTimeout: {
+            type: string;
+            minimum: number;
+        };
+        rebalanceTimeout: {
+            type: string;
+            minimum: number;
+        };
+        heartbeatInterval: {
+            type: string;
+            minimum: number;
+        };
+        protocols: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    name: {
+                        type: string;
+                        pattern: string;
+                    };
+                    version: {
+                        type: string;
+                        minimum: number;
+                    };
+                    topics: {
+                        type: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                    metadata: {
+                        oneOf: ({
+                            type: string;
+                            buffer?: undefined;
+                        } | {
+                            buffer: boolean;
+                            type?: undefined;
+                        })[];
+                    };
+                };
+            };
+        };
+        topics: {
+            type: string;
+            items: {
+                type: string;
+                pattern: string;
+            };
+        };
+        mode: {
+            type: string;
+            enum: ("latest" | "earliest" | "committed" | "manual")[];
+        };
+        fallbackMode: {
+            type: string;
+            enum: ("latest" | "earliest" | "fail")[];
+        };
+        offsets: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    topic: {
+                        type: string;
+                        pattern: string;
+                    };
+                    partition: {
+                        type: string;
+                        minimum: number;
+                    };
+                    offset: {
+                        bigint: boolean;
+                    };
+                };
+                required: string[];
+                additionalProperties: boolean;
+            };
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const consumerOptionsSchema: {
+    type: string;
+    properties: {
+        autocommit: {
+            oneOf: ({
+                type: string;
+                minimum?: undefined;
+            } | {
+                type: string;
+                minimum: number;
+            })[];
+        };
+        minBytes: {
+            type: string;
+            minimum: number;
+        };
+        maxBytes: {
+            type: string;
+            minimum: number;
+        };
+        maxWaitTime: {
+            type: string;
+            minimum: number;
+        };
+        isolationLevel: {
+            type: string;
+            enum: string[];
+        };
+        deserializers: {
+            type: string;
+            properties: {
+                key: {
+                    function: boolean;
+                };
+                value: {
+                    function: boolean;
+                };
+                headerKey: {
+                    function: boolean;
+                };
+                headerValue: {
+                    function: boolean;
+                };
+            };
+            additionalProperties: boolean;
+        };
+        highWaterMark: {
+            type: string;
+            minimum: number;
+        };
+        sessionTimeout: {
+            type: string;
+            minimum: number;
+        };
+        rebalanceTimeout: {
+            type: string;
+            minimum: number;
+        };
+        heartbeatInterval: {
+            type: string;
+            minimum: number;
+        };
+        protocols: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    name: {
+                        type: string;
+                        pattern: string;
+                    };
+                    version: {
+                        type: string;
+                        minimum: number;
+                    };
+                    topics: {
+                        type: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                    metadata: {
+                        oneOf: ({
+                            type: string;
+                            buffer?: undefined;
+                        } | {
+                            buffer: boolean;
+                            type?: undefined;
+                        })[];
+                    };
+                };
+            };
+        };
+        groupId: {
+            type: string;
+            pattern: string;
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const fetchOptionsSchema: {
+    type: string;
+    properties: {
+        autocommit: {
+            oneOf: ({
+                type: string;
+                minimum?: undefined;
+            } | {
+                type: string;
+                minimum: number;
+            })[];
+        };
+        minBytes: {
+            type: string;
+            minimum: number;
+        };
+        maxBytes: {
+            type: string;
+            minimum: number;
+        };
+        maxWaitTime: {
+            type: string;
+            minimum: number;
+        };
+        isolationLevel: {
+            type: string;
+            enum: string[];
+        };
+        deserializers: {
+            type: string;
+            properties: {
+                key: {
+                    function: boolean;
+                };
+                value: {
+                    function: boolean;
+                };
+                headerKey: {
+                    function: boolean;
+                };
+                headerValue: {
+                    function: boolean;
+                };
+            };
+            additionalProperties: boolean;
+        };
+        highWaterMark: {
+            type: string;
+            minimum: number;
+        };
+        sessionTimeout: {
+            type: string;
+            minimum: number;
+        };
+        rebalanceTimeout: {
+            type: string;
+            minimum: number;
+        };
+        heartbeatInterval: {
+            type: string;
+            minimum: number;
+        };
+        protocols: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    name: {
+                        type: string;
+                        pattern: string;
+                    };
+                    version: {
+                        type: string;
+                        minimum: number;
+                    };
+                    topics: {
+                        type: string;
+                        items: {
+                            type: string;
+                        };
+                    };
+                    metadata: {
+                        oneOf: ({
+                            type: string;
+                            buffer?: undefined;
+                        } | {
+                            buffer: boolean;
+                            type?: undefined;
+                        })[];
+                    };
+                };
+            };
+        };
+        node: {
+            type: string;
+            minimum: number;
+        };
+        topics: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    topicId: {
+                        type: string;
+                    };
+                    partitions: {
+                        type: string;
+                        items: {
+                            type: string;
+                            properties: {
+                                partition: {
+                                    type: string;
+                                };
+                                currentLeaderEpoch: {
+                                    type: string;
+                                };
+                                fetchOffset: {
+                                    bigint: boolean;
+                                };
+                                lastFetchedEpoch: {
+                                    type: string;
+                                };
+                                partitionMaxBytes: {
+                                    type: string;
+                                };
+                            };
+                            required: string[];
+                        };
+                    };
+                };
+                required: string[];
+            };
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const commitOptionsSchema: {
+    type: string;
+    properties: {
+        offsets: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    leaderEpoch: {
+                        type: string;
+                    };
+                    topic: {
+                        type: string;
+                        pattern: string;
+                    };
+                    partition: {
+                        type: string;
+                        minimum: number;
+                    };
+                    offset: {
+                        bigint: boolean;
+                    };
+                };
+                required: string[];
+                additionalProperties: boolean;
+            };
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const listCommitsOptionsSchema: {
+    type: string;
+    properties: {
+        topics: {
+            type: string;
+            items: {
+                type: string;
+                properties: {
+                    topic: {
+                        type: string;
+                        pattern: string;
+                    };
+                    partitions: {
+                        type: string;
+                        items: {
+                            type: string;
+                            minimum: number;
+                        };
+                    };
+                };
+                required: string[];
+                additionalProperties: boolean;
+            };
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const listOffsetsOptionsSchema: {
+    type: string;
+    properties: {
+        topics: {
+            type: string;
+            items: {
+                type: string;
+                pattern: string;
+            };
+        };
+        isolationLevel: {
+            type: string;
+            enum: string[];
+        };
+        timestamp: {
+            bigint: boolean;
+        };
+    };
+    required: string[];
+    additionalProperties: boolean;
+};
+export declare const groupOptionsValidator: import("ajv").ValidateFunction<unknown>;
+export declare const consumeOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const consumerOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const fetchOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const commitOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const listCommitsOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const listOffsetsOptionsValidator: import("ajv").ValidateFunction<{
+    [x: string]: {};
+}>;
+export declare const defaultConsumerOptions: Partial<ConsumerOptions<Buffer, Buffer, Buffer, Buffer>>;

package/dist/clients/consumer/options.js

@@ -0,0 +1,177 @@
+import { FetchIsolationLevels } from "../../apis/enumerations.js";
+import { ajv } from "../../utils.js";
+import { idProperty, topicWithPartitionAndOffsetProperties } from "../base/options.js";
+import { serdeProperties } from "../serde.js";
+import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
+const groupOptionsProperties = {
+    sessionTimeout: { type: 'number', minimum: 0 },
+    rebalanceTimeout: { type: 'number', minimum: 0 },
+    heartbeatInterval: { type: 'number', minimum: 0 },
+    protocols: {
+        type: 'array',
+        items: {
+            type: 'object',
+            properties: {
+                name: idProperty,
+                version: { type: 'number', minimum: 0 },
+                topics: {
+                    type: 'array',
+                    items: { type: 'string' }
+                },
+                metadata: { oneOf: [{ type: 'string' }, { buffer: true }] }
+            }
+        }
+    }
+};
+const consumeOptionsProperties = {
+    autocommit: { oneOf: [{ type: 'boolean' }, { type: 'number', minimum: 100 }] },
+    minBytes: { type: 'number', minimum: 0 },
+    maxBytes: { type: 'number', minimum: 0 },
+    maxWaitTime: { type: 'number', minimum: 0 },
+    isolationLevel: { type: 'string', enum: Object.keys(FetchIsolationLevels) },
+    deserializers: serdeProperties,
+    highWaterMark: { type: 'number', minimum: 1 }
+};
+export const groupOptionsSchema = {
+    type: 'object',
+    properties: groupOptionsProperties,
+    additionalProperties: true // This is needed as we might forward options from consume
+};
+export const consumeOptionsSchema = {
+    type: 'object',
+    properties: {
+        topics: { type: 'array', items: idProperty },
+        mode: { type: 'string', enum: Object.values(MessagesStreamModes) },
+        fallbackMode: { type: 'string', enum: Object.values(MessagesStreamFallbackModes) },
+        offsets: {
+            type: 'array',
+            items: {
+                type: 'object',
+                properties: topicWithPartitionAndOffsetProperties,
+                required: ['topic', 'partition', 'offset'],
+                additionalProperties: false
+            }
+        },
+        ...groupOptionsProperties,
+        ...consumeOptionsProperties
+    },
+    required: ['topics'],
+    additionalProperties: false
+};
+export const consumerOptionsSchema = {
+    type: 'object',
+    properties: {
+        groupId: idProperty,
+        ...groupOptionsProperties,
+        ...consumeOptionsProperties
+    },
+    required: ['groupId'],
+    additionalProperties: true
+};
+export const fetchOptionsSchema = {
+    type: 'object',
+    properties: {
+        node: { type: 'number', minimum: 0 },
+        topics: {
+            type: 'array',
+            items: {
+                type: 'object',
+                properties: {
+                    topicId: { type: 'string' },
+                    partitions: {
+                        type: 'array',
+                        items: {
+                            type: 'object',
+                            properties: {
+                                partition: { type: 'integer' },
+                                currentLeaderEpoch: { type: 'integer' },
+                                fetchOffset: { bigint: true },
+                                lastFetchedEpoch: { type: 'integer' },
+                                partitionMaxBytes: { type: 'integer' }
+                            },
+                            required: ['partition', 'currentLeaderEpoch', 'fetchOffset', 'lastFetchedEpoch', 'partitionMaxBytes']
+                        }
+                    }
+                },
+                required: ['topicId', 'partitions']
+            }
+        },
+        ...groupOptionsProperties,
+        ...consumeOptionsProperties
+    },
+    required: ['node', 'topics'],
+    additionalProperties: false
+};
+export const commitOptionsSchema = {
+    type: 'object',
+    properties: {
+        offsets: {
+            type: 'array',
+            items: {
+                type: 'object',
+                properties: {
+                    ...topicWithPartitionAndOffsetProperties,
+                    leaderEpoch: { type: 'integer' }
+                },
+                required: ['topic', 'partition', 'offset', 'leaderEpoch'],
+                additionalProperties: false
+            }
+        }
+    },
+    required: ['offsets'],
+    additionalProperties: false
+};
+export const listCommitsOptionsSchema = {
+    type: 'object',
+    properties: {
+        topics: {
+            type: 'array',
+            items: {
+                type: 'object',
+                properties: {
+                    topic: idProperty,
+                    partitions: {
+                        type: 'array',
+                        items: {
+                            type: 'number',
+                            minimum: 0
+                        }
+                    }
+                },
+                required: ['topic', 'partitions'],
+                additionalProperties: false
+            }
+        }
+    },
+    required: ['topics'],
+    additionalProperties: false
+};
+export const listOffsetsOptionsSchema = {
+    type: 'object',
+    properties: {
+        topics: { type: 'array', items: idProperty },
+        isolationLevel: { type: 'string', enum: Object.keys(FetchIsolationLevels) },
+        timestamp: { bigint: true }
+    },
+    required: ['topics'],
+    additionalProperties: false
+};
+export const groupOptionsValidator = ajv.compile(groupOptionsSchema);
+export const consumeOptionsValidator = ajv.compile(consumeOptionsSchema);
+export const consumerOptionsValidator = ajv.compile(consumerOptionsSchema);
+export const fetchOptionsValidator = ajv.compile(fetchOptionsSchema);
+export const commitOptionsValidator = ajv.compile(commitOptionsSchema);
+export const listCommitsOptionsValidator = ajv.compile(listCommitsOptionsSchema);
+export const listOffsetsOptionsValidator = ajv.compile(listOffsetsOptionsSchema);
+export const defaultConsumerOptions = {
+    autocommit: true,
+    sessionTimeout: 60_000, // One minute
+    rebalanceTimeout: 102_000, // Two minutes,
+    heartbeatInterval: 3000,
+    protocols: [{ name: 'roundrobin', version: 1 }],
+    minBytes: 1,
+    maxBytes: 1_048_576 * 10, // 10 MB
+    maxWaitTime: 5_000,
+    isolationLevel: 'READ_COMMITTED',
+    highWaterMark: 1024
+};

package/dist/clients/consumer/topics-map.d.ts

@@ -0,0 +1,8 @@
+export declare class TopicsMap extends Map<string, number> {
+    #private;
+    get current(): string[];
+    track(topic: string): boolean;
+    trackAll(...topics: string[]): boolean[];
+    untrack(topic: string): boolean;
+    untrackAll(...topics: string[]): boolean[];
+}