@platformatic/rdkafka 4.0.0

package/examples/producer-cluster.md

@@ -0,0 +1,103 @@
+```js
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var cluster = require('cluster');
+var numCPUs = 6;
+var Kafka = require('../');
+
+if (cluster.isMaster) {
+  // Fork workers.
+  for (var i = 0; i < numCPUs; i++) {
+    cluster.fork();
+  }
+
+  var exited_workers = 0;
+
+  cluster.on('exit', function(worker, code, signal) {
+    exited_workers++;
+    if (exited_workers === numCPUs - 1) {
+      process.exit();
+    }
+  });
+} else {
+  // Configure client
+  var producer = new Kafka.Producer({
+    'client.id': 'kafka',
+    'metadata.broker.list': 'localhost:9092',
+    'compression.codec': 'none',
+    'retry.backoff.ms': 200,
+    'message.send.max.retries': 10,
+    'socket.keepalive.enable': true,
+    'queue.buffering.max.messages': 100000,
+    'queue.buffering.max.ms': 1000,
+    'batch.num.messages': 1000000,
+    'dr_cb': true
+  });
+
+  producer.setPollInterval(100);
+
+  var total = 0;
+  var totalSent = 0;
+  var max = 20000;
+  var errors = 0;
+  var started = Date.now();
+
+  var sendMessage = function() {
+    var ret = producer.sendMessage({
+      topic: 'librdtesting-01',
+      message: Buffer.from('message ' + total)
+    }, function() {
+    });
+    total++;
+    if (total >= max) {
+    } else {
+      setImmediate(sendMessage);
+    }
+  };
+
+  var verified_received = 0;
+  var exitNextTick = false;
+  var errorsArr = [];
+
+  var t = setInterval(function() {
+    producer.poll();
+
+    if (exitNextTick) {
+      clearInterval(t);
+      return setTimeout(function() {
+        console.log('[%d] Received: %d, Errors: %d, Total: %d', process.pid, verified_received, errors, total);
+        // console.log('[%d] Finished sending %d in %d seconds', process.pid, total, parseInt((Date.now() - started) / 1000));
+        if (errors > 0) {
+          console.error(errorsArr[0]);
+          return process.exitCode = 1;
+        }
+        process.exitCode = 0;
+        setTimeout(process.exit, 1000);
+      }, 2000);
+    }
+
+    if (verified_received + errors === max) {
+      exitNextTick = true;
+    }
+
+  }, 1000);
+  producer.connect()
+    .on('event.error', function(e) {
+      errors++;
+      errorsArr.push(e);
+    })
+    .on('delivery-report', function() {
+      verified_received++;
+    })
+    .on('ready', sendMessage);
+
+
+}
+```

package/examples/producer.md

@@ -0,0 +1,73 @@
+```js
+/*
+ * node-rdkafka - Node.js wrapper for RdKafka C/C++ library
+ *
+ * Copyright (c) 2016 Blizzard Entertainment
+ *
+ * This software may be modified and distributed under the terms
+ * of the MIT license.  See the LICENSE.txt file for details.
+ */
+
+var Kafka = require('../');
+
+var producer = new Kafka.Producer({
+  //'debug' : 'all',
+  'metadata.broker.list': 'localhost:9092',
+  'dr_cb': true  //delivery report callback
+});
+
+var topicName = 'test';
+
+//logging debug messages, if debug is enabled
+producer.on('event.log', function(log) {
+  console.log(log);
+});
+
+//logging all errors
+producer.on('event.error', function(err) {
+  console.error('Error from producer');
+  console.error(err);
+});
+
+//counter to stop this sample after maxMessages are sent
+var counter = 0;
+var maxMessages = 10;
+
+producer.on('delivery-report', function(err, report) {
+  console.log('delivery-report: ' + JSON.stringify(report));
+  counter++;
+});
+
+//Wait for the ready event before producing
+producer.on('ready', function(arg) {
+  console.log('producer ready.' + JSON.stringify(arg));
+
+  for (var i = 0; i < maxMessages; i++) {
+    var value = Buffer.from('value-' +i);
+    var key = "key-"+i;
+    // if partition is set to -1, librdkafka will use the default partitioner
+    var partition = -1;
+    var headers = [
+      { header: "header value" }
+    ]
+    producer.produce(topicName, partition, value, key, Date.now(), "", headers);
+  }
+
+  //need to keep polling for a while to ensure the delivery reports are received
+  var pollLoop = setInterval(function() {
+      producer.poll();
+      if (counter === maxMessages) {
+        clearInterval(pollLoop);
+        producer.disconnect();
+      }
+    }, 1000);
+
+});
+
+producer.on('disconnected', function(arg) {
+  console.log('producer disconnected. ' + JSON.stringify(arg));
+});
+
+//starting the producer
+producer.connect();
+```

package/index.d.ts

@@ -0,0 +1,357 @@
+import { Readable, ReadableOptions, Writable, WritableOptions } from 'stream';
+import { EventEmitter } from 'events';
+import {
+    GlobalConfig,
+    TopicConfig,
+    ConsumerGlobalConfig,
+    ConsumerTopicConfig,
+    ProducerGlobalConfig,
+    ProducerTopicConfig,
+} from './config';
+
+export * from './config';
+export * from './errors';
+
+export interface LibrdKafkaError {
+    message: string;
+    code: number;
+    errno: number;
+    origin: string;
+    stack?: string;
+    isFatal?: boolean;
+    isRetriable?: boolean;
+    isTxnRequiresAbort?: boolean;
+}
+
+export interface ReadyInfo {
+    name: string;
+}
+
+export interface ClientMetrics {
+    connectionOpened: number;
+}
+
+export interface MetadataOptions {
+    topic?: string;
+    allTopics?: boolean;
+    timeout?: number;
+}
+
+export interface BrokerMetadata {
+    id: number;
+    host: string;
+    port: number;
+}
+
+export interface PartitionMetadata {
+    id: number;
+    leader: number;
+    replicas: number[];
+    isrs: number[];
+}
+
+export interface TopicMetadata {
+    name: string;
+    partitions: PartitionMetadata[];
+}
+
+export interface Metadata {
+    orig_broker_id: number;
+    orig_broker_name: string;
+    topics: TopicMetadata[];
+    brokers: BrokerMetadata[];
+}
+
+export interface WatermarkOffsets{
+    lowOffset: number;
+    highOffset: number;
+}
+
+export interface TopicPartition {
+    topic: string;
+    partition: number;
+}
+
+export interface TopicPartitionOffset extends TopicPartition{
+    offset: number;
+}
+
+export type TopicPartitionTime = TopicPartitionOffset;
+
+export type EofEvent = TopicPartitionOffset;
+
+export type Assignment = TopicPartition | TopicPartitionOffset;
+
+export interface DeliveryReport extends TopicPartitionOffset {
+    value?: MessageValue;
+    size: number;
+    key?: MessageKey;
+    timestamp?: number;
+    opaque?: any;
+}
+
+export type NumberNullUndefined = number | null | undefined;
+
+export type MessageKey = Buffer | string | null | undefined;
+export type MessageHeader = { [key: string]: string | Buffer };
+export type MessageValue = Buffer | null;
+export type SubscribeTopic = string | RegExp;
+export type SubscribeTopicList = SubscribeTopic[];
+
+export interface Message extends TopicPartitionOffset {
+    value: MessageValue;
+    size: number;
+    topic: string;
+    key?: MessageKey;
+    timestamp?: number;
+    headers?: MessageHeader[];
+    opaque?: any;
+}
+
+export interface ReadStreamOptions extends ReadableOptions {
+    topics: SubscribeTopicList | SubscribeTopic | ((metadata: Metadata) => SubscribeTopicList);
+    waitInterval?: number;
+    fetchSize?: number;
+    objectMode?: boolean;
+    highWaterMark?: number;
+    autoClose?: boolean;
+    streamAsBatch?: boolean;
+    connectOptions?: any;
+    initOauthBearerToken?: string;
+}
+
+export interface WriteStreamOptions extends WritableOptions {
+    encoding?: string;
+    objectMode?: boolean;
+    topic?: string;
+    autoClose?: boolean;
+    pollInterval?: number;
+    connectOptions?: any;
+}
+
+export interface ProducerStream extends Writable {
+    producer: Producer;
+    connect(metadataOptions?: MetadataOptions): void;
+    close(cb?: () => void): void;
+}
+
+export interface ConsumerStream extends Readable {
+    consumer: KafkaConsumer;
+    connect(options: ConsumerGlobalConfig): void;
+    refreshOauthBearerToken(tokenStr: string): void;
+    close(cb?: () => void): void;
+}
+
+export type KafkaClientEvents = 'disconnected' | 'ready' | 'connection.failure' | 'event.error' | 'event.stats' | 'event.log' | 'event.event' | 'event.throttle';
+export type KafkaConsumerEvents = 'data' | 'partition.eof' | 'rebalance' | 'rebalance.error' | 'subscribed' | 'unsubscribed' | 'unsubscribe' | 'offset.commit' | KafkaClientEvents;
+export type KafkaProducerEvents = 'delivery-report' | KafkaClientEvents;
+
+type EventListenerMap = {
+    // ### Client
+    // connectivity events
+    'disconnected': (metrics: ClientMetrics) => void,
+    'ready': (info: ReadyInfo, metadata: Metadata) => void,
+    'connection.failure': (error: LibrdKafkaError, metrics: ClientMetrics) => void,
+    // event messages
+    'event.error': (error: LibrdKafkaError) => void,
+    'event.stats': (eventData: any) => void,
+    'event.log': (eventData: any) => void,
+    'event.event': (eventData: any) => void,
+    'event.throttle': (eventData: any) => void,
+    // ### Consumer only
+    // domain events
+    'data': (arg: Message) => void,
+    'partition.eof': (arg: EofEvent) => void,
+    'rebalance': (err: LibrdKafkaError, assignments: TopicPartition[]) => void,
+    'rebalance.error': (err: Error) => void,
+    // connectivity events
+    'subscribed': (topics: SubscribeTopicList) => void,
+    'unsubscribe': () => void,
+    'unsubscribed': () => void,
+    // offsets
+    'offset.commit': (error: LibrdKafkaError, topicPartitions: TopicPartitionOffset[]) => void,
+    // ### Producer only
+    // delivery
+    'delivery-report': (error: LibrdKafkaError, report: DeliveryReport) => void,
+}
+
+type EventListener<K extends string> = K extends keyof EventListenerMap ? EventListenerMap[K] : never;
+
+export abstract class Client<Events extends string> extends EventEmitter {
+    constructor(globalConf: GlobalConfig, SubClientType: any, topicConf: TopicConfig);
+
+    connect(metadataOptions?: MetadataOptions, cb?: (err: LibrdKafkaError, data: Metadata) => any): this;
+
+    setOauthBearerToken(tokenStr: string): this;
+
+    getClient(): any;
+
+    connectedTime(): number;
+
+    getLastError(): LibrdKafkaError;
+
+    disconnect(cb?: (err: any, data: ClientMetrics) => any): this;
+    disconnect(timeout: number, cb?: (err: any, data: ClientMetrics) => any): this;
+
+    isConnected(): boolean;
+
+    getMetadata(metadataOptions?: MetadataOptions, cb?: (err: LibrdKafkaError, data: Metadata) => any): any;
+
+    queryWatermarkOffsets(topic: string, partition: number, timeout: number, cb?: (err: LibrdKafkaError, offsets: WatermarkOffsets) => any): any;
+    queryWatermarkOffsets(topic: string, partition: number, cb?: (err: LibrdKafkaError, offsets: WatermarkOffsets) => any): any;
+
+    on<E extends Events>(event: E, listener: EventListener<E>): this;
+    once<E extends Events>(event: E, listener: EventListener<E>): this;
+}
+
+export class KafkaConsumer extends Client<KafkaConsumerEvents> {
+    constructor(conf: ConsumerGlobalConfig, topicConf: ConsumerTopicConfig);
+
+    assign(assignments: Assignment[]): this;
+    incrementalAssign(assignments: Assignment[]): this;
+
+    assignments(): Assignment[];
+
+    commit(topicPartition: TopicPartitionOffset | TopicPartitionOffset[]): this;
+    commit(): this;
+
+    commitMessage(msg: TopicPartitionOffset): this;
+
+    commitMessageSync(msg: TopicPartitionOffset): this;
+
+    commitSync(topicPartition: TopicPartitionOffset | TopicPartitionOffset[] | null): this;
+
+    committed(toppars: TopicPartition[], timeout: number, cb: (err: LibrdKafkaError, topicPartitions: TopicPartitionOffset[]) => void): this;
+    committed(timeout: number, cb: (err: LibrdKafkaError, topicPartitions: TopicPartitionOffset[]) => void): this;
+
+    consume(number: number, cb?: (err: LibrdKafkaError, messages: Message[]) => void): void;
+    consume(cb: (err: LibrdKafkaError, messages: Message[]) => void): void;
+    consume(): void;
+
+    getWatermarkOffsets(topic: string, partition: number): WatermarkOffsets;
+
+    offsetsStore(topicPartitions: TopicPartitionOffset[]): any;
+
+    pause(topicPartitions: TopicPartition[]): any;
+
+    position(toppars?: TopicPartition[]): TopicPartitionOffset[];
+
+    resume(topicPartitions: TopicPartition[]): any;
+
+    seek(toppar: TopicPartitionOffset, timeout: number | null, cb: (err: LibrdKafkaError) => void): this;
+
+    setDefaultConsumeTimeout(timeoutMs: number): void;
+
+    setDefaultConsumeLoopTimeoutDelay(timeoutMs: number): void;
+
+    subscribe(topics: SubscribeTopicList): this;
+
+    subscription(): string[];
+
+    unassign(): this;
+    incrementalUnassign(assignments: Assignment[]): this;
+
+    unsubscribe(): this;
+
+    offsetsForTimes(topicPartitions: TopicPartitionTime[], timeout: number, cb?: (err: LibrdKafkaError, offsets: TopicPartitionOffset[]) => any): void;
+    offsetsForTimes(topicPartitions: TopicPartitionTime[], cb?: (err: LibrdKafkaError, offsets: TopicPartitionOffset[]) => any): void;
+
+    rebalanceProtocol(): string;
+
+    static createReadStream(conf: ConsumerGlobalConfig, topicConfig: ConsumerTopicConfig, streamOptions: ReadStreamOptions | number): ConsumerStream;
+}
+
+export class Producer extends Client<KafkaProducerEvents> {
+    constructor(conf: ProducerGlobalConfig, topicConf?: ProducerTopicConfig);
+
+    flush(timeout?: NumberNullUndefined, cb?: (err: LibrdKafkaError) => void): this;
+
+    poll(): this;
+
+    produce(topic: string, partition: NumberNullUndefined, message: MessageValue, key?: MessageKey, timestamp?: NumberNullUndefined, opaque?: any, headers?: MessageHeader[]): any;
+
+    setPollInterval(interval: number): this;
+
+    static createWriteStream(conf: ProducerGlobalConfig, topicConf: ProducerTopicConfig, streamOptions: WriteStreamOptions): ProducerStream;
+
+    initTransactions(cb: (err: LibrdKafkaError) => void): void;
+    initTransactions(timeout: number, cb: (err: LibrdKafkaError) => void): void;
+    beginTransaction(cb: (err: LibrdKafkaError) => void): void;
+    commitTransaction(cb: (err: LibrdKafkaError) => void): void;
+    commitTransaction(timeout: number, cb: (err: LibrdKafkaError) => void): void;
+    abortTransaction(cb: (err: LibrdKafkaError) => void): void;
+    abortTransaction(timeout: number, cb: (err: LibrdKafkaError) => void): void;
+    sendOffsetsToTransaction(offsets: TopicPartitionOffset[], consumer: KafkaConsumer, cb: (err: LibrdKafkaError) => void): void;
+    sendOffsetsToTransaction(offsets: TopicPartitionOffset[], consumer: KafkaConsumer, timeout: number, cb: (err: LibrdKafkaError) => void): void;
+}
+
+export class HighLevelProducer extends Producer {
+  produce(topic: string, partition: NumberNullUndefined, message: any, key: any, timestamp: NumberNullUndefined, callback: (err: any, offset?: NumberNullUndefined) => void): any;
+  produce(topic: string, partition: NumberNullUndefined, message: any, key: any, timestamp: NumberNullUndefined, headers: MessageHeader[], callback: (err: any, offset?: NumberNullUndefined) => void): any;
+
+  setKeySerializer(serializer: (key: any, cb: (err: any, key: MessageKey) => void) => void): void;
+  setKeySerializer(serializer: (key: any) => MessageKey | Promise<MessageKey>): void;
+  setValueSerializer(serializer: (value: any, cb: (err: any, value: MessageValue) => void) => void): void;
+  setValueSerializer(serializer: (value: any) => MessageValue | Promise<MessageValue>): void;
+}
+
+export const features: string[];
+
+export const librdkafkaVersion: string;
+
+export function createReadStream(conf: ConsumerGlobalConfig, topicConf: ConsumerTopicConfig, streamOptions: ReadStreamOptions | number): ConsumerStream;
+
+export function createWriteStream(conf: ProducerGlobalConfig, topicConf: ProducerTopicConfig, streamOptions: WriteStreamOptions): ProducerStream;
+
+export interface NewTopic {
+    topic: string;
+    num_partitions: number;
+    replication_factor: number;
+    config?: {
+        'cleanup.policy'?: 'delete' | 'compact' | 'delete,compact' | 'compact,delete';
+        'compression.type'?: 'gzip' | 'snappy' | 'lz4' | 'zstd' | 'uncompressed' | 'producer';
+        'delete.retention.ms'?: string;
+        'file.delete.delay.ms'?: string;
+        'flush.messages'?: string;
+        'flush.ms'?: string;
+        'follower.replication.throttled.replicas'?: string;
+        'index.interval.bytes'?: string;
+        'leader.replication.throttled.replicas'?: string;
+        'max.compaction.lag.ms'?: string;
+        'max.message.bytes'?: string;
+        'message.format.version'?: string;
+        'message.timestamp.difference.max.ms'?: string;
+        'message.timestamp.type'?: string;
+        'min.cleanable.dirty.ratio'?: string;
+        'min.compaction.lag.ms'?: string;
+        'min.insync.replicas'?: string;
+        'preallocate'?: string;
+        'retention.bytes'?: string;
+        'retention.ms'?: string;
+        'segment.bytes'?: string;
+        'segment.index.bytes'?: string;
+        'segment.jitter.ms'?: string;
+        'segment.ms'?: string;
+        'unclean.leader.election.enable'?: string;
+        'message.downconversion.enable'?: string;
+    } | { [cfg: string]: string; };
+}
+
+export interface IAdminClient {
+    refreshOauthBearerToken(tokenStr: string): void;
+
+    createTopic(topic: NewTopic, cb?: (err: LibrdKafkaError) => void): void;
+    createTopic(topic: NewTopic, timeout?: number, cb?: (err: LibrdKafkaError) => void): void;
+
+    deleteTopic(topic: string, cb?: (err: LibrdKafkaError) => void): void;
+    deleteTopic(topic: string, timeout?: number, cb?: (err: LibrdKafkaError) => void): void;
+
+    createPartitions(topic: string, desiredPartitions: number, cb?: (err: LibrdKafkaError) => void): void;
+    createPartitions(topic: string, desiredPartitions: number, timeout?: number, cb?: (err: LibrdKafkaError) => void): void;
+
+    disconnect(): void;
+}
+
+export abstract class AdminClient {
+    static create(conf: GlobalConfig, initOauthBearerToken?: string): IAdminClient;
+}