@platformatic/kafka 2.0.1 → 2.2.0

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -15,6 +15,10 @@ export declare const groupOptionsProperties: {
     type: string;
     minimum: number;
   };
+  heartbeatStallTimeout: {
+    type: string;
+    minimum: number;
+  };
   groupProtocol: {
     type: string;
     enum: ("consumer" | "classic")[];
@@ -53,6 +57,9 @@ export declare const groupOptionsProperties: {
       };
     };
   };
+  protocolsMetadata: {
+    function: boolean;
+  };
   partitionAssigner: {
     function: boolean;
   };
@@ -157,6 +164,10 @@ export declare const groupOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -195,6 +206,9 @@ export declare const groupOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -277,6 +291,10 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -315,6 +333,9 @@ export declare const consumeOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -363,6 +384,9 @@ export declare const consumeOptionsSchema: {
     onCorruptedMessage: {
       function: boolean;
     };
+    onDeserializationError: {
+      function: boolean;
+    };
   };
   required: string[];
   additionalProperties: boolean;
@@ -443,6 +467,10 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -481,6 +509,9 @@ export declare const consumerOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -570,6 +601,10 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -608,6 +643,9 @@ export declare const fetchOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -1,10 +1,11 @@
+import { type Callback } from "../../apis/definitions";
 import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
 import { type GroupProtocols } from "../../apis/enumerations";
 import { type ConnectionPool } from "../../network/connection-pool";
 import { type KafkaRecord, type Message } from "../../protocol/records";
 import { type SchemaRegistry } from "../../registries/abstract";
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
-import { type BeforeDeserializationHook, type Deserializers } from "../serde";
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type Deserializers } from "../serde";
 export interface GroupProtocolSubscription {
   name: string;
   version: number;
@@ -27,8 +28,37 @@ export type OffsetsWithTimestamps = Map<string, Map<number, {
   offset: bigint;
   timestamp: bigint;
 }>>;
-export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
+export interface PreferredReadReplica {
+  node: number;
+  expiresAt: number;
+}
+export declare const DeserializationErrorActions: {
+  readonly FAIL: "fail";
+  readonly SKIP: "skip";
+  readonly CONTINUE: "continue";
+};
+export type DeserializationErrorAction = (typeof DeserializationErrorActions)[keyof typeof DeserializationErrorActions];
+export interface DeserializationErrorContext {
+  error: unknown;
+  payloadType: BeforeHookPayloadType;
+  record: KafkaRecord;
+  topic: string;
+  partition: number;
+  offset: bigint;
+  timestamp: bigint;
+  commit: Message['commit'];
+}
+export interface MessageDeserializationError {
+  error: unknown;
+  payloadType: BeforeHookPayloadType;
+}
+export type DeserializationErrorHandler = (context: DeserializationErrorContext) => DeserializationErrorAction;
+/**
+ * @deprecated Use `DeserializationErrorHandler` instead.
+ */
+export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
   readonly LATEST: "latest";
   readonly EARLIEST: "earliest";
@@ -52,7 +82,9 @@ export interface GroupOptions {
   sessionTimeout?: number;
   rebalanceTimeout?: number;
   heartbeatInterval?: number;
+  heartbeatStallTimeout?: number;
   protocols?: GroupProtocolSubscription[];
+  protocolsMetadata?: GroupProtocolsMetadataCallback;
   partitionAssigner?: GroupPartitionsAssigner;
   assignmentUserData?: Buffer;
 }
@@ -61,6 +93,7 @@ export interface ConsumerGroupOptions {
   groupInstanceId?: string;
   groupRemoteAssignor?: string;
   rebalanceTimeout?: number;
+  heartbeatStallTimeout?: number;
   assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
@@ -82,6 +115,10 @@ export interface StreamOptions {
   fallbackMode?: MessagesStreamFallbackModeValue;
   maxFetches?: number;
   offsets?: TopicWithPartitionAndOffset[];
+  onDeserializationError?: DeserializationErrorHandler;
+  /**
+   * @deprecated Use `onDeserializationError` instead.
+   */
   onCorruptedMessage?: CorruptedMessageHandler;
 }
 export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;
@@ -133,4 +170,9 @@ export interface ConsumerHeartbeatPayload {
 }
 export interface ConsumerHeartbeatErrorPayload extends ConsumerHeartbeatPayload {
   error: Error;
+}
+export interface ConsumerHeartbeatStalledPayload {
+  lastError?: Error;
+  lastHeartbeat: Date | null;
+  stalledFor: number;
 }

package/dist/typescript-4/dist/clients/consumer/utils.d.ts

@@ -0,0 +1,2 @@
+export declare function partitionKey(topic: string, partition: number): string;
+export declare function nextFetchEpoch(epoch: number): number;

package/dist/typescript-4/dist/clients/producer/producer.d.ts

@@ -13,7 +13,7 @@ export declare class Producer<Key = Buffer, Value = Buffer, HeaderKey = Buffer,
   get producerId(): bigint | undefined;
   get producerEpoch(): number | undefined;
   get transaction(): Transaction<Key, Value, HeaderKey, HeaderValue> | undefined;
-  get coordinatorId(): number;
+  get coordinatorId(): number | undefined;
   get streamsCount(): number;
   close(force: boolean | CallbackWithPromise<void>, callback?: CallbackWithPromise<void>): void;
   close(force?: boolean): Promise<void>;

package/dist/typescript-4/dist/clients/serde.d.ts

@@ -17,7 +17,7 @@ export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
   headerValue: Deserializer<HeaderValue>;
 }
 export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
-export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
+export type BeforeDeserializationHook = (payload: Buffer | null, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
 export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
 export declare function stringSerializer(data?: string): Buffer | undefined;
 export declare function stringDeserializer(data?: string | Buffer): string | undefined;

package/dist/typescript-4/dist/errors.d.ts

@@ -17,6 +17,7 @@ export declare class GenericError extends Error {
     ...rest
   }?: ErrorProperties);
   findBy<ErrorType extends GenericError = GenericError>(property: string, value: unknown): ErrorType | null;
+  toString(): string;
 }
 export declare class MultipleErrors extends AggregateError {
   code: string;
@@ -31,7 +32,9 @@ export declare class MultipleErrors extends AggregateError {
     ...rest
   }?: ErrorProperties);
   findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
+  toString(): string;
 }
+export declare function findErrorBy<ErrorType extends GenericError = GenericError>(error: Error | null | undefined, property: string, value: unknown): ErrorType | null;
 export * from "./protocol/errors";
 export declare class AuthenticationError extends GenericError {
   static code: ErrorCode;
@@ -39,6 +42,7 @@ export declare class AuthenticationError extends GenericError {
 }
 export declare class NetworkError extends GenericError {
   static code: ErrorCode;
+  static isRetryable(error: Error | null | undefined): boolean;
   constructor(message: string, properties?: ErrorProperties);
 }
 export declare class ProtocolError extends GenericError {

package/dist/typescript-4/dist/protocol/records.d.ts

@@ -33,11 +33,14 @@ export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown
   partition: number;
   timestamp: string;
   offset: string;
+  leaderEpoch: number;
   metadata: Record<string, unknown>;
 }
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
   headers: Map<HeaderKey, HeaderValue>;
+  headerEntries: Array<[HeaderKey, HeaderValue]>;
   offset: bigint;
+  leaderEpoch: number;
   metadata: Record<string, unknown>;
   commit(callback?: (error?: Error) => void): void | Promise<void>;
   toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;
@@ -71,6 +74,7 @@ export interface KafkaRecord {
 export interface MessageToConsume extends KafkaRecord {
   topic: string;
   partition: number;
+  metadata?: Record<string, unknown>;
 }
 export interface RecordsBatch {
   firstOffset: bigint;

package/dist/typescript-4/dist/registries/abstract.d.ts

@@ -4,7 +4,7 @@ import { type MessageToProduce } from "../protocol/records";
 export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
   get(id: Id): Schema | undefined;
   fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
-  getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+  getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
   getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
   getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
 export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
   get(_: Id): Schema | undefined;
   fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
-  getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+  getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
   getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
   getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;

package/dist/typescript-4/dist/registries/confluent-schema-registry.d.ts

@@ -1,16 +1,32 @@
-import { type ValidateFunction } from 'ajv';
+import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
+import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
 import { type Root } from 'protobufjs';
 import { type Callback } from "../apis/definitions";
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from "../clients/serde";
-import { type CredentialProvider } from "../index";
+import { type CredentialProvider, UserError } from "../index";
 import { type MessageToConsume, type MessageToProduce } from "../protocol/records";
 import { AbstractSchemaRegistry } from "./abstract";
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export type UndiciDispatcher = NonNullable<RequestInit['dispatcher']>;
+export interface UndiciAgentOptions {
+  connect: TLSConnectionOptions;
+}
+export type UndiciAgentConstructor = new (options: UndiciAgentOptions) => UndiciDispatcher;
+export declare function createUndiciAgent(options: UndiciAgentOptions): UndiciDispatcher;
 export interface ConfluentSchemaRegistryMetadata {
-  schemas?: Record<BeforeHookPayloadType, number>;
+  schemas?: Partial<Record<BeforeHookPayloadType, number>>;
+}
+export interface ConfluentSchemaRegistryConsumedMetadata {
+  lengths?: {
+    key?: number;
+    value?: number;
+  };
 }
 export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export type ConfluentSchemaRegistryHeaders = Record<string, string>;
+export type ConfluentSchemaRegistryHeadersProvider = () => ConfluentSchemaRegistryHeaders | Promise<ConfluentSchemaRegistryHeaders>;
+export type ConfluentSchemaRegistryRetryDelayGetter = (attempt: number, retries: number, error: Error) => number;
 export interface ConfluentSchemaRegistryOptions {
   url: string;
   auth?: {
@@ -18,19 +34,56 @@ export interface ConfluentSchemaRegistryOptions {
     password?: string | CredentialProvider;
     token?: string | CredentialProvider;
   };
+  headers?: ConfluentSchemaRegistryHeaders | ConfluentSchemaRegistryHeadersProvider;
+  timeout?: number;
+  retries?: number;
+  retryDelay?: number | ConfluentSchemaRegistryRetryDelayGetter;
+  tls?: TLSConnectionOptions;
+  ssl?: TLSConnectionOptions;
   protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
   jsonValidateSend?: boolean;
+  jsonAjvOptions?: Options;
+}
+export declare const defaultConfluentSchemaRegistryOptions: {
+  timeout: number;
+  retries: number;
+  retryDelay: number;
+};
+export declare const retriableSchemaRegistryStatuses: number[];
+export type SchemaValidationPhase = 'serialization' | 'deserialization';
+export interface SchemaValidationErrorProperties {
+  schemaId: number;
+  schemaType: 'json';
+  phase: SchemaValidationPhase;
+  payloadType: BeforeHookPayloadType;
+  type: BeforeHookPayloadType;
+  data: unknown;
+  headers?: ReadonlyMap<unknown, unknown>;
+  validationErrors: ErrorObject[] | null | undefined;
+}
+export declare class SchemaValidationError extends UserError {
+  readonly schemaId: number;
+  readonly schemaType: 'json';
+  readonly phase: SchemaValidationPhase;
+  readonly payloadType: BeforeHookPayloadType;
+  readonly type: BeforeHookPayloadType;
+  readonly data: unknown;
+  readonly headers?: ReadonlyMap<unknown, unknown>;
+  readonly validationErrors: ErrorObject[] | null | undefined;
+  constructor(message: string, properties: SchemaValidationErrorProperties);
 }
 export interface Schema {
   id: number;
   type: 'avro' | 'protobuf' | 'json';
   schema: Type | Root | ValidateFunction;
 }
+export declare function trackConsumedSchemaId(message: MessageToConsume | undefined, type: BeforeHookPayloadType, schemaId: number): void;
+export declare function trackConsumedPayloadLength(message: MessageToConsume | undefined, type: 'key' | 'value', data: Buffer): void;
 export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
   #private;
   constructor(options: ConfluentSchemaRegistryOptions);
-  getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+  getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
   get(id: number): Schema | undefined;
   fetchSchema(id: number, callback: Callback<void>): Promise<void>;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;

package/dist/typescript-4/dist/symbols.d.ts

@@ -1,6 +1,8 @@
 export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
+export declare const kGetFetchNode: unique symbol;
+export declare const kDeserializationError: unique symbol;
 export declare const kTransaction: unique symbol;
 export declare const kTransactionPrepare: unique symbol;
 export declare const kTransactionAddPartitions: unique symbol;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.0.1";
+export const version = "2.2.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.0.1",
+  "version": "2.2.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -28,6 +28,7 @@
     "@platformatic/dynamic-buffer": "^0.3.1",
     "@platformatic/wasm-utils": "^0.2.1",
     "ajv": "^8.17.1",
+    "ajv-draft-04": "^1.0.0",
     "avsc": "^5.7.9",
     "debug": "^4.4.3",
     "fastq": "^1.19.1",
@@ -89,12 +90,20 @@
     "postbuild": "./scripts/node scripts/postbuild.ts",
     "lint": "eslint --cache",
     "typecheck": "tsc -p . --noEmit",
-    "format": "prettier -w benchmarks playground src test",
+    "format": "prettier -w benchmarks playground regression src test",
     "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
-    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:ci": "./scripts/node --test --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-lines=90 --test-coverage-branches=90 --test-coverage-functions=90 --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:integrity": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/integrity/*.test.ts'",
+    "test:performance": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/performance/*.test.ts'",
+    "test:regression": "npm run test:integrity",
+    "test:regression:all": "npm run test:integrity && npm run test:performance",
+    "test:all": "npm test && npm run test:regression:all",
+    "test:compat": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/integration/*.compat-test.ts'",
     "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
     "test:docker:up": "docker-compose up -d --wait",
     "test:docker:down": "docker-compose down",
+    "regression:baseline:compare": "npm run test:performance",
+    "regression:baseline:update": "./scripts/node regression/helpers/run-benchmarks.ts",
     "ci": "npm run build && npm run lint && npm run test:ci",
     "generate:apis": "node scripts/generate-apis.ts",
     "generate:errors": "node scripts/generate-errors.ts",