@platformatic/kafka 2.1.0 → 2.2.0

package/dist/typescript-4/dist/apis/telemetry/list-config-resources-v1.d.ts

@@ -0,0 +1,16 @@
+import { type Reader } from "../../protocol/reader";
+import { Writer } from "../../protocol/writer";
+import { type ConfigResourceTypeValue } from "../enumerations";
+export type ListConfigResourcesRequest = Parameters<typeof createRequest>;
+export interface ListConfigResourcesResource {
+  resourceName: string;
+  resourceType: ConfigResourceTypeValue;
+}
+export interface ListConfigResourcesResponse {
+  throttleTimeMs: number;
+  errorCode: number;
+  configResources: ListConfigResourcesResource[];
+}
+export declare function createRequest(resourceTypes?: ConfigResourceTypeValue[]): Writer;
+export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): ListConfigResourcesResponse;
+export declare const api: import("../definitions").API<[resourceTypes?: ConfigResourceTypeValue[] | undefined], ListConfigResourcesResponse>;

package/dist/typescript-4/dist/apis/telemetry/push-telemetry-v0.d.ts

@@ -1,10 +1,11 @@
 import { type Reader } from "../../protocol/reader";
 import { Writer } from "../../protocol/writer";
+import { type TelemetryCompressionType } from "./compression-types";
 export type PushTelemetryRequest = Parameters<typeof createRequest>;
 export interface PushTelemetryResponse {
   throttleTimeMs: number;
   errorCode: number;
 }
-export declare function createRequest(clientInstanceId: string, subscriptionId: number, terminating: boolean, compressionType: number, metrics: Buffer): Writer;
+export declare function createRequest(clientInstanceId: string, subscriptionId: number, terminating: boolean, compressionType: TelemetryCompressionType, metrics: Buffer): Writer;
 export declare function parseResponse(_correlationId: number, apiKey: number, apiVersion: number, reader: Reader): PushTelemetryResponse;
-export declare const api: import("../definitions").API<[clientInstanceId: string, subscriptionId: number, terminating: boolean, compressionType: number, metrics: Buffer], PushTelemetryResponse>;
+export declare const api: import("../definitions").API<[clientInstanceId: string, subscriptionId: number, terminating: boolean, compressionType: TelemetryCompressionType, metrics: Buffer], PushTelemetryResponse>;

package/dist/typescript-4/dist/clients/admin/options.d.ts

@@ -14,8 +14,48 @@ export declare const createTopicOptionsSchema: {
     topics: {
       type: string;
       items: {
-        type: string;
-        pattern: string;
+        oneOf: ({
+          type: string;
+          pattern: string;
+        } | {
+          type: string;
+          properties: {
+            topic: {
+              type: string;
+              pattern: string;
+            };
+            partitions: {
+              type: string;
+            };
+            replicas: {
+              type: string;
+            };
+            assignments: {
+              type: string;
+              items: {
+                type: string;
+                properties: {
+                  partition: {
+                    type: string;
+                    minimum: number;
+                  };
+                  brokers: {
+                    type: string;
+                    items: {
+                      type: string;
+                    };
+                    minItems: number;
+                  };
+                };
+                required: string[];
+                additionalProperties: boolean;
+              };
+              minItems: number;
+            };
+          };
+          required: string[];
+          additionalProperties: boolean;
+        })[];
       };
     };
     partitions: {
@@ -46,6 +86,22 @@ export declare const createTopicOptionsSchema: {
       };
       minItems: number;
     };
+    configs: {
+      type: string;
+      items: {
+        type: string;
+        properties: {
+          name: {
+            type: string;
+          };
+          value: {
+            type: string[];
+          };
+        };
+        required: string[];
+        additionalProperties: boolean;
+      };
+    };
   };
   required: string[];
   additionalProperties: boolean;
@@ -129,7 +185,7 @@ export declare const listGroupsOptionsSchema: {
       items: {
         type: string;
         enumeration: {
-          allowed: readonly ["PREPARING_REBALANCE", "COMPLETING_REBALANCE", "STABLE", "DEAD", "EMPTY"];
+          allowed: ("Unknown" | "PreparingRebalance" | "CompletingRebalance" | "Stable" | "Dead" | "Empty" | "Assigning" | "Reconciling" | "NotReady" | "PREPARING_REBALANCE" | "COMPLETING_REBALANCE" | "STABLE" | "DEAD" | "EMPTY")[];
           errorMessage: string;
         };
       };
@@ -708,7 +764,7 @@ export declare const createAclsOptionsSchema: {
           };
           operation: {
             type: string;
-            enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 3 | 5 | 6 | 9 | 11 | 12 | 13 | 14 | 15)[];
+            enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 12 | 3 | 5 | 6 | 9 | 11 | 13 | 14 | 15)[];
           };
           permissionType: {
             type: string;
@@ -767,7 +823,7 @@ export declare const describeAclsOptionsSchema: {
         };
         operation: {
           type: string;
-          enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 3 | 5 | 6 | 9 | 11 | 12 | 13 | 14 | 15)[];
+          enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 12 | 3 | 5 | 6 | 9 | 11 | 13 | 14 | 15)[];
         };
         permissionType: {
           type: string;
@@ -826,7 +882,7 @@ export declare const deleteAclsOptionsSchema: {
           };
           operation: {
             type: string;
-            enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 3 | 5 | 6 | 9 | 11 | 12 | 13 | 14 | 15)[];
+            enum: (0 | 1 | 2 | 4 | 8 | 7 | 10 | 12 | 3 | 5 | 6 | 9 | 11 | 13 | 14 | 15)[];
           };
           permissionType: {
             type: string;

package/dist/typescript-4/dist/clients/admin/types.d.ts

@@ -33,7 +33,7 @@ export interface GroupMember {
 export interface GroupBase {
   id: string;
   state: ConsumerGroupStateValue;
-  groupType: string;
+  groupType?: string;
   protocolType: string;
 }
 export interface Group extends Omit<GroupBase, 'groupType'> {
@@ -42,8 +42,14 @@ export interface Group extends Omit<GroupBase, 'groupType'> {
   authorizedOperations: number;
 }
 export interface AdminOptions extends BaseOptions {}
+export interface CreateTopicsTopicOptions {
+  topic: string;
+  partitions?: number;
+  replicas?: number;
+  assignments?: BrokerAssignment[];
+}
 export interface CreateTopicsOptions {
-  topics: string[];
+  topics: (string | CreateTopicsTopicOptions)[];
   partitions?: number;
   replicas?: number;
   assignments?: BrokerAssignment[];

package/dist/typescript-4/dist/clients/base/base.d.ts

@@ -1,7 +1,7 @@
 import { type ValidateFunction } from 'ajv';
 import { type CallbackWithPromise } from "../../apis/callbacks";
 import { type API, type Callback } from "../../apis/definitions";
-import { type ApiVersionsResponseApi } from "../../apis/metadata/api-versions-v3";
+import { type ApiVersionsResponseApi } from "../../apis/metadata/api-versions-v1";
 import { type ClientType } from "../../diagnostic";
 import { TypedEventEmitter, type TypedEvents } from "../../events";
 import { ConnectionPool, type ConnectionPoolEventPayload } from "../../network/connection-pool";
@@ -84,11 +84,11 @@ export declare class Base<OptionsType extends BaseOptions = BaseOptions, EventsT
   [kMetadata](options: MetadataOptions, callback: CallbackWithPromise<ClusterMetadata>): void;
   [kCheckNotClosed](callback: CallbackWithPromise<any>): boolean;
   clearMetadata(): void;
-  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
+  [kPerformWithRetry]<ReturnType>(operationId: string, operation: (callback: Callback<ReturnType>, attempt: number) => void, callback: CallbackWithPromise<ReturnType>, attempt?: number, errors?: Error[], shouldSkipRetry?: (e: Error) => boolean): void | Promise<ReturnType>;
   [kPerformDeduplicated]<ReturnType>(operationId: string, operation: (callback: CallbackWithPromise<ReturnType>) => void, callback: CallbackWithPromise<ReturnType>): void | Promise<ReturnType>;
   [kGetApi]<RequestArguments extends Array<unknown>, ResponseType>(name: string, callback: Callback<API<RequestArguments, ResponseType>>): void;
   [kGetConnection](broker: Broker, callback: Callback<Connection>): void;
-  [kGetBootstrapConnection](callback: Callback<Connection>): void;
+  [kGetBootstrapConnection](callback: Callback<Connection>, attempt?: number): void;
   [kValidateOptions](target: unknown, validator: ValidateFunction<unknown>, targetName: string, throwOnErrors?: boolean): Error | null;
   [kInspect](...args: unknown[]): void;
   [kFormatValidationErrors](validator: ValidateFunction<unknown>, targetName: string): string;

package/dist/typescript-4/dist/clients/consumer/consumer.d.ts

@@ -6,7 +6,7 @@ import { Base, type BaseEvents, kCreateConnectionPool } from "../base/base";
 import { type ClusterMetadata } from "../base/types";
 import { MessagesStream } from "./messages-stream";
 import { TopicsMap } from "./topics-map";
-import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
+import { type CommitOptions, type ConsumeOptions, type ConsumerGroupJoinPayload, type ConsumerGroupLeavePayload, type ConsumerGroupRebalancePayload, type ConsumerHeartbeatErrorPayload, type ConsumerHeartbeatPayload, type ConsumerHeartbeatStalledPayload, type ConsumerOptions, type FetchOptions, type GetLagOptions, type GroupAssignment, type GroupOptions, type ListCommitsOptions, type ListOffsetsOptions, type Offsets, type OffsetsWithTimestamps } from "./types";
 export interface ConsumerEvents extends BaseEvents {
   'consumer:group:join': (payload: ConsumerGroupJoinPayload) => void;
   'consumer:group:leave': (payload: ConsumerGroupLeavePayload) => void;
@@ -16,6 +16,7 @@ export interface ConsumerEvents extends BaseEvents {
   'consumer:heartbeat:cancel': (payload: ConsumerHeartbeatPayload) => void;
   'consumer:heartbeat:end': (payload?: ConsumerHeartbeatPayload) => void;
   'consumer:heartbeat:error': (payload: ConsumerHeartbeatErrorPayload) => void;
+  'consumer:heartbeat:stalled': (payload: ConsumerHeartbeatStalledPayload) => void;
   'consumer:lag': (lag: Offsets) => void;
   'consumer:lag:error': (error: Error) => void;
 }

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -15,6 +15,10 @@ export declare const groupOptionsProperties: {
     type: string;
     minimum: number;
   };
+  heartbeatStallTimeout: {
+    type: string;
+    minimum: number;
+  };
   groupProtocol: {
     type: string;
     enum: ("consumer" | "classic")[];
@@ -53,6 +57,9 @@ export declare const groupOptionsProperties: {
       };
     };
   };
+  protocolsMetadata: {
+    function: boolean;
+  };
   partitionAssigner: {
     function: boolean;
   };
@@ -157,6 +164,10 @@ export declare const groupOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -195,6 +206,9 @@ export declare const groupOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -277,6 +291,10 @@ export declare const consumeOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -315,6 +333,9 @@ export declare const consumeOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -363,6 +384,9 @@ export declare const consumeOptionsSchema: {
     onCorruptedMessage: {
       function: boolean;
     };
+    onDeserializationError: {
+      function: boolean;
+    };
   };
   required: string[];
   additionalProperties: boolean;
@@ -443,6 +467,10 @@ export declare const consumerOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -481,6 +509,9 @@ export declare const consumerOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };
@@ -570,6 +601,10 @@ export declare const fetchOptionsSchema: {
       type: string;
       minimum: number;
     };
+    heartbeatStallTimeout: {
+      type: string;
+      minimum: number;
+    };
     groupProtocol: {
       type: string;
       enum: ("consumer" | "classic")[];
@@ -608,6 +643,9 @@ export declare const fetchOptionsSchema: {
         };
       };
     };
+    protocolsMetadata: {
+      function: boolean;
+    };
     partitionAssigner: {
       function: boolean;
     };

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -1,10 +1,11 @@
+import { type Callback } from "../../apis/definitions";
 import { type FetchRequestTopic } from "../../apis/consumer/fetch-v17";
 import { type GroupProtocols } from "../../apis/enumerations";
 import { type ConnectionPool } from "../../network/connection-pool";
 import { type KafkaRecord, type Message } from "../../protocol/records";
 import { type SchemaRegistry } from "../../registries/abstract";
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
-import { type BeforeDeserializationHook, type Deserializers } from "../serde";
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type Deserializers } from "../serde";
 export interface GroupProtocolSubscription {
   name: string;
   version: number;
@@ -31,8 +32,33 @@ export interface PreferredReadReplica {
   node: number;
   expiresAt: number;
 }
-export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit']) => boolean;
+export declare const DeserializationErrorActions: {
+  readonly FAIL: "fail";
+  readonly SKIP: "skip";
+  readonly CONTINUE: "continue";
+};
+export type DeserializationErrorAction = (typeof DeserializationErrorActions)[keyof typeof DeserializationErrorActions];
+export interface DeserializationErrorContext {
+  error: unknown;
+  payloadType: BeforeHookPayloadType;
+  record: KafkaRecord;
+  topic: string;
+  partition: number;
+  offset: bigint;
+  timestamp: bigint;
+  commit: Message['commit'];
+}
+export interface MessageDeserializationError {
+  error: unknown;
+  payloadType: BeforeHookPayloadType;
+}
+export type DeserializationErrorHandler = (context: DeserializationErrorContext) => DeserializationErrorAction;
+/**
+ * @deprecated Use `DeserializationErrorHandler` instead.
+ */
+export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
+export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
 export declare const MessagesStreamModes: {
   readonly LATEST: "latest";
   readonly EARLIEST: "earliest";
@@ -56,7 +82,9 @@ export interface GroupOptions {
   sessionTimeout?: number;
   rebalanceTimeout?: number;
   heartbeatInterval?: number;
+  heartbeatStallTimeout?: number;
   protocols?: GroupProtocolSubscription[];
+  protocolsMetadata?: GroupProtocolsMetadataCallback;
   partitionAssigner?: GroupPartitionsAssigner;
   assignmentUserData?: Buffer;
 }
@@ -65,6 +93,7 @@ export interface ConsumerGroupOptions {
   groupInstanceId?: string;
   groupRemoteAssignor?: string;
   rebalanceTimeout?: number;
+  heartbeatStallTimeout?: number;
   assignmentUserData?: Buffer;
 }
 export interface ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> {
@@ -86,6 +115,10 @@ export interface StreamOptions {
   fallbackMode?: MessagesStreamFallbackModeValue;
   maxFetches?: number;
   offsets?: TopicWithPartitionAndOffset[];
+  onDeserializationError?: DeserializationErrorHandler;
+  /**
+   * @deprecated Use `onDeserializationError` instead.
+   */
   onCorruptedMessage?: CorruptedMessageHandler;
 }
 export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;
@@ -137,4 +170,9 @@ export interface ConsumerHeartbeatPayload {
 }
 export interface ConsumerHeartbeatErrorPayload extends ConsumerHeartbeatPayload {
   error: Error;
+}
+export interface ConsumerHeartbeatStalledPayload {
+  lastError?: Error;
+  lastHeartbeat: Date | null;
+  stalledFor: number;
 }

package/dist/typescript-4/dist/clients/consumer/utils.d.ts

@@ -1 +1,2 @@
-export declare function partitionKey(topic: string, partition: number): string;
+export declare function partitionKey(topic: string, partition: number): string;
+export declare function nextFetchEpoch(epoch: number): number;

package/dist/typescript-4/dist/clients/serde.d.ts

@@ -17,7 +17,7 @@ export interface Deserializers<Key, Value, HeaderKey, HeaderValue> {
   headerValue: Deserializer<HeaderValue>;
 }
 export type BeforeHookPayloadType = 'key' | 'value' | 'headerKey' | 'headerValue';
-export type BeforeDeserializationHook = (payload: Buffer, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
+export type BeforeDeserializationHook = (payload: Buffer | null, type: BeforeHookPayloadType, message: MessageToConsume, callback: Callback<void>) => void | Promise<void>;
 export type BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue> = (payload: unknown, type: BeforeHookPayloadType, message: MessageToProduce<Key, Value, HeaderKey, HeaderValue>, callback: Callback<void>) => void | Promise<void>;
 export declare function stringSerializer(data?: string): Buffer | undefined;
 export declare function stringDeserializer(data?: string | Buffer): string | undefined;

package/dist/typescript-4/dist/errors.d.ts

@@ -17,6 +17,7 @@ export declare class GenericError extends Error {
     ...rest
   }?: ErrorProperties);
   findBy<ErrorType extends GenericError = GenericError>(property: string, value: unknown): ErrorType | null;
+  toString(): string;
 }
 export declare class MultipleErrors extends AggregateError {
   code: string;
@@ -31,7 +32,9 @@ export declare class MultipleErrors extends AggregateError {
     ...rest
   }?: ErrorProperties);
   findBy<ErrorType extends GenericError | MultipleErrors = MultipleErrors>(property: string, value: unknown): ErrorType | null;
+  toString(): string;
 }
+export declare function findErrorBy<ErrorType extends GenericError = GenericError>(error: Error | null | undefined, property: string, value: unknown): ErrorType | null;
 export * from "./protocol/errors";
 export declare class AuthenticationError extends GenericError {
   static code: ErrorCode;
@@ -39,6 +42,7 @@ export declare class AuthenticationError extends GenericError {
 }
 export declare class NetworkError extends GenericError {
   static code: ErrorCode;
+  static isRetryable(error: Error | null | undefined): boolean;
   constructor(message: string, properties?: ErrorProperties);
 }
 export declare class ProtocolError extends GenericError {

package/dist/typescript-4/dist/protocol/records.d.ts

@@ -33,11 +33,14 @@ export interface MessageJSON<Key = unknown, Value = unknown, HeaderKey = unknown
   partition: number;
   timestamp: string;
   offset: string;
+  leaderEpoch: number;
   metadata: Record<string, unknown>;
 }
 export interface Message<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends Required<MessageBase<Key, Value>> {
   headers: Map<HeaderKey, HeaderValue>;
+  headerEntries: Array<[HeaderKey, HeaderValue]>;
   offset: bigint;
+  leaderEpoch: number;
   metadata: Record<string, unknown>;
   commit(callback?: (error?: Error) => void): void | Promise<void>;
   toJSON(): MessageJSON<Key, Value, HeaderKey, HeaderValue>;
@@ -71,6 +74,7 @@ export interface KafkaRecord {
 export interface MessageToConsume extends KafkaRecord {
   topic: string;
   partition: number;
+  metadata?: Record<string, unknown>;
 }
 export interface RecordsBatch {
   firstOffset: bigint;

package/dist/typescript-4/dist/registries/abstract.d.ts

@@ -4,7 +4,7 @@ import { type MessageToProduce } from "../protocol/records";
 export interface SchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> {
   get(id: Id): Schema | undefined;
   fetch(id: Id, callback?: (error?: Error) => void): void | Promise<void>;
-  getSchemaId(payload: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
+  getSchemaId(payload: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): Id;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
   getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
   getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;
@@ -14,7 +14,7 @@ export declare function runAsyncSeries<V>(operation: (item: V, cb: Callback<void
 export declare class AbstractSchemaRegistry<Id = unknown, Schema = unknown, Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> implements SchemaRegistry<Id, Schema, Key, Value, HeaderKey, HeaderValue> {
   get(_: Id): Schema | undefined;
   fetch(_i: unknown, _c?: (error?: Error) => void): void | Promise<void>;
-  getSchemaId(_p: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
+  getSchemaId(_p: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, _t?: BeforeHookPayloadType): Id;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;
   getDeserializers(): Deserializers<Key, Value, HeaderKey, HeaderValue>;
   getBeforeSerializationHook(): BeforeSerializationHook<Key, Value, HeaderKey, HeaderValue>;

package/dist/typescript-4/dist/registries/confluent-schema-registry.d.ts

@@ -1,16 +1,32 @@
-import { type ValidateFunction } from 'ajv';
+import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
+import { type ConnectionOptions as TLSConnectionOptions } from 'node:tls';
 import { type Root } from 'protobufjs';
 import { type Callback } from "../apis/definitions";
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from "../clients/serde";
-import { type CredentialProvider } from "../index";
+import { type CredentialProvider, UserError } from "../index";
 import { type MessageToConsume, type MessageToProduce } from "../protocol/records";
 import { AbstractSchemaRegistry } from "./abstract";
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
+export type UndiciDispatcher = NonNullable<RequestInit['dispatcher']>;
+export interface UndiciAgentOptions {
+  connect: TLSConnectionOptions;
+}
+export type UndiciAgentConstructor = new (options: UndiciAgentOptions) => UndiciDispatcher;
+export declare function createUndiciAgent(options: UndiciAgentOptions): UndiciDispatcher;
 export interface ConfluentSchemaRegistryMetadata {
-  schemas?: Record<BeforeHookPayloadType, number>;
+  schemas?: Partial<Record<BeforeHookPayloadType, number>>;
+}
+export interface ConfluentSchemaRegistryConsumedMetadata {
+  lengths?: {
+    key?: number;
+    value?: number;
+  };
 }
 export type ConfluentSchemaRegistryProtobufTypeMapper = (id: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume) => string;
+export type ConfluentSchemaRegistryHeaders = Record<string, string>;
+export type ConfluentSchemaRegistryHeadersProvider = () => ConfluentSchemaRegistryHeaders | Promise<ConfluentSchemaRegistryHeaders>;
+export type ConfluentSchemaRegistryRetryDelayGetter = (attempt: number, retries: number, error: Error) => number;
 export interface ConfluentSchemaRegistryOptions {
   url: string;
   auth?: {
@@ -18,19 +34,56 @@ export interface ConfluentSchemaRegistryOptions {
     password?: string | CredentialProvider;
     token?: string | CredentialProvider;
   };
+  headers?: ConfluentSchemaRegistryHeaders | ConfluentSchemaRegistryHeadersProvider;
+  timeout?: number;
+  retries?: number;
+  retryDelay?: number | ConfluentSchemaRegistryRetryDelayGetter;
+  tls?: TLSConnectionOptions;
+  ssl?: TLSConnectionOptions;
   protobufTypeMapper?: ConfluentSchemaRegistryProtobufTypeMapper;
   jsonValidateSend?: boolean;
+  jsonAjvOptions?: Options;
+}
+export declare const defaultConfluentSchemaRegistryOptions: {
+  timeout: number;
+  retries: number;
+  retryDelay: number;
+};
+export declare const retriableSchemaRegistryStatuses: number[];
+export type SchemaValidationPhase = 'serialization' | 'deserialization';
+export interface SchemaValidationErrorProperties {
+  schemaId: number;
+  schemaType: 'json';
+  phase: SchemaValidationPhase;
+  payloadType: BeforeHookPayloadType;
+  type: BeforeHookPayloadType;
+  data: unknown;
+  headers?: ReadonlyMap<unknown, unknown>;
+  validationErrors: ErrorObject[] | null | undefined;
+}
+export declare class SchemaValidationError extends UserError {
+  readonly schemaId: number;
+  readonly schemaType: 'json';
+  readonly phase: SchemaValidationPhase;
+  readonly payloadType: BeforeHookPayloadType;
+  readonly type: BeforeHookPayloadType;
+  readonly data: unknown;
+  readonly headers?: ReadonlyMap<unknown, unknown>;
+  readonly validationErrors: ErrorObject[] | null | undefined;
+  constructor(message: string, properties: SchemaValidationErrorProperties);
 }
 export interface Schema {
   id: number;
   type: 'avro' | 'protobuf' | 'json';
   schema: Type | Root | ValidateFunction;
 }
+export declare function trackConsumedSchemaId(message: MessageToConsume | undefined, type: BeforeHookPayloadType, schemaId: number): void;
+export declare function trackConsumedPayloadLength(message: MessageToConsume | undefined, type: 'key' | 'value', data: Buffer): void;
 export declare function defaultProtobufTypeMapper(_: number, type: BeforeHookPayloadType, context: ConfluentSchemaRegistryMessageToProduce | MessageToConsume): string;
 export declare class ConfluentSchemaRegistry<Key = Buffer, Value = Buffer, HeaderKey = Buffer, HeaderValue = Buffer> extends AbstractSchemaRegistry<number | undefined, Schema, Key, Value, HeaderKey, HeaderValue> {
   #private;
   constructor(options: ConfluentSchemaRegistryOptions);
-  getSchemaId(message: Buffer | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
+  getSchemaId(message: Buffer | null | MessageToProduce<Key, Value, HeaderKey, HeaderValue>, type?: BeforeHookPayloadType): number | undefined;
   get(id: number): Schema | undefined;
   fetchSchema(id: number, callback: Callback<void>): Promise<void>;
   getSerializers(): Serializers<Key, Value, HeaderKey, HeaderValue>;

package/dist/typescript-4/dist/symbols.d.ts

@@ -2,6 +2,7 @@ export declare const kInstance: unique symbol;
 export declare const kRefreshOffsetsAndFetch: unique symbol;
 export declare const kAutocommit: unique symbol;
 export declare const kGetFetchNode: unique symbol;
+export declare const kDeserializationError: unique symbol;
 export declare const kTransaction: unique symbol;
 export declare const kTransactionPrepare: unique symbol;
 export declare const kTransactionAddPartitions: unique symbol;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.1.0";
+export const version = "2.2.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -28,6 +28,7 @@
     "@platformatic/dynamic-buffer": "^0.3.1",
     "@platformatic/wasm-utils": "^0.2.1",
     "ajv": "^8.17.1",
+    "ajv-draft-04": "^1.0.0",
     "avsc": "^5.7.9",
     "debug": "^4.4.3",
     "fastq": "^1.19.1",
@@ -91,12 +92,13 @@
     "typecheck": "tsc -p . --noEmit",
     "format": "prettier -w benchmarks playground regression src test",
     "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
-    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:ci": "./scripts/node --test --experimental-test-coverage --test-coverage-include='src/**' --test-coverage-lines=90 --test-coverage-branches=90 --test-coverage-functions=90 --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
     "test:integrity": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/integrity/*.test.ts'",
     "test:performance": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'regression/performance/*.test.ts'",
     "test:regression": "npm run test:integrity",
     "test:regression:all": "npm run test:integrity && npm run test:performance",
     "test:all": "npm test && npm run test:regression:all",
+    "test:compat": "./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/integration/*.compat-test.ts'",
     "test:memory": "NODE_OPTIONS='--expose-gc' ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/**/*.memory-test.ts'",
     "test:docker:up": "docker-compose up -d --wait",
     "test:docker:down": "docker-compose down",