@platformatic/kafka 2.7.0 → 2.8.0

package/dist/clients/consumer/consumer.js

@@ -178,6 +178,10 @@ export class Consumer extends Base {
             callback(validationError);
             return callback[kCallbackPromise];
         }
+        if (options.onCorruptedMessage && options.onDeserializationError) {
+            callback(new UserError('Cannot specify both onCorruptedMessage and onDeserializationError.'));
+            return callback[kCallbackPromise];
+        }
         options.autocommit ??= this[kOptions].autocommit ?? true;
         options.maxBytes ??= this[kOptions].maxBytes;
         options.maxBytesPerPartition ??= this[kOptions].maxBytesPerPartition ?? options.maxBytes;

package/dist/clients/consumer/messages-stream.js

@@ -9,7 +9,7 @@ import { kAutocommit, kGetFetchNode, kInstance, kRefreshOffsetsAndFetch } from "
 import { kConnections, kCreateConnectionPool, kInspect, kPrometheus } from "../base/base.js";
 import { ensureMetric } from "../metrics.js";
 import { defaultConsumerOptions } from "./options.js";
-import { MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
+import { DeserializationErrorActions, MessagesStreamFallbackModes, MessagesStreamModes } from "./types.js";
 import { partitionKey } from "./utils.js";
 // Don't move this function as being in the same file will enable V8 to remove.
 // For futher info, ask Matteo.
@@ -61,6 +61,7 @@ export class MessagesStream extends Readable {
     #closeCallbacks;
     #metricsConsumedMessages;
     #corruptedMessageHandler;
+    #deserializationErrorHandler;
     #context;
     #onConsumerGroupJoin;
     #onBrokerDisconnect;
@@ -83,9 +84,12 @@ export class MessagesStream extends Readable {
     */
     [kConnections];
     constructor(consumer, options) {
-        const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, registry, beforeDeserialization, 
+        const { autocommit, mode, fallbackMode, maxFetches, offsets, context, deserializers, onCorruptedMessage, onDeserializationError, registry, beforeDeserialization, 
         // The options below are only destructured to avoid being part of structuredClone below
         partitionAssigner: _partitionAssigner, protocolsMetadata: _protocolsMetadata, ...otherOptions } = options;
+        if (onCorruptedMessage && onDeserializationError) {
+            throw new UserError('Cannot specify both onCorruptedMessage and onDeserializationError.');
+        }
         if (offsets && mode !== MessagesStreamModes.MANUAL) {
             throw new UserError('Cannot specify offsets when the stream mode is not MANUAL.');
         }
@@ -124,6 +128,7 @@ export class MessagesStream extends Readable {
         this.#closed = false;
         this.#closeCallbacks = [];
         this.#corruptedMessageHandler = onCorruptedMessage ?? defaultCorruptedMessageHandler;
+        this.#deserializationErrorHandler = onDeserializationError;
         this.#context = context;
         this.#onConsumerGroupJoin = () => {
             this.#offsetsCommitted.clear();
@@ -605,18 +610,51 @@ export class MessagesStream extends Readable {
                             : this.#commit.bind(this, topic, partition, offset + 1n, leaderEpoch);
                         try {
                             const headers = new Map();
-                            for (const [headerKey, headerValue] of record.headers) {
-                                headers.set(
-                                /* c8 ignore next 2 - Hard to test */
-                                headerKeyDeserializer(headerKey ?? undefined, messageToConsume), headerValueDeserializer(headerValue ?? undefined, messageToConsume));
+                            let payloadType = 'headerKey';
+                            let deserializedKey;
+                            let deserializedValue;
+                            try {
+                                for (const [headerKey, headerValue] of record.headers) {
+                                    payloadType = 'headerKey';
+                                    /* c8 ignore next - Hard to test */
+                                    const deserializedHeaderKey = headerKeyDeserializer(headerKey ?? undefined, messageToConsume);
+                                    payloadType = 'headerValue';
+                                    /* c8 ignore next - Hard to test */
+                                    const deserializedHeaderValue = headerValueDeserializer(headerValue ?? undefined, messageToConsume);
+                                    headers.set(deserializedHeaderKey, deserializedHeaderValue);
+                                }
+                                payloadType = 'key';
+                                /* c8 ignore next - Hard to test */
+                                deserializedKey = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
+                                payloadType = 'value';
+                                deserializedValue = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
+                            }
+                            catch (error) {
+                                const context = {
+                                    error,
+                                    payloadType,
+                                    record,
+                                    topic,
+                                    partition,
+                                    offset,
+                                    timestamp: firstTimestamp + record.timestampDelta,
+                                    commit: commit
+                                };
+                                const shouldDestroy = this.#deserializationErrorHandler
+                                    ? this.#deserializationErrorHandler(context) !== DeserializationErrorActions.SKIP
+                                    : this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit, error);
+                                if (shouldDestroy) {
+                                    diagnosticContext.error = error;
+                                    consumerReceivesChannel.error.publish(diagnosticContext);
+                                    this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
+                                    return;
+                                }
+                                continue;
                             }
-                            /* c8 ignore next 2 - Hard to test */
-                            const key = keyDeserializer(record.key ?? undefined, headers, messageToConsume);
-                            const value = valueDeserializer(record.value ?? undefined, headers, messageToConsume);
                             this.#metricsConsumedMessages?.inc();
                             const message = {
-                                key,
-                                value,
+                                key: deserializedKey,
+                                value: deserializedValue,
                                 headers,
                                 topic,
                                 partition,
@@ -632,13 +670,10 @@ export class MessagesStream extends Readable {
                             consumerReceivesChannel.asyncEnd.publish(diagnosticContext);
                         }
                         catch (error) {
-                            const shouldDestroy = this.#corruptedMessageHandler(record, topic, partition, firstTimestamp, firstOffset, commit, error);
-                            if (shouldDestroy) {
-                                diagnosticContext.error = error;
-                                consumerReceivesChannel.error.publish(diagnosticContext);
-                                this.destroy(new UserError('Failed to deserialize a message.', { cause: error }));
-                                return;
-                            }
+                            diagnosticContext.error = error;
+                            consumerReceivesChannel.error.publish(diagnosticContext);
+                            this.destroy(new UserError('Failed to process a message.', { cause: error }));
+                            return;
                         }
                         finally {
                             consumerReceivesChannel.end.publish(diagnosticContext);

package/dist/clients/consumer/options.d.ts

@@ -372,6 +372,9 @@ export declare const consumeOptionsSchema: {
         onCorruptedMessage: {
             function: boolean;
         };
+        onDeserializationError: {
+            function: boolean;
+        };
     };
     required: string[];
     additionalProperties: boolean;

package/dist/clients/consumer/options.js

@@ -90,6 +90,7 @@ export const consumeOptionsSchema = {
             }
         },
         onCorruptedMessage: { function: true },
+        onDeserializationError: { function: true },
         ...groupOptionsProperties,
         ...consumeOptionsProperties
     },

package/dist/clients/consumer/types.d.ts

@@ -5,7 +5,7 @@ import { type ConnectionPool } from '../../network/connection-pool.ts';
 import { type KafkaRecord, type Message } from '../../protocol/records.ts';
 import { type SchemaRegistry } from '../../registries/abstract.ts';
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from '../base/types.ts';
-import { type BeforeDeserializationHook, type Deserializers } from '../serde.ts';
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type Deserializers } from '../serde.ts';
 export interface GroupProtocolSubscription {
     name: string;
     version: number;
@@ -32,6 +32,25 @@ export interface PreferredReadReplica {
     node: number;
     expiresAt: number;
 }
+export declare const DeserializationErrorActions: {
+    readonly FAIL: "fail";
+    readonly SKIP: "skip";
+};
+export type DeserializationErrorAction = (typeof DeserializationErrorActions)[keyof typeof DeserializationErrorActions];
+export interface DeserializationErrorContext {
+    error: unknown;
+    payloadType: BeforeHookPayloadType;
+    record: KafkaRecord;
+    topic: string;
+    partition: number;
+    offset: bigint;
+    timestamp: bigint;
+    commit: Message['commit'];
+}
+export type DeserializationErrorHandler = (context: DeserializationErrorContext) => DeserializationErrorAction;
+/**
+ * @deprecated Use `DeserializationErrorHandler` instead.
+ */
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
 export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
@@ -89,6 +108,10 @@ export interface StreamOptions {
     fallbackMode?: MessagesStreamFallbackModeValue;
     maxFetches?: number;
     offsets?: TopicWithPartitionAndOffset[];
+    onDeserializationError?: DeserializationErrorHandler;
+    /**
+     * @deprecated Use `onDeserializationError` instead.
+     */
     onCorruptedMessage?: CorruptedMessageHandler;
 }
 export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;

package/dist/clients/consumer/types.js

@@ -1,3 +1,7 @@
+export const DeserializationErrorActions = {
+    FAIL: 'fail',
+    SKIP: 'skip'
+};
 export const MessagesStreamModes = {
     LATEST: 'latest',
     EARLIEST: 'earliest',

package/dist/registries/confluent-schema-registry.d.ts

@@ -1,9 +1,9 @@
-import { type Options, type ValidateFunction } from 'ajv';
+import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
 import { type Root } from 'protobufjs';
 import { type Callback } from '../apis/definitions.ts';
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from '../clients/serde.ts';
-import { type CredentialProvider } from '../index.ts';
+import { type CredentialProvider, UserError } from '../index.ts';
 import { type MessageToConsume, type MessageToProduce } from '../protocol/records.ts';
 import { AbstractSchemaRegistry } from './abstract.ts';
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
@@ -22,6 +22,28 @@ export interface ConfluentSchemaRegistryOptions {
     jsonValidateSend?: boolean;
     jsonAjvOptions?: Options;
 }
+export type SchemaValidationPhase = 'serialization' | 'deserialization';
+export interface SchemaValidationErrorProperties {
+    schemaId: number;
+    schemaType: 'json';
+    phase: SchemaValidationPhase;
+    payloadType: BeforeHookPayloadType;
+    type: BeforeHookPayloadType;
+    data: unknown;
+    headers?: ReadonlyMap<unknown, unknown>;
+    validationErrors: ErrorObject[] | null | undefined;
+}
+export declare class SchemaValidationError extends UserError {
+    readonly schemaId: number;
+    readonly schemaType: 'json';
+    readonly phase: SchemaValidationPhase;
+    readonly payloadType: BeforeHookPayloadType;
+    readonly type: BeforeHookPayloadType;
+    readonly data: unknown;
+    readonly headers?: ReadonlyMap<unknown, unknown>;
+    readonly validationErrors: ErrorObject[] | null | undefined;
+    constructor(message: string, properties: SchemaValidationErrorProperties);
+}
 export interface Schema {
     id: number;
     type: 'avro' | 'protobuf' | 'json';

package/dist/registries/confluent-schema-registry.js

@@ -1,3 +1,4 @@
+import { Ajv } from 'ajv';
 import { Ajv2020 } from 'ajv/dist/2020.js';
 import avro from 'avsc';
 import { createRequire } from 'node:module';
@@ -6,6 +7,14 @@ import { EMPTY_BUFFER, UnsupportedFormatError, UserError } from "../index.js";
 import { getCredential } from "../protocol/sasl/utils.js";
 import { AbstractSchemaRegistry } from "./abstract.js";
 const require = createRequire(import.meta.url);
+const AjvDraft04 = require('ajv-draft-04');
+const draft06MetaSchema = require('ajv/dist/refs/json-schema-draft-06.json');
+export class SchemaValidationError extends UserError {
+    constructor(message, properties) {
+        super(message, properties);
+        this.name = 'SchemaValidationError';
+    }
+}
 /* c8 ignore next 8 */
 export function defaultProtobufTypeMapper(_, type, context) {
     // Confluent Schema Registry convention
@@ -18,6 +27,8 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
     #protobufTypeMapper;
     #jsonValidateSend;
     #jsonAjv;
+    #jsonAjvDraft7;
+    #jsonAjvDraft04;
     #pendingFetches;
     #auth;
     constructor(options) {
@@ -26,7 +37,11 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
         this.#schemas = new Map();
         this.#protobufTypeMapper = options.protobufTypeMapper ?? defaultProtobufTypeMapper;
         this.#jsonValidateSend = options.jsonValidateSend ?? false;
-        this.#jsonAjv = new Ajv2020({ allErrors: true, coerceTypes: false, strict: true, ...options.jsonAjvOptions });
+        const jsonAjvOptions = { allErrors: true, coerceTypes: false, strict: true, ...options.jsonAjvOptions };
+        this.#jsonAjv = new Ajv2020(jsonAjvOptions);
+        this.#jsonAjvDraft7 = new Ajv(jsonAjvOptions);
+        this.#jsonAjvDraft7.addMetaSchema(draft06MetaSchema);
+        this.#jsonAjvDraft04 = new AjvDraft04(jsonAjvOptions);
         this.#auth = options.auth;
         this.#pendingFetches = new Map();
     }
@@ -145,10 +160,24 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
                 this.#protobufParse ??= this.#loadProtobuf();
                 this.#schemas.set(id, { id, type: 'protobuf', schema: this.#protobufParse(schema).root });
                 break;
-            case 'JSON':
-                this.#schemas.set(id, { id, type: 'json', schema: this.#jsonAjv.compile(JSON.parse(schema)) });
+            case 'JSON': {
+                const jsonSchema = JSON.parse(schema);
+                this.#schemas.set(id, { id, type: 'json', schema: this.#getJsonAjv(jsonSchema).compile(jsonSchema) });
                 break;
+            }
+        }
+    }
+    #getJsonAjv(schema) {
+        if (typeof schema !== 'object' || schema === null || typeof schema.$schema !== 'string') {
+            return this.#jsonAjv;
+        }
+        if (schema.$schema.includes('draft-04')) {
+            return this.#jsonAjvDraft04;
+        }
+        if (schema.$schema.includes('draft-06') || schema.$schema.includes('draft-07')) {
+            return this.#jsonAjvDraft7;
         }
+        return this.#jsonAjv;
     }
     #schemaSerializer(type, fallbackSerializer, data, headers, message) {
         /* c8 ignore next 3 - Hard to test */
@@ -183,7 +212,16 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
                     const validate = schema.schema;
                     const valid = validate(data);
                     if (!valid) {
-                        throw new UserError(`JSON Schema validation failed before serialization: ${this.#jsonAjv.errorsText(validate.errors)}`, { type, data, headers, validationErrors: validate.errors });
+                        throw new SchemaValidationError(`JSON Schema validation failed before serialization: ${this.#jsonAjv.errorsText(validate.errors)}`, {
+                            schemaId,
+                            schemaType: 'json',
+                            phase: 'serialization',
+                            payloadType: type,
+                            type,
+                            data,
+                            headers,
+                            validationErrors: validate.errors
+                        });
                     }
                 }
                 encodedMessage = Buffer.from(JSON.stringify(data));
@@ -222,7 +260,16 @@ export class ConfluentSchemaRegistry extends AbstractSchemaRegistry {
                 const validate = schema.schema;
                 const valid = validate(parsed);
                 if (!valid) {
-                    throw new UserError(`JSON Schema validation failed before deserialization: ${this.#jsonAjv.errorsText(validate.errors)}`, { type, data: parsed, headers, validationErrors: validate.errors });
+                    throw new SchemaValidationError(`JSON Schema validation failed before deserialization: ${this.#jsonAjv.errorsText(validate.errors)}`, {
+                        schemaId,
+                        schemaType: 'json',
+                        phase: 'deserialization',
+                        payloadType: type,
+                        type,
+                        data: parsed,
+                        headers,
+                        validationErrors: validate.errors
+                    });
                 }
                 return parsed;
             }

package/dist/typescript-4/dist/clients/consumer/options.d.ts

@@ -372,6 +372,9 @@ export declare const consumeOptionsSchema: {
     onCorruptedMessage: {
       function: boolean;
     };
+    onDeserializationError: {
+      function: boolean;
+    };
   };
   required: string[];
   additionalProperties: boolean;

package/dist/typescript-4/dist/clients/consumer/types.d.ts

@@ -5,7 +5,7 @@ import { type ConnectionPool } from "../../network/connection-pool";
 import { type KafkaRecord, type Message } from "../../protocol/records";
 import { type SchemaRegistry } from "../../registries/abstract";
 import { type BaseOptions, type ClusterMetadata, type TopicWithPartitionAndOffset } from "../base/types";
-import { type BeforeDeserializationHook, type Deserializers } from "../serde";
+import { type BeforeDeserializationHook, type BeforeHookPayloadType, type Deserializers } from "../serde";
 export interface GroupProtocolSubscription {
   name: string;
   version: number;
@@ -32,6 +32,25 @@ export interface PreferredReadReplica {
   node: number;
   expiresAt: number;
 }
+export declare const DeserializationErrorActions: {
+  readonly FAIL: "fail";
+  readonly SKIP: "skip";
+};
+export type DeserializationErrorAction = (typeof DeserializationErrorActions)[keyof typeof DeserializationErrorActions];
+export interface DeserializationErrorContext {
+  error: unknown;
+  payloadType: BeforeHookPayloadType;
+  record: KafkaRecord;
+  topic: string;
+  partition: number;
+  offset: bigint;
+  timestamp: bigint;
+  commit: Message['commit'];
+}
+export type DeserializationErrorHandler = (context: DeserializationErrorContext) => DeserializationErrorAction;
+/**
+ * @deprecated Use `DeserializationErrorHandler` instead.
+ */
 export type CorruptedMessageHandler = (record: KafkaRecord, topic: string, partition: number, firstTimestamp: bigint, firstOffset: bigint, commit: Message['commit'], error: unknown) => boolean;
 export type GroupPartitionsAssigner = (current: string, members: Map<string, ExtendedGroupProtocolSubscription>, topics: Set<string>, metadata: ClusterMetadata) => GroupPartitionsAssignments[];
 export type GroupProtocolsMetadataCallback = (protocols: GroupProtocolSubscription[], topics: string[], metadata: ClusterMetadata, callback: Callback<Buffer>) => Buffer | Promise<Buffer> | undefined;
@@ -89,6 +108,10 @@ export interface StreamOptions {
   fallbackMode?: MessagesStreamFallbackModeValue;
   maxFetches?: number;
   offsets?: TopicWithPartitionAndOffset[];
+  onDeserializationError?: DeserializationErrorHandler;
+  /**
+   * @deprecated Use `onDeserializationError` instead.
+   */
   onCorruptedMessage?: CorruptedMessageHandler;
 }
 export type ConsumeOptions<Key, Value, HeaderKey, HeaderValue> = StreamOptions & ConsumeBaseOptions<Key, Value, HeaderKey, HeaderValue> & GroupOptions;

package/dist/typescript-4/dist/registries/confluent-schema-registry.d.ts

@@ -1,9 +1,9 @@
-import { type Options, type ValidateFunction } from 'ajv';
+import { type ErrorObject, type Options, type ValidateFunction } from 'ajv';
 import { type Type } from 'avsc';
 import { type Root } from 'protobufjs';
 import { type Callback } from "../apis/definitions";
 import { type BeforeDeserializationHook, type BeforeHookPayloadType, type BeforeSerializationHook, type Deserializers, type Serializers } from "../clients/serde";
-import { type CredentialProvider } from "../index";
+import { type CredentialProvider, UserError } from "../index";
 import { type MessageToConsume, type MessageToProduce } from "../protocol/records";
 import { AbstractSchemaRegistry } from "./abstract";
 type ConfluentSchemaRegistryMessageToProduce = MessageToProduce<unknown, unknown, unknown, unknown>;
@@ -22,6 +22,28 @@ export interface ConfluentSchemaRegistryOptions {
   jsonValidateSend?: boolean;
   jsonAjvOptions?: Options;
 }
+export type SchemaValidationPhase = 'serialization' | 'deserialization';
+export interface SchemaValidationErrorProperties {
+  schemaId: number;
+  schemaType: 'json';
+  phase: SchemaValidationPhase;
+  payloadType: BeforeHookPayloadType;
+  type: BeforeHookPayloadType;
+  data: unknown;
+  headers?: ReadonlyMap<unknown, unknown>;
+  validationErrors: ErrorObject[] | null | undefined;
+}
+export declare class SchemaValidationError extends UserError {
+  readonly schemaId: number;
+  readonly schemaType: 'json';
+  readonly phase: SchemaValidationPhase;
+  readonly payloadType: BeforeHookPayloadType;
+  readonly type: BeforeHookPayloadType;
+  readonly data: unknown;
+  readonly headers?: ReadonlyMap<unknown, unknown>;
+  readonly validationErrors: ErrorObject[] | null | undefined;
+  constructor(message: string, properties: SchemaValidationErrorProperties);
+}
 export interface Schema {
   id: number;
   type: 'avro' | 'protobuf' | 'json';

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "2.7.0";
+export const version = "2.8.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "2.7.0",
+  "version": "2.8.0",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -28,6 +28,7 @@
     "@platformatic/dynamic-buffer": "^0.3.1",
     "@platformatic/wasm-utils": "^0.2.1",
     "ajv": "^8.17.1",
+    "ajv-draft-04": "^1.0.0",
     "avsc": "^5.7.9",
     "debug": "^4.4.3",
     "fastq": "^1.19.1",