@platformatic/kafka 1.26.0 → 1.27.0-alpha.1

package/dist/protocol/compression.js

@@ -1,8 +1,7 @@
-import { createRequire } from 'node:module';
 import zlib from 'node:zlib';
 import { UnsupportedCompressionError } from "../errors.js";
 import { DynamicBuffer } from "./dynamic-buffer.js";
-const require = createRequire(import.meta.url);
+import { lz4Compress, lz4Decompress, snappyCompress, snappyDecompress } from "./native.js";
 const { zstdCompressSync, zstdDecompressSync, gzipSync, gunzipSync } = zlib;
 export const CompressionAlgorithms = {
     NONE: 'none',
@@ -13,34 +12,12 @@ export const CompressionAlgorithms = {
 };
 export const allowedCompressionsAlgorithms = Object.values(CompressionAlgorithms);
 function ensureBuffer(data) {
-    return DynamicBuffer.isDynamicBuffer(data) ? data.slice() : data;
-}
-let snappyCompressSync;
-let snappyDecompressSync;
-let lz4CompressFrameSync;
-let lz4DecompressFrameSync;
-function loadSnappy() {
-    try {
-        const snappy = require('snappy');
-        snappyCompressSync = snappy.compressSync;
-        snappyDecompressSync = snappy.uncompressSync;
-        /* c8 ignore next 5 - In tests snappy is always available */
-    }
-    catch (e) {
-        throw new UnsupportedCompressionError('Cannot load snappy module, which is an optionalDependency. Please check your local installation.');
-    }
-}
-function loadLZ4() {
-    try {
-        const lz4 = require('lz4-napi');
-        lz4CompressFrameSync = lz4.compressFrameSync;
-        lz4DecompressFrameSync = lz4.decompressFrameSync;
-        /* c8 ignore next 5 - In tests lz4-napi is always available */
-    }
-    catch (e) {
-        throw new UnsupportedCompressionError('Cannot load lz4-napi module, which is an optionalDependency. Please check your local installation.');
-    }
+    return DynamicBuffer.isDynamicBuffer(data) ? data.buffer : data;
 }
+const snappyCompressSync = snappyCompress;
+const snappyDecompressSync = snappyDecompress;
+const lz4CompressFrameSync = lz4Compress;
+const lz4DecompressFrameSync = lz4Decompress;
 export const compressionsAlgorithms = {
     /* c8 ignore next 8 - 'none' is actually never used but this is to please Typescript */
     none: {
@@ -65,17 +42,9 @@ export const compressionsAlgorithms = {
     },
     snappy: {
         compressSync(data) {
-            /* c8 ignore next 4 - In tests snappy is always available */
-            if (!snappyCompressSync) {
-                loadSnappy();
-            }
             return snappyCompressSync(ensureBuffer(data));
         },
         decompressSync(data) {
-            /* c8 ignore next 4 - In tests snappy is always available */
-            if (!snappyDecompressSync) {
-                loadSnappy();
-            }
             return snappyDecompressSync(ensureBuffer(data));
         },
         bitmask: 2,
@@ -83,17 +52,9 @@ export const compressionsAlgorithms = {
     },
     lz4: {
         compressSync(data) {
-            /* c8 ignore next 4 - In tests lz4-napi is always available */
-            if (!lz4CompressFrameSync) {
-                loadLZ4();
-            }
             return lz4CompressFrameSync(ensureBuffer(data));
         },
         decompressSync(data) {
-            /* c8 ignore next 4 - In tests lz4-napi is always available */
-            if (!lz4DecompressFrameSync) {
-                loadLZ4();
-            }
             return lz4DecompressFrameSync(ensureBuffer(data));
         },
         bitmask: 3,

package/dist/protocol/crc32c.js

@@ -1,6 +1,7 @@
 // Based on the work from: https://github.com/tulios/kafkajs/blob/master/src/protocol/recordBatch/crc32C/crc32C.js
 import { createRequire } from 'node:module';
 import { DynamicBuffer } from "./dynamic-buffer.js";
+import { crc32c as wasmCRC32C } from "./native.js";
 /* prettier-ignore */
 const CRC = [
     0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4,
@@ -85,9 +86,7 @@ export function loadNativeCRC32C() {
     }
 }
 export function jsCRC32C(data) {
-    const bytes = DynamicBuffer.isDynamicBuffer(data)
-        ? data.buffer
-        : new Uint8Array(data);
+    const bytes = DynamicBuffer.isDynamicBuffer(data) ? data.buffer : data;
     let crc = 0xffffffff;
     for (let i = 0, len = bytes.length; i < len; ++i) {
         crc = CRC[(crc ^ bytes[i]) & 0xff] ^ (crc >>> 8);
@@ -95,4 +94,4 @@ export function jsCRC32C(data) {
     return (crc ^ 0xffffffff) >>> 0;
 }
 /* c8 ignore next - Hard to test */
-export const crc32c = loadNativeCRC32C() ?? jsCRC32C;
+export const crc32c = loadNativeCRC32C() ?? wasmCRC32C;

package/dist/protocol/native.d.ts

@@ -0,0 +1,8 @@
+import { DynamicBuffer } from './dynamic-buffer.ts';
+export declare function prepareInput(data: Buffer | Uint8Array | DynamicBuffer): void;
+export declare function prepareOutput(combined: bigint): Buffer;
+export declare function crc32c(data: Buffer | Uint8Array | DynamicBuffer): number;
+export declare function lz4Compress(data: Buffer | DynamicBuffer): Buffer;
+export declare function lz4Decompress(data: Buffer | DynamicBuffer): Buffer;
+export declare function snappyCompress(data: Buffer | DynamicBuffer): Buffer;
+export declare function snappyDecompress(data: Buffer | DynamicBuffer): Buffer;

package/dist/protocol/native.js

@@ -0,0 +1,48 @@
+import { readFileSync } from 'node:fs';
+import { DynamicBuffer } from "./dynamic-buffer.js";
+// @ts-expect-error - Buffer[Symbol.species] is not typed in Node.js types
+const FastBuffer = Buffer[Symbol.species];
+const wasm = readFileSync(new URL('../../dist/native.wasm', import.meta.url));
+const wasmModule = new WebAssembly.Module(wasm);
+const instance = new WebAssembly.Instance(wasmModule);
+const { alloc, crc32c: _crc32c, dealloc, lz4_compress: _lz4Compress, lz4_decompress: _lz4Decompress, memory, snappy_compress: _snappyCompress, snappy_decompress: _snappyDecompress } = instance.exports;
+let currentBufferLength = 4096;
+let currentBufferOffset = alloc(currentBufferLength);
+let currentBuffer = new Uint8Array(memory.buffer, currentBufferOffset, currentBufferLength);
+export function prepareInput(data) {
+    const input = DynamicBuffer.isDynamicBuffer(data) ? data.buffer : data;
+    if (currentBuffer.length < data.length) {
+        dealloc(currentBufferOffset, currentBufferLength);
+        currentBufferLength = data.length;
+        currentBufferOffset = alloc(currentBufferLength);
+        currentBuffer = new Uint8Array(memory.buffer, currentBufferOffset, currentBufferLength);
+    }
+    currentBuffer.set(input);
+}
+export function prepareOutput(combined) {
+    const len = Number(BigInt.asUintN(32, combined));
+    const ptr = Number(combined >> 32n);
+    const output = Buffer.from(new FastBuffer(memory.buffer, ptr, len));
+    dealloc(ptr, len);
+    return output;
+}
+export function crc32c(data) {
+    prepareInput(data);
+    return _crc32c(currentBufferOffset, data.length) >>> 0;
+}
+export function lz4Compress(data) {
+    prepareInput(data);
+    return prepareOutput(_lz4Compress(currentBufferOffset, data.length));
+}
+export function lz4Decompress(data) {
+    prepareInput(data);
+    return prepareOutput(_lz4Decompress(currentBufferOffset, data.length));
+}
+export function snappyCompress(data) {
+    prepareInput(data);
+    return prepareOutput(_snappyCompress(currentBufferOffset, data.length));
+}
+export function snappyDecompress(data) {
+    prepareInput(data);
+    return prepareOutput(_snappyDecompress(currentBufferOffset, data.length));
+}

package/dist/typescript-4/dist/protocol/native.d.ts

@@ -0,0 +1,8 @@
+import { DynamicBuffer } from "./dynamic-buffer";
+export declare function prepareInput(data: Buffer | Uint8Array | DynamicBuffer): void;
+export declare function prepareOutput(combined: bigint): Buffer;
+export declare function crc32c(data: Buffer | Uint8Array | DynamicBuffer): number;
+export declare function lz4Compress(data: Buffer | DynamicBuffer): Buffer;
+export declare function lz4Decompress(data: Buffer | DynamicBuffer): Buffer;
+export declare function snappyCompress(data: Buffer | DynamicBuffer): Buffer;
+export declare function snappyDecompress(data: Buffer | DynamicBuffer): Buffer;

package/dist/version.js

@@ -1,2 +1,2 @@
 export const name = "@platformatic/kafka";
-export const version = "1.26.0";
+export const version = "1.27.0-alpha.1";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@platformatic/kafka",
-  "version": "1.26.0",
+  "version": "1.27.0-alpha.1",
   "description": "Modern and performant client for Apache Kafka",
   "homepage": "https://github.com/platformatic/kafka",
   "author": "Platformatic Inc. <oss@platformatic.dev> (https://platformatic.dev)",
@@ -32,9 +32,7 @@
     "scule": "^1.3.0"
   },
   "optionalDependencies": {
-    "@node-rs/crc32": "^1.10.6",
-    "lz4-napi": "^2.9.0",
-    "snappy": "^7.3.3"
+    "@node-rs/crc32": "^1.10.6"
   },
   "devDependencies": {
     "@babel/generator": "^7.28.5",
@@ -44,11 +42,12 @@
     "@confluentinc/kafka-javascript": "^1.5.0",
     "@opentelemetry/instrumentation": "^0.210.0",
     "@platformatic/rdkafka": "^4.0.0",
+    "@swc-node/register": "^1.11.1",
     "@types/babel__generator": "^7.27.0",
     "@types/babel__traverse": "^7.28.0",
     "@types/debug": "^4.1.12",
     "@types/kerberos": "^1.1.5",
-    "@types/node": "^22.18.5",
+    "@types/node": "^22.18.8",
     "@types/semver": "^7.7.1",
     "@watchable/unpromise": "^1.0.2",
     "avsc": "^5.7.9",
@@ -62,12 +61,14 @@
     "json5": "^2.2.3",
     "kafkajs": "^2.2.4",
     "kerberos": "^2.2.2",
+    "lz4-napi": "^2.9.0",
     "neostandard": "^0.12.2",
     "parse5": "^7.3.0",
     "prettier": "^3.6.2",
     "prettier-plugin-space-before-function-paren": "^0.0.8",
     "prom-client": "^15.1.3",
     "semver": "^7.7.2",
+    "snappy": "^7.3.3",
     "table": "^6.9.0",
     "typescript": "^5.9.2"
   },
@@ -83,16 +84,16 @@
   },
   "scripts": {
     "build": "rm -rf dist && tsc -p tsconfig.base.json",
-    "postbuild": "node scripts/postbuild.ts",
+    "build:native": "cd native && docker compose up",
+    "postbuild": "./scripts/node scripts/postbuild.ts && npm run build:native",
     "lint": "eslint --cache",
     "typecheck": "tsc -p . --noEmit",
     "format": "prettier -w benchmarks playground src test",
-    "test": "c8 -c test/config/c8-local.json node --env-file=test/config/env --no-warnings --test --test-timeout=600000 'test/**/*.test.ts'",
-    "test:ci": "c8 -c test/config/c8-ci.json node --env-file=test/config/env --no-warnings --test --test-timeout=600000 'test/**/*.test.ts'",
+    "test": "c8 -c test/config/c8-local.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
+    "test:ci": "c8 -c test/config/c8-ci.json ./scripts/node --test --test-reporter=cleaner-spec-reporter 'test/*.test.ts' 'test/*/*.test.ts' 'test/*/*/*.test.ts'",
     "test:docker:up": "docker-compose up -d --wait",
     "test:docker:down": "docker-compose down",
     "ci": "npm run build && npm run lint && npm run test:ci",
-    "ci:v20": "npm run lint && rm -rf dist && tsc -p tsconfig.json && node dist/scripts/postbuild.js && cd dist && node --env-file=../test/config/env --no-warnings --test --test-timeout=600000",
     "generate:apis": "node scripts/generate-apis.ts",
     "generate:errors": "node scripts/generate-errors.ts",
     "create:api": "node scripts/create-api.ts"