@addmaple/lz4 0.1.0 → 0.1.2

package/README.md

@@ -37,11 +37,12 @@ const compressed = await compress(input);
 ### `init()`
 Initialize the WASM module.
 
-### `compress(input, options?)`
+### `compress(input)`
 - `input`: `Uint8Array`
-- `options.level`: 1 or 9 (default: 1)
 - Returns: `Promise<Uint8Array>`
 
+Note: LZ4 is a single-speed algorithm optimized for maximum throughput. Unlike Brotli/Gzip, it doesn't have compression levels.
+
 ## Sponsor
 
 Development of this module was sponsored by [addmaple.com](https://addmaple.com) — a modern data analysis platform.

package/dist/browser-inline.d.ts

@@ -1,2 +1,5 @@
-export function init(imports?: WebAssembly.Imports): Promise<void>;
+export interface InitOptions {
+  backend?: 'auto' | 'simd' | 'base';
+}
+export function init(imports?: WebAssembly.Imports, opts?: InitOptions): Promise<void>;
 export * from "./custom.js";

package/dist/browser-inline.js

@@ -1,19 +1,26 @@
 import { setInstance, registerInit } from "./core.js";
-import { instantiateWithFallback } from "./util.js";
+import { instantiateWithBackend } from "./util.js";
 
-import { wasmBytes as simdBytes } from "./wasm-inline/lz4.simd.wasm.js";
-import { wasmBytes as baseBytes } from "./wasm-inline/lz4.base.wasm.js";
+import { wasmBytes as _simdBytes } from "./wasm-inline/lz4.simd.wasm.js";
+import { wasmBytes as _baseBytes } from "./wasm-inline/lz4.base.wasm.js";
 
-async function getWasmBytes() {
-  return { simdBytes, baseBytes };
+async function getSimdBytes() {
+  return _simdBytes;
+}
+
+async function getBaseBytes() {
+  return _baseBytes;
 }
 
 
 let _ready = null;
-export function init(imports = {}) {
-  return (_ready ??= (async () => {
-    const { simdBytes, baseBytes } = await getWasmBytes();
-    const { instance } = await instantiateWithFallback(simdBytes, baseBytes, imports);
+let _backend = null;
+export function init(imports = {}, opts = {}) {
+  const backend = opts.backend || 'auto';
+  if (_ready && _backend === backend) return _ready;
+  _backend = backend;
+  return (_ready = (async () => {
+    const { instance } = await instantiateWithBackend({ getSimdBytes, getBaseBytes, imports, backend });
     setInstance(instance);
   })());
 }

package/dist/browser.d.ts

@@ -1,2 +1,5 @@
-export function init(imports?: WebAssembly.Imports): Promise<void>;
+export interface InitOptions {
+  backend?: 'auto' | 'simd' | 'base';
+}
+export function init(imports?: WebAssembly.Imports, opts?: InitOptions): Promise<void>;
 export * from "./custom.js";

package/dist/browser.js

@@ -1,21 +1,28 @@
 import { setInstance, registerInit } from "./core.js";
-import { instantiateWithFallback } from "./util.js";
+import { instantiateWithBackend } from "./util.js";
 
 const simdUrl = new URL("./wasm/lz4.simd.wasm", import.meta.url);
 const baseUrl = new URL("./wasm/lz4.base.wasm", import.meta.url);
 
-async function getWasmBytes() {
-  const [simdRes, baseRes] = await Promise.all([fetch(simdUrl), fetch(baseUrl)]);
-  const [simdBytes, baseBytes] = await Promise.all([simdRes.arrayBuffer(), baseRes.arrayBuffer()]);
-  return { simdBytes, baseBytes };
+async function getSimdBytes() {
+  const res = await fetch(simdUrl);
+  return res.arrayBuffer();
+}
+
+async function getBaseBytes() {
+  const res = await fetch(baseUrl);
+  return res.arrayBuffer();
 }
 
 
 let _ready = null;
-export function init(imports = {}) {
-  return (_ready ??= (async () => {
-    const { simdBytes, baseBytes } = await getWasmBytes();
-    const { instance } = await instantiateWithFallback(simdBytes, baseBytes, imports);
+let _backend = null;
+export function init(imports = {}, opts = {}) {
+  const backend = opts.backend || 'auto';
+  if (_ready && _backend === backend) return _ready;
+  _backend = backend;
+  return (_ready = (async () => {
+    const { instance } = await instantiateWithBackend({ getSimdBytes, getBaseBytes, imports, backend });
     setInstance(instance);
   })());
 }

package/dist/core.d.ts

@@ -6,5 +6,13 @@ export function memoryU8(): Uint8Array;
 export function alloc(len: number): number;
 export function free(ptr: number, len: number): void;
 
-export function compress_level_1(input: WasmInput): Promise<Uint8Array>;
-export function compress_level_9(input: WasmInput): Promise<Uint8Array>;
+export function compress_lz4_block(input: WasmInput): Promise<Uint8Array>;
+export function decompress_lz4_block(input: WasmInput): Promise<Uint8Array>;
+export function compress_lz4(input: WasmInput): Promise<Uint8Array>;
+export function create_compressor(input: WasmInput): Promise<number>;
+export function compress_chunk(input: WasmInput): Promise<Uint8Array>;
+export function destroy_compressor(input: WasmInput): Promise<Uint8Array>;
+export function decompress_lz4(input: WasmInput): Promise<Uint8Array>;
+export function create_decompressor(input: WasmInput): Promise<number>;
+export function decompress_chunk(input: WasmInput): Promise<Uint8Array>;
+export function destroy_decompressor(input: WasmInput): Promise<Uint8Array>;

package/dist/core.js

@@ -18,7 +18,7 @@ export function wasmExports() {
 let _ready = null;
 export function registerInit(fn) { _initFn = fn; }
 
-async function ensureReady() {
+export async function ensureReady() {
   if (_ready) return _ready;
   if (!_initFn) throw new Error("init not registered");
   _ready = _initFn();
@@ -45,6 +45,40 @@ function toBytes(input) {
   throw new TypeError("Expected a TypedArray or ArrayBuffer");
 }
 
+function scalarSize(type) {
+  switch (type) {
+    case "f64": return 8;
+    case "f32":
+    case "i32":
+    case "u32": return 4;
+    case "i16":
+    case "u16": return 2;
+    case "i8":
+    case "u8": return 1;
+    case "u32_array":
+    case "i32_array":
+    case "f32_array": return 1024 * 1024;
+    default: return 0;
+  }
+}
+
+function decodeReturn(view, type) {
+  switch (type) {
+    case "f32": return view.getFloat32(0, true);
+    case "f64": return view.getFloat64(0, true);
+    case "i32": return view.getInt32(0, true);
+    case "u32": return view.getUint32(0, true);
+    case "i16": return view.getInt16(0, true);
+    case "u16": return view.getUint16(0, true);
+    case "i8": return view.getInt8(0);
+    case "u8": return view.getUint8(0);
+    case "u32_array": return new Uint32Array(view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength));
+    case "i32_array": return new Int32Array(view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength));
+    case "f32_array": return new Float32Array(view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength));
+    default: return null;
+  }
+}
+
 function callWasm(abi, input, outLen, reuse) {
   if (!_inst) throw new Error("WASM instance not initialized");
   const view = toBytes(input);
@@ -79,12 +113,27 @@ function callWasm(abi, input, outLen, reuse) {
   return { inPtr, outPtr, len, outLen, written };
 }
 
-async function compress_level_1(input) {
+async function compress_lz4_block(input) {
   await ensureReady();
   const view = toBytes(input);
   const len = view.byteLength;
   const outLen = len + 1024;
-  const { outPtr, written, inPtr } = callWasm("compress_lz4_level_1", view, outLen, null);
+  const { outPtr, written, inPtr } = callWasm("compress_lz4_block", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { compress_lz4_block };
+
+async function decompress_lz4_block(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = outLen;
+  const { outPtr, written, inPtr } = callWasm("decompress_lz4_block", view, outLen, null);
 
   const result = memoryU8().slice(outPtr, outPtr + written);
 
@@ -92,14 +141,121 @@ async function compress_level_1(input) {
   free(outPtr, outLen);
   return result;
 }
-export { compress_level_1 };
+export { decompress_lz4_block };
 
-async function compress_level_9(input) {
+async function compress_lz4(input) {
   await ensureReady();
   const view = toBytes(input);
   const len = view.byteLength;
   const outLen = len + 1024;
-  const { outPtr, written, inPtr } = callWasm("compress_lz4_level_9", view, outLen, null);
+  const { outPtr, written, inPtr } = callWasm("compress_lz4", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { compress_lz4 };
+
+async function create_compressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = (scalarSize('u32') || 4);
+  const { outPtr, written, inPtr } = callWasm("create_compressor", view, outLen, null);
+
+  const retView = new DataView(memoryU8().buffer, outPtr, written);
+  const result = decodeReturn(retView, "u32");
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { create_compressor };
+
+async function compress_chunk(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = len + 1024;
+  const { outPtr, written, inPtr } = callWasm("compress_chunk", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { compress_chunk };
+
+async function destroy_compressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = Math.max(len, 4);
+  const { outPtr, written, inPtr } = callWasm("destroy_compressor", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { destroy_compressor };
+
+async function decompress_lz4(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = len * 10;
+  const { outPtr, written, inPtr } = callWasm("decompress_lz4", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { decompress_lz4 };
+
+async function create_decompressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = (scalarSize('u32') || 4);
+  const { outPtr, written, inPtr } = callWasm("create_decompressor", view, outLen, null);
+
+  const retView = new DataView(memoryU8().buffer, outPtr, written);
+  const result = decodeReturn(retView, "u32");
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { create_decompressor };
+
+async function decompress_chunk(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = len * 4;
+  const { outPtr, written, inPtr } = callWasm("decompress_chunk", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { decompress_chunk };
+
+async function destroy_decompressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = Math.max(len, 4);
+  const { outPtr, written, inPtr } = callWasm("destroy_decompressor", view, outLen, null);
 
   const result = memoryU8().slice(outPtr, outPtr + written);
 
@@ -107,4 +263,4 @@ async function compress_level_9(input) {
   free(outPtr, outLen);
   return result;
 }
-export { compress_level_9 };
+export { destroy_decompressor };

package/dist/custom.js

@@ -1,14 +1,339 @@
-import { compress_level_1, compress_level_9, wasmExports } from './core.js';
+import { 
+  compress_lz4, 
+  compress_lz4_block,
+  decompress_lz4,
+  decompress_lz4_block,
+  wasmExports,
+  alloc,
+  free,
+  memoryU8,
+  ensureReady
+} from './core.js';
 
-export async function compress(input, options = {}) {
-  const level = options.level ?? 1;
+function toBytes(input) {
+  if (input instanceof Uint8Array) return input;
+  if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
+  if (input instanceof ArrayBuffer) return new Uint8Array(input);
+  throw new TypeError("Expected a TypedArray or ArrayBuffer");
+}
+
+// ============================================================================
+// Block API - Maximum speed, no frame overhead
+// Output is NOT compatible with standard LZ4 tools (lz4 CLI, etc.)
+// Use when you control both compression and decompression
+// ============================================================================
+
+/**
+ * Compress using raw LZ4 block format (maximum speed)
+ * ~5x faster than frame API due to no checksumming overhead
+ * @param {Uint8Array} input - Data to compress
+ * @returns {Promise<Uint8Array>} Compressed data (raw block format)
+ */
+export async function compressBlock(input) {
+  try {
+    return compress_lz4_block(input);
+  } catch (error) {
+    throw new Error(`Block compression failed: ${error.message}`);
+  }
+}
+
+/**
+ * Decompress raw LZ4 block format
+ * @param {Uint8Array} input - Compressed data (raw block format)
+ * @param {number} originalSize - Original uncompressed size (REQUIRED)
+ * @returns {Promise<Uint8Array>} Decompressed data
+ */
+export async function decompressBlock(input, originalSize) {
+  if (typeof originalSize !== 'number' || originalSize <= 0) {
+    throw new Error('decompressBlock requires originalSize parameter');
+  }
+  await ensureReady();
+  
+  const view = toBytes(input);
+  const len = view.byteLength;
   
+  const inPtr = alloc(len);
+  const outPtr = alloc(originalSize);
+  
+  try {
+    memoryU8().set(view, inPtr);
+    const written = wasmExports().decompress_lz4_block(inPtr, len, outPtr, originalSize);
+    
+    if (written < 0) {
+      throw new Error('Block decompression failed');
+    }
+    
+    const result = memoryU8().slice(outPtr, outPtr + written);
+    return result;
+  } finally {
+    free(inPtr, len);
+    free(outPtr, originalSize);
+  }
+}
+
+// ============================================================================
+// Packed Block API - Block format with size prefix (self-contained)
+// Perfect for network transfer when you control both client and server
+// Format: [4 bytes: original size (little-endian)] + [compressed data]
+// ============================================================================
+
+/**
+ * Compress with size prefix - ready for network transfer
+ * Output includes original size, so decompression doesn't need it separately
+ * @param {Uint8Array} input - Data to compress
+ * @returns {Promise<Uint8Array>} [4-byte size prefix] + [compressed block]
+ */
+export async function compressPacked(input) {
+  const view = toBytes(input);
+  const compressed = await compressBlock(view);
+  
+  // Prepend 4-byte little-endian size
+  const result = new Uint8Array(4 + compressed.length);
+  const dataView = new DataView(result.buffer);
+  dataView.setUint32(0, view.length, true); // little-endian
+  result.set(compressed, 4);
+  
+  return result;
+}
+
+/**
+ * Decompress packed format (with size prefix)
+ * @param {Uint8Array} input - Packed compressed data (from compressPacked)
+ * @returns {Promise<Uint8Array>} Decompressed data
+ */
+export async function decompressPacked(input) {
+  const view = toBytes(input);
+  if (view.length < 4) {
+    throw new Error('Invalid packed data: too short');
+  }
+  
+  const dataView = new DataView(view.buffer, view.byteOffset, view.byteLength);
+  const originalSize = dataView.getUint32(0, true); // little-endian
+  const compressed = view.subarray(4);
+  
+  return decompressBlock(compressed, originalSize);
+}
+
+// ============================================================================
+// Frame API - Standard LZ4 frame format (compatible with lz4 CLI tools)
+// Includes headers and checksums for integrity
+// ============================================================================
+
+/**
+ * Compress using standard LZ4 frame format
+ * Compatible with lz4 CLI and other standard tools
+ * @param {Uint8Array} input - Data to compress
+ * @returns {Promise<Uint8Array>} Compressed data (LZ4 frame format)
+ */
+export async function compress(input) {
   try {
-    if (level <= 1) return compress_level_1(input);
-    return compress_level_9(input);
+    return compress_lz4(input);
   } catch (error) {
     throw new Error(`Compression failed: ${error.message}`);
   }
 }
 
+// Streaming compression API
+// 
+// Note: We don't use wasm-bindgen-lite's createTransformStream() helper here because
+// compression/decompression requires stateful streaming:
+// - Compression: Must accumulate input chunks until finish() is called
+// - Decompression: Must handle partial frames and buffer incomplete data
+// 
+// createTransformStream() is designed for stateless transformations where each chunk
+// is processed independently. Instead, we use a manual handle-based approach that:
+// 1. Creates compressor/decompressor handles in Rust (stateful)
+// 2. Maintains state between chunks via handles
+// 3. Manually manages WASM memory allocation/freeing
+export class StreamingCompressor {
+  constructor() {
+    this._initPromise = ensureReady();
+    this.handle = null;
+  }
+
+  async _ensureInit() {
+    await this._initPromise;
+    if (this.handle === null) {
+      this.handle = wasmExports().create_compressor();
+      if (this.handle === 0) {
+        throw new Error('Failed to create compressor');
+      }
+    }
+  }
+
+  async compressChunk(input, finish = false) {
+    await this._ensureInit();
+    if (this.handle === 0) {
+      throw new Error('Compressor already destroyed');
+    }
+    
+    const view = toBytes(input);
+    const len = view.byteLength;
+    const outLen = len + 1024; // LZ4 compression typically produces smaller output
+    
+    const inPtr = alloc(len);
+    const outPtr = alloc(outLen);
+    
+    try {
+      memoryU8().set(view, inPtr);
+      const written = wasmExports().compress_chunk(this.handle, inPtr, len, outPtr, outLen, finish ? 1 : 0);
+      
+      if (written < 0) {
+        if (written === -1) {
+          throw new Error('Compression failed');
+        } else {
+          // Negative value indicates needed buffer size
+          free(outPtr, outLen);
+          const neededLen = -written;
+          const newOutPtr = alloc(neededLen);
+          memoryU8().set(view, inPtr);
+          const retryWritten = wasmExports().compress_chunk(this.handle, inPtr, len, newOutPtr, neededLen, finish ? 1 : 0);
+          if (retryWritten < 0) {
+            free(newOutPtr, neededLen);
+            throw new Error('Compression failed after retry');
+          }
+          const result = memoryU8().slice(newOutPtr, newOutPtr + retryWritten);
+          free(newOutPtr, neededLen);
+          free(inPtr, len);
+          if (finish) {
+            this.handle = 0;
+          }
+          return result;
+        }
+      }
+      
+      if (written === 0) {
+        // No output yet (buffering)
+        free(outPtr, outLen);
+        free(inPtr, len);
+        return new Uint8Array(0);
+      }
+      
+      const result = memoryU8().slice(outPtr, outPtr + written);
+      free(outPtr, outLen);
+      free(inPtr, len);
+      
+      if (finish) {
+        this.handle = 0;
+      }
+      
+      return result;
+    } catch (error) {
+      free(outPtr, outLen);
+      free(inPtr, len);
+      throw new Error(`Compression failed: ${error.message}`);
+    }
+  }
+
+  async destroy() {
+    await this._ensureInit();
+    if (this.handle !== 0 && this.handle !== null) {
+      wasmExports().destroy_compressor(this.handle);
+      this.handle = 0;
+    }
+  }
+}
+
+// Streaming decompression API
+// 
+// See note above about why we use manual handles instead of createTransformStream()
+export class StreamingDecompressor {
+  constructor() {
+    this._initPromise = ensureReady();
+    this.handle = null;
+  }
+
+  async _ensureInit() {
+    await this._initPromise;
+    if (this.handle === null) {
+      this.handle = wasmExports().create_decompressor();
+      if (this.handle === 0) {
+        throw new Error('Failed to create decompressor');
+      }
+    }
+  }
+
+  async decompressChunk(input, finish = false) {
+    await this._ensureInit();
+    if (this.handle === 0) {
+      throw new Error('Decompressor already destroyed');
+    }
+    
+    const view = toBytes(input);
+    const len = view.byteLength;
+    const outLen = len * 4; // Decompressed data is typically larger
+    
+    const inPtr = alloc(len);
+    const outPtr = alloc(outLen);
+    
+    try {
+      memoryU8().set(view, inPtr);
+      const written = wasmExports().decompress_chunk(this.handle, inPtr, len, outPtr, outLen, finish ? 1 : 0);
+      
+      if (written < 0) {
+        if (written === -1) {
+          throw new Error('Decompression failed');
+        } else {
+          // Negative value indicates needed buffer size
+          free(outPtr, outLen);
+          const neededLen = -written;
+          const newOutPtr = alloc(neededLen);
+          memoryU8().set(view, inPtr);
+          const retryWritten = wasmExports().decompress_chunk(this.handle, inPtr, len, newOutPtr, neededLen, finish ? 1 : 0);
+          if (retryWritten < 0) {
+            free(newOutPtr, neededLen);
+            throw new Error('Decompression failed after retry');
+          }
+          const result = memoryU8().slice(newOutPtr, newOutPtr + retryWritten);
+          free(newOutPtr, neededLen);
+          free(inPtr, len);
+          if (finish) {
+            this.handle = 0;
+          }
+          return result;
+        }
+      }
+      
+      if (written === 0) {
+        // No output yet (buffering)
+        free(outPtr, outLen);
+        free(inPtr, len);
+        return new Uint8Array(0);
+      }
+      
+      const result = memoryU8().slice(outPtr, outPtr + written);
+      free(outPtr, outLen);
+      free(inPtr, len);
+      
+      if (finish) {
+        this.handle = 0;
+      }
+      
+      return result;
+    } catch (error) {
+      free(outPtr, outLen);
+      free(inPtr, len);
+      throw new Error(`Decompression failed: ${error.message}`);
+    }
+  }
+
+  async destroy() {
+    await this._ensureInit();
+    if (this.handle !== 0 && this.handle !== null) {
+      wasmExports().destroy_decompressor(this.handle);
+      this.handle = 0;
+    }
+  }
+}
+
+// One-shot decompression
+export async function decompress(input) {
+  try {
+    return decompress_lz4(input);
+  } catch (error) {
+    throw new Error(`Decompression failed: ${error.message}`);
+  }
+}
+
 export { wasmExports };
+

package/dist/node-inline.d.ts

@@ -1,2 +1,5 @@
-export function init(imports?: WebAssembly.Imports): Promise<void>;
+export interface InitOptions {
+  backend?: 'auto' | 'simd' | 'base';
+}
+export function init(imports?: WebAssembly.Imports, opts?: InitOptions): Promise<void>;
 export * from "./custom.js";