@addmaple/lz4 0.1.1 → 0.2.0

package/README.md

@@ -4,6 +4,24 @@ Fast LZ4 compression in the browser and Node.js using Rust + WASM.
 
 **2.5x-3.5x faster** than `lz4js`.
 
+## Implementation (Rust)
+
+This package is backed by these Rust crates in the `wasm-fast-compress` repo:
+
+- `codec-lz4` (this repo): high-level codec wrapper
+- `lz4_flex` (Git fork, branch `wasm-simd`): core LZ4 implementation with WASM SIMD128 hot paths
+
+## SIMD acceleration (how it works)
+
+- We build **two WASM binaries**:
+  - `lz4.base.wasm`: compiled without `+simd128`
+  - `lz4.simd.wasm`: compiled with `-C target-feature=+simd128`
+- At runtime, the JS loader detects SIMD support and loads the best binary automatically.
+
+On wasm32, the SIMD build benefits from:
+- `lz4_flex` explicit `wasm32 + simd128` intrinsics for match finding / copying hot paths
+- additional LLVM autovectorization where applicable
+
 ## Installation
 
 ```bash
@@ -15,12 +33,81 @@ npm install @addmaple/lz4
 ```javascript
 import { init, compress } from '@addmaple/lz4';
 
+// Optional: call init() to avoid first-call latency.
+// If you skip init(), the first compress/decompress call will lazy-initialize.
 await init();
 
 const input = new TextEncoder().encode('hello world');
 const compressed = await compress(input);
 ```
 
+### Lazy init (init() optional)
+
+```javascript
+import { compress } from '@addmaple/lz4';
+
+const input = new TextEncoder().encode('hello world');
+const compressed = await compress(input); // triggers lazy init on first call
+```
+
+### Streaming compression + decompression
+
+For chunked input (e.g. streaming over the network), use the handle-based streaming helpers:
+
+```javascript
+import { init, StreamingCompressor, StreamingDecompressor } from '@addmaple/lz4';
+
+// Optional: init() is not required; streaming helpers also lazy-init.
+await init();
+
+// Compress
+const enc = new StreamingCompressor();
+const c1 = await enc.compressChunk(chunk1, false);
+const c2 = await enc.compressChunk(chunk2, false);
+const c3 = await enc.compressChunk(chunk3, true); // finish
+
+// Decompress (finish must be true to produce output for frame format)
+const dec = new StreamingDecompressor();
+await dec.decompressChunk(c1, false);
+await dec.decompressChunk(c2, false);
+const plain = await dec.decompressChunk(c3, true);
+```
+
+### Streaming to `fetch()` (ergonomic)
+
+If you want to upload a `File`/`Blob` with LZ4 compression, you can pipe it through the built-in stream helper:
+
+```javascript
+import { createCompressionStream } from '@addmaple/lz4';
+
+const body = file.stream().pipeThrough(createCompressionStream());
+
+await fetch('/upload', {
+  method: 'POST',
+  headers: {
+    // Not a standard encoding token like gzip; your server must explicitly support this.
+    'Content-Encoding': 'lz4',
+  },
+  body,
+  // Needed for streaming request bodies in some runtimes (notably Node fetch).
+  duplex: 'half',
+});
+```
+
+### Streaming decompression from `fetch()` (ergonomic)
+
+```javascript
+import { createDecompressionStream } from '@addmaple/lz4';
+
+const res = await fetch('/download');
+if (!res.body) throw new Error('No response body');
+
+const decompressed = res.body.pipeThrough(createDecompressionStream());
+
+// Example: read it all (or pipe somewhere else)
+const buf = await new Response(decompressed).arrayBuffer();
+```
+
 ### Inline (Zero-latency)
 
 WASM bytes embedded directly in JS — no separate file fetching:

package/dist/browser-inline.d.ts

@@ -1,2 +1,5 @@
-export function init(imports?: WebAssembly.Imports): Promise<void>;
+export interface InitOptions {
+  backend?: 'auto' | 'simd' | 'base';
+}
+export function init(imports?: WebAssembly.Imports, opts?: InitOptions): Promise<void>;
 export * from "./custom.js";

package/dist/browser-inline.js

@@ -1,19 +1,26 @@
 import { setInstance, registerInit } from "./core.js";
-import { instantiateWithFallback } from "./util.js";
+import { instantiateWithBackend } from "./util.js";
 
-import { wasmBytes as simdBytes } from "./wasm-inline/lz4.simd.wasm.js";
-import { wasmBytes as baseBytes } from "./wasm-inline/lz4.base.wasm.js";
+import { wasmBytes as _simdBytes } from "./wasm-inline/lz4.simd.wasm.js";
+import { wasmBytes as _baseBytes } from "./wasm-inline/lz4.base.wasm.js";
 
-async function getWasmBytes() {
-  return { simdBytes, baseBytes };
+async function getSimdBytes() {
+  return _simdBytes;
+}
+
+async function getBaseBytes() {
+  return _baseBytes;
 }
 
 
 let _ready = null;
-export function init(imports = {}) {
-  return (_ready ??= (async () => {
-    const { simdBytes, baseBytes } = await getWasmBytes();
-    const { instance } = await instantiateWithFallback(simdBytes, baseBytes, imports);
+let _backend = null;
+export function init(imports = {}, opts = {}) {
+  const backend = opts.backend || 'auto';
+  if (_ready && _backend === backend) return _ready;
+  _backend = backend;
+  return (_ready = (async () => {
+    const { instance } = await instantiateWithBackend({ getSimdBytes, getBaseBytes, imports, backend });
     setInstance(instance);
   })());
 }

package/dist/browser.d.ts

@@ -1,2 +1,5 @@
-export function init(imports?: WebAssembly.Imports): Promise<void>;
+export interface InitOptions {
+  backend?: 'auto' | 'simd' | 'base';
+}
+export function init(imports?: WebAssembly.Imports, opts?: InitOptions): Promise<void>;
 export * from "./custom.js";

package/dist/browser.js

@@ -1,21 +1,28 @@
 import { setInstance, registerInit } from "./core.js";
-import { instantiateWithFallback } from "./util.js";
+import { instantiateWithBackend } from "./util.js";
 
 const simdUrl = new URL("./wasm/lz4.simd.wasm", import.meta.url);
 const baseUrl = new URL("./wasm/lz4.base.wasm", import.meta.url);
 
-async function getWasmBytes() {
-  const [simdRes, baseRes] = await Promise.all([fetch(simdUrl), fetch(baseUrl)]);
-  const [simdBytes, baseBytes] = await Promise.all([simdRes.arrayBuffer(), baseRes.arrayBuffer()]);
-  return { simdBytes, baseBytes };
+async function getSimdBytes() {
+  const res = await fetch(simdUrl);
+  return res.arrayBuffer();
+}
+
+async function getBaseBytes() {
+  const res = await fetch(baseUrl);
+  return res.arrayBuffer();
 }
 
 
 let _ready = null;
-export function init(imports = {}) {
-  return (_ready ??= (async () => {
-    const { simdBytes, baseBytes } = await getWasmBytes();
-    const { instance } = await instantiateWithFallback(simdBytes, baseBytes, imports);
+let _backend = null;
+export function init(imports = {}, opts = {}) {
+  const backend = opts.backend || 'auto';
+  if (_ready && _backend === backend) return _ready;
+  _backend = backend;
+  return (_ready = (async () => {
+    const { instance } = await instantiateWithBackend({ getSimdBytes, getBaseBytes, imports, backend });
     setInstance(instance);
   })());
 }

package/dist/core.d.ts

@@ -6,4 +6,13 @@ export function memoryU8(): Uint8Array;
 export function alloc(len: number): number;
 export function free(ptr: number, len: number): void;
 
-export function compress_lz4(input: WasmInput): Promise<Uint8Array>;
+export function compress_lz4_block(input: WasmInput): Promise<Uint8Array>;
+export function decompress_lz4_block(input: WasmInput): Promise<Uint8Array>;
+export function compress_lz4(input: WasmInput): Promise<Uint8Array>;
+export function create_compressor(input: WasmInput): Promise<number>;
+export function compress_chunk(input: WasmInput): Promise<Uint8Array>;
+export function destroy_compressor(input: WasmInput): Promise<Uint8Array>;
+export function decompress_lz4(input: WasmInput): Promise<Uint8Array>;
+export function create_decompressor(input: WasmInput): Promise<number>;
+export function decompress_chunk(input: WasmInput): Promise<Uint8Array>;
+export function destroy_decompressor(input: WasmInput): Promise<Uint8Array>;

package/dist/core.js

@@ -18,7 +18,7 @@ export function wasmExports() {
 let _ready = null;
 export function registerInit(fn) { _initFn = fn; }
 
-async function ensureReady() {
+export async function ensureReady() {
   if (_ready) return _ready;
   if (!_initFn) throw new Error("init not registered");
   _ready = _initFn();
@@ -42,7 +42,42 @@ function toBytes(input) {
   if (input instanceof Uint8Array) return input;
   if (ArrayBuffer.isView(input)) return new Uint8Array(input.buffer, input.byteOffset, input.byteLength);
   if (input instanceof ArrayBuffer) return new Uint8Array(input);
-  throw new TypeError("Expected a TypedArray or ArrayBuffer");
+  if (typeof input === 'string') return new TextEncoder().encode(input);
+  throw new TypeError("Expected a TypedArray, ArrayBuffer, or string");
+}
+
+function scalarSize(type) {
+  switch (type) {
+    case "f64": return 8;
+    case "f32":
+    case "i32":
+    case "u32": return 4;
+    case "i16":
+    case "u16": return 2;
+    case "i8":
+    case "u8": return 1;
+    case "u32_array":
+    case "i32_array":
+    case "f32_array": return 1024 * 1024;
+    default: return 0;
+  }
+}
+
+function decodeReturn(view, type) {
+  switch (type) {
+    case "f32": return view.getFloat32(0, true);
+    case "f64": return view.getFloat64(0, true);
+    case "i32": return view.getInt32(0, true);
+    case "u32": return view.getUint32(0, true);
+    case "i16": return view.getInt16(0, true);
+    case "u16": return view.getUint16(0, true);
+    case "i8": return view.getInt8(0);
+    case "u8": return view.getUint8(0);
+    case "u32_array": return new Uint32Array(view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength));
+    case "i32_array": return new Int32Array(view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength));
+    case "f32_array": return new Float32Array(view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength));
+    default: return null;
+  }
 }
 
 function callWasm(abi, input, outLen, reuse) {
@@ -79,6 +114,36 @@ function callWasm(abi, input, outLen, reuse) {
   return { inPtr, outPtr, len, outLen, written };
 }
 
+async function compress_lz4_block(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = len + 1024;
+  const { outPtr, written, inPtr } = callWasm("compress_lz4_block", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { compress_lz4_block };
+
+async function decompress_lz4_block(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = outLen;
+  const { outPtr, written, inPtr } = callWasm("decompress_lz4_block", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { decompress_lz4_block };
+
 async function compress_lz4(input) {
   await ensureReady();
   const view = toBytes(input);
@@ -93,3 +158,110 @@ async function compress_lz4(input) {
   return result;
 }
 export { compress_lz4 };
+
+async function create_compressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = (scalarSize('u32') || 4);
+  const { outPtr, written, inPtr } = callWasm("create_compressor", view, outLen, null);
+
+  const retView = new DataView(memoryU8().buffer, outPtr, written);
+  const result = decodeReturn(retView, "u32");
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { create_compressor };
+
+async function compress_chunk(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = len + 1024;
+  const { outPtr, written, inPtr } = callWasm("compress_chunk", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { compress_chunk };
+
+async function destroy_compressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = Math.max(len, 4);
+  const { outPtr, written, inPtr } = callWasm("destroy_compressor", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { destroy_compressor };
+
+async function decompress_lz4(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = len * 10;
+  const { outPtr, written, inPtr } = callWasm("decompress_lz4", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { decompress_lz4 };
+
+async function create_decompressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = (scalarSize('u32') || 4);
+  const { outPtr, written, inPtr } = callWasm("create_decompressor", view, outLen, null);
+
+  const retView = new DataView(memoryU8().buffer, outPtr, written);
+  const result = decodeReturn(retView, "u32");
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { create_decompressor };
+
+async function decompress_chunk(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = len * 4;
+  const { outPtr, written, inPtr } = callWasm("decompress_chunk", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { decompress_chunk };
+
+async function destroy_decompressor(input) {
+  await ensureReady();
+  const view = toBytes(input);
+  const len = view.byteLength;
+  const outLen = Math.max(len, 4);
+  const { outPtr, written, inPtr } = callWasm("destroy_decompressor", view, outLen, null);
+
+  const result = memoryU8().slice(outPtr, outPtr + written);
+
+  free(inPtr, len);
+  free(outPtr, outLen);
+  return result;
+}
+export { destroy_decompressor };