@hpcc-js/wasm-zstd 1.8.0 → 1.9.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hpcc-js/wasm-zstd",
-  "version": "1.8.0",
+  "version": "1.9.1",
   "description": "hpcc-js - WASM zstd",
   "type": "module",
   "exports": {
@@ -18,6 +18,8 @@
   ],
   "scripts": {
     "clean": "rimraf coverage dist dist-test types .nyc_output",
+    "build-cpp": "cmake --build ../../build --target zstdlib",
+    "build-cpp-watch": "chokidar 'src-cpp/**.*' -c 'npm run build-cpp'",
     "gen-types": "tsc --project tsconfig.json --emitDeclarationOnly",
     "gen-types-watch": "npm run gen-types -- --watch",
     "bundle": "node esbuild.js",
@@ -26,7 +28,7 @@
     "build-dev": "run-p gen-types bundle-dev",
     "build": "run-p gen-types bundle",
     "lint-skypack": "npx -y @skypack/package-check",
-    "lint-eslint": "eslint src/**/*.ts tests/**/*.ts",
+    "lint-eslint": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
     "lint": "run-p lint-eslint",
     "test-browser": "vitest run --project browser",
     "test-node": "vitest run --project node",
@@ -36,7 +38,8 @@
     "update-major": "npx -y npm-check-updates -u"
   },
   "devDependencies": {
-    "@hpcc-js/esbuild-plugins": "1.7.0"
+    "@hpcc-js/esbuild-plugins": "1.7.0",
+    "@hpcc-js/wasm-util": "1.0.0"
   },
   "keywords": [
     "graphviz",
@@ -54,5 +57,5 @@
   },
   "homepage": "https://hpcc-systems.github.io/hpcc-js-wasm/",
   "license": "Apache-2.0",
-  "gitHead": "3fa48df3d5a7a6c43bf4f0c8839f1f9df27080da"
+  "gitHead": "11395f5acfc865accde71905509bb45d5c79b11b"
 }

package/src/zstd.ts

@@ -1,6 +1,8 @@
 // @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin
-import load, { reset } from "../../../build/packages/zstd/src-cpp/zstdlib.wasm";
-import { WasmLibrary } from "./wasm-library.ts";
+import load, { reset } from "../../../build/packages/zstd/zstdlib.wasm";
+import type { MainModule, zstd } from "../types/zstdlib.js";
+import { MainModuleEx } from "@hpcc-js/wasm-util";
+type ZstdExports = MainModule["zstd"];
 
 //  Ref:  http://facebook.github.io/zstd/zstd_manual.html
 //  Ref:  https://github.com/facebook/zstd
@@ -24,10 +26,14 @@ let g_zstd: Promise<Zstd>;
  * const decompressed_data = zstd.decompress(compressed_data);
  * ```
  */
-export class Zstd extends WasmLibrary {
-
-    private constructor(_module: any) {
-        super(_module, _module.zstd.prototype);
+export class Zstd extends MainModuleEx<MainModule> {
+    private _zstdClass: ZstdExports;
+    private _zstd: zstd;
+
+    private constructor(_module: MainModule) {
+        super(_module);
+        this._zstdClass = _module.zstd;
+        this._zstd = new this._zstdClass();
     }
 
     /**
@@ -41,9 +47,7 @@ export class Zstd extends WasmLibrary {
      */
     static load(): Promise<Zstd> {
         if (!g_zstd) {
-            g_zstd = load().then((module: any) => {
-                return new Zstd(module)
-            });
+            g_zstd = (load() as Promise<MainModule>).then((module) => new Zstd(module));
         }
         return g_zstd;
     }
@@ -59,7 +63,22 @@ export class Zstd extends WasmLibrary {
      * @returns The Zstd c++ version
      */
     version(): string {
-        return this._exports.version();
+        return this._zstdClass.version();
+    }
+
+    /**
+     * Resets the internal compression/decompression state.
+     */
+    reset(): void {
+        this._zstd.reset();
+    }
+
+    /**
+     * Sets the compression level for streaming compression.
+     * @param level Compression level (use minCLevel() to maxCLevel())
+     */
+    setCompressionLevel(level: number): void {
+        this._zstd.setCompressionLevel(level);
     }
 
     /**
@@ -72,19 +91,76 @@ export class Zstd extends WasmLibrary {
      * :::
      */
     compress(data: Uint8Array, compressionLevel: number = this.defaultCLevel()): Uint8Array {
-        const uncompressed = this.uint8_heapu8(data);
+        const uncompressed = this.dataToHeap(data);
 
-        const compressedSize = this._exports.compressBound(data.length);
-        const compressed = this.malloc_heapu8(compressedSize);
-        compressed.size = this._exports.compress(compressed.ptr, compressedSize, uncompressed.ptr, uncompressed.size, compressionLevel);
+        const compressedSize = this._zstdClass.compressBound(data.length);
+        const compressed = this.malloc(compressedSize);
+        compressed.size = this._zstdClass.compress(compressed.ptr, compressedSize, uncompressed.ptr, uncompressed.size, compressionLevel);
         /* istanbul ignore if  */
-        if (this._exports.isError(compressed.size)) {
-            console.error(this._exports.getErrorName(compressed.size));
+        if (this._zstdClass.isError(compressed.size)) {
+            console.error(this._zstdClass.getErrorName(compressed.size));
+        }
+        const retVal = this.heapToUint8Array(compressed);
+
+        this.free(compressed);
+        this.free(uncompressed);
+        return retVal;
+    }
+
+    /**
+     * Compresses a chunk of data in streaming mode.
+     * Call reset() before the first chunk, then compressChunk() for each chunk, and finally compressEnd().
+     * @param data Chunk of data to be compressed
+     * @returns Compressed chunk data
+     */
+    compressChunk(data: Uint8Array): Uint8Array {
+        const uncompressed = this.dataToHeap(data);
+        // For streaming compression, we need enough space for:
+        // 1. The compressed data (compressBound gives worst case)
+        // 2. Additional overhead for frame headers and internal buffering
+        // Use compressBound + CStreamOutSize to ensure we have enough
+        const boundSize = this._zstdClass.compressBound(data.length);
+        const streamOutSize = this._zstdClass.CStreamOutSize();
+        const compressedSize = boundSize + streamOutSize;
+        const compressed = this.malloc(compressedSize);
+
+        compressed.size = this._zstd.compressChunk(compressed.ptr, compressedSize, uncompressed.ptr, uncompressed.size);
+
+        // Check for errors before trying to use the size
+        if (this._zstdClass.isError(compressed.size)) {
+            const errorName = this._zstdClass.getErrorName(compressed.size);
+            this.free(compressed);
+            this.free(uncompressed);
+            throw new Error(`compressChunk failed: ${errorName} (data.length=${data.length}, compressedSize=${compressedSize})`);
         }
-        const retVal = this.heapu8_uint8(compressed);
 
-        this.free_heapu8(compressed);
-        this.free_heapu8(uncompressed);
+        const retVal = this.heapToUint8Array(compressed);
+
+        this.free(compressed);
+        this.free(uncompressed);
+        return retVal;
+    }
+
+    /**
+     * Finishes the streaming compression and returns any remaining compressed data.
+     * @returns Final compressed data
+     */
+    compressEnd(): Uint8Array {
+        const compressedSize = this._zstdClass.CStreamOutSize(); // Recommended buffer size for output
+        const compressed = this.malloc(compressedSize);
+
+        compressed.size = this._zstd.compressEnd(compressed.ptr, compressedSize);
+
+        // Check for errors before trying to use the size
+        if (this._zstdClass.isError(compressed.size)) {
+            const errorName = this._zstdClass.getErrorName(compressed.size);
+            this.free(compressed);
+            throw new Error(`compressEnd failed: ${errorName} (compressedSize=${compressedSize})`);
+        }
+
+        const retVal = this.heapToUint8Array(compressed);
+
+        this.free(compressed);
         return retVal;
     }
 
@@ -93,23 +169,62 @@ export class Zstd extends WasmLibrary {
      * @returns Uncompressed data.
      */
     decompress(compressedData: Uint8Array): Uint8Array {
-        const compressed = this.uint8_heapu8(compressedData);
-        const uncompressedSize = this._exports.getFrameContentSize(compressed.ptr, compressed.size);
+        const compressed = this.dataToHeap(compressedData);
+        let uncompressedSize = this._zstdClass.getFrameContentSize(compressed.ptr, compressed.size);
+
+        // Check if size is unknown (happens with streaming compression)
+        // ZSTD_CONTENTSIZE_UNKNOWN is (uint64_t)-1, which becomes a very large number in JS
+        // Using BigInt to avoid precision loss warning
+        const CONTENTSIZE_UNKNOWN = BigInt("0xFFFFFFFFFFFFFFFF");
+
         /* istanbul ignore if  */
-        if (this._exports.isError(uncompressedSize)) {
-            console.error(this._exports.getErrorName(uncompressedSize));
+        if (this._zstdClass.isError(uncompressedSize)) {
+            const errorName = this._zstdClass.getErrorName(uncompressedSize);
+            this.free(compressed);
+            throw new Error(`Failed to get frame content size: ${errorName}`);
         }
-        const uncompressed = this.malloc_heapu8(uncompressedSize);
 
-        uncompressed.size = this._exports.decompress(uncompressed.ptr, uncompressedSize, compressed.ptr, compressed.size);
+        // If content size is unknown, use a reasonable upper bound
+        // For safety, use decompression bound or a multiple of compressed size
+        if (BigInt(uncompressedSize) >= CONTENTSIZE_UNKNOWN || uncompressedSize === 0) {
+            // Use a heuristic: decompressed data is typically 2-10x compressed size for text/structured data
+            // Allocate generously to avoid buffer overflow
+            uncompressedSize = Math.max(compressed.size * 20, 1024 * 1024); // At least 1MB or 20x compressed
+        }
+
+        const uncompressed = this.malloc(uncompressedSize);
+
+        uncompressed.size = this._zstdClass.decompress(uncompressed.ptr, uncompressedSize, compressed.ptr, compressed.size);
         /* istanbul ignore if  */
-        if (this._exports.isError(uncompressed.size)) {
-            console.error(this._exports.getErrorName(uncompressed.size));
+        if (this._zstdClass.isError(uncompressed.size)) {
+            const errorName = this._zstdClass.getErrorName(uncompressed.size);
+            this.free(uncompressed);
+            this.free(compressed);
+            throw new Error(`Decompression failed: ${errorName}`);
         }
-        const retVal = this.heapu8_uint8(uncompressed);
+        const retVal = this.heapToUint8Array(uncompressed);
+
+        this.free(uncompressed);
+        this.free(compressed);
+        return retVal;
+    }
+
+    /**
+     * Decompresses a chunk of data in streaming mode.
+     * Call reset() before the first chunk, then decompressChunk() for each chunk.
+     * @param compressedData Chunk of compressed data
+     * @param outputSize Expected output size for this chunk
+     * @returns Decompressed chunk data
+     */
+    decompressChunk(compressedData: Uint8Array, outputSize: number): Uint8Array {
+        const compressed = this.dataToHeap(compressedData);
+        const uncompressed = this.malloc(outputSize);
+
+        uncompressed.size = this._zstd.decompressChunk(uncompressed.ptr, outputSize, compressed.ptr, compressed.size);
+        const retVal = this.heapToUint8Array(uncompressed);
 
-        this.free_heapu8(uncompressed);
-        this.free_heapu8(compressed);
+        this.free(uncompressed);
+        this.free(compressed);
         return retVal;
     }
 
@@ -117,14 +232,14 @@ export class Zstd extends WasmLibrary {
      * @returns Default compression level (see notes above above).
      */
     defaultCLevel(): number {
-        return this._exports.defaultCLevel();
+        return this._zstdClass.defaultCLevel();
     }
 
     minCLevel(): number {
-        return this._exports.minCLevel();
+        return this._zstdClass.minCLevel();
     }
 
     maxCLevel(): number {
-        return this._exports.maxCLevel();
+        return this._zstdClass.maxCLevel();
     }
 }

package/types/zstd.d.ts

@@ -1,4 +1,5 @@
-import { WasmLibrary } from "./wasm-library.ts";
+import type { MainModule } from "../types/zstdlib.js";
+import { MainModuleEx } from "@hpcc-js/wasm-util";
 /**
  * The Zstandard WASM library, provides a simplified wrapper around the Zstandard c++ library.
  *
@@ -16,7 +17,9 @@ import { WasmLibrary } from "./wasm-library.ts";
  * const decompressed_data = zstd.decompress(compressed_data);
  * ```
  */
-export declare class Zstd extends WasmLibrary {
+export declare class Zstd extends MainModuleEx<MainModule> {
+    private _zstdClass;
+    private _zstd;
     private constructor();
     /**
      * Compiles and instantiates the raw wasm.
@@ -36,6 +39,15 @@ export declare class Zstd extends WasmLibrary {
      * @returns The Zstd c++ version
      */
     version(): string;
+    /**
+     * Resets the internal compression/decompression state.
+     */
+    reset(): void;
+    /**
+     * Sets the compression level for streaming compression.
+     * @param level Compression level (use minCLevel() to maxCLevel())
+     */
+    setCompressionLevel(level: number): void;
     /**
      * @param data Data to be compressed
      * @param compressionLevel Compression v Speed tradeoff, when omitted it will default to `zstd.defaultCLevel()` which is currently 3.
@@ -46,11 +58,31 @@ export declare class Zstd extends WasmLibrary {
      * :::
      */
     compress(data: Uint8Array, compressionLevel?: number): Uint8Array;
+    /**
+     * Compresses a chunk of data in streaming mode.
+     * Call reset() before the first chunk, then compressChunk() for each chunk, and finally compressEnd().
+     * @param data Chunk of data to be compressed
+     * @returns Compressed chunk data
+     */
+    compressChunk(data: Uint8Array): Uint8Array;
+    /**
+     * Finishes the streaming compression and returns any remaining compressed data.
+     * @returns Final compressed data
+     */
+    compressEnd(): Uint8Array;
     /**
      * @param compressedData Data to be compressed
      * @returns Uncompressed data.
      */
     decompress(compressedData: Uint8Array): Uint8Array;
+    /**
+     * Decompresses a chunk of data in streaming mode.
+     * Call reset() before the first chunk, then decompressChunk() for each chunk.
+     * @param compressedData Chunk of compressed data
+     * @param outputSize Expected output size for this chunk
+     * @returns Decompressed chunk data
+     */
+    decompressChunk(compressedData: Uint8Array, outputSize: number): Uint8Array;
     /**
      * @returns Default compression level (see notes above above).
      */

package/types/zstdlib.d.ts

@@ -0,0 +1,64 @@
+// TypeScript bindings for emscripten-generated code.  Automatically generated at compile time.
+declare namespace RuntimeExports {
+    /**
+     * Given a pointer 'ptr' to a null-terminated UTF8-encoded string in the
+     * emscripten HEAP, returns a copy of that string as a Javascript String object.
+     *
+     * @param {number} ptr
+     * @param {number=} maxBytesToRead - An optional length that specifies the
+     *   maximum number of bytes to read. You can omit this parameter to scan the
+     *   string until the first 0 byte. If maxBytesToRead is passed, and the string
+     *   at [ptr, ptr+maxBytesToReadr[ contains a null byte in the middle, then the
+     *   string will cut short at that byte index.
+     * @param {boolean=} ignoreNul - If true, the function will not stop on a NUL character.
+     * @return {string}
+     */
+    function UTF8ToString(ptr: number, maxBytesToRead?: number | undefined, ignoreNul?: boolean | undefined): string;
+    function lengthBytesUTF8(str: any): number;
+    function stringToUTF8(str: any, outPtr: any, maxBytesToWrite: any): any;
+    let HEAPU8: any;
+}
+interface WasmModule {
+  _malloc(_0: number): number;
+  _free(_0: number): void;
+}
+
+export interface ClassHandle {
+  isAliasOf(other: ClassHandle): boolean;
+  delete(): void;
+  deleteLater(): this;
+  isDeleted(): boolean;
+  // @ts-ignore - If targeting lower than ESNext, this symbol might not exist.
+  [Symbol.dispose](): void;
+  clone(): this;
+}
+export interface zstd extends ClassHandle {
+  reset(): void;
+  setCompressionLevel(_0: number): void;
+  compressChunk(_0: number, _1: number, _2: number, _3: number): number;
+  compressEnd(_0: number, _1: number): number;
+  decompressChunk(_0: number, _1: number, _2: number, _3: number): number;
+}
+
+interface EmbindModule {
+  zstd: {
+    new(): zstd;
+    minCLevel(): number;
+    maxCLevel(): number;
+    defaultCLevel(): number;
+    malloc(_0: number): number;
+    free(_0: number): void;
+    compress(_0: number, _1: number, _2: number, _3: number, _4: number): number;
+    decompress(_0: number, _1: number, _2: number, _3: number): number;
+    findFrameCompressedSize(_0: number, _1: number): number;
+    compressBound(_0: number): number;
+    CStreamOutSize(): number;
+    isError(_0: number): number;
+    getFrameContentSize(_0: number, _1: number): number;
+    version(): string;
+    getErrorName(_0: number): string;
+  };
+}
+
+export type MainModule = WasmModule & typeof RuntimeExports & EmbindModule;
+export default function MainModuleFactory (options?: unknown): Promise<MainModule>;

package/src/wasm-library.ts

@@ -1,59 +0,0 @@
-export type PTR = number;
-export interface HeapU8 {
-    ptr: PTR;
-    size: number;
-}
-
-/**
- * Base class to simplify moving data into and out of Wasm memory.
- */
-export class WasmLibrary {
-
-    protected _module: any;
-    protected _exports: any;
-
-    protected constructor(_module: any, _export: any) {
-        this._module = _module;
-        this._exports = _export;
-    }
-
-    protected malloc_heapu8(size: number): HeapU8 {
-        const ptr: PTR = this._exports.malloc(size);
-        return {
-            ptr,
-            size
-        };
-    }
-
-    protected free_heapu8(data: HeapU8) {
-        this._exports.free(data.ptr);
-    }
-
-    protected uint8_heapu8(data: Uint8Array): HeapU8 {
-        const retVal = this.malloc_heapu8(data.byteLength);
-        this._module.HEAPU8.set(data, retVal.ptr);
-        return retVal;
-    }
-
-    protected heapu8_view(data: HeapU8): Uint8Array {
-        return this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size);
-    }
-
-    protected heapu8_uint8(data: HeapU8): Uint8Array {
-        return new Uint8Array([...this.heapu8_view(data)]);
-    }
-
-    protected string_heapu8(str: string): HeapU8 {
-        const data = Uint8Array.from(str, x => x.charCodeAt(0));
-        return this.uint8_heapu8(data);
-    }
-
-    protected heapu8_string(data: HeapU8): string {
-        const retVal = Array.from({ length: data.size });
-        const submodule = this._module.HEAPU8.subarray(data.ptr, data.ptr + data.size);
-        submodule.forEach((c: number, i: number) => {
-            retVal[i] = String.fromCharCode(c);
-        });
-        return retVal.join("");
-    }
-}

package/types/wasm-library.d.ts

@@ -1,20 +0,0 @@
-export type PTR = number;
-export interface HeapU8 {
-    ptr: PTR;
-    size: number;
-}
-/**
- * Base class to simplify moving data into and out of Wasm memory.
- */
-export declare class WasmLibrary {
-    protected _module: any;
-    protected _exports: any;
-    protected constructor(_module: any, _export: any);
-    protected malloc_heapu8(size: number): HeapU8;
-    protected free_heapu8(data: HeapU8): void;
-    protected uint8_heapu8(data: Uint8Array): HeapU8;
-    protected heapu8_view(data: HeapU8): Uint8Array;
-    protected heapu8_uint8(data: HeapU8): Uint8Array;
-    protected string_heapu8(str: string): HeapU8;
-    protected heapu8_string(data: HeapU8): string;
-}