@loaders.gl/compression 4.2.0-alpha.6 → 4.2.0-beta.2

package/src/lib/lzo-compression.ts

@@ -1,6 +1,10 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // LZO
 // import {loadLibrary} from '@loaders.gl/worker-utils';
-import {toBuffer} from '@loaders.gl/loader-utils';
+import {registerJSModules, getJSModule, toBuffer} from '@loaders.gl/loader-utils';
 
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';
@@ -12,8 +16,6 @@ import {Compression} from './compression';
 // const LZO_WASM_JS_URL = './node_modules/lzo-wasm/lzo-wasm.js';
 // const LZO_WASM_WASM_URL = './node_modules/lzo-wasm/lzo-wasm.wasm';
 
-let lzo;
-
 /**
  * Lempel-Ziv-Oberheimer compression / decompression
  */
@@ -31,20 +33,18 @@ export class LZOCompression extends Compression {
   constructor(options: CompressionOptions) {
     super(options);
     this.options = options;
-
-    lzo = lzo || this.options?.modules?.lzo;
-    if (!lzo) {
-      throw new Error(this.name);
-    }
+    registerJSModules(options?.modules);
   }
 
-  async preload() {
+  async preload(modules: Record<string, any> = {}): Promise<void> {
+    registerJSModules(modules);
     // await loadLibrary(LZO_WASM_JS_URL);
     // await loadLibrary(LZO_WASM_WASM_URL);
   }
 
   async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
     await this.preload();
+    const lzo = getJSModule('lzo', this.name);
     // const inputArray = new Uint8Array(input);
     const inputBuffer = toBuffer(input);
     return lzo.compress(inputBuffer).buffer;
@@ -53,6 +53,7 @@ export class LZOCompression extends Compression {
   async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
     try {
       await this.preload();
+      const lzo = getJSModule('lzo', this.name);
       // const inputArray = new Uint8Array(input);
       const inputBuffer = toBuffer(input);
       return lzo.decompress(inputBuffer).buffer;

package/src/lib/no-compression.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // NO COMPRESSION
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';

package/src/lib/snappy-compression.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // SNAPPY (aka ZIPPY)
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';

package/src/lib/zstd-compression.ts

@@ -1,9 +1,18 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // ZSTD
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';
+import {registerJSModules} from '@loaders.gl/loader-utils';
+import {checkJSModule, getJSModule, getJSModuleOrNull} from '@loaders.gl/loader-utils';
+
 // import {ZstdCodec} from 'zstd-codec'; // https://bundlephobia.com/package/zstd-codec
 
-let ZstdCodec;
+const CHUNK_SIZE = 1000000; // Tested value
+
+let zstdPromise: Promise<any>;
 let zstd;
 
 /**
@@ -23,31 +32,47 @@ export class ZstdCompression extends Compression {
   constructor(options: CompressionOptions) {
     super(options);
     this.options = options;
-
-    ZstdCodec = this.options?.modules?.['zstd-codec'];
-    if (!ZstdCodec) {
-      // eslint-disable-next-line no-console
-      console.warn(`${this.name} library not installed`);
-    }
+    registerJSModules(options?.modules);
   }
 
-  async preload(): Promise<void> {
-    if (!zstd && ZstdCodec) {
-      zstd = await new Promise((resolve) => ZstdCodec.run((zstd) => resolve(zstd)));
+  async preload(modules: Record<string, any> = {}): Promise<void> {
+    registerJSModules(modules);
+    checkJSModule('zstd-codec', this.name);
+    const ZstdCodec = getJSModuleOrNull('zstd-codec');
+    // eslint-disable-next-line  @typescript-eslint/no-misused-promises
+    if (!zstdPromise && ZstdCodec) {
+      zstdPromise = new Promise((resolve) => ZstdCodec.run((zstd) => resolve(zstd)));
+      zstd = await zstdPromise;
     }
   }
 
   compressSync(input: ArrayBuffer): ArrayBuffer {
+    getJSModule('zstd-codec', this.name);
     const simpleZstd = new zstd.Simple();
     const inputArray = new Uint8Array(input);
     return simpleZstd.compress(inputArray).buffer;
   }
 
   decompressSync(input: ArrayBuffer): ArrayBuffer {
+    getJSModule('zstd-codec', this.name);
     const simpleZstd = new zstd.Simple();
     // var ddict = new zstd.Dict.Decompression(dictData);
     // var jsonBytes = simpleZstd.decompressUsingDict(jsonZstData, ddict);
     const inputArray = new Uint8Array(input);
     return simpleZstd.decompress(inputArray).buffer;
   }
+
+  async decompress(input: ArrayBuffer, size?: number): Promise<ArrayBuffer> {
+    await this.preload();
+    const simpleZstd = new zstd.Streaming();
+    const inputArray = new Uint8Array(input);
+
+    const chunks: ArrayBuffer[] = [];
+    for (let i = 0; i <= inputArray.length; i += CHUNK_SIZE) {
+      chunks.push(inputArray.subarray(i, i + CHUNK_SIZE));
+    }
+
+    const decompressResult = await simpleZstd.decompressChunks(chunks);
+    return decompressResult.buffer;
+  }
 }

package/src/types.ts

@@ -1 +1,5 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 export type {Compression, CompressionOptions} from './lib/compression';

package/src/workers/compression-worker-node.ts

@@ -1 +1,5 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 import './compression-worker';

package/src/workers/compression-worker.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 import {createWorker} from '@loaders.gl/worker-utils';
 
 // Compressors