@loaders.gl/compression 4.2.0-alpha.5 → 4.2.0-beta.1

package/src/lib/brotli-compression.ts

@@ -1,7 +1,13 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // BROTLI
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';
 import {isBrowser, toArrayBuffer} from '@loaders.gl/loader-utils';
+import {registerJSModules, getJSModule, getJSModuleOrNull} from '@loaders.gl/loader-utils';
+
 import type brotliNamespace from 'brotli';
 // import brotli from 'brotli';  // https://bundlephobia.com/package/brotli
 import {BrotliDecode} from '../brotli/decode';
@@ -25,7 +31,7 @@ const DEFAULT_BROTLI_OPTIONS = {
   }
 };
 
-let brotli: typeof brotliNamespace;
+type Brotli = typeof brotliNamespace;
 
 /**
  * brotli compression / decompression
@@ -40,18 +46,15 @@ export class BrotliCompression extends Compression {
   constructor(options: BrotliCompressionOptions) {
     super(options);
     this.options = options;
+    registerJSModules(options?.modules);
   }
 
   /**
    * brotli is an injectable dependency due to big size
    * @param options
    */
-  async preload(): Promise<void> {
-    brotli = brotli || this.options?.modules?.brotli;
-    if (!brotli) {
-      // eslint-disable-next-line no-console
-      console.warn(`${this.name} library not installed`);
-    }
+  async preload(modules: Record<string, any> = {}): Promise<void> {
+    registerJSModules(modules);
   }
 
   async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
@@ -72,10 +75,7 @@ export class BrotliCompression extends Compression {
     const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli};
     const inputArray = new Uint8Array(input);
 
-    if (!brotli) {
-      throw new Error('brotli compression: brotli module not installed');
-    }
-
+    const brotli = getJSModule<Brotli>('brotli', this.name);
     // @ts-ignore brotli types state that only Buffers are accepted...
     const outputArray = brotli.compress(inputArray, brotliOptions);
     return outputArray.buffer;
@@ -100,6 +100,7 @@ export class BrotliCompression extends Compression {
     const brotliOptions = {...DEFAULT_BROTLI_OPTIONS.brotli, ...this.options?.brotli};
     const inputArray = new Uint8Array(input);
 
+    const brotli = getJSModuleOrNull<Brotli>('brotli');
     if (brotli) {
       // @ts-ignore brotli types state that only Buffers are accepted...
       const outputArray = brotli.decompress(inputArray, brotliOptions);

package/src/lib/compression.ts

@@ -1,5 +1,10 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // Compression interface
 import {concatenateArrayBuffersAsync} from '@loaders.gl/loader-utils';
+import {registerJSModules} from '@loaders.gl/loader-utils';
 
 /** Compression options */
 export type CompressionOptions = {
@@ -20,7 +25,8 @@ export abstract class Compression {
   }
 
   /** Preloads any dynamic libraries. May enable sync functions */
-  async preload(): Promise<void> {
+  async preload(modules: Record<string, any> = {}): Promise<void> {
+    registerJSModules(modules);
     return;
   }
 

package/src/lib/deflate-compression.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // DEFLATE
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';

package/src/lib/gzip-compression.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // GZIP
 // import {isBrowser} from '@loaders.gl/loader-utils';
 import type {CompressionOptions} from './compression';

package/src/lib/lz4-compression.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // Copyright (c) 2012 Pierre Curto
 
 // Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -23,13 +27,13 @@
 
 // LZ4
 import {toArrayBuffer} from '@loaders.gl/loader-utils';
+import {registerJSModules, getJSModule} from '@loaders.gl/loader-utils';
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';
+
 // import lz4js from 'lz4js'; // https://bundlephobia.com/package/lz4
 const LZ4_MAGIC_NUMBER = 0x184d2204;
 
-let lz4js;
-
 /**
  * LZ4 compression / decompression
  */
@@ -44,13 +48,15 @@ export class LZ4Compression extends Compression {
     super(options);
     this.options = options;
 
-    lz4js = lz4js || this.options?.modules?.lz4js;
-    if (!lz4js) {
-      throw new Error(this.name);
-    }
+    registerJSModules(options?.modules);
+  }
+
+  async preload(modules: Record<string, any> = {}): Promise<void> {
+    registerJSModules(modules);
   }
 
   compressSync(input: ArrayBuffer): ArrayBuffer {
+    const lz4js = getJSModule('lz4js', this.name);
     const inputArray = new Uint8Array(input);
     return lz4js.compress(inputArray).buffer;
   }
@@ -62,6 +68,7 @@ export class LZ4Compression extends Compression {
    * If data provided without magic number we will parse it as block
    */
   decompressSync(data: ArrayBuffer, maxSize?: number): ArrayBuffer {
+    const lz4js = getJSModule('lz4js', this.name);
     try {
       const isMagicNumberExists = this.checkMagicNumber(data);
       const inputArray = new Uint8Array(data);

package/src/lib/lzo-compression.ts

@@ -1,6 +1,10 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // LZO
 // import {loadLibrary} from '@loaders.gl/worker-utils';
-import {toBuffer} from '@loaders.gl/loader-utils';
+import {registerJSModules, getJSModule, toBuffer} from '@loaders.gl/loader-utils';
 
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';
@@ -12,8 +16,6 @@ import {Compression} from './compression';
 // const LZO_WASM_JS_URL = './node_modules/lzo-wasm/lzo-wasm.js';
 // const LZO_WASM_WASM_URL = './node_modules/lzo-wasm/lzo-wasm.wasm';
 
-let lzo;
-
 /**
  * Lempel-Ziv-Oberheimer compression / decompression
  */
@@ -31,20 +33,18 @@ export class LZOCompression extends Compression {
   constructor(options: CompressionOptions) {
     super(options);
     this.options = options;
-
-    lzo = lzo || this.options?.modules?.lzo;
-    if (!lzo) {
-      throw new Error(this.name);
-    }
+    registerJSModules(options?.modules);
   }
 
-  async preload() {
+  async preload(modules: Record<string, any> = {}): Promise<void> {
+    registerJSModules(modules);
     // await loadLibrary(LZO_WASM_JS_URL);
     // await loadLibrary(LZO_WASM_WASM_URL);
   }
 
   async compress(input: ArrayBuffer): Promise<ArrayBuffer> {
     await this.preload();
+    const lzo = getJSModule('lzo', this.name);
     // const inputArray = new Uint8Array(input);
     const inputBuffer = toBuffer(input);
     return lzo.compress(inputBuffer).buffer;
@@ -53,6 +53,7 @@ export class LZOCompression extends Compression {
   async decompress(input: ArrayBuffer): Promise<ArrayBuffer> {
     try {
       await this.preload();
+      const lzo = getJSModule('lzo', this.name);
       // const inputArray = new Uint8Array(input);
       const inputBuffer = toBuffer(input);
       return lzo.decompress(inputBuffer).buffer;

package/src/lib/no-compression.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // NO COMPRESSION
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';

package/src/lib/snappy-compression.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // SNAPPY (aka ZIPPY)
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';

package/src/lib/zstd-compression.ts

@@ -1,9 +1,18 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 // ZSTD
 import type {CompressionOptions} from './compression';
 import {Compression} from './compression';
+import {registerJSModules} from '@loaders.gl/loader-utils';
+import {checkJSModule, getJSModule, getJSModuleOrNull} from '@loaders.gl/loader-utils';
+
 // import {ZstdCodec} from 'zstd-codec'; // https://bundlephobia.com/package/zstd-codec
 
-let ZstdCodec;
+const CHUNK_SIZE = 1000000; // Tested value
+
+let zstdPromise: Promise<any>;
 let zstd;
 
 /**
@@ -23,31 +32,47 @@ export class ZstdCompression extends Compression {
   constructor(options: CompressionOptions) {
     super(options);
     this.options = options;
-
-    ZstdCodec = this.options?.modules?.['zstd-codec'];
-    if (!ZstdCodec) {
-      // eslint-disable-next-line no-console
-      console.warn(`${this.name} library not installed`);
-    }
+    registerJSModules(options?.modules);
   }
 
-  async preload(): Promise<void> {
-    if (!zstd && ZstdCodec) {
-      zstd = await new Promise((resolve) => ZstdCodec.run((zstd) => resolve(zstd)));
+  async preload(modules: Record<string, any> = {}): Promise<void> {
+    registerJSModules(modules);
+    checkJSModule('zstd-codec', this.name);
+    const ZstdCodec = getJSModuleOrNull('zstd-codec');
+    // eslint-disable-next-line  @typescript-eslint/no-misused-promises
+    if (!zstdPromise && ZstdCodec) {
+      zstdPromise = new Promise((resolve) => ZstdCodec.run((zstd) => resolve(zstd)));
+      zstd = await zstdPromise;
     }
   }
 
   compressSync(input: ArrayBuffer): ArrayBuffer {
+    getJSModule('zstd-codec', this.name);
     const simpleZstd = new zstd.Simple();
     const inputArray = new Uint8Array(input);
     return simpleZstd.compress(inputArray).buffer;
   }
 
   decompressSync(input: ArrayBuffer): ArrayBuffer {
+    getJSModule('zstd-codec', this.name);
     const simpleZstd = new zstd.Simple();
     // var ddict = new zstd.Dict.Decompression(dictData);
     // var jsonBytes = simpleZstd.decompressUsingDict(jsonZstData, ddict);
     const inputArray = new Uint8Array(input);
     return simpleZstd.decompress(inputArray).buffer;
   }
+
+  async decompress(input: ArrayBuffer, size?: number): Promise<ArrayBuffer> {
+    await this.preload();
+    const simpleZstd = new zstd.Streaming();
+    const inputArray = new Uint8Array(input);
+
+    const chunks: ArrayBuffer[] = [];
+    for (let i = 0; i <= inputArray.length; i += CHUNK_SIZE) {
+      chunks.push(inputArray.subarray(i, i + CHUNK_SIZE));
+    }
+
+    const decompressResult = await simpleZstd.decompressChunks(chunks);
+    return decompressResult.buffer;
+  }
 }

package/src/types.ts

@@ -1 +1,5 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 export type {Compression, CompressionOptions} from './lib/compression';

package/src/workers/compression-worker-node.ts

@@ -1 +1,5 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 import './compression-worker';

package/src/workers/compression-worker.ts

@@ -1,3 +1,7 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
 import {createWorker} from '@loaders.gl/worker-utils';
 
 // Compressors