@loaders.gl/compression 3.1.0-alpha.5 → 3.1.0-beta.4

package/dist/lib/lz4-compression.js

@@ -0,0 +1,141 @@
+"use strict";
+// Copyright (c) 2012 Pierre Curto
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LZ4Compression = void 0;
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+/* eslint-disable complexity */
+/* eslint-disable max-statements */
+// LZ4
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const compression_1 = require("./compression");
+// import lz4js from 'lz4js'; // https://bundlephobia.com/package/lz4
+const LZ4_MAGIC_NUMBER = 0x184d2204;
+let lz4js;
+/**
+ * LZ4 compression / decompression
+ */
+class LZ4Compression extends compression_1.Compression {
+    constructor(options) {
+        super(options);
+        this.name = 'lz4';
+        this.extensions = ['lz4'];
+        this.contentEncodings = ['x-lz4'];
+        this.isSupported = true;
+        this.options = options;
+        lz4js = lz4js || this.options?.modules?.lz4js;
+        if (!lz4js) {
+            throw new Error(this.name);
+        }
+    }
+    compressSync(input) {
+        const inputArray = new Uint8Array(input);
+        return lz4js.compress(inputArray).buffer;
+    }
+    /**
+     * Decompresses an ArrayBuffer containing an Lz4 frame. maxSize is optional; if not
+     * provided, a maximum size will be determined by examining the data. The
+     * returned ArrayBuffer will always be perfectly sized.
+     * If data provided without magic number we will parse it as block
+     */
+    decompressSync(data, maxSize) {
+        try {
+            const isMagicNumberExists = this.checkMagicNumber(data);
+            const inputArray = new Uint8Array(data);
+            if (isMagicNumberExists) {
+                return lz4js.decompress(inputArray, maxSize).buffer;
+            }
+            if (!maxSize) {
+                const error = new Error('Need to provide maxSize');
+                throw this.improveError(error);
+            }
+            let uncompressed = new Uint8Array(maxSize);
+            const uncompressedSize = this.decodeBlock(inputArray, uncompressed);
+            uncompressed = uncompressed.slice(0, uncompressedSize);
+            return (0, loader_utils_1.toArrayBuffer)(uncompressed);
+        }
+        catch (error) {
+            throw this.improveError(error);
+        }
+    }
+    /**
+     * Decode lz4 file as block
+     * Solution taken from here
+     * https://github.com/pierrec/node-lz4/blob/0dac687262403fd34f905b963da7220692f2a4a1/lib/binding.js#L25
+     * @param input
+     * @param output
+     * @param startIndex
+     * @param endIndex
+     */
+    decodeBlock(data, output, startIndex, endIndex) {
+        startIndex = startIndex || 0;
+        endIndex = endIndex || data.length - startIndex;
+        let uncompressedSize = 0;
+        // Process each sequence in the incoming data
+        for (let index = startIndex; index < endIndex;) {
+            const token = data[index++];
+            // Literals
+            let literalsLength = token >> 4;
+            if (literalsLength > 0) {
+                // length of literals
+                let length = literalsLength + 240;
+                while (length === 255) {
+                    length = data[index++];
+                    literalsLength += length;
+                }
+                // Copy the literals
+                const end = index + literalsLength;
+                while (index < end) {
+                    output[uncompressedSize++] = data[index++];
+                }
+                // End of buffer?
+                if (index === endIndex) {
+                    return uncompressedSize;
+                }
+            }
+            // Match copy
+            // 2 bytes offset (little endian)
+            const offset = data[index++] | (data[index++] << 8);
+            // 0 is an invalid offset value
+            if (offset === 0 || offset > uncompressedSize) {
+                return -(index - 2);
+            }
+            // length of match copy
+            let matchLength = token & 0xf;
+            let length = matchLength + 240;
+            while (length === 255) {
+                length = data[index++];
+                matchLength += length;
+            }
+            // Copy the match
+            let pos = uncompressedSize - offset; // position of the match copy in the current output
+            const end = uncompressedSize + matchLength + 4; // minmatch = 4
+            while (uncompressedSize < end) {
+                output[uncompressedSize++] = output[pos++];
+            }
+        }
+        return uncompressedSize;
+    }
+    /**
+     * Compare file magic with lz4 magic number
+     * @param input
+     */
+    checkMagicNumber(data) {
+        const magic = new Uint32Array(data.slice(0, 4));
+        return magic[0] === LZ4_MAGIC_NUMBER;
+    }
+}
+exports.LZ4Compression = LZ4Compression;

package/dist/lib/lzo-compression.d.ts

@@ -18,3 +18,4 @@ export declare class LZOCompression extends Compression {
     compress(input: ArrayBuffer): Promise<ArrayBuffer>;
     decompress(input: ArrayBuffer): Promise<ArrayBuffer>;
 }
+//# sourceMappingURL=lzo-compression.d.ts.map

package/dist/lib/lzo-compression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lzo-compression.d.ts","sourceRoot":"","sources":["../../src/lib/lzo-compression.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,eAAe,CAAC;AACtD,OAAO,EAAC,WAAW,EAAC,MAAM,eAAe,CAAC;AAW1C;;GAEG;AACH,qBAAa,cAAe,SAAQ,WAAW;IAC7C,QAAQ,CAAC,IAAI,SAAS;IACtB,QAAQ,CAAC,UAAU,UAAM;IACzB,QAAQ,CAAC,gBAAgB,UAAM;IAC/B,QAAQ,CAAC,WAAW,SAAS;IAC7B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IAErC;;;OAGG;gBACS,OAAO,EAAE,kBAAkB;IAUjC,OAAO;IAKP,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;IAOlD,UAAU,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;CAY3D"}

package/dist/lib/lzo-compression.js

@@ -0,0 +1,58 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LZOCompression = void 0;
+// LZO
+// import {loadLibrary} from '@loaders.gl/worker-utils';
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const compression_1 = require("./compression");
+// import {isBrowser} from '@loaders.gl/loader-utils';
+// import lzo from 'lzo'; // https://bundlephobia.com/package/lzo
+// import {decompress} from 'lzo-wasm';
+// const LZO_WASM_JS_URL = './node_modules/lzo-wasm/lzo-wasm.js';
+// const LZO_WASM_WASM_URL = './node_modules/lzo-wasm/lzo-wasm.wasm';
+let lzo;
+/**
+ * Lempel-Ziv-Oberheimer compression / decompression
+ */
+class LZOCompression extends compression_1.Compression {
+    /**
+     * lzo is an injectable dependency due to big size
+     * @param options
+     */
+    constructor(options) {
+        super(options);
+        this.name = 'lzo';
+        this.extensions = [];
+        this.contentEncodings = [];
+        this.isSupported = false; // !isBrowser;
+        this.options = options;
+        lzo = lzo || this.options?.modules?.lzo;
+        if (!lzo) {
+            throw new Error(this.name);
+        }
+    }
+    async preload() {
+        // await loadLibrary(LZO_WASM_JS_URL);
+        // await loadLibrary(LZO_WASM_WASM_URL);
+    }
+    async compress(input) {
+        await this.preload();
+        // const inputArray = new Uint8Array(input);
+        const inputBuffer = (0, loader_utils_1.toBuffer)(input);
+        return lzo.compress(inputBuffer).buffer;
+    }
+    async decompress(input) {
+        try {
+            await this.preload();
+            // const inputArray = new Uint8Array(input);
+            const inputBuffer = (0, loader_utils_1.toBuffer)(input);
+            return lzo.decompress(inputBuffer).buffer;
+        }
+        catch (error) {
+            // TODO - solve SharedArrayBuffer issues
+            // return decompress(input);
+            throw error;
+        }
+    }
+}
+exports.LZOCompression = LZOCompression;

package/dist/lib/no-compression.d.ts

@@ -15,3 +15,4 @@ export declare class NoCompression extends Compression {
     compressBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<ArrayBuffer>;
     decompressBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<ArrayBuffer>;
 }
+//# sourceMappingURL=no-compression.d.ts.map

package/dist/lib/no-compression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"no-compression.d.ts","sourceRoot":"","sources":["../../src/lib/no-compression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,eAAe,CAAC;AACtD,OAAO,EAAC,WAAW,EAAC,MAAM,eAAe,CAAC;AAE1C;;GAEG;AACH,qBAAa,aAAc,SAAQ,WAAW;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAkB;IACvC,QAAQ,CAAC,UAAU,EAAE,MAAM,EAAE,CAAM;IACnC,QAAQ,CAAC,gBAAgB,EAAE,MAAM,EAAE,CAAM;IACzC,QAAQ,CAAC,WAAW,QAAQ;IAE5B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;gBAEzB,OAAO,CAAC,EAAE,kBAAkB;IAKxC,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW;IAI7C,cAAc,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW;IAIxC,eAAe,CACpB,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,WAAW,CAAC;IAItB,iBAAiB,CACtB,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,WAAW,CAAC;CAG9B"}

package/dist/lib/no-compression.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NoCompression = void 0;
+const compression_1 = require("./compression");
+/**
+ * Applies no compression.
+ */
+class NoCompression extends compression_1.Compression {
+    constructor(options) {
+        super(options);
+        this.name = 'uncompressed';
+        this.extensions = [];
+        this.contentEncodings = [];
+        this.isSupported = true;
+        this.options = options || {};
+    }
+    compressSync(input) {
+        return input;
+    }
+    decompressSync(input) {
+        return input;
+    }
+    async *compressBatches(asyncIterator) {
+        return yield* asyncIterator;
+    }
+    async *decompressBatches(asyncIterator) {
+        return yield* asyncIterator;
+    }
+}
+exports.NoCompression = NoCompression;

package/dist/lib/snappy-compression.d.ts

@@ -13,3 +13,4 @@ export declare class SnappyCompression extends Compression {
     compressSync(input: ArrayBuffer): ArrayBuffer;
     decompressSync(input: ArrayBuffer): ArrayBuffer;
 }
+//# sourceMappingURL=snappy-compression.d.ts.map

package/dist/lib/snappy-compression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"snappy-compression.d.ts","sourceRoot":"","sources":["../../src/lib/snappy-compression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,eAAe,CAAC;AACtD,OAAO,EAAC,WAAW,EAAC,MAAM,eAAe,CAAC;AAG1C;;GAEG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAY;IACjC,QAAQ,CAAC,UAAU,UAAM;IACzB,QAAQ,CAAC,gBAAgB,UAAM;IAC/B,QAAQ,CAAC,WAAW,QAAQ;IAC5B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;gBAEzB,OAAO,CAAC,EAAE,kBAAkB;IAKxC,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW;IAK7C,cAAc,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW;CAIhD"}

package/dist/lib/snappy-compression.js

@@ -0,0 +1,27 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SnappyCompression = void 0;
+const compression_1 = require("./compression");
+const snappyjs_1 = require("snappyjs"); // https://bundlephobia.com/package/snappy
+/**
+ * Snappy/zippy compression / decompression
+ */
+class SnappyCompression extends compression_1.Compression {
+    constructor(options) {
+        super(options);
+        this.name = 'snappy';
+        this.extensions = [];
+        this.contentEncodings = [];
+        this.isSupported = true;
+        this.options = options || {};
+    }
+    compressSync(input) {
+        // Accepts arrayBuffer - https://github.com/zhipeng-jia/snappyjs#usage
+        return (0, snappyjs_1.compress)(input);
+    }
+    decompressSync(input) {
+        // Accepts arrayBuffer - https://github.com/zhipeng-jia/snappyjs#usage
+        return (0, snappyjs_1.uncompress)(input);
+    }
+}
+exports.SnappyCompression = SnappyCompression;

package/dist/lib/zstd-compression.d.ts

@@ -18,3 +18,4 @@ export declare class ZstdCompression extends Compression {
     compressSync(input: ArrayBuffer): ArrayBuffer;
     decompressSync(input: ArrayBuffer): ArrayBuffer;
 }
+//# sourceMappingURL=zstd-compression.d.ts.map

package/dist/lib/zstd-compression.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"zstd-compression.d.ts","sourceRoot":"","sources":["../../src/lib/zstd-compression.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,eAAe,CAAC;AACtD,OAAO,EAAC,WAAW,EAAC,MAAM,eAAe,CAAC;AAM1C;;GAEG;AACH,qBAAa,eAAgB,SAAQ,WAAW;IAC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAU;IAC/B,QAAQ,CAAC,UAAU,UAAM;IACzB,QAAQ,CAAC,gBAAgB,UAAM;IAC/B,QAAQ,CAAC,WAAW,QAAQ;IAC5B,QAAQ,CAAC,OAAO,EAAE,kBAAkB,CAAC;IAErC;;;OAGG;gBACS,OAAO,EAAE,kBAAkB;IAUjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAI9B,YAAY,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW;IAM7C,cAAc,CAAC,KAAK,EAAE,WAAW,GAAG,WAAW;CAOhD"}

package/dist/lib/zstd-compression.js

@@ -0,0 +1,44 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ZstdCompression = void 0;
+const compression_1 = require("./compression");
+// import {ZstdCodec} from 'zstd-codec'; // https://bundlephobia.com/package/zstd-codec
+let ZstdCodec;
+let zstd;
+/**
+ * Zstandard compression / decompression
+ */
+class ZstdCompression extends compression_1.Compression {
+    /**
+     * zstd-codec is an injectable dependency due to big size
+     * @param options
+     */
+    constructor(options) {
+        super(options);
+        this.name = 'zstd';
+        this.extensions = [];
+        this.contentEncodings = [];
+        this.isSupported = true;
+        this.options = options;
+        ZstdCodec = this.options?.modules?.['zstd-codec'];
+        if (!ZstdCodec) {
+            console.warn(`${this.name} library not installed`);
+        }
+    }
+    async preload() {
+        zstd = zstd || (await new Promise((resolve) => ZstdCodec.run((zstd) => resolve(zstd))));
+    }
+    compressSync(input) {
+        const simpleZstd = new zstd.Simple();
+        const inputArray = new Uint8Array(input);
+        return simpleZstd.compress(inputArray).buffer;
+    }
+    decompressSync(input) {
+        const simpleZstd = new zstd.Simple();
+        // var ddict = new zstd.Dict.Decompression(dictData);
+        // var jsonBytes = simpleZstd.decompressUsingDict(jsonZstData, ddict);
+        const inputArray = new Uint8Array(input);
+        return simpleZstd.decompress(inputArray).buffer;
+    }
+}
+exports.ZstdCompression = ZstdCompression;

package/dist/types.d.ts

@@ -1 +1,2 @@
 export type { Compression, CompressionOptions } from './lib/compression';
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,YAAY,EAAC,WAAW,EAAE,kBAAkB,EAAC,MAAM,mBAAmB,CAAC"}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/workers/worker.d.ts

@@ -1 +1,2 @@
 export {};
+//# sourceMappingURL=worker.d.ts.map

package/dist/workers/worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/workers/worker.ts"],"names":[],"mappings":""}

package/dist/workers/worker.js

@@ -0,0 +1,77 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const worker_utils_1 = require("@loaders.gl/worker-utils");
+// Compressors
+const no_compression_1 = require("../lib/no-compression");
+const brotli_compression_1 = require("../lib/brotli-compression");
+const deflate_compression_1 = require("../lib/deflate-compression");
+const gzip_compression_1 = require("../lib/gzip-compression");
+const lz4_compression_1 = require("../lib/lz4-compression");
+// import {LZOCompression} from '../lib/lzo-compression';
+const snappy_compression_1 = require("../lib/snappy-compression");
+const zstd_compression_1 = require("../lib/zstd-compression");
+// Import big dependencies
+// import brotli from 'brotli'; - brotli has problems with decompress in browsers
+// import brotliDecompress from 'brotli/decompress';
+const lz4js_1 = __importDefault(require("lz4js"));
+// import lzo from 'lzo';
+// import {ZstdCodec} from 'zstd-codec';
+// Inject large dependencies through Compression constructor options
+const modules = {
+    // brotli has problems with decompress in browsers
+    // brotli: {
+    //   decompress: brotliDecompress,
+    //   compress: () => {
+    //     throw new Error('brotli compress');
+    //   }
+    // },
+    lz4js: lz4js_1.default
+    // lzo,
+    // 'zstd-codec': ZstdCodec
+};
+/** @type {Compression[]} */
+const COMPRESSIONS = [
+    new no_compression_1.NoCompression({ modules }),
+    new brotli_compression_1.BrotliCompression({ modules }),
+    new deflate_compression_1.DeflateCompression({ modules }),
+    new gzip_compression_1.GZipCompression({ modules }),
+    // new LZOCompression({modules}),
+    new lz4_compression_1.LZ4Compression({ modules }),
+    new snappy_compression_1.SnappyCompression({ modules }),
+    new zstd_compression_1.ZstdCompression({ modules })
+];
+(0, worker_utils_1.createWorker)(async (data, options = {}) => {
+    const operation = getOperation(String(options?.operation));
+    const compression = getCompression(String(options?.compression));
+    // @ts-ignore
+    switch (operation) {
+        case 'compress':
+            return await compression.compress(data);
+        case 'decompress':
+            return await compression.decompress(data);
+        default:
+            throw new Error('invalid option');
+    }
+});
+function getOperation(operation) {
+    switch (operation) {
+        case 'compress':
+        case 'deflate':
+            return 'compress';
+        case 'decompress':
+        case 'inflate':
+            return 'decompress';
+        default:
+            throw new Error(`@loaders.gl/compression: Unsupported operation ${operation}. Expected 'compress' or 'decompress'`);
+    }
+}
+function getCompression(name) {
+    const Compression = COMPRESSIONS.find((compression_) => name === compression_.name);
+    if (!Compression) {
+        throw new Error(`@loaders.gl/compression: Unsupported compression ${name}`);
+    }
+    return Compression;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/compression",
-  "version": "3.1.0-alpha.5",
+  "version": "3.1.0-beta.4",
   "description": "Decompression and compression plugins for loaders.gl",
   "license": "MIT",
   "publishConfig": {
@@ -33,14 +33,13 @@
   },
   "scripts": {
     "pre-build": "npm run build-bundle && npm run build-worker",
-    "post-build": "tsc",
-    "build-bundle": "webpack --display=minimal --config ../../scripts/webpack/bundle.js",
-    "build-worker": "webpack --entry ./src/workers/worker.ts --output ./dist/compression-worker.js --config ../../scripts/webpack/worker.js"
+    "build-bundle": "esbuild src/bundle.ts --outfile=dist/dist.min.js --bundle --minify --sourcemap --external:{fs,path,crypto}",
+    "build-worker": "esbuild src/workers/worker.ts --outfile=dist/compression-worker.js --bundle --minify --sourcemap --external:{fs,path,crypto}"
   },
   "dependencies": {
     "@babel/runtime": "^7.3.1",
-    "@loaders.gl/loader-utils": "3.1.0-alpha.5",
-    "@loaders.gl/worker-utils": "3.1.0-alpha.5",
+    "@loaders.gl/loader-utils": "3.1.0-beta.4",
+    "@loaders.gl/worker-utils": "3.1.0-beta.4",
     "@types/brotli": "^1.3.0",
     "@types/pako": "^1.0.1",
     "lzo-wasm": "^0.0.4",
@@ -59,5 +58,5 @@
     "lzo": "^0.4.11",
     "zstd-codec": "^0.1"
   },
-  "gitHead": "352241dd910a8c6307a235dadbe154ca915b885b"
+  "gitHead": "5c7c74215416b30fcea98f5cecc820c8a21023b3"
 }