@loaders.gl/tile-converter 4.2.0-alpha.4 → 4.2.0-alpha.6

package/dist/lib/utils/write-queue.js

@@ -1,96 +1,82 @@
 import { Queue } from "./queue.js";
 import process from 'process';
-const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024;
+/** Memory limit size is based on testing */
+const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
 export default class WriteQueue extends Queue {
-  constructor(conversionDump) {
-    let listeningInterval = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 2000;
-    let writeConcurrency = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 400;
-    super();
-    this.intervalId = void 0;
-    this.conversionDump = void 0;
-    this.writePromise = null;
-    this.fileMap = {};
-    this.listeningInterval = void 0;
-    this.writeConcurrency = void 0;
-    this.conversionDump = conversionDump;
-    this.listeningInterval = listeningInterval;
-    this.writeConcurrency = writeConcurrency;
-  }
-  async enqueue(val) {
-    let writeImmediately = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : false;
-    if (writeImmediately) {
-      const {
-        archiveKey,
-        writePromise
-      } = val;
-      const result = await writePromise();
-      if (archiveKey && result) {
-        this.fileMap[archiveKey] = result;
-      }
-    } else {
-      super.enqueue(val);
-      if (process.memoryUsage().rss > MEMORY_LIMIT) {
-        await this.startWrite();
-      }
+    intervalId; // eslint-disable-line no-undef
+    conversionDump;
+    writePromise = null;
+    fileMap = {};
+    listeningInterval;
+    writeConcurrency;
+    constructor(conversionDump, listeningInterval = 2000, writeConcurrency = 400) {
+        super();
+        this.conversionDump = conversionDump;
+        this.listeningInterval = listeningInterval;
+        this.writeConcurrency = writeConcurrency;
+    }
+    async enqueue(val, writeImmediately = false) {
+        if (writeImmediately) {
+            const { archiveKey, writePromise } = val;
+            const result = await writePromise();
+            if (archiveKey && result) {
+                this.fileMap[archiveKey] = result;
+            }
+        }
+        else {
+            super.enqueue(val);
+            /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */
+            if (process.memoryUsage().rss > MEMORY_LIMIT) {
+                await this.startWrite();
+            }
+        }
+    }
+    startListening() {
+        this.intervalId = setInterval(() => this.startWrite.bind(this), this.listeningInterval);
     }
-  }
-  startListening() {
-    this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
-  }
-  stopListening() {
-    if (this.intervalId) {
-      clearInterval(this.intervalId);
+    stopListening() {
+        if (this.intervalId) {
+            clearInterval(this.intervalId);
+        }
     }
-  }
-  async startWrite() {
-    if (!this.writePromise) {
-      this.writePromise = this.doWrite();
+    async startWrite() {
+        if (!this.writePromise) {
+            this.writePromise = this.doWrite();
+        }
+        await this.writePromise;
+        this.writePromise = null;
     }
-    await this.writePromise;
-    this.writePromise = null;
-  }
-  async finalize() {
-    this.stopListening();
-    await this.startWrite();
-  }
-  async doWrite() {
-    while (this.length) {
-      const promises = [];
-      const archiveKeys = [];
-      const changedRecords = [];
-      for (let i = 0; i < this.writeConcurrency; i++) {
-        const item = this.dequeue();
-        if (!item) {
-          break;
+    async finalize() {
+        this.stopListening();
+        await this.startWrite();
+    }
+    async doWrite() {
+        while (this.length) {
+            const promises = [];
+            const archiveKeys = [];
+            const changedRecords = [];
+            for (let i = 0; i < this.writeConcurrency; i++) {
+                const item = this.dequeue();
+                if (!item) {
+                    break;
+                }
+                const { archiveKey, sourceId, outputId, resourceType, writePromise } = item;
+                archiveKeys.push(archiveKey);
+                changedRecords.push({ sourceId, outputId, resourceType });
+                const promise = writePromise();
+                promises.push(promise);
+            }
+            const writeResults = await Promise.allSettled(promises);
+            this.updateFileMap(archiveKeys, writeResults);
+            await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults);
         }
-        const {
-          archiveKey,
-          sourceId,
-          outputId,
-          resourceType,
-          writePromise
-        } = item;
-        archiveKeys.push(archiveKey);
-        changedRecords.push({
-          sourceId,
-          outputId,
-          resourceType
-        });
-        const promise = writePromise();
-        promises.push(promise);
-      }
-      const writeResults = await Promise.allSettled(promises);
-      this.updateFileMap(archiveKeys, writeResults);
-      await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults);
     }
-  }
-  updateFileMap(archiveKeys, writeResults) {
-    for (let i = 0; i < archiveKeys.length; i++) {
-      const archiveKey = archiveKeys[i];
-      if (archiveKey && 'value' in writeResults[i]) {
-        this.fileMap[archiveKey] = writeResults[i].value;
-      }
+    updateFileMap(archiveKeys, writeResults) {
+        for (let i = 0; i < archiveKeys.length; i++) {
+            const archiveKey = archiveKeys[i];
+            if (archiveKey && 'value' in writeResults[i]) {
+                this.fileMap[archiveKey] = writeResults[i].value;
+            }
+        }
     }
-  }
 }
-//# sourceMappingURL=write-queue.js.map

package/dist/pgm-loader.js

@@ -1,18 +1,22 @@
 import { Geoid, parsePGM } from '@math.gl/geoid';
-const VERSION = typeof "4.2.0-alpha.4" !== 'undefined' ? "4.2.0-alpha.4" : 'latest';
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof "4.2.0-alpha.5" !== 'undefined' ? "4.2.0-alpha.5" : 'latest';
 export { Geoid };
+/**
+ * Loader for PGM - Netpbm grayscale image format
+ */
 export const PGMLoader = {
-  name: 'PGM - Netpbm grayscale image format',
-  id: 'pgm',
-  module: 'tile-converter',
-  version: VERSION,
-  mimeTypes: ['image/x-portable-graymap'],
-  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), (options === null || options === void 0 ? void 0 : options.pgm) || {}),
-  extensions: ['pgm'],
-  options: {
-    pgm: {
-      cubic: false
+    name: 'PGM - Netpbm grayscale image format',
+    id: 'pgm',
+    module: 'tile-converter',
+    version: VERSION,
+    mimeTypes: ['image/x-portable-graymap'],
+    parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options?.pgm || {}),
+    extensions: ['pgm'],
+    options: {
+        pgm: {
+            cubic: false
+        }
     }
-  }
 };
-//# sourceMappingURL=pgm-loader.js.map

package/dist/slpk-extractor/slpk-extractor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"slpk-extractor.d.ts","sourceRoot":"","sources":["../../src/slpk-extractor/slpk-extractor.ts"],"names":[],"mappings":"AAuBA;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,aAAa;IAChC;;;;;OAKG;IACU,OAAO,CAAC,OAAO,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IA8BtF;;;OAGG;IAEH,OAAO,CAAC,iBAAiB;YAYX,MAAM;YAWN,SAAS;CASxB"}
+{"version":3,"file":"slpk-extractor.d.ts","sourceRoot":"","sources":["../../src/slpk-extractor/slpk-extractor.ts"],"names":[],"mappings":"AAuBA;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,aAAa;IAChC;;;;;OAKG;IACU,OAAO,CAAC,OAAO,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IA+BtF;;;OAGG;IAEH,OAAO,CAAC,iBAAiB;YAYX,MAAM;YAWN,SAAS;CASxB"}

package/dist/slpk-extractor/slpk-extractor.js

@@ -5,59 +5,69 @@ import { FileHandleFile } from '@loaders.gl/loader-utils';
 import { parseZipLocalFileHeader } from '@loaders.gl/zip';
 import { GZipCompression } from '@loaders.gl/compression';
 import { writeFile } from "../lib/utils/file-utils.js";
-const indexNames = ['3dSceneLayer.json.gz', '3dNodeIndexDocument.json.gz', 'sharedResource.json.gz'];
+const indexNames = [
+    '3dSceneLayer.json.gz',
+    '3dNodeIndexDocument.json.gz',
+    'sharedResource.json.gz'
+];
+/**
+ * Converter from slpk to i3s
+ */
 export default class SLPKExtractor {
-  async extract(options) {
-    if (isBrowser) {
-      console.log(BROWSER_ERROR_MESSAGE);
-      return BROWSER_ERROR_MESSAGE;
+    /**
+     * Extract slpk to i3s
+     * @param options
+     * @param options.inputUrl the url to read SLPK file
+     * @param options.outputPath the output filename
+     */
+    async extract(options) {
+        if (isBrowser) {
+            // eslint-disable-next-line no-console
+            console.log(BROWSER_ERROR_MESSAGE);
+            return BROWSER_ERROR_MESSAGE;
+        }
+        const { inputUrl } = options;
+        const provider = new FileHandleFile(inputUrl);
+        let localHeader = await parseZipLocalFileHeader(0n, provider);
+        while (localHeader) {
+            await this.writeFile(await this.unGzip({
+                name: this.correctIndexNames(localHeader.fileName),
+                data: await provider.slice(localHeader.fileDataOffset, localHeader.fileDataOffset + localHeader.compressedSize)
+            }), options.outputPath);
+            localHeader = await parseZipLocalFileHeader(localHeader.fileDataOffset + localHeader?.compressedSize, provider);
+        }
+        return 'success';
     }
-    const {
-      inputUrl
-    } = options;
-    const provider = new FileHandleFile(inputUrl);
-    let localHeader = await parseZipLocalFileHeader(0n, provider);
-    while (localHeader) {
-      var _localHeader;
-      await this.writeFile(await this.unGzip({
-        name: this.correctIndexNames(localHeader.fileName),
-        data: await provider.slice(localHeader.fileDataOffset, localHeader.fileDataOffset + localHeader.compressedSize)
-      }), options.outputPath);
-      localHeader = await parseZipLocalFileHeader(localHeader.fileDataOffset + ((_localHeader = localHeader) === null || _localHeader === void 0 ? void 0 : _localHeader.compressedSize), provider);
+    /**
+     * Defines file name and path for i3s format
+     * @param fileName initial file name and path
+     */
+    correctIndexNames(fileName) {
+        if (indexNames.includes(path.filename(path.join('/', fileName)))) {
+            return path.join(path.dirname(fileName), 'index.json.gz');
+        }
+        // finds path with name part and extention part
+        const parts = /^(.*\/[^\/\.]*)(\..+)$/.exec(fileName);
+        if (!parts) {
+            return null;
+        }
+        return `${parts?.at(1)}/index${parts?.at(2)}`;
     }
-    return 'success';
-  }
-  correctIndexNames(fileName) {
-    if (indexNames.includes(path.filename(path.join('/', fileName)))) {
-      return path.join(path.dirname(fileName), 'index.json.gz');
+    async unGzip(file) {
+        if (/\.gz$/.test(file.name ?? '')) {
+            const compression = new GZipCompression();
+            const decompressedData = await compression.decompress(file.data);
+            return { data: decompressedData, name: (file.name ?? '').slice(0, -3) };
+        }
+        return Promise.resolve(file);
     }
-    let parts = /^(.*\/[^\/\.]*)(\..+)$/.exec(fileName);
-    if (!parts) {
-      return null;
+    async writeFile(options, outputPath) {
+        if (!options.name) {
+            return;
+        }
+        const finalPath = path.join(outputPath, options.name);
+        const dirName = path.dirname(finalPath);
+        const fileName = path.filename(finalPath);
+        await writeFile(dirName, options.data, fileName);
     }
-    return `${parts === null || parts === void 0 ? void 0 : parts.at(1)}/index${parts === null || parts === void 0 ? void 0 : parts.at(2)}`;
-  }
-  async unGzip(file) {
-    var _file$name;
-    if (/\.gz$/.test((_file$name = file.name) !== null && _file$name !== void 0 ? _file$name : '')) {
-      var _file$name2;
-      const compression = new GZipCompression();
-      const decompressedData = await compression.decompress(file.data);
-      return {
-        data: decompressedData,
-        name: ((_file$name2 = file.name) !== null && _file$name2 !== void 0 ? _file$name2 : '').slice(0, -3)
-      };
-    }
-    return Promise.resolve(file);
-  }
-  async writeFile(options, outputPath) {
-    if (!options.name) {
-      return;
-    }
-    const finalPath = path.join(outputPath, options.name);
-    const dirName = path.dirname(finalPath);
-    const fileName = path.filename(finalPath);
-    await writeFile(dirName, options.data, fileName);
-  }
 }
-//# sourceMappingURL=slpk-extractor.js.map

package/dist/slpk-extractor-cli.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"slpk-extractor-cli.d.ts","sourceRoot":"","sources":["../src/slpk-extractor-cli.ts"],"names":[],"mappings":"AACA,OAAO,uBAAuB,CAAC;AAI/B,KAAK,qBAAqB,GAAG;IAC3B,wFAAwF;IACxF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;gCAC4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAGF,MAAM,MAAM,8BAA8B,GAAG,qBAAqB,GAAG;IACnE,gBAAgB;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB;gCAC4B;IAC5B,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC"}
+{"version":3,"file":"slpk-extractor-cli.d.ts","sourceRoot":"","sources":["../src/slpk-extractor-cli.ts"],"names":[],"mappings":"AAAA,OAAO,uBAAuB,CAAC;AAI/B,KAAK,qBAAqB,GAAG;IAC3B,wFAAwF;IACxF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;gCAC4B;IAC5B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAGF,MAAM,MAAM,8BAA8B,GAAG,qBAAqB,GAAG;IACnE,gBAAgB;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB;gCAC4B;IAC5B,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC"}

package/dist/slpk-extractor-cli.js

@@ -1,74 +1,105 @@
 import '@loaders.gl/polyfills';
 import SLPKExtractor from "./slpk-extractor/slpk-extractor.js";
 import { getURLValue, validateOptionsWithEqual } from "./lib/utils/cli-utils.js";
+/**
+ * CLI entry
+ * @returns
+ */
 async function main() {
-  const [,, ...args] = process.argv;
-  if (args.length === 0) {
-    printHelp();
-  }
-  const validatedOptionsArr = validateOptionsWithEqual(args);
-  const options = parseOptions(validatedOptionsArr);
-  const validatedOptions = validateOptions(options);
-  await convert(validatedOptions);
+    const [, , ...args] = process.argv;
+    if (args.length === 0) {
+        printHelp();
+    }
+    const validatedOptionsArr = validateOptionsWithEqual(args);
+    const options = parseOptions(validatedOptionsArr);
+    const validatedOptions = validateOptions(options);
+    await convert(validatedOptions);
 }
-main().catch(error => {
-  console.log(error);
-  process.exit(1);
+main().catch((error) => {
+    console.log(error); // eslint-disable-line no-console
+    process.exit(1); // eslint-disable-line no-process-exit
 });
+/**
+ * Output for `npx slpk-extractor --help`
+ */
 function printHelp() {
-  console.log('cli: converter slpk to I3S...');
-  console.log('--output [Output folder, default: "data" folder]');
-  console.log('--tileset [SLPK file]');
-  process.exit(0);
+    // eslint-disable-next-line no-console
+    console.log('cli: converter slpk to I3S...');
+    // eslint-disable-next-line no-console
+    console.log('--output [Output folder, default: "data" folder]');
+    // eslint-disable-next-line no-console
+    console.log('--tileset [SLPK file]');
+    process.exit(0); // eslint-disable-line no-process-exit
 }
+/**
+ * Run extraction process
+ * @param options validated slpk-extractor options
+ */
 async function convert(options) {
-  console.log(`------------------------------------------------`);
-  console.log(`Starting conversion of SLPK`);
-  console.log(`------------------------------------------------`);
-  const slpkExtractor = new SLPKExtractor();
-  slpkExtractor.extract({
-    inputUrl: options.tileset,
-    outputPath: options.output
-  });
+    // eslint-disable-next-line no-console
+    console.log('------------------------------------------------');
+    // eslint-disable-next-line no-console
+    console.log('Starting conversion of SLPK');
+    // eslint-disable-next-line no-console
+    console.log('------------------------------------------------');
+    const slpkExtractor = new SLPKExtractor();
+    slpkExtractor.extract({
+        inputUrl: options.tileset,
+        outputPath: options.output
+    });
 }
+// OPTIONS
+/**
+ * Validate input options of the CLI command
+ * @param options - input options of the CLI command
+ * @returns validated options
+ */
 function validateOptions(options) {
-  const mandatoryOptionsWithExceptions = {
-    output: () => console.log('Missed: --output [Output path name]'),
-    tileset: () => console.log('Missed: --tileset [SLPK file]')
-  };
-  const exceptions = [];
-  for (const mandatoryOption in mandatoryOptionsWithExceptions) {
-    const optionValue = options[mandatoryOption];
-    if (!optionValue) {
-      exceptions.push(mandatoryOptionsWithExceptions[mandatoryOption]);
+    const mandatoryOptionsWithExceptions = {
+        // eslint-disable-next-line no-console
+        output: () => console.log('Missed: --output [Output path name]'),
+        // eslint-disable-next-line no-console
+        tileset: () => console.log('Missed: --tileset [SLPK file]')
+    };
+    const exceptions = [];
+    for (const mandatoryOption in mandatoryOptionsWithExceptions) {
+        const optionValue = options[mandatoryOption];
+        if (!optionValue) {
+            exceptions.push(mandatoryOptionsWithExceptions[mandatoryOption]);
+        }
+    }
+    if (exceptions.length) {
+        exceptions.forEach((exeption) => exeption());
+        process.exit(1); // eslint-disable-line no-process-exit
     }
-  }
-  if (exceptions.length) {
-    exceptions.forEach(exeption => exeption());
-    process.exit(1);
-  }
-  return options;
+    return options;
 }
+/**
+ * Parse option from the cli arguments array
+ * @param args
+ * @returns
+ */
 function parseOptions(args) {
-  const opts = {};
-  args.forEach((arg, index) => {
-    if (arg.indexOf('--') === 0) {
-      switch (arg) {
-        case '--tileset':
-          opts.tileset = getURLValue(index, args);
-          break;
-        case '--output':
-          opts.output = getURLValue(index, args);
-          break;
-        case '--help':
-          printHelp();
-          break;
-        default:
-          console.warn(`Unknown option ${arg}`);
-          process.exit(0);
-      }
-    }
-  });
-  return opts;
+    const opts = {};
+    // eslint-disable-next-line complexity
+    args.forEach((arg, index) => {
+        if (arg.indexOf('--') === 0) {
+            switch (arg) {
+                case '--tileset':
+                    opts.tileset = getURLValue(index, args);
+                    break;
+                case '--output':
+                    opts.output = getURLValue(index, args);
+                    break;
+                case '--help':
+                    printHelp();
+                    break;
+                default:
+                    // eslint-disable-next-line no-console
+                    console.warn(`Unknown option ${arg}`);
+                    process.exit(0); // eslint-disable-line
+            }
+        }
+    });
+    return opts;
 }
-//# sourceMappingURL=slpk-extractor-cli.js.map