@loaders.gl/tile-converter 4.0.0-alpha.23 → 4.0.0-alpha.24

package/dist/lib/utils/file-utils.js

@@ -1,139 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAbsoluteFilePath = exports.removeFile = exports.removeDir = exports.isFileExists = exports.openJson = exports.writeFileForSlpk = exports.writeFile = void 0;
-const core_1 = require("@loaders.gl/core");
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const fs_1 = require("fs");
-const path_1 = require("path");
-const compress_util_1 = require("./compress-util");
-/**
- * Write a file with data and name fileName to path
- *
- * @param path - output path
- * @param data - file content
- * @param fileName - name of output file (default: index.json)
- */
-async function writeFile(path, data, fileName = 'index.json') {
-    let toWriteData;
-    if (data instanceof Promise) {
-        toWriteData = new Uint8Array(await data);
-    }
-    else if (data instanceof ArrayBuffer) {
-        toWriteData = new Uint8Array(data);
-    }
-    else {
-        toWriteData = data;
-    }
-    await fs_1.promises.mkdir(path, { recursive: true });
-    const pathFile = (0, path_1.join)(path, fileName);
-    try {
-        await fs_1.promises.writeFile(pathFile, toWriteData);
-    }
-    catch (err) {
-        throw err;
-    }
-    console.log(`${pathFile} saved.`); // eslint-disable-line
-    return pathFile;
-}
-exports.writeFile = writeFile;
-/**
- * Write a file with data and name fileName to path - specific one for further packaging into slpk
- *
- * @param path - output path
- * @param data - file content
- * @param fileName - name of output file (default: index.json)
- * @param compress - if need to compress file with gzip (default: true)
- * @param compressList - if set - the file should be added to this list and compressed in the end of conversion
- */
-async function writeFileForSlpk(path, data, fileName = 'index.json', compress = true, compressList) {
-    const pathFile = await writeFile(path, data, fileName);
-    if (compress) {
-        if (compressList) {
-            if (!compressList.includes(pathFile)) {
-                compressList.push(pathFile);
-                return `${pathFile}.gz`;
-            }
-            else {
-                return null;
-            }
-        }
-        else {
-            const pathGzFile = await (0, compress_util_1.compressFileWithGzip)(pathFile);
-            // After compression, we don't need an uncompressed file
-            await removeFile(pathFile);
-            return pathGzFile;
-        }
-    }
-    return pathFile;
-}
-exports.writeFileForSlpk = writeFileForSlpk;
-/**
- * Open json file
- * @param path - path to the file
- * @param fileName - file name
- * @returns object
- */
-async function openJson(path, fileName) {
-    return new Promise((resolve, reject) => {
-        let count = 0;
-        console.log(`load ${path}/${fileName}.`); // eslint-disable-line
-        const intervalId = setInterval(() => {
-            const pathFile = (0, path_1.join)(path, fileName);
-            (0, core_1.load)(pathFile, loader_utils_1.JSONLoader)
-                .then((result) => {
-                clearInterval(intervalId);
-                resolve(result);
-            })
-                .catch(() => {
-                count++;
-                if (count > 100) {
-                    clearInterval(intervalId);
-                    reject(new Error(`Cannon load ${path}/${fileName}.`));
-                }
-            });
-        }, 200);
-    });
-}
-exports.openJson = openJson;
-/**
- * Check if the file exists
- * @param fileName - full name of file
- * @returns true if file exists, otherwise - false
- */
-async function isFileExists(fileName) {
-    try {
-        await fs_1.promises.stat(fileName);
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-exports.isFileExists = isFileExists;
-/**
- * Remove dir with path
- *
- * @param path
- */
-function removeDir(path) {
-    // (node:35607) [DEP0147] DeprecationWarning: In future versions of Node.js, fs.rmdir(path, { recursive: true }) will be removed. Use fs.rm(path, { recursive: true }) instead
-    return fs_1.promises.rm(path, { recursive: true });
-}
-exports.removeDir = removeDir;
-/**
- * Remove file with path
- *
- * @param path
- */
-function removeFile(path) {
-    return fs_1.promises.unlink(path);
-}
-exports.removeFile = removeFile;
-/**
- * Generates absolute file path
- * @param filePath
- */
-function getAbsoluteFilePath(filePath) {
-    return (0, path_1.isAbsolute)(filePath) ? filePath : (0, path_1.join)(process.cwd(), filePath);
-}
-exports.getAbsoluteFilePath = getAbsoluteFilePath;

package/dist/lib/utils/geometry-utils.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateSyntheticIndices = void 0;
-/**
- * luma.gl can not work without indices now:
- * https://github.com/visgl/luma.gl/blob/d8cad75b9f8ca3e578cf078ed9d19e619c2ddbc9/modules/experimental/src/gltf/gltf-instantiator.js#L115
- * This method generates syntetic indices array: [0, 1, 2, 3, .... , vertexCount-1]
- * @param {number} vertexCount - vertex count in the geometry
- * @returns {Uint32Array} indices array.
- */
-const generateSyntheticIndices = (vertexCount) => {
-    const result = new Uint32Array(vertexCount);
-    for (let index = 0; index < vertexCount; index++) {
-        result[index] = index;
-    }
-    return result;
-};
-exports.generateSyntheticIndices = generateSyntheticIndices;

package/dist/lib/utils/lod-conversion-utils.js

@@ -1,76 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.convertScreenThresholdToGeometricError = exports.convertGeometricErrorToScreenThreshold = void 0;
-// https://cesium.com/docs/cesiumjs-ref-doc/Cesium3DTileset.html
-const DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR = 16;
-/**
- * Do conversion from geometric error to screen threshold
- *
- * In 3DTiles we have HLOD logic and parent tile also has bigger lodMetric value then its children.
- * In I3s we have reverse logic related to maxError. Parent has lower maxError than its child.
- * In nodes where are no children tile.lodMetricValue is 0. This is because of logic of HLOD in 3DTiles
- * 3DTiles spec:
- * https://github.com/CesiumGS/3d-tiles/tree/master/specification#geometric-error
- * I3S spec:
- * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/lodSelection.cmn.md
- * To avoid infinity values when we do calculations of maxError we shold replace 0 with value which allows us
- * to make child maxError bigger than his parent maxError.
- *
- * @param tile - 3d-tiles tile JSON
- * @param coordinates - node converted coordinates
- * @returns An array of LOD metrics in format compatible with i3s 3DNodeIndexDocument.lodSelection
- * @example
- *  [
-        {
-            "metricType": "maxScreenThresholdSQ",
-            "maxError": 870638.071285568
-        },
-        {
-            "metricType": "maxScreenThreshold",
-            "maxError": 1052.8679031638949
-        }
-    ]
- */
-function convertGeometricErrorToScreenThreshold(tile, coordinates) {
-    const lodSelection = [];
-    const boundingVolume = tile.boundingVolume;
-    const lodMetricValue = tile.lodMetricValue || 0.1;
-    const maxScreenThreshold = {
-        metricType: 'maxScreenThreshold',
-        maxError: (coordinates.mbs[3] * 2 * DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR) / lodMetricValue
-    };
-    const maxScreenThresholdSQ = {
-        metricType: 'maxScreenThresholdSQ',
-        maxError: Math.PI * 0.25 * maxScreenThreshold.maxError * maxScreenThreshold.maxError
-    };
-    if (boundingVolume.constructor.name === 'OrientedBoundingBox') {
-        lodSelection.push(maxScreenThresholdSQ);
-        lodSelection.push(maxScreenThreshold);
-    }
-    else {
-        lodSelection.push(maxScreenThreshold);
-        lodSelection.push(maxScreenThresholdSQ);
-    }
-    return lodSelection;
-}
-exports.convertGeometricErrorToScreenThreshold = convertGeometricErrorToScreenThreshold;
-/**
- * Convert LOD metric from "Screen Threshold" to "Screen Space Error"
- * @param node - i3s node data
- * @returns lod metric in 3d-tiles format
- */
-function convertScreenThresholdToGeometricError(node) {
-    const metricData = node.lodSelection?.find((metric) => metric.metricType === 'maxScreenThreshold');
-    let maxError = metricData?.maxError;
-    if (!maxError) {
-        const sqMetricData = node.lodSelection?.find((metric) => metric.metricType === 'maxScreenThresholdSQ');
-        if (sqMetricData) {
-            maxError = Math.sqrt(sqMetricData.maxError / (Math.PI * 0.25));
-        }
-    }
-    if (!maxError) {
-        maxError = DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR;
-    }
-    return (node.mbs[3] * 2 * DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR) / maxError;
-}
-exports.convertScreenThresholdToGeometricError = convertScreenThresholdToGeometricError;

package/dist/lib/utils/queue.js

@@ -1,18 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Queue = void 0;
-class Queue extends Array {
-    enqueue(val) {
-        this.push(val);
-    }
-    dequeue() {
-        return this.shift();
-    }
-    peek() {
-        return this[0];
-    }
-    isEmpty() {
-        return this.length === 0;
-    }
-}
-exports.Queue = Queue;

package/dist/lib/utils/statistic-utills.js

@@ -1,64 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.calculateFilesSize = exports.timeConverter = void 0;
-const path_1 = require("path");
-const fs_1 = require("fs");
-const file_utils_1 = require("./file-utils");
-function timeConverter(time) {
-    const nanoSecondsInMillisecond = 1e6;
-    let timeInSeconds = time[0];
-    const hours = Math.floor(timeInSeconds / 3600);
-    timeInSeconds = timeInSeconds - hours * 3600;
-    const minutes = Math.floor(timeInSeconds / 60);
-    timeInSeconds = timeInSeconds - minutes * 60;
-    const seconds = Math.floor(timeInSeconds);
-    const milliseconds = time[1] / nanoSecondsInMillisecond;
-    let result = '';
-    if (hours) {
-        result += `${hours}h `;
-    }
-    if (minutes) {
-        result += `${minutes}m `;
-    }
-    if (seconds) {
-        result += `${seconds}s`;
-    }
-    if (!result) {
-        result += `${milliseconds}ms`;
-    }
-    return result;
-}
-exports.timeConverter = timeConverter;
-async function calculateFilesSize(params) {
-    const { slpk, outputPath, tilesetName } = params;
-    const fullOutputPath = (0, file_utils_1.getAbsoluteFilePath)(outputPath);
-    try {
-        if (slpk) {
-            const slpkPath = (0, path_1.join)(fullOutputPath, `${tilesetName}.slpk`);
-            const stat = await fs_1.promises.stat(slpkPath);
-            return stat.size;
-        }
-        const directoryPath = (0, path_1.join)(fullOutputPath, tilesetName);
-        const totalSize = await getTotalFilesSize(directoryPath);
-        return totalSize;
-    }
-    catch (error) {
-        console.log('Calculate file sizes error: ', error); // eslint-disable-line
-        return null;
-    }
-}
-exports.calculateFilesSize = calculateFilesSize;
-async function getTotalFilesSize(dirPath) {
-    let totalFileSize = 0;
-    const files = await fs_1.promises.readdir(dirPath);
-    for (const file of files) {
-        const fileStat = await fs_1.promises.stat((0, path_1.join)(dirPath, file));
-        if (fileStat.isDirectory()) {
-            totalFileSize += await getTotalFilesSize((0, path_1.join)(dirPath, file));
-        }
-        else {
-            totalFileSize += fileStat.size;
-        }
-    }
-    return totalFileSize;
-}

package/dist/lib/utils/write-queue.js

@@ -1,80 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const queue_1 = require("./queue");
-const process_1 = __importDefault(require("process"));
-/** Memory limit size is based on testing */
-const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
-class WriteQueue extends queue_1.Queue {
-    constructor(listeningInterval = 2000, writeConcurrency = 400) {
-        super();
-        this.writePromise = null;
-        this.fileMap = {};
-        this.listeningInterval = listeningInterval;
-        this.writeConcurrency = writeConcurrency;
-    }
-    async enqueue(val, writeImmediately = false) {
-        if (writeImmediately) {
-            const { archiveKey, writePromise } = val;
-            const result = await writePromise();
-            if (archiveKey && result) {
-                this.fileMap[archiveKey] = result;
-            }
-        }
-        else {
-            super.enqueue(val);
-            /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */
-            if (process_1.default.memoryUsage().rss > MEMORY_LIMIT) {
-                await this.startWrite();
-            }
-        }
-    }
-    startListening() {
-        this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
-    }
-    stopListening() {
-        if (this.intervalId) {
-            clearInterval(this.intervalId);
-        }
-    }
-    async startWrite() {
-        if (!this.writePromise) {
-            this.writePromise = this.doWrite();
-        }
-        await this.writePromise;
-        this.writePromise = null;
-    }
-    async finalize() {
-        this.stopListening();
-        await this.startWrite();
-    }
-    async doWrite() {
-        while (this.length) {
-            const promises = [];
-            const archiveKeys = [];
-            for (let i = 0; i < this.writeConcurrency; i++) {
-                const item = this.dequeue();
-                if (!item) {
-                    break;
-                }
-                const { archiveKey, writePromise } = item;
-                archiveKeys.push(archiveKey);
-                const promise = writePromise();
-                promises.push(promise);
-            }
-            const writeResults = await Promise.allSettled(promises);
-            this.updateFileMap(archiveKeys, writeResults);
-        }
-    }
-    updateFileMap(archiveKeys, writeResults) {
-        for (let i = 0; i < archiveKeys.length; i++) {
-            const archiveKey = archiveKeys[i];
-            if (archiveKey && 'value' in writeResults[i]) {
-                this.fileMap[archiveKey] = writeResults[i].value;
-            }
-        }
-    }
-}
-exports.default = WriteQueue;

package/dist/pgm-loader.js

@@ -1,24 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PGMLoader = void 0;
-const geoid_1 = require("@math.gl/geoid");
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-/**
- * Loader for PGM - Netpbm grayscale image format
- */
-exports.PGMLoader = {
-    name: 'PGM - Netpbm grayscale image format',
-    id: 'pgm',
-    module: 'tile-converter',
-    version: VERSION,
-    mimeTypes: ['image/x-portable-graymap'],
-    // @ts-expect-error LoaderOptions does not have cubic parameter
-    parse: async (arrayBuffer, options) => (0, geoid_1.parsePGM)(new Uint8Array(arrayBuffer), options),
-    extensions: ['pgm'],
-    options: {
-        // TODO - use pgm namespace
-        cubic: false
-    }
-};

package/dist/slpk-extractor/slpk-extractor.js

@@ -1,75 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const core_1 = require("@loaders.gl/core");
-const constants_1 = require("../constants");
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const loader_utils_2 = require("@loaders.gl/loader-utils");
-const zip_1 = require("@loaders.gl/zip");
-const compression_1 = require("@loaders.gl/compression");
-const file_utils_1 = require("../lib/utils/file-utils");
-const indexNames = [
-    '3dSceneLayer.json.gz',
-    '3dNodeIndexDocument.json.gz',
-    'sharedResource.json.gz'
-];
-/**
- * Converter from slpk to i3s
- */
-class SLPKExtractor {
-    /**
-     * Extract slpk to i3s
-     * @param options
-     * @param options.inputUrl the url to read SLPK file
-     * @param options.outputPath the output filename
-     */
-    async extract(options) {
-        if (core_1.isBrowser) {
-            console.log(constants_1.BROWSER_ERROR_MESSAGE);
-            return constants_1.BROWSER_ERROR_MESSAGE;
-        }
-        const { inputUrl } = options;
-        const provider = await loader_utils_2.FileHandleFile.from(inputUrl);
-        let localHeader = await (0, zip_1.parseZipLocalFileHeader)(0n, provider);
-        while (localHeader) {
-            await this.writeFile(await this.unGzip({
-                name: this.correctIndexNames(localHeader.fileName),
-                data: await provider.slice(localHeader.fileDataOffset, localHeader.fileDataOffset + localHeader.compressedSize)
-            }), options.outputPath);
-            localHeader = await (0, zip_1.parseZipLocalFileHeader)(localHeader.fileDataOffset + localHeader?.compressedSize, provider);
-        }
-        return 'success';
-    }
-    /**
-     * Defines file name and path for i3s format
-     * @param fileName initial file name and path
-     */
-    correctIndexNames(fileName) {
-        if (indexNames.includes(loader_utils_1.path.filename(loader_utils_1.path.join('/', fileName)))) {
-            return loader_utils_1.path.join(loader_utils_1.path.dirname(fileName), 'index.json.gz');
-        }
-        // finds path with name part and extention part
-        let parts = /^(.*\/[^\/\.]*)(\..+)$/.exec(fileName);
-        if (!parts) {
-            return null;
-        }
-        return `${parts?.at(1)}/index${parts?.at(2)}`;
-    }
-    async unGzip(file) {
-        if (/\.gz$/.test(file.name ?? '')) {
-            const compression = new compression_1.GZipCompression();
-            const decompressedData = await compression.decompress(file.data);
-            return { data: decompressedData, name: (file.name ?? '').slice(0, -3) };
-        }
-        return Promise.resolve(file);
-    }
-    async writeFile(options, outputPath) {
-        if (!options.name) {
-            return;
-        }
-        const finalPath = loader_utils_1.path.join(outputPath, options.name);
-        const dirName = loader_utils_1.path.dirname(finalPath);
-        const fileName = loader_utils_1.path.filename(finalPath);
-        await (0, file_utils_1.writeFile)(dirName, options.data, fileName);
-    }
-}
-exports.default = SLPKExtractor;

package/dist/slpk-extractor-cli.js

@@ -1,102 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-/* eslint-disable no-console */
-require("@loaders.gl/polyfills");
-const slpk_extractor_1 = __importDefault(require("./slpk-extractor/slpk-extractor"));
-const cli_utils_1 = require("./lib/utils/cli-utils");
-/**
- * CLI entry
- * @returns
- */
-async function main() {
-    const [, , ...args] = process.argv;
-    if (args.length === 0) {
-        printHelp();
-    }
-    const validatedOptionsArr = (0, cli_utils_1.validateOptionsWithEqual)(args);
-    const options = parseOptions(validatedOptionsArr);
-    const validatedOptions = validateOptions(options);
-    await convert(validatedOptions);
-}
-main().catch((error) => {
-    console.log(error);
-    process.exit(1); // eslint-disable-line
-});
-/**
- * Output for `npx slpk-extractor --help`
- */
-function printHelp() {
-    console.log('cli: converter slpk to I3S...');
-    console.log('--output [Output folder, default: "data" folder]');
-    console.log('--tileset [SLPK file]');
-    process.exit(0); // eslint-disable-line
-}
-/**
- * Run extraction process
- * @param options validated slpk-extractor options
- */
-async function convert(options) {
-    console.log(`------------------------------------------------`); // eslint-disable-line
-    console.log(`Starting conversion of SLPK`); // eslint-disable-line
-    console.log(`------------------------------------------------`); // eslint-disable-line
-    const slpkExtractor = new slpk_extractor_1.default();
-    slpkExtractor.extract({
-        inputUrl: options.tileset,
-        outputPath: options.output
-    });
-}
-// OPTIONS
-/**
- * Validate input options of the CLI command
- * @param options - input options of the CLI command
- * @returns validated options
- */
-function validateOptions(options) {
-    const mandatoryOptionsWithExceptions = {
-        output: () => console.log('Missed: --output [Output path name]'),
-        tileset: () => console.log('Missed: --tileset [SLPK file]')
-    };
-    const exceptions = [];
-    for (const mandatoryOption in mandatoryOptionsWithExceptions) {
-        const optionValue = options[mandatoryOption];
-        if (!optionValue) {
-            exceptions.push(mandatoryOptionsWithExceptions[mandatoryOption]);
-        }
-    }
-    if (exceptions.length) {
-        exceptions.forEach((exeption) => exeption());
-        process.exit(1);
-    }
-    return options;
-}
-/**
- * Parse option from the cli arguments array
- * @param args
- * @returns
- */
-function parseOptions(args) {
-    const opts = {};
-    // eslint-disable-next-line complexity
-    args.forEach((arg, index) => {
-        if (arg.indexOf('--') === 0) {
-            switch (arg) {
-                case '--tileset':
-                    opts.tileset = (0, cli_utils_1.getURLValue)(index, args);
-                    break;
-                case '--output':
-                    opts.output = (0, cli_utils_1.getURLValue)(index, args);
-                    break;
-                case '--help':
-                    printHelp();
-                    break;
-                default:
-                    console.warn(`Unknown option ${arg}`);
-                    process.exit(0); // eslint-disable-line
-            }
-        }
-    });
-    return opts;
-}