@loaders.gl/tile-converter 4.4.0-alpha.1 → 4.4.0-alpha.9

package/dist/lib/utils/statistic-utils.js

@@ -1,88 +0,0 @@
-import { join } from 'path';
-import { promises as fs } from 'fs';
-import { getAbsoluteFilePath } from "./file-utils.js";
-/**
- * Converts time value to string.
- * @param time - high-resolution real time in a [seconds, nanoseconds] tuple Array, or a value on milliseconds.
- * @returns string representation of the time
- */
-export function timeConverter(time) {
-    if (typeof time === 'number') {
-        // time - real time in milli-seconds
-        const milliSecondsInSecond = 1e3;
-        const timeInSeconds = Math.floor(time / milliSecondsInSecond);
-        const milliseconds = time - timeInSeconds * milliSecondsInSecond;
-        return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);
-    }
-    // time - high-resolution real time in a [seconds, nanoseconds] tuple Array
-    const nanoSecondsInMillisecond = 1e6;
-    const timeInSeconds = time[0];
-    const milliseconds = time[1] / nanoSecondsInMillisecond;
-    return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);
-}
-function timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds) {
-    const hours = Math.floor(timeInSeconds / 3600);
-    timeInSeconds = timeInSeconds - hours * 3600;
-    const minutes = Math.floor(timeInSeconds / 60);
-    timeInSeconds = timeInSeconds - minutes * 60;
-    const seconds = Math.floor(timeInSeconds);
-    let result = '';
-    if (hours) {
-        result += `${hours}h `;
-    }
-    if (minutes) {
-        result += `${minutes}m `;
-    }
-    if (seconds) {
-        result += `${seconds}s`;
-    }
-    if (!result) {
-        result += `${Math.floor(milliseconds)}ms`;
-    }
-    return result;
-}
-/**
- * Calculates overall datatset size
- * @param params - params object
- * @param params.slpk - if dataset is in slpk format
- * @param params.outputPath - path to the folder containing the dataset
- * @param params.tilesetName - tileset name
- * @returns dataset size in bytes
- */
-export async function calculateDatasetSize(params) {
-    const { slpk, outputPath, tilesetName } = params;
-    const fullOutputPath = getAbsoluteFilePath(outputPath);
-    try {
-        if (slpk) {
-            const slpkPath = join(fullOutputPath, `${tilesetName}.slpk`);
-            const stat = await fs.stat(slpkPath);
-            return stat.size;
-        }
-        const directoryPath = join(fullOutputPath, tilesetName);
-        const totalSize = await getDirectorySize(directoryPath);
-        return totalSize;
-    }
-    catch (error) {
-        console.log('Calculate file sizes error: ', error); // eslint-disable-line
-        return null;
-    }
-}
-/**
- * Calculates directory size
- * @param dirPath - path to the directory
- * @returns directory size in bytes
- */
-async function getDirectorySize(dirPath) {
-    let totalFileSize = 0;
-    const files = await fs.readdir(dirPath);
-    for (const file of files) {
-        const fileStat = await fs.stat(join(dirPath, file));
-        if (fileStat.isDirectory()) {
-            totalFileSize += await getDirectorySize(join(dirPath, file));
-        }
-        else {
-            totalFileSize += fileStat.size;
-        }
-    }
-    return totalFileSize;
-}

package/dist/lib/utils/write-queue.d.ts

@@ -1,44 +0,0 @@
-import { Queue } from "./queue.js";
-import { ConversionDump } from "./conversion-dump.js";
-export type WriteQueueItem = {
-    archiveKey?: string;
-    sourceId?: string;
-    outputId?: number;
-    resourceType?: string;
-    /**
-     * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);
-     * Arguments for this call are specified in writeQueue.enqueue call like this:
-     * await writeQueue.enqueue({
-     *     archiveKey: `nodePages/xxx.json.gz`,
-     *     writePromise: () => writeFileForSlpk(slpkPath, data, `xxx.json`)
-     * });
-     * Note, a function like writeFileForSlpk should NOT be called when initializing the object for enqueue().
-     * If he function is called, the promise will be created
-     * and the function will allocate resources (file descriptors) for file writing.
-     * It will be done for ALL items in the queue, which is not supposed to happen.
-     * That's why the function should be passed as
-     *   writePromise: () => writeFileForSlpk(slpkPath, content, `xxx.json`)
-     * instead of
-     *  writePromise: writeFileForSlpk(slpkPath, content, `xxx.json`) // INCORRECT !
-     */
-    writePromise: () => Promise<string | null>;
-};
-export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
-    private intervalId?;
-    private conversionDump;
-    writePromise: Promise<void> | null;
-    fileMap: {
-        [key: string]: string;
-    };
-    listeningInterval: number;
-    writeConcurrency: number;
-    constructor(conversionDump: ConversionDump, listeningInterval?: number, writeConcurrency?: number);
-    enqueue(val: T, writeImmediately?: boolean): Promise<void>;
-    startListening(): void;
-    stopListening(): void;
-    startWrite(): Promise<void>;
-    finalize(): Promise<void>;
-    private doWrite;
-    private updateFileMap;
-}
-//# sourceMappingURL=write-queue.d.ts.map

package/dist/lib/utils/write-queue.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"write-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/write-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,EAAC,mBAAgB;AAE9B,OAAO,EAAC,cAAc,EAAC,6BAA0B;AAKjD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;;;;;;;;OAeG;IACH,YAAY,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,CAAC,SAAS,cAAc,CAAE,SAAQ,KAAK,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,UAAU,CAAC,CAAe;IAClC,OAAO,CAAC,cAAc,CAAiB;IAChC,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;IAC1C,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAM;IACtC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;gBAG9B,cAAc,EAAE,cAAc,EAC9B,iBAAiB,GAAE,MAAa,EAChC,gBAAgB,GAAE,MAAY;IAQ1B,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,gBAAgB,GAAE,OAAe;IAgBvD,cAAc;IAId,aAAa;IAMP,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAKjB,OAAO;IAsBrB,OAAO,CAAC,aAAa;CAWtB"}

package/dist/lib/utils/write-queue.js

@@ -1,82 +0,0 @@
-import { Queue } from "./queue.js";
-import process from 'process';
-/** Memory limit size is based on testing */
-const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
-export default class WriteQueue extends Queue {
-    intervalId; // eslint-disable-line no-undef
-    conversionDump;
-    writePromise = null;
-    fileMap = {};
-    listeningInterval;
-    writeConcurrency;
-    constructor(conversionDump, listeningInterval = 2000, writeConcurrency = 400) {
-        super();
-        this.conversionDump = conversionDump;
-        this.listeningInterval = listeningInterval;
-        this.writeConcurrency = writeConcurrency;
-    }
-    async enqueue(val, writeImmediately = false) {
-        if (writeImmediately) {
-            const { archiveKey, writePromise } = val;
-            const result = await writePromise();
-            if (archiveKey && result) {
-                this.fileMap[archiveKey] = result;
-            }
-        }
-        else {
-            super.enqueue(val);
-            /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */
-            if (process.memoryUsage().rss > MEMORY_LIMIT) {
-                await this.startWrite();
-            }
-        }
-    }
-    startListening() {
-        this.intervalId = setInterval(() => this.startWrite.bind(this), this.listeningInterval);
-    }
-    stopListening() {
-        if (this.intervalId) {
-            clearInterval(this.intervalId);
-        }
-    }
-    async startWrite() {
-        if (!this.writePromise) {
-            this.writePromise = this.doWrite();
-        }
-        await this.writePromise;
-        this.writePromise = null;
-    }
-    async finalize() {
-        this.stopListening();
-        await this.startWrite();
-    }
-    async doWrite() {
-        while (this.length) {
-            const promises = [];
-            const archiveKeys = [];
-            const changedRecords = [];
-            for (let i = 0; i < this.writeConcurrency; i++) {
-                const item = this.dequeue();
-                if (!item) {
-                    break;
-                }
-                const { archiveKey, sourceId, outputId, resourceType, writePromise } = item;
-                archiveKeys.push(archiveKey);
-                changedRecords.push({ sourceId, outputId, resourceType });
-                const promise = writePromise();
-                promises.push(promise);
-            }
-            const writeResults = await Promise.allSettled(promises);
-            this.updateFileMap(archiveKeys, writeResults);
-            await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults);
-        }
-    }
-    updateFileMap(archiveKeys, writeResults) {
-        for (let i = 0; i < archiveKeys.length; i++) {
-            const archiveKey = archiveKeys[i];
-            if (archiveKey && 'value' in writeResults[i]) {
-                this.fileMap[archiveKey] = writeResults[i].value;
-            }
-        }
-    }
-}

package/dist/pgm-loader.d.ts

@@ -1,28 +0,0 @@
-import type { LoaderOptions } from '@loaders.gl/loader-utils';
-import { Geoid } from '@math.gl/geoid';
-export { Geoid };
-export type PGMLoaderOptions = LoaderOptions & {
-    pgm?: {
-        cubic?: boolean;
-    };
-};
-/**
- * Loader for PGM - Netpbm grayscale image format
- */
-export declare const PGMLoader: {
-    readonly dataType: Geoid;
-    readonly batchType: never;
-    readonly name: "PGM - Netpbm grayscale image format";
-    readonly id: "pgm";
-    readonly module: "tile-converter";
-    readonly version: any;
-    readonly mimeTypes: ["image/x-portable-graymap"];
-    readonly parse: (arrayBuffer: ArrayBuffer, options: PGMLoaderOptions | undefined) => Promise<Geoid>;
-    readonly extensions: ["pgm"];
-    readonly options: {
-        readonly pgm: {
-            readonly cubic: false;
-        };
-    };
-};
-//# sourceMappingURL=pgm-loader.d.ts.map

package/dist/pgm-loader.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"pgm-loader.d.ts","sourceRoot":"","sources":["../src/pgm-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAmB,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,EAAC,KAAK,EAAW,MAAM,gBAAgB,CAAC;AAM/C,OAAO,EAAC,KAAK,EAAC,CAAC;AAEf,MAAM,MAAM,gBAAgB,GAAG,aAAa,GAAG;IAC7C,GAAG,CAAC,EAAE;QACJ,KAAK,CAAC,EAAE,OAAO,CAAC;KACjB,CAAC;CACH,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,SAAS;uBACS,KAAK;wBACf,KAAK;;;;;;;;;;;;;CAc2C,CAAC"}

package/dist/pgm-loader.js

@@ -1,24 +0,0 @@
-import { Geoid, parsePGM } from '@math.gl/geoid';
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.4.0-alpha.0" !== 'undefined' ? "4.4.0-alpha.0" : 'latest';
-export { Geoid };
-/**
- * Loader for PGM - Netpbm grayscale image format
- */
-export const PGMLoader = {
-    dataType: null,
-    batchType: null,
-    name: 'PGM - Netpbm grayscale image format',
-    id: 'pgm',
-    module: 'tile-converter',
-    version: VERSION,
-    mimeTypes: ['image/x-portable-graymap'],
-    parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options?.pgm || {}),
-    extensions: ['pgm'],
-    options: {
-        pgm: {
-            cubic: false
-        }
-    }
-};