@loaders.gl/tile-converter 4.1.0-alpha.9 → 4.2.0-alpha.1

package/dist/lib/utils/statistic-utills.js.map

@@ -1 +1 @@
-{"version":3,"file":"statistic-utills.js","names":["join","promises","fs","getAbsoluteFilePath","timeConverter","time","milliSecondsInSecond","timeInSeconds","Math","floor","milliseconds","timeConverterFromSecondsAndMilliseconds","nanoSecondsInMillisecond","hours","minutes","seconds","result","calculateFilesSize","params","slpk","outputPath","tilesetName","fullOutputPath","slpkPath","stat","size","directoryPath","totalSize","getTotalFilesSize","error","console","log","dirPath","totalFileSize","files","readdir","file","fileStat","isDirectory"],"sources":["../../../src/lib/utils/statistic-utills.ts"],"sourcesContent":["import {join} from 'path';\nimport {promises as fs} from 'fs';\nimport {getAbsoluteFilePath} from './file-utils';\n\n/**\n * Converts time value to string.\n * @param time - high-resolution real time in a [seconds, nanoseconds] tuple Array, or a value on milliseconds.\n * @returns string representation of the time\n */\nexport function timeConverter(time: number | [number, number]): string {\n  if (typeof time === 'number') {\n    // time - real time in milli-seconds\n    const milliSecondsInSecond = 1e3;\n    const timeInSeconds = Math.floor(time / milliSecondsInSecond);\n    const milliseconds = time - timeInSeconds * milliSecondsInSecond;\n    return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);\n  } else {\n    // time - high-resolution real time in a [seconds, nanoseconds] tuple Array\n    const nanoSecondsInMillisecond = 1e6;\n    const timeInSeconds = time[0];\n    const milliseconds = time[1] / nanoSecondsInMillisecond;\n    return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);\n  }\n}\n\nfunction timeConverterFromSecondsAndMilliseconds(timeInSeconds: number, milliseconds: number) {\n  const hours = Math.floor(timeInSeconds / 3600);\n  timeInSeconds = timeInSeconds - hours * 3600;\n  const minutes = Math.floor(timeInSeconds / 60);\n  timeInSeconds = timeInSeconds - minutes * 60;\n  const seconds = Math.floor(timeInSeconds);\n  let result = '';\n\n  if (hours) {\n    result += `${hours}h `;\n  }\n\n  if (minutes) {\n    result += `${minutes}m `;\n  }\n\n  if (seconds) {\n    result += `${seconds}s`;\n  }\n\n  if (!result) {\n    result += `${milliseconds}ms`;\n  }\n\n  return result;\n}\n\nexport async function calculateFilesSize(params) {\n  const {slpk, outputPath, tilesetName} = params;\n  const fullOutputPath = getAbsoluteFilePath(outputPath);\n\n  try {\n    if (slpk) {\n      const slpkPath = join(fullOutputPath, `${tilesetName}.slpk`);\n      const stat = await fs.stat(slpkPath);\n      return stat.size;\n    }\n\n    const directoryPath = join(fullOutputPath, tilesetName);\n    const totalSize = await getTotalFilesSize(directoryPath);\n    return totalSize;\n  } catch (error) {\n    console.log('Calculate file sizes error: ', error); // eslint-disable-line\n    return null;\n  }\n}\n\nasync function getTotalFilesSize(dirPath) {\n  let totalFileSize = 0;\n\n  const files = await fs.readdir(dirPath);\n\n  for (const file of files) {\n    const fileStat = await fs.stat(join(dirPath, file));\n    if (fileStat.isDirectory()) {\n      totalFileSize += await getTotalFilesSize(join(dirPath, file));\n    } else {\n      totalFileSize += fileStat.size;\n    }\n  }\n  return totalFileSize;\n}\n"],"mappings":"AAAA,SAAQA,IAAI,QAAO,MAAM;AACzB,SAAQC,QAAQ,IAAIC,EAAE,QAAO,IAAI;AAAC,SAC1BC,mBAAmB;AAO3B,OAAO,SAASC,aAAaA,CAACC,IAA+B,EAAU;EACrE,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;IAE5B,MAAMC,oBAAoB,GAAG,GAAG;IAChC,MAAMC,aAAa,GAAGC,IAAI,CAACC,KAAK,CAACJ,IAAI,GAAGC,oBAAoB,CAAC;IAC7D,MAAMI,YAAY,GAAGL,IAAI,GAAGE,aAAa,GAAGD,oBAAoB;IAChE,OAAOK,uCAAuC,CAACJ,aAAa,EAAEG,YAAY,CAAC;EAC7E,CAAC,MAAM;IAEL,MAAME,wBAAwB,GAAG,GAAG;IACpC,MAAML,aAAa,GAAGF,IAAI,CAAC,CAAC,CAAC;IAC7B,MAAMK,YAAY,GAAGL,IAAI,CAAC,CAAC,CAAC,GAAGO,wBAAwB;IACvD,OAAOD,uCAAuC,CAACJ,aAAa,EAAEG,YAAY,CAAC;EAC7E;AACF;AAEA,SAASC,uCAAuCA,CAACJ,aAAqB,EAAEG,YAAoB,EAAE;EAC5F,MAAMG,KAAK,GAAGL,IAAI,CAACC,KAAK,CAACF,aAAa,GAAG,IAAI,CAAC;EAC9CA,aAAa,GAAGA,aAAa,GAAGM,KAAK,GAAG,IAAI;EAC5C,MAAMC,OAAO,GAAGN,IAAI,CAACC,KAAK,CAACF,aAAa,GAAG,EAAE,CAAC;EAC9CA,aAAa,GAAGA,aAAa,GAAGO,OAAO,GAAG,EAAE;EAC5C,MAAMC,OAAO,GAAGP,IAAI,CAACC,KAAK,CAACF,aAAa,CAAC;EACzC,IAAIS,MAAM,GAAG,EAAE;EAEf,IAAIH,KAAK,EAAE;IACTG,MAAM,IAAK,GAAEH,KAAM,IAAG;EACxB;EAEA,IAAIC,OAAO,EAAE;IACXE,MAAM,IAAK,GAAEF,OAAQ,IAAG;EAC1B;EAEA,IAAIC,OAAO,EAAE;IACXC,MAAM,IAAK,GAAED,OAAQ,GAAE;EACzB;EAEA,IAAI,CAACC,MAAM,EAAE;IACXA,MAAM,IAAK,GAAEN,YAAa,IAAG;EAC/B;EAEA,OAAOM,MAAM;AACf;AAEA,OAAO,eAAeC,kBAAkBA,CAACC,MAAM,EAAE;EAC/C,MAAM;IAACC,IAAI;IAAEC,UAAU;IAAEC;EAAW,CAAC,GAAGH,MAAM;EAC9C,MAAMI,cAAc,GAAGnB,mBAAmB,CAACiB,UAAU,CAAC;EAEtD,IAAI;IACF,IAAID,IAAI,EAAE;MACR,MAAMI,QAAQ,GAAGvB,IAAI,CAACsB,cAAc,EAAG,GAAED,WAAY,OAAM,CAAC;MAC5D,MAAMG,IAAI,GAAG,MAAMtB,EAAE,CAACsB,IAAI,CAACD,QAAQ,CAAC;MACpC,OAAOC,IAAI,CAACC,IAAI;IAClB;IAEA,MAAMC,aAAa,GAAG1B,IAAI,CAACsB,cAAc,EAAED,WAAW,CAAC;IACvD,MAAMM,SAAS,GAAG,MAAMC,iBAAiB,CAACF,aAAa,CAAC;IACxD,OAAOC,SAAS;EAClB,CAAC,CAAC,OAAOE,KAAK,EAAE;IACdC,OAAO,CAACC,GAAG,CAAC,8BAA8B,EAAEF,KAAK,CAAC;IAClD,OAAO,IAAI;EACb;AACF;AAEA,eAAeD,iBAAiBA,CAACI,OAAO,EAAE;EACxC,IAAIC,aAAa,GAAG,CAAC;EAErB,MAAMC,KAAK,GAAG,MAAMhC,EAAE,CAACiC,OAAO,CAACH,OAAO,CAAC;EAEvC,KAAK,MAAMI,IAAI,IAAIF,KAAK,EAAE;IACxB,MAAMG,QAAQ,GAAG,MAAMnC,EAAE,CAACsB,IAAI,CAACxB,IAAI,CAACgC,OAAO,EAAEI,IAAI,CAAC,CAAC;IACnD,IAAIC,QAAQ,CAACC,WAAW,CAAC,CAAC,EAAE;MAC1BL,aAAa,IAAI,MAAML,iBAAiB,CAAC5B,IAAI,CAACgC,OAAO,EAAEI,IAAI,CAAC,CAAC;IAC/D,CAAC,MAAM;MACLH,aAAa,IAAII,QAAQ,CAACZ,IAAI;IAChC;EACF;EACA,OAAOQ,aAAa;AACtB"}
+{"version":3,"file":"statistic-utills.js","names":["join","promises","fs","getAbsoluteFilePath","timeConverter","time","milliSecondsInSecond","timeInSeconds","Math","floor","milliseconds","timeConverterFromSecondsAndMilliseconds","nanoSecondsInMillisecond","hours","minutes","seconds","result","calculateFilesSize","params","slpk","outputPath","tilesetName","fullOutputPath","slpkPath","stat","size","directoryPath","totalSize","getTotalFilesSize","error","console","log","dirPath","totalFileSize","files","readdir","file","fileStat","isDirectory"],"sources":["../../../src/lib/utils/statistic-utills.ts"],"sourcesContent":["import {join} from 'path';\nimport {promises as fs} from 'fs';\nimport {getAbsoluteFilePath} from './file-utils';\n\n/**\n * Converts time value to string.\n * @param time - high-resolution real time in a [seconds, nanoseconds] tuple Array, or a value on milliseconds.\n * @returns string representation of the time\n */\nexport function timeConverter(time: number | [number, number]): string {\n  if (typeof time === 'number') {\n    // time - real time in milli-seconds\n    const milliSecondsInSecond = 1e3;\n    const timeInSeconds = Math.floor(time / milliSecondsInSecond);\n    const milliseconds = time - timeInSeconds * milliSecondsInSecond;\n    return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);\n  } else {\n    // time - high-resolution real time in a [seconds, nanoseconds] tuple Array\n    const nanoSecondsInMillisecond = 1e6;\n    const timeInSeconds = time[0];\n    const milliseconds = time[1] / nanoSecondsInMillisecond;\n    return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);\n  }\n}\n\nfunction timeConverterFromSecondsAndMilliseconds(timeInSeconds: number, milliseconds: number) {\n  const hours = Math.floor(timeInSeconds / 3600);\n  timeInSeconds = timeInSeconds - hours * 3600;\n  const minutes = Math.floor(timeInSeconds / 60);\n  timeInSeconds = timeInSeconds - minutes * 60;\n  const seconds = Math.floor(timeInSeconds);\n  let result = '';\n\n  if (hours) {\n    result += `${hours}h `;\n  }\n\n  if (minutes) {\n    result += `${minutes}m `;\n  }\n\n  if (seconds) {\n    result += `${seconds}s`;\n  }\n\n  if (!result) {\n    result += `${Math.floor(milliseconds)}ms`;\n  }\n\n  return result;\n}\n\nexport async function calculateFilesSize(params) {\n  const {slpk, outputPath, tilesetName} = params;\n  const fullOutputPath = getAbsoluteFilePath(outputPath);\n\n  try {\n    if (slpk) {\n      const slpkPath = join(fullOutputPath, `${tilesetName}.slpk`);\n      const stat = await fs.stat(slpkPath);\n      return stat.size;\n    }\n\n    const directoryPath = join(fullOutputPath, tilesetName);\n    const totalSize = await getTotalFilesSize(directoryPath);\n    return totalSize;\n  } catch (error) {\n    console.log('Calculate file sizes error: ', error); // eslint-disable-line\n    return null;\n  }\n}\n\nasync function getTotalFilesSize(dirPath) {\n  let totalFileSize = 0;\n\n  const files = await fs.readdir(dirPath);\n\n  for (const file of files) {\n    const fileStat = await fs.stat(join(dirPath, file));\n    if (fileStat.isDirectory()) {\n      totalFileSize += await getTotalFilesSize(join(dirPath, file));\n    } else {\n      totalFileSize += fileStat.size;\n    }\n  }\n  return totalFileSize;\n}\n"],"mappings":"AAAA,SAAQA,IAAI,QAAO,MAAM;AACzB,SAAQC,QAAQ,IAAIC,EAAE,QAAO,IAAI;AAAC,SAC1BC,mBAAmB;AAO3B,OAAO,SAASC,aAAaA,CAACC,IAA+B,EAAU;EACrE,IAAI,OAAOA,IAAI,KAAK,QAAQ,EAAE;IAE5B,MAAMC,oBAAoB,GAAG,GAAG;IAChC,MAAMC,aAAa,GAAGC,IAAI,CAACC,KAAK,CAACJ,IAAI,GAAGC,oBAAoB,CAAC;IAC7D,MAAMI,YAAY,GAAGL,IAAI,GAAGE,aAAa,GAAGD,oBAAoB;IAChE,OAAOK,uCAAuC,CAACJ,aAAa,EAAEG,YAAY,CAAC;EAC7E,CAAC,MAAM;IAEL,MAAME,wBAAwB,GAAG,GAAG;IACpC,MAAML,aAAa,GAAGF,IAAI,CAAC,CAAC,CAAC;IAC7B,MAAMK,YAAY,GAAGL,IAAI,CAAC,CAAC,CAAC,GAAGO,wBAAwB;IACvD,OAAOD,uCAAuC,CAACJ,aAAa,EAAEG,YAAY,CAAC;EAC7E;AACF;AAEA,SAASC,uCAAuCA,CAACJ,aAAqB,EAAEG,YAAoB,EAAE;EAC5F,MAAMG,KAAK,GAAGL,IAAI,CAACC,KAAK,CAACF,aAAa,GAAG,IAAI,CAAC;EAC9CA,aAAa,GAAGA,aAAa,GAAGM,KAAK,GAAG,IAAI;EAC5C,MAAMC,OAAO,GAAGN,IAAI,CAACC,KAAK,CAACF,aAAa,GAAG,EAAE,CAAC;EAC9CA,aAAa,GAAGA,aAAa,GAAGO,OAAO,GAAG,EAAE;EAC5C,MAAMC,OAAO,GAAGP,IAAI,CAACC,KAAK,CAACF,aAAa,CAAC;EACzC,IAAIS,MAAM,GAAG,EAAE;EAEf,IAAIH,KAAK,EAAE;IACTG,MAAM,IAAK,GAAEH,KAAM,IAAG;EACxB;EAEA,IAAIC,OAAO,EAAE;IACXE,MAAM,IAAK,GAAEF,OAAQ,IAAG;EAC1B;EAEA,IAAIC,OAAO,EAAE;IACXC,MAAM,IAAK,GAAED,OAAQ,GAAE;EACzB;EAEA,IAAI,CAACC,MAAM,EAAE;IACXA,MAAM,IAAK,GAAER,IAAI,CAACC,KAAK,CAACC,YAAY,CAAE,IAAG;EAC3C;EAEA,OAAOM,MAAM;AACf;AAEA,OAAO,eAAeC,kBAAkBA,CAACC,MAAM,EAAE;EAC/C,MAAM;IAACC,IAAI;IAAEC,UAAU;IAAEC;EAAW,CAAC,GAAGH,MAAM;EAC9C,MAAMI,cAAc,GAAGnB,mBAAmB,CAACiB,UAAU,CAAC;EAEtD,IAAI;IACF,IAAID,IAAI,EAAE;MACR,MAAMI,QAAQ,GAAGvB,IAAI,CAACsB,cAAc,EAAG,GAAED,WAAY,OAAM,CAAC;MAC5D,MAAMG,IAAI,GAAG,MAAMtB,EAAE,CAACsB,IAAI,CAACD,QAAQ,CAAC;MACpC,OAAOC,IAAI,CAACC,IAAI;IAClB;IAEA,MAAMC,aAAa,GAAG1B,IAAI,CAACsB,cAAc,EAAED,WAAW,CAAC;IACvD,MAAMM,SAAS,GAAG,MAAMC,iBAAiB,CAACF,aAAa,CAAC;IACxD,OAAOC,SAAS;EAClB,CAAC,CAAC,OAAOE,KAAK,EAAE;IACdC,OAAO,CAACC,GAAG,CAAC,8BAA8B,EAAEF,KAAK,CAAC;IAClD,OAAO,IAAI;EACb;AACF;AAEA,eAAeD,iBAAiBA,CAACI,OAAO,EAAE;EACxC,IAAIC,aAAa,GAAG,CAAC;EAErB,MAAMC,KAAK,GAAG,MAAMhC,EAAE,CAACiC,OAAO,CAACH,OAAO,CAAC;EAEvC,KAAK,MAAMI,IAAI,IAAIF,KAAK,EAAE;IACxB,MAAMG,QAAQ,GAAG,MAAMnC,EAAE,CAACsB,IAAI,CAACxB,IAAI,CAACgC,OAAO,EAAEI,IAAI,CAAC,CAAC;IACnD,IAAIC,QAAQ,CAACC,WAAW,CAAC,CAAC,EAAE;MAC1BL,aAAa,IAAI,MAAML,iBAAiB,CAAC5B,IAAI,CAACgC,OAAO,EAAEI,IAAI,CAAC,CAAC;IAC/D,CAAC,MAAM;MACLH,aAAa,IAAII,QAAQ,CAACZ,IAAI;IAChC;EACF;EACA,OAAOQ,aAAa;AACtB"}

package/dist/lib/utils/write-queue.d.ts

@@ -1,6 +1,10 @@
 import { Queue } from './queue';
+import { ConversionDump } from './conversion-dump';
 export type WriteQueueItem = {
     archiveKey?: string;
+    sourceId?: string;
+    outputId?: number;
+    resourceType?: string;
     /**
      * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);
      * Arguments for this call are specified in writeQueue.enqueue call like this:
@@ -21,13 +25,14 @@ export type WriteQueueItem = {
 };
 export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
     private intervalId?;
+    private conversionDump;
     writePromise: Promise<void> | null;
     fileMap: {
         [key: string]: string;
     };
     listeningInterval: number;
     writeConcurrency: number;
-    constructor(listeningInterval?: number, writeConcurrency?: number);
+    constructor(conversionDump: ConversionDump, listeningInterval?: number, writeConcurrency?: number);
     enqueue(val: T, writeImmediately?: boolean): Promise<void>;
     startListening(): void;
     stopListening(): void;

package/dist/lib/utils/write-queue.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"write-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/write-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,EAAC,MAAM,SAAS,CAAC;AAM9B,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;;;;;;;;;;;;OAeG;IACH,YAAY,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,CAAC,SAAS,cAAc,CAAE,SAAQ,KAAK,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,UAAU,CAAC,CAAiB;IAC7B,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;IAC1C,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAM;IACtC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;gBAEpB,iBAAiB,GAAE,MAAa,EAAE,gBAAgB,GAAE,MAAY;IAMtE,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,gBAAgB,GAAE,OAAe;IAgBvD,cAAc;IAId,aAAa;IAMP,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAKjB,OAAO;IAmBrB,OAAO,CAAC,aAAa;CAWtB"}
+{"version":3,"file":"write-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/write-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,EAAC,MAAM,SAAS,CAAC;AAE9B,OAAO,EAAC,cAAc,EAAC,MAAM,mBAAmB,CAAC;AAKjD,MAAM,MAAM,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB;;;;;;;;;;;;;;;OAeG;IACH,YAAY,EAAE,MAAM,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;CAC5C,CAAC;AAEF,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,CAAC,SAAS,cAAc,CAAE,SAAQ,KAAK,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,UAAU,CAAC,CAAiB;IACpC,OAAO,CAAC,cAAc,CAAiB;IAChC,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;IAC1C,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAM;IACtC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;gBAG9B,cAAc,EAAE,cAAc,EAC9B,iBAAiB,GAAE,MAAa,EAChC,gBAAgB,GAAE,MAAY;IAQ1B,OAAO,CAAC,GAAG,EAAE,CAAC,EAAE,gBAAgB,GAAE,OAAe;IAgBvD,cAAc;IAId,aAAa;IAMP,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ3B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAKjB,OAAO;IAsBrB,OAAO,CAAC,aAAa;CAWtB"}

package/dist/lib/utils/write-queue.js

@@ -2,15 +2,17 @@ import { Queue } from "./queue.js";
 import process from 'process';
 const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024;
 export default class WriteQueue extends Queue {
-  constructor() {
-    let listeningInterval = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 2000;
-    let writeConcurrency = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 400;
+  constructor(conversionDump) {
+    let listeningInterval = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 2000;
+    let writeConcurrency = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 400;
     super();
     this.intervalId = void 0;
+    this.conversionDump = void 0;
     this.writePromise = null;
     this.fileMap = {};
     this.listeningInterval = void 0;
     this.writeConcurrency = void 0;
+    this.conversionDump = conversionDump;
     this.listeningInterval = listeningInterval;
     this.writeConcurrency = writeConcurrency;
   }
@@ -55,6 +57,7 @@ export default class WriteQueue extends Queue {
     while (this.length) {
       const promises = [];
       const archiveKeys = [];
+      const changedRecords = [];
       for (let i = 0; i < this.writeConcurrency; i++) {
         const item = this.dequeue();
         if (!item) {
@@ -62,14 +65,23 @@ export default class WriteQueue extends Queue {
         }
         const {
           archiveKey,
+          sourceId,
+          outputId,
+          resourceType,
           writePromise
         } = item;
         archiveKeys.push(archiveKey);
+        changedRecords.push({
+          sourceId,
+          outputId,
+          resourceType
+        });
         const promise = writePromise();
         promises.push(promise);
       }
       const writeResults = await Promise.allSettled(promises);
       this.updateFileMap(archiveKeys, writeResults);
+      await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults);
     }
   }
   updateFileMap(archiveKeys, writeResults) {

package/dist/lib/utils/write-queue.js.map

@@ -1 +1 @@
-{"version":3,"file":"write-queue.js","names":["Queue","process","MEMORY_LIMIT","WriteQueue","constructor","listeningInterval","arguments","length","undefined","writeConcurrency","intervalId","writePromise","fileMap","enqueue","val","writeImmediately","archiveKey","result","memoryUsage","rss","startWrite","startListening","setInterval","bind","stopListening","clearInterval","doWrite","finalize","promises","archiveKeys","i","item","dequeue","push","promise","writeResults","Promise","allSettled","updateFileMap","value"],"sources":["../../../src/lib/utils/write-queue.ts"],"sourcesContent":["import {Queue} from './queue';\nimport process from 'process';\n\n/** Memory limit size is based on testing */\nconst MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB\n\nexport type WriteQueueItem = {\n  archiveKey?: string;\n  /**\n   * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);\n   * Arguments for this call are specified in writeQueue.enqueue call like this:\n   * await writeQueue.enqueue({\n   *     archiveKey: `nodePages/xxx.json.gz`,\n   *     writePromise: () => writeFileForSlpk(slpkPath, data, `xxx.json`)\n   * });\n   * Note, a function like writeFileForSlpk should NOT be called when initializing the object for enqueue().\n   * If he function is called, the promise will be created\n   * and the function will allocate resources (file descriptors) for file writing.\n   * It will be done for ALL items in the queue, which is not supposed to happen.\n   * That's why the function should be passed as\n   *   writePromise: () => writeFileForSlpk(slpkPath, content, `xxx.json`)\n   * instead of\n   *  writePromise: writeFileForSlpk(slpkPath, content, `xxx.json`) // INCORRECT !\n   */\n  writePromise: () => Promise<string | null>;\n};\n\nexport default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {\n  private intervalId?: NodeJS.Timeout;\n  public writePromise: Promise<void> | null = null;\n  public fileMap: {[key: string]: string} = {};\n  public listeningInterval: number;\n  public writeConcurrency: number;\n\n  constructor(listeningInterval: number = 2000, writeConcurrency: number = 400) {\n    super();\n    this.listeningInterval = listeningInterval;\n    this.writeConcurrency = writeConcurrency;\n  }\n\n  async enqueue(val: T, writeImmediately: boolean = false) {\n    if (writeImmediately) {\n      const {archiveKey, writePromise} = val as WriteQueueItem;\n      const result = await writePromise();\n      if (archiveKey && result) {\n        this.fileMap[archiveKey] = result;\n      }\n    } else {\n      super.enqueue(val);\n      /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */\n      if (process.memoryUsage().rss > MEMORY_LIMIT) {\n        await this.startWrite();\n      }\n    }\n  }\n\n  startListening() {\n    this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);\n  }\n\n  stopListening() {\n    if (this.intervalId) {\n      clearInterval(this.intervalId);\n    }\n  }\n\n  async startWrite(): Promise<void> {\n    if (!this.writePromise) {\n      this.writePromise = this.doWrite();\n    }\n    await this.writePromise;\n    this.writePromise = null;\n  }\n\n  async finalize(): Promise<void> {\n    this.stopListening();\n    await this.startWrite();\n  }\n\n  private async doWrite(): Promise<void> {\n    while (this.length) {\n      const promises: Promise<string | null>[] = [];\n      const archiveKeys: (string | undefined)[] = [];\n      for (let i = 0; i < this.writeConcurrency; i++) {\n        const item = this.dequeue();\n        if (!item) {\n          break;\n        }\n        const {archiveKey, writePromise} = item as WriteQueueItem;\n        archiveKeys.push(archiveKey);\n        const promise = writePromise();\n        promises.push(promise);\n      }\n      const writeResults = await Promise.allSettled(promises);\n      this.updateFileMap(archiveKeys, writeResults);\n    }\n  }\n\n  private updateFileMap(\n    archiveKeys: (string | undefined)[],\n    writeResults: PromiseSettledResult<string | null>[]\n  ) {\n    for (let i = 0; i < archiveKeys.length; i++) {\n      const archiveKey = archiveKeys[i];\n      if (archiveKey && 'value' in writeResults[i]) {\n        this.fileMap[archiveKey] = (writeResults[i] as PromiseFulfilledResult<string>).value;\n      }\n    }\n  }\n}\n"],"mappings":"SAAQA,KAAK;AACb,OAAOC,OAAO,MAAM,SAAS;AAG7B,MAAMC,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AAuB3C,eAAe,MAAMC,UAAU,SAAmCH,KAAK,CAAI;EAOzEI,WAAWA,CAAA,EAAmE;IAAA,IAAlEC,iBAAyB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,IAAI;IAAA,IAAEG,gBAAwB,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,GAAG;IAC1E,KAAK,CAAC,CAAC;IAAC,KAPFI,UAAU;IAAA,KACXC,YAAY,GAAyB,IAAI;IAAA,KACzCC,OAAO,GAA4B,CAAC,CAAC;IAAA,KACrCP,iBAAiB;IAAA,KACjBI,gBAAgB;IAIrB,IAAI,CAACJ,iBAAiB,GAAGA,iBAAiB;IAC1C,IAAI,CAACI,gBAAgB,GAAGA,gBAAgB;EAC1C;EAEA,MAAMI,OAAOA,CAACC,GAAM,EAAqC;IAAA,IAAnCC,gBAAyB,GAAAT,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,KAAK;IACrD,IAAIS,gBAAgB,EAAE;MACpB,MAAM;QAACC,UAAU;QAAEL;MAAY,CAAC,GAAGG,GAAqB;MACxD,MAAMG,MAAM,GAAG,MAAMN,YAAY,CAAC,CAAC;MACnC,IAAIK,UAAU,IAAIC,MAAM,EAAE;QACxB,IAAI,CAACL,OAAO,CAACI,UAAU,CAAC,GAAGC,MAAM;MACnC;IACF,CAAC,MAAM;MACL,KAAK,CAACJ,OAAO,CAACC,GAAG,CAAC;MAElB,IAAIb,OAAO,CAACiB,WAAW,CAAC,CAAC,CAACC,GAAG,GAAGjB,YAAY,EAAE;QAC5C,MAAM,IAAI,CAACkB,UAAU,CAAC,CAAC;MACzB;IACF;EACF;EAEAC,cAAcA,CAAA,EAAG;IACf,IAAI,CAACX,UAAU,GAAGY,WAAW,CAAC,IAAI,CAACF,UAAU,CAACG,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAClB,iBAAiB,CAAC;EACnF;EAEAmB,aAAaA,CAAA,EAAG;IACd,IAAI,IAAI,CAACd,UAAU,EAAE;MACnBe,aAAa,CAAC,IAAI,CAACf,UAAU,CAAC;IAChC;EACF;EAEA,MAAMU,UAAUA,CAAA,EAAkB;IAChC,IAAI,CAAC,IAAI,CAACT,YAAY,EAAE;MACtB,IAAI,CAACA,YAAY,GAAG,IAAI,CAACe,OAAO,CAAC,CAAC;IACpC;IACA,MAAM,IAAI,CAACf,YAAY;IACvB,IAAI,CAACA,YAAY,GAAG,IAAI;EAC1B;EAEA,MAAMgB,QAAQA,CAAA,EAAkB;IAC9B,IAAI,CAACH,aAAa,CAAC,CAAC;IACpB,MAAM,IAAI,CAACJ,UAAU,CAAC,CAAC;EACzB;EAEA,MAAcM,OAAOA,CAAA,EAAkB;IACrC,OAAO,IAAI,CAACnB,MAAM,EAAE;MAClB,MAAMqB,QAAkC,GAAG,EAAE;MAC7C,MAAMC,WAAmC,GAAG,EAAE;MAC9C,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG,IAAI,CAACrB,gBAAgB,EAAEqB,CAAC,EAAE,EAAE;QAC9C,MAAMC,IAAI,GAAG,IAAI,CAACC,OAAO,CAAC,CAAC;QAC3B,IAAI,CAACD,IAAI,EAAE;UACT;QACF;QACA,MAAM;UAACf,UAAU;UAAEL;QAAY,CAAC,GAAGoB,IAAsB;QACzDF,WAAW,CAACI,IAAI,CAACjB,UAAU,CAAC;QAC5B,MAAMkB,OAAO,GAAGvB,YAAY,CAAC,CAAC;QAC9BiB,QAAQ,CAACK,IAAI,CAACC,OAAO,CAAC;MACxB;MACA,MAAMC,YAAY,GAAG,MAAMC,OAAO,CAACC,UAAU,CAACT,QAAQ,CAAC;MACvD,IAAI,CAACU,aAAa,CAACT,WAAW,EAAEM,YAAY,CAAC;IAC/C;EACF;EAEQG,aAAaA,CACnBT,WAAmC,EACnCM,YAAmD,EACnD;IACA,KAAK,IAAIL,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGD,WAAW,CAACtB,MAAM,EAAEuB,CAAC,EAAE,EAAE;MAC3C,MAAMd,UAAU,GAAGa,WAAW,CAACC,CAAC,CAAC;MACjC,IAAId,UAAU,IAAI,OAAO,IAAImB,YAAY,CAACL,CAAC,CAAC,EAAE;QAC5C,IAAI,CAAClB,OAAO,CAACI,UAAU,CAAC,GAAImB,YAAY,CAACL,CAAC,CAAC,CAAoCS,KAAK;MACtF;IACF;EACF;AACF"}
+{"version":3,"file":"write-queue.js","names":["Queue","process","MEMORY_LIMIT","WriteQueue","constructor","conversionDump","listeningInterval","arguments","length","undefined","writeConcurrency","intervalId","writePromise","fileMap","enqueue","val","writeImmediately","archiveKey","result","memoryUsage","rss","startWrite","startListening","setInterval","bind","stopListening","clearInterval","doWrite","finalize","promises","archiveKeys","changedRecords","i","item","dequeue","sourceId","outputId","resourceType","push","promise","writeResults","Promise","allSettled","updateFileMap","updateConvertedTilesDump","value"],"sources":["../../../src/lib/utils/write-queue.ts"],"sourcesContent":["import {Queue} from './queue';\nimport process from 'process';\nimport {ConversionDump} from './conversion-dump';\n\n/** Memory limit size is based on testing */\nconst MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB\n\nexport type WriteQueueItem = {\n  archiveKey?: string;\n  sourceId?: string;\n  outputId?: number;\n  resourceType?: string;\n  /**\n   * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);\n   * Arguments for this call are specified in writeQueue.enqueue call like this:\n   * await writeQueue.enqueue({\n   *     archiveKey: `nodePages/xxx.json.gz`,\n   *     writePromise: () => writeFileForSlpk(slpkPath, data, `xxx.json`)\n   * });\n   * Note, a function like writeFileForSlpk should NOT be called when initializing the object for enqueue().\n   * If he function is called, the promise will be created\n   * and the function will allocate resources (file descriptors) for file writing.\n   * It will be done for ALL items in the queue, which is not supposed to happen.\n   * That's why the function should be passed as\n   *   writePromise: () => writeFileForSlpk(slpkPath, content, `xxx.json`)\n   * instead of\n   *  writePromise: writeFileForSlpk(slpkPath, content, `xxx.json`) // INCORRECT !\n   */\n  writePromise: () => Promise<string | null>;\n};\n\nexport default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {\n  private intervalId?: NodeJS.Timeout;\n  private conversionDump: ConversionDump;\n  public writePromise: Promise<void> | null = null;\n  public fileMap: {[key: string]: string} = {};\n  public listeningInterval: number;\n  public writeConcurrency: number;\n\n  constructor(\n    conversionDump: ConversionDump,\n    listeningInterval: number = 2000,\n    writeConcurrency: number = 400\n  ) {\n    super();\n    this.conversionDump = conversionDump;\n    this.listeningInterval = listeningInterval;\n    this.writeConcurrency = writeConcurrency;\n  }\n\n  async enqueue(val: T, writeImmediately: boolean = false) {\n    if (writeImmediately) {\n      const {archiveKey, writePromise} = val as WriteQueueItem;\n      const result = await writePromise();\n      if (archiveKey && result) {\n        this.fileMap[archiveKey] = result;\n      }\n    } else {\n      super.enqueue(val);\n      /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */\n      if (process.memoryUsage().rss > MEMORY_LIMIT) {\n        await this.startWrite();\n      }\n    }\n  }\n\n  startListening() {\n    this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);\n  }\n\n  stopListening() {\n    if (this.intervalId) {\n      clearInterval(this.intervalId);\n    }\n  }\n\n  async startWrite(): Promise<void> {\n    if (!this.writePromise) {\n      this.writePromise = this.doWrite();\n    }\n    await this.writePromise;\n    this.writePromise = null;\n  }\n\n  async finalize(): Promise<void> {\n    this.stopListening();\n    await this.startWrite();\n  }\n\n  private async doWrite(): Promise<void> {\n    while (this.length) {\n      const promises: Promise<string | null>[] = [];\n      const archiveKeys: (string | undefined)[] = [];\n      const changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[] = [];\n      for (let i = 0; i < this.writeConcurrency; i++) {\n        const item = this.dequeue();\n        if (!item) {\n          break;\n        }\n        const {archiveKey, sourceId, outputId, resourceType, writePromise} = item as WriteQueueItem;\n        archiveKeys.push(archiveKey);\n        changedRecords.push({sourceId, outputId, resourceType});\n        const promise = writePromise();\n        promises.push(promise);\n      }\n      const writeResults = await Promise.allSettled(promises);\n      this.updateFileMap(archiveKeys, writeResults);\n      await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults);\n    }\n  }\n\n  private updateFileMap(\n    archiveKeys: (string | undefined)[],\n    writeResults: PromiseSettledResult<string | null>[]\n  ) {\n    for (let i = 0; i < archiveKeys.length; i++) {\n      const archiveKey = archiveKeys[i];\n      if (archiveKey && 'value' in writeResults[i]) {\n        this.fileMap[archiveKey] = (writeResults[i] as PromiseFulfilledResult<string>).value;\n      }\n    }\n  }\n}\n"],"mappings":"SAAQA,KAAK;AACb,OAAOC,OAAO,MAAM,SAAS;AAI7B,MAAMC,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;AA0B3C,eAAe,MAAMC,UAAU,SAAmCH,KAAK,CAAI;EAQzEI,WAAWA,CACTC,cAA8B,EAG9B;IAAA,IAFAC,iBAAyB,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,IAAI;IAAA,IAChCG,gBAAwB,GAAAH,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,GAAG;IAE9B,KAAK,CAAC,CAAC;IAAC,KAZFI,UAAU;IAAA,KACVN,cAAc;IAAA,KACfO,YAAY,GAAyB,IAAI;IAAA,KACzCC,OAAO,GAA4B,CAAC,CAAC;IAAA,KACrCP,iBAAiB;IAAA,KACjBI,gBAAgB;IAQrB,IAAI,CAACL,cAAc,GAAGA,cAAc;IACpC,IAAI,CAACC,iBAAiB,GAAGA,iBAAiB;IAC1C,IAAI,CAACI,gBAAgB,GAAGA,gBAAgB;EAC1C;EAEA,MAAMI,OAAOA,CAACC,GAAM,EAAqC;IAAA,IAAnCC,gBAAyB,GAAAT,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAG,KAAK;IACrD,IAAIS,gBAAgB,EAAE;MACpB,MAAM;QAACC,UAAU;QAAEL;MAAY,CAAC,GAAGG,GAAqB;MACxD,MAAMG,MAAM,GAAG,MAAMN,YAAY,CAAC,CAAC;MACnC,IAAIK,UAAU,IAAIC,MAAM,EAAE;QACxB,IAAI,CAACL,OAAO,CAACI,UAAU,CAAC,GAAGC,MAAM;MACnC;IACF,CAAC,MAAM;MACL,KAAK,CAACJ,OAAO,CAACC,GAAG,CAAC;MAElB,IAAId,OAAO,CAACkB,WAAW,CAAC,CAAC,CAACC,GAAG,GAAGlB,YAAY,EAAE;QAC5C,MAAM,IAAI,CAACmB,UAAU,CAAC,CAAC;MACzB;IACF;EACF;EAEAC,cAAcA,CAAA,EAAG;IACf,IAAI,CAACX,UAAU,GAAGY,WAAW,CAAC,IAAI,CAACF,UAAU,CAACG,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAClB,iBAAiB,CAAC;EACnF;EAEAmB,aAAaA,CAAA,EAAG;IACd,IAAI,IAAI,CAACd,UAAU,EAAE;MACnBe,aAAa,CAAC,IAAI,CAACf,UAAU,CAAC;IAChC;EACF;EAEA,MAAMU,UAAUA,CAAA,EAAkB;IAChC,IAAI,CAAC,IAAI,CAACT,YAAY,EAAE;MACtB,IAAI,CAACA,YAAY,GAAG,IAAI,CAACe,OAAO,CAAC,CAAC;IACpC;IACA,MAAM,IAAI,CAACf,YAAY;IACvB,IAAI,CAACA,YAAY,GAAG,IAAI;EAC1B;EAEA,MAAMgB,QAAQA,CAAA,EAAkB;IAC9B,IAAI,CAACH,aAAa,CAAC,CAAC;IACpB,MAAM,IAAI,CAACJ,UAAU,CAAC,CAAC;EACzB;EAEA,MAAcM,OAAOA,CAAA,EAAkB;IACrC,OAAO,IAAI,CAACnB,MAAM,EAAE;MAClB,MAAMqB,QAAkC,GAAG,EAAE;MAC7C,MAAMC,WAAmC,GAAG,EAAE;MAC9C,MAAMC,cAA+E,GAAG,EAAE;MAC1F,KAAK,IAAIC,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAG,IAAI,CAACtB,gBAAgB,EAAEsB,CAAC,EAAE,EAAE;QAC9C,MAAMC,IAAI,GAAG,IAAI,CAACC,OAAO,CAAC,CAAC;QAC3B,IAAI,CAACD,IAAI,EAAE;UACT;QACF;QACA,MAAM;UAAChB,UAAU;UAAEkB,QAAQ;UAAEC,QAAQ;UAAEC,YAAY;UAAEzB;QAAY,CAAC,GAAGqB,IAAsB;QAC3FH,WAAW,CAACQ,IAAI,CAACrB,UAAU,CAAC;QAC5Bc,cAAc,CAACO,IAAI,CAAC;UAACH,QAAQ;UAAEC,QAAQ;UAAEC;QAAY,CAAC,CAAC;QACvD,MAAME,OAAO,GAAG3B,YAAY,CAAC,CAAC;QAC9BiB,QAAQ,CAACS,IAAI,CAACC,OAAO,CAAC;MACxB;MACA,MAAMC,YAAY,GAAG,MAAMC,OAAO,CAACC,UAAU,CAACb,QAAQ,CAAC;MACvD,IAAI,CAACc,aAAa,CAACb,WAAW,EAAEU,YAAY,CAAC;MAC7C,MAAM,IAAI,CAACnC,cAAc,CAACuC,wBAAwB,CAACb,cAAc,EAAES,YAAY,CAAC;IAClF;EACF;EAEQG,aAAaA,CACnBb,WAAmC,EACnCU,YAAmD,EACnD;IACA,KAAK,IAAIR,CAAC,GAAG,CAAC,EAAEA,CAAC,GAAGF,WAAW,CAACtB,MAAM,EAAEwB,CAAC,EAAE,EAAE;MAC3C,MAAMf,UAAU,GAAGa,WAAW,CAACE,CAAC,CAAC;MACjC,IAAIf,UAAU,IAAI,OAAO,IAAIuB,YAAY,CAACR,CAAC,CAAC,EAAE;QAC5C,IAAI,CAACnB,OAAO,CAACI,UAAU,CAAC,GAAIuB,YAAY,CAACR,CAAC,CAAC,CAAoCa,KAAK;MACtF;IACF;EACF;AACF"}

package/dist/pgm-loader.js

@@ -1,5 +1,5 @@
 import { Geoid, parsePGM } from '@math.gl/geoid';
-const VERSION = typeof "4.1.0-alpha.9" !== 'undefined' ? "4.1.0-alpha.9" : 'latest';
+const VERSION = typeof "4.2.0-alpha.1" !== 'undefined' ? "4.2.0-alpha.1" : 'latest';
 export { Geoid };
 export const PGMLoader = {
   name: 'PGM - Netpbm grayscale image format',