@loaders.gl/tile-converter 3.2.5 → 3.2.6

package/dist/i3s-converter/helpers/node-pages.js

@@ -184,14 +184,14 @@ class NodePages {
             for (const [index, nodePage] of this.nodePages.entries()) {
                 const nodePageStr = JSON.stringify(nodePage);
                 const slpkPath = (0, path_1.join)(layers0Path, 'nodepages');
-                writeQueue.enqueue({
+                await writeQueue.enqueue({
                     archiveKey: `nodePages/${index.toString()}.json.gz`,
                     writePromise: this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
                 });
             }
             const metadata = (0, json_map_transform_1.default)({ nodeCount: this.nodesCounter }, (0, metadata_1.METADATA)());
             const compress = false;
-            writeQueue.enqueue({
+            await writeQueue.enqueue({
                 archiveKey: 'metadata.json',
                 writePromise: this.writeFile(layers0Path, JSON.stringify(metadata), 'metadata.json', compress)
             });
@@ -200,7 +200,7 @@ class NodePages {
             for (const [index, nodePage] of this.nodePages.entries()) {
                 const nodePageStr = JSON.stringify(nodePage);
                 const nodePagePath = (0, path_1.join)(layers0Path, 'nodepages', index.toString());
-                writeQueue.enqueue({ writePromise: this.writeFile(nodePagePath, nodePageStr) });
+                await writeQueue.enqueue({ writePromise: this.writeFile(nodePagePath, nodePageStr) });
             }
         }
     }

package/dist/i3s-converter/i3s-converter.js

@@ -240,13 +240,13 @@ class I3SConverter {
             const [child] = await this._createNode(root0, sourceRootTile, parentId, 0);
             const childPath = (0, path_1.join)(this.layers0Path, 'nodes', child.path);
             if (this.options.slpk) {
-                this.writeQueue.enqueue({
+                await this.writeQueue.enqueue({
                     archiveKey: 'nodes/1/3dNodeIndexDocument.json.gz',
                     writePromise: (0, file_utils_1.writeFileForSlpk)(childPath, JSON.stringify(child), '3dNodeIndexDocument.json')
                 });
             }
             else {
-                this.writeQueue.enqueue({ writePromise: (0, file_utils_1.writeFile)(childPath, JSON.stringify(child)) });
+                await this.writeQueue.enqueue({ writePromise: (0, file_utils_1.writeFile)(childPath, JSON.stringify(child)) });
             }
         }
         else {
@@ -264,13 +264,13 @@ class I3SConverter {
      */
     async _writeLayers0() {
         if (this.options.slpk) {
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 archiveKey: '3dSceneLayer.json.gz',
                 writePromise: (0, file_utils_1.writeFileForSlpk)(this.layers0Path, JSON.stringify(this.layers0), '3dSceneLayer.json')
             });
         }
         else {
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 writePromise: (0, file_utils_1.writeFile)(this.layers0Path, JSON.stringify(this.layers0))
             });
         }
@@ -280,13 +280,13 @@ class I3SConverter {
      */
     async _writeNodeIndexDocument(root0, nodePath, rootPath) {
         if (this.options.slpk) {
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 archiveKey: `nodes/${nodePath}/3dNodeIndexDocument.json.gz`,
                 writePromise: (0, file_utils_1.writeFileForSlpk)(rootPath, JSON.stringify(root0), '3dNodeIndexDocument.json')
             });
         }
         else {
-            this.writeQueue.enqueue({ writePromise: (0, file_utils_1.writeFile)(rootPath, JSON.stringify(root0)) });
+            await this.writeQueue.enqueue({ writePromise: (0, file_utils_1.writeFile)(rootPath, JSON.stringify(root0)) });
         }
     }
     /**
@@ -636,28 +636,28 @@ class I3SConverter {
     async _writeGeometries(geometryBuffer, compressedGeometry, childPath, slpkChildPath) {
         if (this.options.slpk) {
             const slpkGeometryPath = (0, path_1.join)(childPath, 'geometries');
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 archiveKey: `${slpkChildPath}/geometries/0.bin.gz`,
                 writePromise: (0, file_utils_1.writeFileForSlpk)(slpkGeometryPath, geometryBuffer, '0.bin')
             });
         }
         else {
             const geometryPath = (0, path_1.join)(childPath, 'geometries/0/');
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 writePromise: (0, file_utils_1.writeFile)(geometryPath, geometryBuffer, 'index.bin')
             });
         }
         if (this.options.draco) {
             if (this.options.slpk) {
                 const slpkCompressedGeometryPath = (0, path_1.join)(childPath, 'geometries');
-                this.writeQueue.enqueue({
+                await this.writeQueue.enqueue({
                     archiveKey: `${slpkChildPath}/geometries/1.bin.gz`,
                     writePromise: (0, file_utils_1.writeFileForSlpk)(slpkCompressedGeometryPath, compressedGeometry, '1.bin')
                 });
             }
             else {
                 const compressedGeometryPath = (0, path_1.join)(childPath, 'geometries/1/');
-                this.writeQueue.enqueue({
+                await this.writeQueue.enqueue({
                     writePromise: (0, file_utils_1.writeFile)(compressedGeometryPath, compressedGeometry, 'index.bin')
                 });
             }
@@ -679,14 +679,14 @@ class I3SConverter {
         const sharedDataStr = JSON.stringify(sharedData);
         if (this.options.slpk) {
             const slpkSharedPath = (0, path_1.join)(childPath, 'shared');
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 archiveKey: `${slpkChildPath}/shared/sharedResource.json.gz`,
                 writePromise: (0, file_utils_1.writeFileForSlpk)(slpkSharedPath, sharedDataStr, 'sharedResource.json')
             });
         }
         else {
             const sharedPath = (0, path_1.join)(childPath, 'shared/');
-            this.writeQueue.enqueue({ writePromise: (0, file_utils_1.writeFile)(sharedPath, sharedDataStr) });
+            await this.writeQueue.enqueue({ writePromise: (0, file_utils_1.writeFile)(sharedPath, sharedDataStr) });
         }
     }
     /**
@@ -744,14 +744,14 @@ class I3SConverter {
         if (this.options.slpk) {
             const slpkTexturePath = (0, path_1.join)(childPath, 'textures');
             const compress = false;
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 archiveKey: `${slpkChildPath}/textures/${name}.${format}`,
                 writePromise: (0, file_utils_1.writeFileForSlpk)(slpkTexturePath, textureData, `${name}.${format}`, compress)
             });
         }
         else {
             const texturePath = (0, path_1.join)(childPath, `textures/${name}/`);
-            this.writeQueue.enqueue({
+            await this.writeQueue.enqueue({
                 writePromise: (0, file_utils_1.writeFile)(texturePath, textureData, `index.${format}`)
             });
         }
@@ -769,14 +769,14 @@ class I3SConverter {
                 const fileBuffer = new Uint8Array(attributes[index]);
                 if (this.options.slpk) {
                     const slpkAttributesPath = (0, path_1.join)(childPath, 'attributes', folderName);
-                    this.writeQueue.enqueue({
+                    await this.writeQueue.enqueue({
                         archiveKey: `${slpkChildPath}/attributes/${folderName}.bin.gz`,
                         writePromise: (0, file_utils_1.writeFileForSlpk)(slpkAttributesPath, fileBuffer, '0.bin')
                     });
                 }
                 else {
                     const attributesPath = (0, path_1.join)(childPath, `attributes/${folderName}/0`);
-                    this.writeQueue.enqueue({
+                    await this.writeQueue.enqueue({
                         writePromise: (0, file_utils_1.writeFile)(attributesPath, fileBuffer, 'index.bin')
                     });
                 }

package/dist/lib/utils/write-queue.d.ts

@@ -12,6 +12,7 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
     listeningInterval: number;
     writeConcurrency: number;
     constructor(listeningInterval?: number, writeConcurrency?: number);
+    enqueue(val: T): Promise<void>;
     startListening(): void;
     stopListening(): void;
     startWrite(): Promise<void>;

package/dist/lib/utils/write-queue.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"write-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/write-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,EAAC,MAAM,SAAS,CAAC;AAE9B,oBAAY,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/B,CAAC;AAEF,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,CAAC,SAAS,cAAc,CAAE,SAAQ,KAAK,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,UAAU,CAAC,CAAiB;IAC7B,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;IAC1C,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAM;IACtC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;gBAEpB,iBAAiB,GAAE,MAAa,EAAE,gBAAgB,GAAE,MAAY;IAM5E,cAAc;IAId,aAAa;IAMP,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAKjB,OAAO;IAmBrB,OAAO,CAAC,aAAa;CAStB"}
+{"version":3,"file":"write-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/write-queue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,KAAK,EAAC,MAAM,SAAS,CAAC;AAM9B,oBAAY,cAAc,GAAG;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/B,CAAC;AAEF,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,CAAC,SAAS,cAAc,CAAE,SAAQ,KAAK,CAAC,CAAC,CAAC;IACxE,OAAO,CAAC,UAAU,CAAC,CAAiB;IAC7B,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAQ;IAC1C,OAAO,EAAE;QAAC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAA;KAAC,CAAM;IACtC,iBAAiB,EAAE,MAAM,CAAC;IAC1B,gBAAgB,EAAE,MAAM,CAAC;gBAEpB,iBAAiB,GAAE,MAAa,EAAE,gBAAgB,GAAE,MAAY;IAMtE,OAAO,CAAC,GAAG,EAAE,CAAC;IAQpB,cAAc;IAId,aAAa;IAMP,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAW3B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAKjB,OAAO;IAmBrB,OAAO,CAAC,aAAa;CAStB"}

package/dist/lib/utils/write-queue.js

@@ -1,6 +1,12 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const queue_1 = require("./queue");
+const process_1 = __importDefault(require("process"));
+/** Memory limit size is based on testing */
+const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
 class WriteQueue extends queue_1.Queue {
     constructor(listeningInterval = 2000, writeConcurrency = 400) {
         super();
@@ -9,6 +15,13 @@ class WriteQueue extends queue_1.Queue {
         this.listeningInterval = listeningInterval;
         this.writeConcurrency = writeConcurrency;
     }
+    async enqueue(val) {
+        super.enqueue(val);
+        /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */
+        if (process_1.default.memoryUsage().rss > MEMORY_LIMIT) {
+            await this.startWrite();
+        }
+    }
     startListening() {
         this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "3.2.5",
+  "version": "3.2.6",
   "description": "Converter",
   "license": "MIT",
   "publishConfig": {
@@ -43,19 +43,19 @@
     "build-3d-tiles-attributes-worker": "esbuild src/workers/3d-tiles-attributes-worker.ts --outfile=dist/3d-tiles-attributes-worker.js --platform=node --target=esnext,node12 --minify --bundle --sourcemap --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "3.2.5",
-    "@loaders.gl/crypto": "3.2.5",
-    "@loaders.gl/draco": "3.2.5",
-    "@loaders.gl/gltf": "3.2.5",
-    "@loaders.gl/i3s": "3.2.5",
-    "@loaders.gl/images": "3.2.5",
-    "@loaders.gl/loader-utils": "3.2.5",
-    "@loaders.gl/polyfills": "3.2.5",
-    "@loaders.gl/schema": "3.2.5",
-    "@loaders.gl/textures": "3.2.5",
-    "@loaders.gl/tiles": "3.2.5",
-    "@loaders.gl/worker-utils": "3.2.5",
-    "@loaders.gl/zip": "3.2.5",
+    "@loaders.gl/3d-tiles": "3.2.6",
+    "@loaders.gl/crypto": "3.2.6",
+    "@loaders.gl/draco": "3.2.6",
+    "@loaders.gl/gltf": "3.2.6",
+    "@loaders.gl/i3s": "3.2.6",
+    "@loaders.gl/images": "3.2.6",
+    "@loaders.gl/loader-utils": "3.2.6",
+    "@loaders.gl/polyfills": "3.2.6",
+    "@loaders.gl/schema": "3.2.6",
+    "@loaders.gl/textures": "3.2.6",
+    "@loaders.gl/tiles": "3.2.6",
+    "@loaders.gl/worker-utils": "3.2.6",
+    "@loaders.gl/zip": "3.2.6",
     "@luma.gl/engine": "^8.5.4",
     "@math.gl/core": "^3.5.1",
     "@math.gl/culling": "^3.5.1",
@@ -75,5 +75,5 @@
   "peerDependencies": {
     "@loaders.gl/core": "^3.2.0"
   },
-  "gitHead": "1c084200e8b87479d06526c7107e75c6bff8968b"
+  "gitHead": "4c7303904d6d4ea57082cf3c7a66e780b008edb9"
 }

package/src/i3s-converter/helpers/node-pages.ts

@@ -204,14 +204,14 @@ export default class NodePages {
       for (const [index, nodePage] of this.nodePages.entries()) {
         const nodePageStr = JSON.stringify(nodePage);
         const slpkPath = join(layers0Path, 'nodepages');
-        writeQueue.enqueue({
+        await writeQueue.enqueue({
           archiveKey: `nodePages/${index.toString()}.json.gz`,
           writePromise: this.writeFile(slpkPath, nodePageStr, `${index.toString()}.json`)
         });
       }
       const metadata = transform({nodeCount: this.nodesCounter}, metadataTemplate());
       const compress = false;
-      writeQueue.enqueue({
+      await writeQueue.enqueue({
         archiveKey: 'metadata.json',
         writePromise: this.writeFile(
           layers0Path,
@@ -224,7 +224,7 @@ export default class NodePages {
       for (const [index, nodePage] of this.nodePages.entries()) {
         const nodePageStr = JSON.stringify(nodePage);
         const nodePagePath = join(layers0Path, 'nodepages', index.toString());
-        writeQueue.enqueue({writePromise: this.writeFile(nodePagePath, nodePageStr)});
+        await writeQueue.enqueue({writePromise: this.writeFile(nodePagePath, nodePageStr)});
       }
     }
   }

package/src/i3s-converter/i3s-converter.ts

@@ -341,7 +341,7 @@ export default class I3SConverter {
       const childPath = join(this.layers0Path, 'nodes', child.path!);
 
       if (this.options.slpk) {
-        this.writeQueue.enqueue({
+        await this.writeQueue.enqueue({
           archiveKey: 'nodes/1/3dNodeIndexDocument.json.gz',
           writePromise: writeFileForSlpk(
             childPath,
@@ -350,7 +350,7 @@ export default class I3SConverter {
           )
         });
       } else {
-        this.writeQueue.enqueue({writePromise: writeFile(childPath, JSON.stringify(child))});
+        await this.writeQueue.enqueue({writePromise: writeFile(childPath, JSON.stringify(child))});
       }
     } else {
       await this._addChildrenWithNeighborsAndWriteFile({
@@ -368,7 +368,7 @@ export default class I3SConverter {
    */
   private async _writeLayers0(): Promise<void> {
     if (this.options.slpk) {
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         archiveKey: '3dSceneLayer.json.gz',
         writePromise: writeFileForSlpk(
           this.layers0Path,
@@ -377,7 +377,7 @@ export default class I3SConverter {
         )
       });
     } else {
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         writePromise: writeFile(this.layers0Path, JSON.stringify(this.layers0))
       });
     }
@@ -392,12 +392,12 @@ export default class I3SConverter {
     rootPath: string
   ): Promise<void> {
     if (this.options.slpk) {
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         archiveKey: `nodes/${nodePath}/3dNodeIndexDocument.json.gz`,
         writePromise: writeFileForSlpk(rootPath, JSON.stringify(root0), '3dNodeIndexDocument.json')
       });
     } else {
-      this.writeQueue.enqueue({writePromise: writeFile(rootPath, JSON.stringify(root0))});
+      await this.writeQueue.enqueue({writePromise: writeFile(rootPath, JSON.stringify(root0))});
     }
   }
 
@@ -856,13 +856,13 @@ export default class I3SConverter {
   ): Promise<void> {
     if (this.options.slpk) {
       const slpkGeometryPath = join(childPath, 'geometries');
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         archiveKey: `${slpkChildPath}/geometries/0.bin.gz`,
         writePromise: writeFileForSlpk(slpkGeometryPath, geometryBuffer, '0.bin')
       });
     } else {
       const geometryPath = join(childPath, 'geometries/0/');
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         writePromise: writeFile(geometryPath, geometryBuffer, 'index.bin')
       });
     }
@@ -870,13 +870,13 @@ export default class I3SConverter {
     if (this.options.draco) {
       if (this.options.slpk) {
         const slpkCompressedGeometryPath = join(childPath, 'geometries');
-        this.writeQueue.enqueue({
+        await this.writeQueue.enqueue({
           archiveKey: `${slpkChildPath}/geometries/1.bin.gz`,
           writePromise: writeFileForSlpk(slpkCompressedGeometryPath, compressedGeometry, '1.bin')
         });
       } else {
         const compressedGeometryPath = join(childPath, 'geometries/1/');
-        this.writeQueue.enqueue({
+        await this.writeQueue.enqueue({
           writePromise: writeFile(compressedGeometryPath, compressedGeometry, 'index.bin')
         });
       }
@@ -904,13 +904,13 @@ export default class I3SConverter {
     const sharedDataStr = JSON.stringify(sharedData);
     if (this.options.slpk) {
       const slpkSharedPath = join(childPath, 'shared');
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         archiveKey: `${slpkChildPath}/shared/sharedResource.json.gz`,
         writePromise: writeFileForSlpk(slpkSharedPath, sharedDataStr, 'sharedResource.json')
       });
     } else {
       const sharedPath = join(childPath, 'shared/');
-      this.writeQueue.enqueue({writePromise: writeFile(sharedPath, sharedDataStr)});
+      await this.writeQueue.enqueue({writePromise: writeFile(sharedPath, sharedDataStr)});
     }
   }
 
@@ -994,13 +994,13 @@ export default class I3SConverter {
       const slpkTexturePath = join(childPath, 'textures');
       const compress = false;
 
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         archiveKey: `${slpkChildPath}/textures/${name}.${format}`,
         writePromise: writeFileForSlpk(slpkTexturePath, textureData, `${name}.${format}`, compress)
       });
     } else {
       const texturePath = join(childPath, `textures/${name}/`);
-      this.writeQueue.enqueue({
+      await this.writeQueue.enqueue({
         writePromise: writeFile(texturePath, textureData, `index.${format}`)
       });
     }
@@ -1024,13 +1024,13 @@ export default class I3SConverter {
 
         if (this.options.slpk) {
           const slpkAttributesPath = join(childPath, 'attributes', folderName);
-          this.writeQueue.enqueue({
+          await this.writeQueue.enqueue({
             archiveKey: `${slpkChildPath}/attributes/${folderName}.bin.gz`,
             writePromise: writeFileForSlpk(slpkAttributesPath, fileBuffer, '0.bin')
           });
         } else {
           const attributesPath = join(childPath, `attributes/${folderName}/0`);
-          this.writeQueue.enqueue({
+          await this.writeQueue.enqueue({
             writePromise: writeFile(attributesPath, fileBuffer, 'index.bin')
           });
         }

package/src/lib/utils/write-queue.ts

@@ -1,4 +1,8 @@
 import {Queue} from './queue';
+import process from 'process';
+
+/** Memory limit size is based on testing */
+const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
 
 export type WriteQueueItem = {
   archiveKey?: string;
@@ -18,6 +22,14 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
     this.writeConcurrency = writeConcurrency;
   }
 
+  async enqueue(val: T) {
+    super.enqueue(val);
+    /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */
+    if (process.memoryUsage().rss > MEMORY_LIMIT) {
+      await this.startWrite();
+    }
+  }
+
   startListening() {
     this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
   }