@loaders.gl/tile-converter 3.1.8 → 3.2.0-alpha.3

package/dist/esm/i3s-converter/i3s-converter.js

@@ -1,5 +1,5 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { load, encode } from '@loaders.gl/core';
+import { load, encode, fetchFile, getLoaderOptions } from '@loaders.gl/core';
 import { Tileset3D } from '@loaders.gl/tiles';
 import { CesiumIonLoader, Tiles3DLoader } from '@loaders.gl/3d-tiles';
 import { join } from 'path';
@@ -12,7 +12,7 @@ import { writeFile, removeDir, writeFileForSlpk } from '../lib/utils/file-utils'
 import { compressWithChildProcess } from '../lib/utils/compress-util';
 import { calculateFilesSize, timeConverter } from '../lib/utils/statistic-utills';
 import convertB3dmToI3sGeometry from './helpers/geometry-converter';
-import { createBoundingVolumes, convertCommonToI3SExtentCoordinate } from './helpers/coordinate-converter';
+import { createBoundingVolumes, convertBoundingVolumeToI3SFullExtent } from './helpers/coordinate-converter';
 import { createSceneServerPath } from './helpers/create-scene-server-path';
 import { convertGeometricErrorToScreenThreshold } from '../lib/utils/lod-conversion-utils';
 import { PGMLoader } from '../pgm-loader';
@@ -20,8 +20,12 @@ import { LAYERS as layersTemplate } from './json-templates/layers';
 import { NODE as nodeTemplate } from './json-templates/node';
 import { SHARED_RESOURCES as sharedResourcesTemplate } from './json-templates/shared-resources';
 import { validateNodeBoundingVolumes } from './helpers/node-debug';
-import { KTX2BasisUniversalTextureWriter } from '@loaders.gl/textures';
+import { KTX2BasisWriterWorker } from '@loaders.gl/textures';
 import { ImageWriter } from '@loaders.gl/images';
+import { getWorkerURL, WorkerFarm } from '@loaders.gl/worker-utils';
+import { DracoWriterWorker } from '@loaders.gl/draco';
+import WriteQueue from '../lib/utils/write-queue';
+import { I3SAttributesWorker } from '../i3s-attributes-worker';
 const ION_DEFAULT_TOKEN = process.env.IonToken || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiJlYWMxMzcyYy0zZjJkLTQwODctODNlNi01MDRkZmMzMjIxOWIiLCJpZCI6OTYyMCwic2NvcGVzIjpbImFzbCIsImFzciIsImdjIl0sImlhdCI6MTU2Mjg2NjI3M30.1FNiClUyk00YH_nWfSGpiQAjR5V2OvREDq1PJ5QMjWQ';
 const HARDCODED_NODES_PER_PAGE = 64;
 const _3D_TILES = '3DTILES';
@@ -36,8 +40,6 @@ export default class I3SConverter {
   constructor() {
     _defineProperty(this, "nodePages", void 0);
 
-    _defineProperty(this, "fileMap", void 0);
-
     _defineProperty(this, "options", void 0);
 
     _defineProperty(this, "layers0Path", void 0);
@@ -72,8 +74,13 @@ export default class I3SConverter {
 
     _defineProperty(this, "generateBoundingVolumes", void 0);
 
+    _defineProperty(this, "layersHasTexture", void 0);
+
+    _defineProperty(this, "workerSource", {});
+
+    _defineProperty(this, "writeQueue", new WriteQueue());
+
     this.nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE);
-    this.fileMap = {};
     this.options = {};
     this.layers0Path = '';
     this.materialMap = new Map();
@@ -88,6 +95,7 @@ export default class I3SConverter {
     this.validate = false;
     this.generateTextures = false;
     this.generateBoundingVolumes = false;
+    this.layersHasTexture = false;
   }
 
   async convert(options) {
@@ -119,6 +127,8 @@ export default class I3SConverter {
     this.Loader = inputUrl.indexOf(CESIUM_DATASET_PREFIX) !== -1 ? CesiumIonLoader : Tiles3DLoader;
     this.generateTextures = Boolean(generateTextures);
     this.generateBoundingVolumes = Boolean(generateBoundingVolumes);
+    this.writeQueue = new WriteQueue();
+    this.writeQueue.startListening();
     console.log('Loading egm file...');
     this.geoidHeightModel = await load(egmFilePath, PGMLoader);
     console.log('Loading egm file completed!');
@@ -127,31 +137,40 @@ export default class I3SConverter {
       this.nodePages.useWriteFunction(writeFileForSlpk);
     }
 
-    const preloadOptions = await this._fetchPreloadOptions();
-    const tilesetOptions = {
-      loadOptions: {
-        basis: {
-          format: 'rgba32'
+    await this.loadWorkers();
+
+    try {
+      const preloadOptions = await this._fetchPreloadOptions();
+      const tilesetOptions = {
+        loadOptions: {
+          basis: {
+            format: 'rgba32'
+          }
         }
+      };
+
+      if (preloadOptions.headers) {
+        tilesetOptions.loadOptions.fetch = {
+          headers: preloadOptions.headers
+        };
       }
-    };
 
-    if (preloadOptions.headers) {
-      tilesetOptions.loadOptions.fetch = {
-        headers: preloadOptions.headers
-      };
+      Object.assign(tilesetOptions, preloadOptions);
+      const sourceTilesetJson = await load(inputUrl, this.Loader, tilesetOptions.loadOptions);
+      this.sourceTileset = new Tileset3D(sourceTilesetJson, tilesetOptions);
+      await this._createAndSaveTileset(outputPath, tilesetName);
+      await this._finishConversion({
+        slpk: Boolean(slpk),
+        outputPath,
+        tilesetName
+      });
+      return sourceTilesetJson;
+    } catch (error) {
+      throw error;
+    } finally {
+      const workerFarm = WorkerFarm.getWorkerFarm({});
+      workerFarm.destroy();
     }
-
-    Object.assign(tilesetOptions, preloadOptions);
-    const sourceTilesetJson = await load(inputUrl, this.Loader, tilesetOptions.loadOptions);
-    this.sourceTileset = new Tileset3D(sourceTilesetJson, tilesetOptions);
-    await this._createAndSaveTileset(outputPath, tilesetName);
-    await this._finishConversion({
-      slpk: Boolean(slpk),
-      outputPath,
-      tilesetName
-    });
-    return sourceTilesetJson;
   }
 
   async _createAndSaveTileset(outputPath, tilesetName) {
@@ -181,15 +200,24 @@ export default class I3SConverter {
 
     await this._convertNodesTree(root0, sourceRootTile, parentId, boundingVolumes);
     this.layers0.materialDefinitions = this.materialDefinitions;
+
+    if (this.layersHasTexture === false) {
+      this.layers0.store.defaultGeometrySchema.ordering = this.layers0.store.defaultGeometrySchema.ordering.filter(attribute => attribute !== 'uv0');
+    }
+
     await this._writeLayers0();
     createSceneServerPath(tilesetName, this.layers0, tilesetPath);
     await this._writeNodeIndexDocument(root0, 'root', join(this.layers0Path, 'nodes', 'root'));
-    await this.nodePages.save(this.layers0Path, this.fileMap, isCreateSlpk);
+    await this.nodePages.save(this.layers0Path, this.writeQueue, isCreateSlpk);
+    await this.writeQueue.finalize();
     await this._createSlpk(tilesetPath);
   }
 
   _formLayers0(tilesetName) {
-    const extent = convertCommonToI3SExtentCoordinate(this.sourceTileset);
+    var _this$sourceTileset, _this$sourceTileset2, _this$sourceTileset2$;
+
+    const fullExtent = convertBoundingVolumeToI3SFullExtent(((_this$sourceTileset = this.sourceTileset) === null || _this$sourceTileset === void 0 ? void 0 : _this$sourceTileset.boundingVolume) || ((_this$sourceTileset2 = this.sourceTileset) === null || _this$sourceTileset2 === void 0 ? void 0 : (_this$sourceTileset2$ = _this$sourceTileset2.root) === null || _this$sourceTileset2$ === void 0 ? void 0 : _this$sourceTileset2$.boundingVolume));
+    const extent = [fullExtent.xmin, fullExtent.ymin, fullExtent.xmax, fullExtent.ymax];
     const layers0data = {
       version: "{".concat(uuidv4().toUpperCase(), "}"),
       id: 0,
@@ -239,9 +267,14 @@ export default class I3SConverter {
       const childPath = join(this.layers0Path, 'nodes', child.path);
 
       if (this.options.slpk) {
-        this.fileMap['nodes/1/3dNodeIndexDocument.json.gz'] = await writeFileForSlpk(childPath, JSON.stringify(child), '3dNodeIndexDocument.json');
+        this.writeQueue.enqueue({
+          archiveKey: 'nodes/1/3dNodeIndexDocument.json.gz',
+          writePromise: writeFileForSlpk(childPath, JSON.stringify(child), '3dNodeIndexDocument.json')
+        });
       } else {
-        await writeFile(childPath, JSON.stringify(child));
+        this.writeQueue.enqueue({
+          writePromise: writeFile(childPath, JSON.stringify(child))
+        });
       }
     } else {
       await this._addChildrenWithNeighborsAndWriteFile({
@@ -257,17 +290,27 @@ export default class I3SConverter {
 
   async _writeLayers0() {
     if (this.options.slpk) {
-      this.fileMap['3dSceneLayer.json.gz'] = await writeFileForSlpk(this.layers0Path, JSON.stringify(this.layers0), '3dSceneLayer.json');
+      this.writeQueue.enqueue({
+        archiveKey: '3dSceneLayer.json.gz',
+        writePromise: writeFileForSlpk(this.layers0Path, JSON.stringify(this.layers0), '3dSceneLayer.json')
+      });
     } else {
-      await writeFile(this.layers0Path, JSON.stringify(this.layers0));
+      this.writeQueue.enqueue({
+        writePromise: writeFile(this.layers0Path, JSON.stringify(this.layers0))
+      });
     }
   }
 
   async _writeNodeIndexDocument(root0, nodePath, rootPath) {
     if (this.options.slpk) {
-      this.fileMap["nodes/".concat(nodePath, "/3dNodeIndexDocument.json.gz")] = await writeFileForSlpk(rootPath, JSON.stringify(root0), '3dNodeIndexDocument.json');
+      this.writeQueue.enqueue({
+        archiveKey: "nodes/".concat(nodePath, "/3dNodeIndexDocument.json.gz"),
+        writePromise: writeFileForSlpk(rootPath, JSON.stringify(root0), '3dNodeIndexDocument.json')
+      });
     } else {
-      await writeFile(rootPath, JSON.stringify(root0));
+      this.writeQueue.enqueue({
+        writePromise: writeFile(rootPath, JSON.stringify(root0))
+      });
     }
   }
 
@@ -397,6 +440,8 @@ export default class I3SConverter {
     };
 
     for (const resources of resourcesData || [emptyResources]) {
+      this.layersHasTexture = this.layersHasTexture || Boolean(resources.texture);
+
       if (this.generateBoundingVolumes && resources.boundingVolumes) {
         boundingVolumes = resources.boundingVolumes;
       }
@@ -453,7 +498,7 @@ export default class I3SConverter {
       return null;
     }
 
-    const resourcesData = await convertB3dmToI3sGeometry(sourceTile.content, Number(this.nodePages.nodesCounter), this.featuresHashArray, (_this$layers2 = this.layers0) === null || _this$layers2 === void 0 ? void 0 : _this$layers2.attributeStorageInfo, this.options.draco, this.generateBoundingVolumes, this.geoidHeightModel);
+    const resourcesData = await convertB3dmToI3sGeometry(sourceTile.content, Number(this.nodePages.nodesCounter), this.featuresHashArray, (_this$layers2 = this.layers0) === null || _this$layers2 === void 0 ? void 0 : _this$layers2.attributeStorageInfo, this.options.draco, this.generateBoundingVolumes, this.geoidHeightModel, this.workerSource);
     return resourcesData;
   }
 
@@ -588,34 +633,53 @@ export default class I3SConverter {
   async _writeGeometries(geometryBuffer, compressedGeometry, childPath, slpkChildPath) {
     if (this.options.slpk) {
       const slpkGeometryPath = join(childPath, 'geometries');
-      this.fileMap["".concat(slpkChildPath, "/geometries/0.bin.gz")] = await writeFileForSlpk(slpkGeometryPath, geometryBuffer, '0.bin');
+      this.writeQueue.enqueue({
+        archiveKey: "".concat(slpkChildPath, "/geometries/0.bin.gz"),
+        writePromise: writeFileForSlpk(slpkGeometryPath, geometryBuffer, '0.bin')
+      });
     } else {
       const geometryPath = join(childPath, 'geometries/0/');
-      await writeFile(geometryPath, geometryBuffer, 'index.bin');
+      this.writeQueue.enqueue({
+        writePromise: writeFile(geometryPath, geometryBuffer, 'index.bin')
+      });
     }
 
     if (this.options.draco) {
       if (this.options.slpk) {
         const slpkCompressedGeometryPath = join(childPath, 'geometries');
-        this.fileMap["".concat(slpkChildPath, "/geometries/1.bin.gz")] = await writeFileForSlpk(slpkCompressedGeometryPath, compressedGeometry, '1.bin');
+        this.writeQueue.enqueue({
+          archiveKey: "".concat(slpkChildPath, "/geometries/1.bin.gz"),
+          writePromise: writeFileForSlpk(slpkCompressedGeometryPath, compressedGeometry, '1.bin')
+        });
       } else {
         const compressedGeometryPath = join(childPath, 'geometries/1/');
-        await writeFile(compressedGeometryPath, compressedGeometry, 'index.bin');
+        this.writeQueue.enqueue({
+          writePromise: writeFile(compressedGeometryPath, compressedGeometry, 'index.bin')
+        });
       }
     }
   }
 
   async _writeShared(sharedResources, childPath, slpkChildPath, nodePath) {
+    if (!sharedResources) {
+      return;
+    }
+
     sharedResources.nodePath = nodePath;
     const sharedData = transform(sharedResources, sharedResourcesTemplate());
     const sharedDataStr = JSON.stringify(sharedData);
 
     if (this.options.slpk) {
       const slpkSharedPath = join(childPath, 'shared');
-      this.fileMap["".concat(slpkChildPath, "/shared/sharedResource.json.gz")] = await writeFileForSlpk(slpkSharedPath, sharedDataStr, 'sharedResource.json');
+      this.writeQueue.enqueue({
+        archiveKey: "".concat(slpkChildPath, "/shared/sharedResource.json.gz"),
+        writePromise: writeFileForSlpk(slpkSharedPath, sharedDataStr, 'sharedResource.json')
+      });
     } else {
       const sharedPath = join(childPath, 'shared/');
-      await writeFile(sharedPath, sharedDataStr);
+      this.writeQueue.enqueue({
+        writePromise: writeFile(sharedPath, sharedDataStr)
+      });
     }
   }
 
@@ -641,7 +705,11 @@ export default class I3SConverter {
                 name: '1',
                 format: 'ktx2'
               });
-              const ktx2TextureData = new Uint8Array(await encode(texture.image, KTX2BasisUniversalTextureWriter));
+              const ktx2TextureData = encode(texture.image, KTX2BasisWriterWorker, { ...KTX2BasisWriterWorker.options,
+                source: this.workerSource.ktx2,
+                reuseWorkers: true,
+                _nodeWorkers: true
+              });
               await this.writeTextureFile(ktx2TextureData, '1', 'ktx2', childPath, slpkChildPath);
             }
 
@@ -661,7 +729,7 @@ export default class I3SConverter {
                 name: '0',
                 format: 'jpg'
               });
-              const decodedFromKTX2TextureData = new Uint8Array(await encode(texture.image.data[0], ImageWriter));
+              const decodedFromKTX2TextureData = encode(texture.image.data[0], ImageWriter);
               await this.writeTextureFile(decodedFromKTX2TextureData, '0', 'jpg', childPath, slpkChildPath);
             }
           }
@@ -679,27 +747,37 @@ export default class I3SConverter {
     if (this.options.slpk) {
       const slpkTexturePath = join(childPath, 'textures');
       const compress = false;
-      this.fileMap["".concat(slpkChildPath, "/textures/").concat(name, ".").concat(format)] = await writeFileForSlpk(slpkTexturePath, textureData, "".concat(name, ".").concat(format), compress);
+      this.writeQueue.enqueue({
+        archiveKey: "".concat(slpkChildPath, "/textures/").concat(name, ".").concat(format),
+        writePromise: writeFileForSlpk(slpkTexturePath, textureData, "".concat(name, ".").concat(format), compress)
+      });
     } else {
       const texturePath = join(childPath, "textures/".concat(name, "/"));
-      await writeFile(texturePath, textureData, "index.".concat(format));
+      this.writeQueue.enqueue({
+        writePromise: writeFile(texturePath, textureData, "index.".concat(format))
+      });
     }
   }
 
-  async _writeAttributes(attributes, childPath, slpkChildPath) {
+  async _writeAttributes(attributes = [], childPath, slpkChildPath) {
     var _this$layers4, _this$layers4$attribu;
 
-    if (attributes.length && (_this$layers4 = this.layers0) !== null && _this$layers4 !== void 0 && (_this$layers4$attribu = _this$layers4.attributeStorageInfo) !== null && _this$layers4$attribu !== void 0 && _this$layers4$attribu.length) {
+    if (attributes !== null && attributes !== void 0 && attributes.length && (_this$layers4 = this.layers0) !== null && _this$layers4 !== void 0 && (_this$layers4$attribu = _this$layers4.attributeStorageInfo) !== null && _this$layers4$attribu !== void 0 && _this$layers4$attribu.length) {
       for (let index = 0; index < attributes.length; index++) {
         const folderName = this.layers0.attributeStorageInfo[index].key;
         const fileBuffer = new Uint8Array(attributes[index]);
 
         if (this.options.slpk) {
           const slpkAttributesPath = join(childPath, 'attributes', folderName);
-          this.fileMap["".concat(slpkChildPath, "/attributes/").concat(folderName, ".bin.gz")] = await writeFileForSlpk(slpkAttributesPath, fileBuffer, '0.bin');
+          this.writeQueue.enqueue({
+            archiveKey: "".concat(slpkChildPath, "/attributes/").concat(folderName, ".bin.gz"),
+            writePromise: writeFileForSlpk(slpkAttributesPath, fileBuffer, '0.bin')
+          });
         } else {
           const attributesPath = join(childPath, "attributes/".concat(folderName, "/0"));
-          await writeFile(attributesPath, fileBuffer, 'index.bin');
+          this.writeQueue.enqueue({
+            writePromise: writeFile(attributesPath, fileBuffer, 'index.bin')
+          });
         }
       }
     }
@@ -976,5 +1054,32 @@ export default class I3SConverter {
     return ['b3dm', 'glTF'].includes(sourceRootTile === null || sourceRootTile === void 0 ? void 0 : (_sourceRootTile$conte = sourceRootTile.content) === null || _sourceRootTile$conte === void 0 ? void 0 : _sourceRootTile$conte.type);
   }
 
+  async loadWorkers() {
+    console.log("Loading workers source...");
+
+    if (this.options.draco) {
+      const url = getWorkerURL(DracoWriterWorker, { ...getLoaderOptions()
+      });
+      const sourceResponse = await fetchFile(url);
+      const source = await sourceResponse.text();
+      this.workerSource.draco = source;
+    }
+
+    if (this.generateTextures) {
+      const url = getWorkerURL(KTX2BasisWriterWorker, { ...getLoaderOptions()
+      });
+      const sourceResponse = await fetchFile(url);
+      const source = await sourceResponse.text();
+      this.workerSource.ktx2 = source;
+    }
+
+    const i3sAttributesWorkerUrl = getWorkerURL(I3SAttributesWorker, { ...getLoaderOptions()
+    });
+    const sourceResponse = await fetchFile(i3sAttributesWorkerUrl);
+    const source = await sourceResponse.text();
+    this.workerSource.I3SAttributes = source;
+    console.log("Loading workers source completed!");
+  }
+
 }
 //# sourceMappingURL=i3s-converter.js.map