@loaders.gl/tile-converter 4.0.0-alpha.7 → 4.0.0-alpha.9

package/dist/esm/i3s-converter/i3s-converter.js

@@ -1,7 +1,6 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 var _process$env;
 import { load, encode, fetchFile, getLoaderOptions, isBrowser } from '@loaders.gl/core';
-import { Tileset3D } from '@loaders.gl/tiles';
 import { CesiumIonLoader, Tiles3DLoader } from '@loaders.gl/3d-tiles';
 import { join } from 'path';
 import { v4 as uuidv4 } from 'uuid';
@@ -23,6 +22,7 @@ import { SHARED_RESOURCES as sharedResourcesTemplate } from './json-templates/sh
 import { validateNodeBoundingVolumes } from './helpers/node-debug';
 import { KTX2BasisWriterWorker } from '@loaders.gl/textures';
 import { ImageWriter } from '@loaders.gl/images';
+import { GltfPrimitiveModeString } from './types';
 import { getWorkerURL, WorkerFarm } from '@loaders.gl/worker-utils';
 import { DracoWriterWorker } from '@loaders.gl/draco';
 import WriteQueue from '../lib/utils/write-queue';
@@ -30,6 +30,11 @@ import { I3SAttributesWorker } from '../i3s-attributes-worker';
 import { BROWSER_ERROR_MESSAGE } from '../constants';
 import { createdStorageAttribute, createFieldAttribute, createPopupInfo, getAttributeType, getFieldAttributeType } from './helpers/feature-attributes';
 import { NodeIndexDocument } from './helpers/node-index-document';
+import { loadNestedTileset, loadTile3DContent } from './helpers/load-3d-tiles';
+import { Matrix4 } from '@math.gl/core';
+import { createBoundingVolume } from '@loaders.gl/tiles';
+import { traverseDatasetWith } from './helpers/tileset-traversal';
+import { analyzeTileContent, mergePreprocessData } from './helpers/preprocess-3d-tiles';
 const ION_DEFAULT_TOKEN = ((_process$env = process.env) === null || _process$env === void 0 ? void 0 : _process$env.IonToken) || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiJlYWMxMzcyYy0zZjJkLTQwODctODNlNi01MDRkZmMzMjIxOWIiLCJpZCI6OTYyMCwic2NvcGVzIjpbImFzbCIsImFzciIsImdjIl0sImlhdCI6MTU2Mjg2NjI3M30.1FNiClUyk00YH_nWfSGpiQAjR5V2OvREDq1PJ5QMjWQ';
 const HARDCODED_NODES_PER_PAGE = 64;
 const _3D_TILES = '3DTILES';
@@ -54,6 +59,18 @@ export default class I3SConverter {
     _defineProperty(this, "conversionStartTime", [0, 0]);
     _defineProperty(this, "refreshTokenTime", [0, 0]);
     _defineProperty(this, "sourceTileset", null);
+    _defineProperty(this, "loadOptions", {
+      _nodeWorkers: true,
+      reuseWorkers: true,
+      basis: {
+        format: 'rgba32',
+        workerUrl: './modules/textures/dist/basis-worker-node.js'
+      },
+      draco: {
+        workerUrl: './modules/draco/dist/draco-worker-node.js'
+      },
+      fetch: {}
+    });
     _defineProperty(this, "geoidHeightModel", null);
     _defineProperty(this, "Loader", Tiles3DLoader);
     _defineProperty(this, "generateTextures", void 0);
@@ -62,6 +79,9 @@ export default class I3SConverter {
     _defineProperty(this, "workerSource", {});
     _defineProperty(this, "writeQueue", new WriteQueue());
     _defineProperty(this, "compressList", null);
+    _defineProperty(this, "preprocessData", {
+      meshTopologyTypes: new Set()
+    });
     this.nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE, this);
     this.options = {};
     this.layers0Path = '';
@@ -130,50 +150,81 @@ export default class I3SConverter {
     }
     await this.loadWorkers();
     try {
-      var _sourceTilesetJson$ro, _sourceTilesetJson$ro2;
       const preloadOptions = await this._fetchPreloadOptions();
-      const tilesetOptions = {
-        loadOptions: {
-          _nodeWorkers: true,
-          reuseWorkers: true,
-          basis: {
-            format: 'rgba32'
-          }
-        }
-      };
       if (preloadOptions.headers) {
-        tilesetOptions.loadOptions.fetch = {
+        this.loadOptions.fetch = {
           headers: preloadOptions.headers
         };
       }
-      Object.assign(tilesetOptions, preloadOptions);
-      const sourceTilesetJson = await load(inputUrl, this.Loader, tilesetOptions.loadOptions);
-      this.sourceTileset = new Tileset3D(sourceTilesetJson, tilesetOptions);
-      await this._createAndSaveTileset(outputPath, tilesetName, sourceTilesetJson === null || sourceTilesetJson === void 0 ? void 0 : (_sourceTilesetJson$ro = sourceTilesetJson.root) === null || _sourceTilesetJson$ro === void 0 ? void 0 : (_sourceTilesetJson$ro2 = _sourceTilesetJson$ro.boundingVolume) === null || _sourceTilesetJson$ro2 === void 0 ? void 0 : _sourceTilesetJson$ro2.region);
-      await this._finishConversion({
-        slpk: Boolean(slpk),
-        outputPath,
-        tilesetName
-      });
-      return sourceTilesetJson;
+      this.sourceTileset = await load(inputUrl, this.Loader, this.loadOptions);
+      const preprocessResult = await this.preprocessConversion();
+      if (preprocessResult) {
+        await this._createAndSaveTileset(outputPath, tilesetName);
+        await this._finishConversion({
+          slpk: Boolean(slpk),
+          outputPath,
+          tilesetName
+        });
+      }
     } catch (error) {
       throw error;
     } finally {
+      await this.writeQueue.finalize();
       const workerFarm = WorkerFarm.getWorkerFarm({});
       workerFarm.destroy();
     }
+    return 'success';
+  }
+  async preprocessConversion() {
+    console.log("Analyze source tileset");
+    const sourceRootTile = this.sourceTileset.root;
+    await traverseDatasetWith(sourceRootTile, null, this.analyzeTile.bind(this), undefined, this.options.maxDepth);
+    const {
+      meshTopologyTypes
+    } = this.preprocessData;
+    console.log("------------------------------------------------");
+    console.log("Preprocess results:");
+    console.log("glTF mesh topology types: ".concat(Array.from(meshTopologyTypes).join(', ')));
+    console.log("------------------------------------------------");
+    if (!meshTopologyTypes.has(GltfPrimitiveModeString.TRIANGLES) && !meshTopologyTypes.has(GltfPrimitiveModeString.TRIANGLE_STRIP)) {
+      console.log('The tileset is of unsupported mesh topology types. The conversion will be interrupted.');
+      console.log("------------------------------------------------");
+      return false;
+    }
+    return true;
   }
-  async _createAndSaveTileset(outputPath, tilesetName, boundingVolumeRegion) {
+  async analyzeTile(sourceTile, traversalProps) {
+    if (sourceTile.type === 'json') {
+      await loadNestedTileset(this.sourceTileset, sourceTile, this.loadOptions);
+      return null;
+    }
+    if (sourceTile.id) {
+      console.log("[analyze]: ".concat(sourceTile.id));
+    }
+    const tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, {
+      ...this.loadOptions,
+      '3d-tiles': {
+        ...this.loadOptions['3d-tiles'],
+        loadGLTF: false
+      }
+    });
+    const tilePreprocessData = await analyzeTileContent(sourceTile, tileContent);
+    mergePreprocessData(this.preprocessData, tilePreprocessData);
+    return null;
+  }
+  async _createAndSaveTileset(outputPath, tilesetName) {
+    var _this$sourceTileset, _this$sourceTileset$r, _this$sourceTileset$r2;
     const tilesetPath = join("".concat(outputPath), "".concat(tilesetName));
     try {
       await removeDir(tilesetPath);
     } catch (e) {}
     this.layers0Path = join(tilesetPath, 'SceneServer', 'layers', '0');
-    this._formLayers0(tilesetName, boundingVolumeRegion);
     this.materialDefinitions = [];
     this.materialMap = new Map();
     const sourceRootTile = this.sourceTileset.root;
-    const boundingVolumes = createBoundingVolumes(sourceRootTile, this.geoidHeightModel);
+    const sourceBoundingVolume = createBoundingVolume(sourceRootTile.boundingVolume, new Matrix4(sourceRootTile.transform), null);
+    this._formLayers0(tilesetName, sourceBoundingVolume, (_this$sourceTileset = this.sourceTileset) === null || _this$sourceTileset === void 0 ? void 0 : (_this$sourceTileset$r = _this$sourceTileset.root) === null || _this$sourceTileset$r === void 0 ? void 0 : (_this$sourceTileset$r2 = _this$sourceTileset$r.boundingVolume) === null || _this$sourceTileset$r2 === void 0 ? void 0 : _this$sourceTileset$r2.region);
+    const boundingVolumes = createBoundingVolumes(sourceBoundingVolume, this.geoidHeightModel);
     await this.nodePages.push({
       index: 0,
       lodThreshold: 0,
@@ -181,7 +232,10 @@ export default class I3SConverter {
       children: []
     });
     const rootNode = await NodeIndexDocument.createRootNode(boundingVolumes, this);
-    await this._convertNodesTree(rootNode, sourceRootTile);
+    await traverseDatasetWith(sourceRootTile, {
+      transform: new Matrix4(sourceRootTile.transform),
+      parentNodes: [rootNode]
+    }, this.convertTile.bind(this), this.finalizeTile.bind(this), this.options.maxDepth);
     this.layers0.materialDefinitions = this.materialDefinitions;
     this.layers0.geometryDefinitions = transform(this.geometryConfigs.map(config => ({
       geometryConfig: {
@@ -202,9 +256,12 @@ export default class I3SConverter {
     await this.writeQueue.finalize();
     await this._createSlpk(tilesetPath);
   }
-  _formLayers0(tilesetName, boundingVolumeRegion) {
-    var _this$sourceTileset, _this$sourceTileset2, _this$sourceTileset2$;
-    const fullExtent = convertBoundingVolumeToI3SFullExtent(((_this$sourceTileset = this.sourceTileset) === null || _this$sourceTileset === void 0 ? void 0 : _this$sourceTileset.boundingVolume) || ((_this$sourceTileset2 = this.sourceTileset) === null || _this$sourceTileset2 === void 0 ? void 0 : (_this$sourceTileset2$ = _this$sourceTileset2.root) === null || _this$sourceTileset2$ === void 0 ? void 0 : _this$sourceTileset2$.boundingVolume));
+  _formLayers0(tilesetName, sourceBoundingVolume, boundingVolumeRegion) {
+    var _this$sourceTileset2;
+    if (!((_this$sourceTileset2 = this.sourceTileset) !== null && _this$sourceTileset2 !== void 0 && _this$sourceTileset2.root)) {
+      return;
+    }
+    const fullExtent = convertBoundingVolumeToI3SFullExtent(sourceBoundingVolume);
     if (boundingVolumeRegion) {
       fullExtent.zmin = boundingVolumeRegion[4];
       fullExtent.zmax = boundingVolumeRegion[5];
@@ -227,24 +284,6 @@ export default class I3SConverter {
     };
     this.layers0 = transform(layers0data, layersTemplate());
   }
-  async _convertNodesTree(rootNode, sourceRootTile) {
-    await this.sourceTileset._loadTile(sourceRootTile);
-    if (this.isContentSupported(sourceRootTile)) {
-      const childNodes = await this._createNode(rootNode, sourceRootTile, 0);
-      for (const childNode of childNodes) {
-        await childNode.save();
-      }
-      await rootNode.addChildren(childNodes);
-    } else {
-      await this._addChildrenWithNeighborsAndWriteFile({
-        parentNode: rootNode,
-        sourceTiles: sourceRootTile.children,
-        level: 1
-      });
-    }
-    await sourceRootTile.unloadContent();
-    await rootNode.save();
-  }
   async _writeLayers0() {
     if (this.options.slpk) {
       await this.writeQueue.enqueue({
@@ -267,72 +306,58 @@ export default class I3SConverter {
       } catch (e) {}
     }
   }
-  async _addChildrenWithNeighborsAndWriteFile(data) {
-    await this._addChildren(data);
-    await data.parentNode.addNeighbors();
-  }
-  async convertNestedTileset(_ref) {
-    let {
-      parentNode,
-      sourceTile,
-      level
-    } = _ref;
-    await this.sourceTileset._loadTile(sourceTile);
-    await this._addChildren({
-      parentNode,
-      sourceTiles: sourceTile.children,
-      level: level + 1
-    });
-    await sourceTile.unloadContent();
-  }
-  async convertNode(_ref2) {
-    let {
-      parentNode,
-      sourceTile,
-      level
-    } = _ref2;
-    const childNodes = await this._createNode(parentNode, sourceTile, level);
-    await parentNode.addChildren(childNodes);
-  }
-  async _addChildren(data) {
-    const {
-      sourceTiles,
-      parentNode,
-      level
-    } = data;
-    if (this.options.maxDepth && level > this.options.maxDepth) {
-      return;
-    }
-    for (const sourceTile of sourceTiles) {
+  async convertTile(sourceTile, traversalProps) {
+    if (sourceTile.type === 'json' || sourceTile.type === 'empty') {
       if (sourceTile.type === 'json') {
-        await this.convertNestedTileset({
-          parentNode,
-          sourceTile,
-          level
-        });
-      } else {
-        await this.convertNode({
-          parentNode,
-          sourceTile,
-          level
-        });
+        if (sourceTile.id) {
+          console.log("[load]: ".concat(sourceTile.id));
+        }
+        await loadNestedTileset(this.sourceTileset, sourceTile, this.loadOptions);
       }
-      if (sourceTile.id) {
-        console.log(sourceTile.id);
+      return traversalProps;
+    }
+    if (sourceTile.id) {
+      console.log("[convert]: ".concat(sourceTile.id));
+    }
+    const {
+      parentNodes,
+      transform
+    } = traversalProps;
+    let transformationMatrix = transform.clone();
+    if (sourceTile.transform) {
+      transformationMatrix = transformationMatrix.multiplyRight(sourceTile.transform);
+    }
+    const parentNode = parentNodes[0];
+    const childNodes = await this._createNode(parentNode, sourceTile, transformationMatrix);
+    await parentNode.addChildren(childNodes);
+    const newTraversalProps = {
+      transform: transformationMatrix,
+      parentNodes: childNodes
+    };
+    return newTraversalProps;
+  }
+  async finalizeTile(conversionResults, currentTraversalProps) {
+    for (const result of conversionResults) {
+      for (const node of result.parentNodes) {
+        await node.addNeighbors();
       }
     }
+    for (const node of currentTraversalProps.parentNodes) {
+      await node.save();
+    }
   }
-  async _createNode(parentNode, sourceTile, level) {
+  async _createNode(parentNode, sourceTile, transformationMatrix) {
     var _this$layers, _this$layers$attribut;
     this._checkAddRefinementTypeForTile(sourceTile);
     await this._updateTilesetOptions();
-    await this.sourceTileset._loadTile(sourceTile);
-    let boundingVolumes = createBoundingVolumes(sourceTile, this.geoidHeightModel);
-    const propertyTable = getPropertyTable(sourceTile.content);
+    const tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, this.loadOptions);
+    const sourceBoundingVolume = createBoundingVolume(sourceTile.boundingVolume, transformationMatrix, null);
+    let boundingVolumes = createBoundingVolumes(sourceBoundingVolume, this.geoidHeightModel);
+    const propertyTable = getPropertyTable(tileContent);
     if (propertyTable && !((_this$layers = this.layers0) !== null && _this$layers !== void 0 && (_this$layers$attribut = _this$layers.attributeStorageInfo) !== null && _this$layers$attribut !== void 0 && _this$layers$attribut.length)) {
       this._convertPropertyTableToNodeAttributes(propertyTable);
     }
-    const resourcesData = await this._convertResources(sourceTile, parentNode.inPageId, propertyTable);
+    const resourcesData = await this._convertResources(sourceTile, transformationMatrix, sourceBoundingVolume, tileContent, parentNode.inPageId, propertyTable);
     const nodes = [];
     const nodeIds = [];
     const nodesInPage = [];
@@ -373,17 +398,11 @@ export default class I3SConverter {
       nodeIds.push(nodeInPage.index);
       nodesInPage.push(nodeInPage);
     }
-    sourceTile.unloadContent();
-    await this._addChildrenWithNeighborsAndWriteFile({
-      parentNode: nodes[0],
-      sourceTiles: sourceTile.children,
-      level: level + 1
-    });
     return nodes;
   }
-  async _convertResources(sourceTile, parentId, propertyTable) {
+  async _convertResources(sourceTile, transformationMatrix, boundingVolume, tileContent, parentId, propertyTable) {
     var _this$layers2;
-    if (!this.isContentSupported(sourceTile)) {
+    if (!this.isContentSupported(sourceTile) || !tileContent) {
       return null;
     }
     const draftObb = {
@@ -391,7 +410,7 @@ export default class I3SConverter {
       halfSize: [],
       quaternion: []
     };
-    const resourcesData = await convertB3dmToI3sGeometry(sourceTile.content, async () => (await this.nodePages.push({
+    const resourcesData = await convertB3dmToI3sGeometry(tileContent, transformationMatrix, boundingVolume, async () => (await this.nodePages.push({
       index: 0,
       obb: draftObb
     }, parentId)).index, propertyTable, this.featuresHashArray, (_this$layers2 = this.layers0) === null || _this$layers2 === void 0 ? void 0 : _this$layers2.attributeStorageInfo, this.options.draco, this.generateBoundingVolumes, this.options.mergeMaterials, this.geoidHeightModel, this.workerSource);
@@ -710,13 +729,9 @@ export default class I3SConverter {
     }
     this.refreshTokenTime = process.hrtime();
     const preloadOptions = await this._fetchPreloadOptions();
-    this.sourceTileset.options = {
-      ...this.sourceTileset.options,
-      ...preloadOptions
-    };
     if (preloadOptions.headers) {
-      this.sourceTileset.loadOptions.fetch = {
-        ...this.sourceTileset.loadOptions.fetch,
+      this.loadOptions.fetch = {
+        ...this.loadOptions.fetch,
         headers: preloadOptions.headers
       };
       console.log('Authorization Bearer token has been updated');
@@ -730,9 +745,8 @@ export default class I3SConverter {
     }
     this.refinementCounter.tilesCount += 1;
   }
-  isContentSupported(sourceRootTile) {
-    var _sourceRootTile$conte;
-    return ['b3dm', 'glTF'].includes(sourceRootTile === null || sourceRootTile === void 0 ? void 0 : (_sourceRootTile$conte = sourceRootTile.content) === null || _sourceRootTile$conte === void 0 ? void 0 : _sourceRootTile$conte.type);
+  isContentSupported(sourceTile) {
+    return ['b3dm', 'glTF', 'scenegraph'].includes(sourceTile.type || '');
   }
   async loadWorkers() {
     console.log("Loading workers source...");