@loaders.gl/tile-converter 4.0.0-alpha.9 → 4.0.0-beta.1

package/dist/esm/i3s-converter/i3s-converter.js

@@ -1,6 +1,6 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
 var _process$env;
-import { load, encode, fetchFile, getLoaderOptions, isBrowser } from '@loaders.gl/core';
+import { load, encode, isBrowser } from '@loaders.gl/core';
 import { CesiumIonLoader, Tiles3DLoader } from '@loaders.gl/3d-tiles';
 import { join } from 'path';
 import { v4 as uuidv4 } from 'uuid';
@@ -22,20 +22,18 @@ import { SHARED_RESOURCES as sharedResourcesTemplate } from './json-templates/sh
 import { validateNodeBoundingVolumes } from './helpers/node-debug';
 import { KTX2BasisWriterWorker } from '@loaders.gl/textures';
 import { ImageWriter } from '@loaders.gl/images';
-import { GltfPrimitiveModeString } from './types';
-import { getWorkerURL, WorkerFarm } from '@loaders.gl/worker-utils';
-import { DracoWriterWorker } from '@loaders.gl/draco';
+import { GLTFPrimitiveModeString } from './types';
+import { WorkerFarm } from '@loaders.gl/worker-utils';
 import WriteQueue from '../lib/utils/write-queue';
-import { I3SAttributesWorker } from '../i3s-attributes-worker';
 import { BROWSER_ERROR_MESSAGE } from '../constants';
 import { createdStorageAttribute, createFieldAttribute, createPopupInfo, getAttributeType, getFieldAttributeType } from './helpers/feature-attributes';
 import { NodeIndexDocument } from './helpers/node-index-document';
-import { loadNestedTileset, loadTile3DContent } from './helpers/load-3d-tiles';
+import { isNestedTileset, loadNestedTileset, loadTile3DContent, loadFromArchive } from './helpers/load-3d-tiles';
 import { Matrix4 } from '@math.gl/core';
 import { createBoundingVolume } from '@loaders.gl/tiles';
 import { traverseDatasetWith } from './helpers/tileset-traversal';
 import { analyzeTileContent, mergePreprocessData } from './helpers/preprocess-3d-tiles';
-const ION_DEFAULT_TOKEN = ((_process$env = process.env) === null || _process$env === void 0 ? void 0 : _process$env.IonToken) || 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJqdGkiOiJlYWMxMzcyYy0zZjJkLTQwODctODNlNi01MDRkZmMzMjIxOWIiLCJpZCI6OTYyMCwic2NvcGVzIjpbImFzbCIsImFzciIsImdjIl0sImlhdCI6MTU2Mjg2NjI3M30.1FNiClUyk00YH_nWfSGpiQAjR5V2OvREDq1PJ5QMjWQ';
+const ION_DEFAULT_TOKEN = (_process$env = process.env) === null || _process$env === void 0 ? void 0 : _process$env.IonToken;
 const HARDCODED_NODES_PER_PAGE = 64;
 const _3D_TILES = '3DTILES';
 const _3D_OBJECT_LAYER_TYPE = '3DObject';
@@ -62,6 +60,7 @@ export default class I3SConverter {
     _defineProperty(this, "loadOptions", {
       _nodeWorkers: true,
       reuseWorkers: true,
+      useLocalLibraries: true,
       basis: {
         format: 'rgba32',
         workerUrl: './modules/textures/dist/basis-worker-node.js'
@@ -69,7 +68,8 @@ export default class I3SConverter {
       draco: {
         workerUrl: './modules/draco/dist/draco-worker-node.js'
       },
-      fetch: {}
+      fetch: {},
+      modules: {}
     });
     _defineProperty(this, "geoidHeightModel", null);
     _defineProperty(this, "Loader", Tiles3DLoader);
@@ -80,7 +80,8 @@ export default class I3SConverter {
     _defineProperty(this, "writeQueue", new WriteQueue());
     _defineProperty(this, "compressList", null);
     _defineProperty(this, "preprocessData", {
-      meshTopologyTypes: new Set()
+      meshTopologyTypes: new Set(),
+      metadataClasses: new Set()
     });
     this.nodePages = new NodePages(writeFile, HARDCODED_NODES_PER_PAGE, this);
     this.options = {};
@@ -122,7 +123,9 @@ export default class I3SConverter {
       generateTextures,
       generateBoundingVolumes,
       instantNodeWriting = false,
-      mergeMaterials = true
+      mergeMaterials = true,
+      inquirer,
+      metadataClass
     } = options;
     this.options = {
       maxDepth,
@@ -133,7 +136,9 @@ export default class I3SConverter {
       token,
       inputUrl,
       instantNodeWriting,
-      mergeMaterials
+      mergeMaterials,
+      inquirer,
+      metadataClass
     };
     this.compressList = this.options.instantNodeWriting && [] || null;
     this.validate = Boolean(validate);
@@ -148,15 +153,18 @@ export default class I3SConverter {
     if (slpk) {
       this.nodePages.useWriteFunction(writeFileForSlpk);
     }
-    await this.loadWorkers();
     try {
       const preloadOptions = await this._fetchPreloadOptions();
+      let tilesetUrl = inputUrl;
+      if (preloadOptions.url) {
+        tilesetUrl = preloadOptions.url;
+      }
       if (preloadOptions.headers) {
         this.loadOptions.fetch = {
           headers: preloadOptions.headers
         };
       }
-      this.sourceTileset = await load(inputUrl, this.Loader, this.loadOptions);
+      this.sourceTileset = await loadFromArchive(tilesetUrl, this.Loader, this.loadOptions);
       const preprocessResult = await this.preprocessConversion();
       if (preprocessResult) {
         await this._createAndSaveTileset(outputPath, tilesetName);
@@ -180,35 +188,66 @@ export default class I3SConverter {
     const sourceRootTile = this.sourceTileset.root;
     await traverseDatasetWith(sourceRootTile, null, this.analyzeTile.bind(this), undefined, this.options.maxDepth);
     const {
-      meshTopologyTypes
+      meshTopologyTypes,
+      metadataClasses
     } = this.preprocessData;
     console.log("------------------------------------------------");
     console.log("Preprocess results:");
     console.log("glTF mesh topology types: ".concat(Array.from(meshTopologyTypes).join(', ')));
+    if (metadataClasses.size) {
+      console.log("Feature metadata classes have been found: ".concat(Array.from(metadataClasses).join(', ')));
+    } else {
+      console.log('Feature metadata classes have not been found');
+    }
     console.log("------------------------------------------------");
-    if (!meshTopologyTypes.has(GltfPrimitiveModeString.TRIANGLES) && !meshTopologyTypes.has(GltfPrimitiveModeString.TRIANGLE_STRIP)) {
+    if (!meshTopologyTypes.has(GLTFPrimitiveModeString.TRIANGLES) && !meshTopologyTypes.has(GLTFPrimitiveModeString.TRIANGLE_STRIP)) {
       console.log('The tileset is of unsupported mesh topology types. The conversion will be interrupted.');
       console.log("------------------------------------------------");
       return false;
     }
+    if (metadataClasses.size > 1) {
+      var _this$options$metadat;
+      if ((_this$options$metadat = this.options.metadataClass) !== null && _this$options$metadat !== void 0 && _this$options$metadat.length) {
+        console.log("".concat(this.options.metadataClass, " has been selected"));
+      } else if (this.options.inquirer) {
+        const result = await this.options.inquirer.prompt([{
+          name: 'metadataClass',
+          type: 'list',
+          message: 'Select feature metadata data class to convert...',
+          choices: Array.from(metadataClasses)
+        }]);
+        this.options.metadataClass = result.metadataClass;
+        console.log("".concat(result.metadataClass, " has been selected"));
+      } else {
+        console.log("A feature metadata class has not been selected. Start the converter with option \"--metadata-class\". For example, \"npx tile-converter ... --metadata-class ".concat(Array.from(metadataClasses)[0], "\""));
+        console.log("------------------------------------------------");
+        return false;
+      }
+    }
     return true;
   }
   async analyzeTile(sourceTile, traversalProps) {
-    if (sourceTile.type === 'json') {
+    const isTileset = isNestedTileset(sourceTile);
+    if (isTileset) {
       await loadNestedTileset(this.sourceTileset, sourceTile, this.loadOptions);
       return null;
     }
     if (sourceTile.id) {
       console.log("[analyze]: ".concat(sourceTile.id));
     }
-    const tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, {
-      ...this.loadOptions,
-      '3d-tiles': {
-        ...this.loadOptions['3d-tiles'],
-        loadGLTF: false
-      }
-    });
-    const tilePreprocessData = await analyzeTileContent(sourceTile, tileContent);
+    let tileContent = null;
+    try {
+      tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, {
+        ...this.loadOptions,
+        '3d-tiles': {
+          ...this.loadOptions['3d-tiles'],
+          loadGLTF: false
+        }
+      });
+    } catch (error) {
+      console.log("[warning]: Failed to load ".concat(sourceTile.contentUrl, ". An I3S tile with empty content will be added to the output tileset"));
+    }
+    const tilePreprocessData = await analyzeTileContent(tileContent);
     mergePreprocessData(this.preprocessData, tilePreprocessData);
     return null;
   }
@@ -307,8 +346,9 @@ export default class I3SConverter {
     }
   }
   async convertTile(sourceTile, traversalProps) {
-    if (sourceTile.type === 'json' || sourceTile.type === 'empty') {
-      if (sourceTile.type === 'json') {
+    const isTileset = isNestedTileset(sourceTile);
+    if (isTileset || sourceTile.type === 'empty') {
+      if (isTileset) {
         if (sourceTile.id) {
           console.log("[load]: ".concat(sourceTile.id));
         }
@@ -347,14 +387,18 @@ export default class I3SConverter {
     }
   }
   async _createNode(parentNode, sourceTile, transformationMatrix) {
-    var _this$layers, _this$layers$attribut;
     this._checkAddRefinementTypeForTile(sourceTile);
     await this._updateTilesetOptions();
-    const tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, this.loadOptions);
+    let tileContent = null;
+    try {
+      tileContent = await loadTile3DContent(this.sourceTileset, sourceTile, this.loadOptions);
+    } catch (error) {
+      console.log("[warning]: Failed to load ".concat(sourceTile.contentUrl));
+    }
     const sourceBoundingVolume = createBoundingVolume(sourceTile.boundingVolume, transformationMatrix, null);
     let boundingVolumes = createBoundingVolumes(sourceBoundingVolume, this.geoidHeightModel);
-    const propertyTable = getPropertyTable(tileContent);
-    if (propertyTable && !((_this$layers = this.layers0) !== null && _this$layers !== void 0 && (_this$layers$attribut = _this$layers.attributeStorageInfo) !== null && _this$layers$attribut !== void 0 && _this$layers$attribut.length)) {
+    const propertyTable = getPropertyTable(tileContent, this.options.metadataClass);
+    if (propertyTable) {
       this._convertPropertyTableToNodeAttributes(propertyTable);
     }
     const resourcesData = await this._convertResources(sourceTile, transformationMatrix, sourceBoundingVolume, tileContent, parentNode.inPageId, propertyTable);
@@ -401,7 +445,7 @@ export default class I3SConverter {
     return nodes;
   }
   async _convertResources(sourceTile, transformationMatrix, boundingVolume, tileContent, parentId, propertyTable) {
-    var _this$layers2;
+    var _this$layers;
     if (!this.isContentSupported(sourceTile) || !tileContent) {
       return null;
     }
@@ -413,7 +457,7 @@ export default class I3SConverter {
     const resourcesData = await convertB3dmToI3sGeometry(tileContent, transformationMatrix, boundingVolume, async () => (await this.nodePages.push({
       index: 0,
       obb: draftObb
-    }, parentId)).index, propertyTable, this.featuresHashArray, (_this$layers2 = this.layers0) === null || _this$layers2 === void 0 ? void 0 : _this$layers2.attributeStorageInfo, this.options.draco, this.generateBoundingVolumes, this.options.mergeMaterials, this.geoidHeightModel, this.workerSource);
+    }, parentId)).index, propertyTable, this.featuresHashArray, (_this$layers = this.layers0) === null || _this$layers === void 0 ? void 0 : _this$layers.attributeStorageInfo, this.options.draco, this.generateBoundingVolumes, this.options.mergeMaterials, this.geoidHeightModel, this.loadOptions.modules, this.options.metadataClass);
     return resourcesData;
   }
   async _updateNodeInNodePages(maxScreenThresholdSQ, boundingVolumes, sourceTile, parentId, resources) {
@@ -452,6 +496,9 @@ export default class I3SConverter {
     } else {
       node = await this.nodePages.getNodeById(nodeId);
     }
+    if (!nodeInPage.mesh) {
+      console.log("[warning]: node ".concat(node.index, " is created with empty content"));
+    }
     NodePages.updateAll(node, nodeInPage);
     if (meshMaterial) {
       NodePages.updateMaterialByNodeId(node, this._findOrCreateMaterial(meshMaterial));
@@ -560,9 +607,12 @@ export default class I3SConverter {
                 data: arrayToEncode
               }, KTX2BasisWriterWorker, {
                 ...KTX2BasisWriterWorker.options,
-                source: this.workerSource.ktx2,
+                ['ktx2-basis-writer']: {
+                  workerUrl: './modules/textures/dist/ktx2-basis-writer-worker-node.js'
+                },
                 reuseWorkers: true,
-                _nodeWorkers: true
+                _nodeWorkers: true,
+                useLocalLibraries: true
               });
               await this.writeTextureFile(ktx2TextureData, '1', 'ktx2', childPath, slpkChildPath);
             }
@@ -612,12 +662,13 @@ export default class I3SConverter {
     }
   }
   async _writeAttributes() {
-    var _this$layers3, _this$layers3$attribu;
+    var _this$layers2, _this$layers2$attribu;
     let attributes = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : [];
     let childPath = arguments.length > 1 ? arguments[1] : undefined;
     let slpkChildPath = arguments.length > 2 ? arguments[2] : undefined;
-    if (attributes !== null && attributes !== void 0 && attributes.length && (_this$layers3 = this.layers0) !== null && _this$layers3 !== void 0 && (_this$layers3$attribu = _this$layers3.attributeStorageInfo) !== null && _this$layers3$attribu !== void 0 && _this$layers3$attribu.length) {
-      for (let index = 0; index < attributes.length; index++) {
+    if (attributes !== null && attributes !== void 0 && attributes.length && (_this$layers2 = this.layers0) !== null && _this$layers2 !== void 0 && (_this$layers2$attribu = _this$layers2.attributeStorageInfo) !== null && _this$layers2$attribu !== void 0 && _this$layers2$attribu.length) {
+      const minimumLength = attributes.length < this.layers0.attributeStorageInfo.length ? attributes.length : this.layers0.attributeStorageInfo.length;
+      for (let index = 0; index < minimumLength; index++) {
         const folderName = this.layers0.attributeStorageInfo[index].key;
         const fileBuffer = new Uint8Array(attributes[index]);
         if (this.options.slpk) {
@@ -676,16 +727,19 @@ export default class I3SConverter {
       ...propertyTable
     };
     for (const key in propertyTableWithObjectId) {
-      const firstAttribute = propertyTableWithObjectId[key][0];
-      const attributeType = getAttributeType(key, firstAttribute);
-      const storageAttribute = createdStorageAttribute(attributeIndex, key, attributeType);
-      const fieldAttributeType = getFieldAttributeType(attributeType);
-      const fieldAttribute = createFieldAttribute(key, fieldAttributeType);
-      const popupInfo = createPopupInfo(propertyTableWithObjectId);
-      this.layers0.attributeStorageInfo.push(storageAttribute);
-      this.layers0.fields.push(fieldAttribute);
-      this.layers0.popupInfo = popupInfo;
-      this.layers0.layerType = _3D_OBJECT_LAYER_TYPE;
+      const found = this.layers0.attributeStorageInfo.find(element => element.name === key);
+      if (!found) {
+        const firstAttribute = propertyTableWithObjectId[key][0];
+        const attributeType = getAttributeType(key, firstAttribute);
+        const storageAttribute = createdStorageAttribute(attributeIndex, key, attributeType);
+        const fieldAttributeType = getFieldAttributeType(attributeType);
+        const fieldAttribute = createFieldAttribute(key, fieldAttributeType);
+        const popupInfo = createPopupInfo(propertyTableWithObjectId);
+        this.layers0.attributeStorageInfo.push(storageAttribute);
+        this.layers0.fields.push(fieldAttribute);
+        this.layers0.popupInfo = popupInfo;
+        this.layers0.layerType = _3D_OBJECT_LAYER_TYPE;
+      }
       attributeIndex += 1;
     }
   }
@@ -748,31 +802,5 @@ export default class I3SConverter {
   isContentSupported(sourceTile) {
     return ['b3dm', 'glTF', 'scenegraph'].includes(sourceTile.type || '');
   }
-  async loadWorkers() {
-    console.log("Loading workers source...");
-    if (this.options.draco) {
-      const url = getWorkerURL(DracoWriterWorker, {
-        ...getLoaderOptions()
-      });
-      const sourceResponse = await fetchFile(url);
-      const source = await sourceResponse.text();
-      this.workerSource.draco = source;
-    }
-    if (this.generateTextures) {
-      const url = getWorkerURL(KTX2BasisWriterWorker, {
-        ...getLoaderOptions()
-      });
-      const sourceResponse = await fetchFile(url);
-      const source = await sourceResponse.text();
-      this.workerSource.ktx2 = source;
-    }
-    const i3sAttributesWorkerUrl = getWorkerURL(I3SAttributesWorker, {
-      ...getLoaderOptions()
-    });
-    const sourceResponse = await fetchFile(i3sAttributesWorkerUrl);
-    const source = await sourceResponse.text();
-    this.workerSource.I3SAttributes = source;
-    console.log("Loading workers source completed!");
-  }
 }
 //# sourceMappingURL=i3s-converter.js.map