@loaders.gl/tile-converter 4.1.0 → 4.2.0-alpha.1

package/src/i3s-converter/i3s-converter.ts

@@ -17,7 +17,8 @@ import type {
   BoundingVolumes,
   MaxScreenThresholdSQ,
   NodeInPage,
-  Attribute
+  Attribute,
+  Node3DIndexDocument
 } from '@loaders.gl/i3s';
 import {load, encode, isBrowser} from '@loaders.gl/core';
 import {CesiumIonLoader, Tiles3DLoader} from '@loaders.gl/3d-tiles';
@@ -78,7 +79,7 @@ import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-t
 import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles';
 import {Progress} from './helpers/progress';
 import {composeHashFile, createZip} from '@loaders.gl/zip';
-import {ConversionDump, ConversionDumpOptions} from '../lib/utils/conversion-dump';
+import {ConversionDump, ConversionDumpOptions, DumpMetadata} from '../lib/utils/conversion-dump';
 
 const ION_DEFAULT_TOKEN = process.env?.IonToken;
 const HARDCODED_NODES_PER_PAGE = 64;
@@ -259,9 +260,6 @@ export default class I3SConverter {
       this.nodePages.useWriteFunction(writeFileForSlpk);
     }
 
-    //create a dump file with convertion options
-    await this.conversionDump.createDumpFile(options as ConversionDumpOptions);
-
     try {
       const preloadOptions = await this._fetchPreloadOptions();
       let tilesetUrl = inputUrl;
@@ -415,18 +413,49 @@ export default class I3SConverter {
    */
   private async _createAndSaveTileset(outputPath: string, tilesetName: string): Promise<void> {
     const tilesetPath = join(`${outputPath}`, `${tilesetName}`);
+
+    await this.conversionDump.createDump(this.options as ConversionDumpOptions);
+    if (this.conversionDump.restored && this.options.inquirer) {
+      const result = await this.options.inquirer.prompt([
+        {
+          name: 'resumeConversion',
+          type: 'confirm',
+          message:
+            'Dump file of the previous conversion exists, do you want to resume that conversion?'
+        }
+      ]);
+      if (!result.resumeConversion) {
+        this.conversionDump.reset();
+      }
+    }
+
+    this.layers0Path = join(tilesetPath, 'SceneServer', 'layers', '0');
+
     // Removing the tilesetPath needed to exclude erroneous files after conversion
+    const removePath = this.conversionDump.restored
+      ? join(this.layers0Path, 'nodepages')
+      : tilesetPath;
     try {
-      await removeDir(tilesetPath);
+      await removeDir(removePath);
     } catch (e) {
       // do nothing
     }
 
-    this.layers0Path = join(tilesetPath, 'SceneServer', 'layers', '0');
+    if (this.conversionDump.restored && this.conversionDump.attributeMetadataInfo) {
+      this.attributeMetadataInfo.fromObject(this.conversionDump.attributeMetadataInfo);
+    }
 
     this.materialDefinitions = [];
     this.materialMap = new Map();
 
+    if (this.conversionDump.restored && this.conversionDump.materialDefinitions) {
+      for (let i = 0; i < this.conversionDump.materialDefinitions.length; i++) {
+        const hash = md5(JSON.stringify(this.conversionDump.materialDefinitions[i]));
+        this.materialMap.set(hash, i);
+      }
+      this.materialDefinitions = this.conversionDump.materialDefinitions;
+    }
+
     const sourceRootTile: Tiles3DTileJSONPostprocessed = this.sourceTileset!.root!;
     const sourceBoundingVolume = createBoundingVolume(
       sourceRootTile.boundingVolume,
@@ -471,6 +500,10 @@ export default class I3SConverter {
       this.layers0!.layerType = _3D_OBJECT_LAYER_TYPE;
     }
 
+    if (this.conversionDump.restored && this.conversionDump.textureSetDefinitions) {
+      this.layers0!.textureSetDefinitions = this.conversionDump.textureSetDefinitions;
+    }
+
     this.layers0!.materialDefinitions = this.materialDefinitions;
     // @ts-ignore
     this.layers0.geometryDefinitions = transform(
@@ -607,7 +640,13 @@ export default class I3SConverter {
       transformationMatrix = transformationMatrix.multiplyRight(sourceTile.transform);
     }
     const parentNode = parentNodes[0];
-    const childNodes = await this._createNode(parentNode, sourceTile, transformationMatrix);
+    const restoreResult = await this._restoreNode(parentNode, sourceTile, transformationMatrix);
+    let childNodes;
+    if (restoreResult === null) {
+      childNodes = await this._createNode(parentNode, sourceTile, transformationMatrix);
+    } else {
+      childNodes = restoreResult;
+    }
     await parentNode.addChildren(childNodes);
 
     const newTraversalProps: TraversalConversionProps = {
@@ -651,6 +690,114 @@ export default class I3SConverter {
     }
   }
 
+  /**
+   * Generate NodeIndexDocument
+   * @param boundingVolumes - Bounding volumes
+   * @param resources - converted or dumped node resources data
+   * @param parentNode - 3DNodeIndexDocument of parent node
+   * @param sourceTile - source 3DTile data
+   * @param isDumped - indicator if the node is dumped
+   * @return NodeIndexDocument, nodeInPage and node data
+   */
+  private async _generateNodeIndexDocument(
+    boundingVolumes: BoundingVolumes,
+    resources: I3SConvertedResources | DumpMetadata,
+    parentNode: NodeIndexDocument,
+    sourceTile: Tiles3DTileJSONPostprocessed,
+    isDumped: boolean
+  ): Promise<{node: NodeIndexDocument; nodeInPage: NodeInPage; nodeData: Node3DIndexDocument}> {
+    this.layersHasTexture =
+      this.layersHasTexture ||
+      Boolean(
+        ('texture' in resources && resources.texture) ||
+          ('texelCountHint' in resources && resources.texelCountHint)
+      );
+
+    if (this.generateBoundingVolumes && resources.boundingVolumes) {
+      boundingVolumes = resources.boundingVolumes;
+    }
+
+    const lodSelection = convertGeometricErrorToScreenThreshold(sourceTile, boundingVolumes);
+    const maxScreenThresholdSQ = lodSelection.find(
+      (val) => val.metricType === 'maxScreenThresholdSQ'
+    ) || {maxError: 0};
+
+    if (isDumped) {
+      const draftObb = {
+        center: [],
+        halfSize: [],
+        quaternion: []
+      };
+      await this.nodePages.push({index: 0, obb: draftObb}, parentNode.inPageId);
+    }
+
+    const nodeInPage = await this._updateNodeInNodePages(
+      maxScreenThresholdSQ,
+      boundingVolumes,
+      sourceTile,
+      parentNode.inPageId,
+      resources
+    );
+
+    const nodeData = await NodeIndexDocument.createNodeIndexDocument(
+      parentNode,
+      boundingVolumes,
+      lodSelection,
+      nodeInPage,
+      resources
+    );
+
+    const node = await new NodeIndexDocument(nodeInPage.index, this).addData(nodeData);
+    return {node, nodeInPage, nodeData};
+  }
+
+  /**
+   * Restore 3DNodeIndexDocument from a comversion dump file
+   * @param parentNode - 3DNodeIndexDocument of parent node
+   * @param sourceTile - source 3DTile data
+   * @param transformationMatrix - transformation matrix of the current tile, calculated recursively multiplying
+   *                               transform of all parent tiles and transform of the current tile
+   */
+  private async _restoreNode(
+    parentNode: NodeIndexDocument,
+    sourceTile: Tiles3DTileJSONPostprocessed,
+    transformationMatrix: Matrix4
+  ): Promise<null | NodeIndexDocument[]> {
+    this._checkAddRefinementTypeForTile(sourceTile);
+    await this._updateTilesetOptions();
+    if (
+      this.conversionDump.restored &&
+      sourceTile.id &&
+      this.conversionDump.isFileConversionComplete(sourceTile.id)
+    ) {
+      const sourceBoundingVolume = createBoundingVolume(
+        sourceTile.boundingVolume,
+        transformationMatrix,
+        null
+      );
+      let boundingVolumes = createBoundingVolumes(sourceBoundingVolume, this.geoidHeightModel!);
+      const nodes: NodeIndexDocument[] = [];
+      for (const convertedNode of this.conversionDump.tilesConverted[sourceTile.id].nodes) {
+        const {node} = await this._generateNodeIndexDocument(
+          boundingVolumes,
+          {
+            ...(convertedNode.dumpMetadata as DumpMetadata),
+            nodeId: convertedNode.nodeId
+          } as I3SConvertedResources | DumpMetadata,
+          parentNode,
+          sourceTile,
+          true
+        );
+        nodes.push(node);
+      }
+      return nodes;
+    } else if (this.conversionDump.restored && sourceTile.id) {
+      //clear existing record in a dump
+      this.conversionDump.clearDumpRecord(sourceTile.id);
+    }
+    return null;
+  }
+
   /**
    * Convert tile to one or more I3S nodes
    * @param parentNode - 3DNodeIndexDocument of parent node
@@ -684,6 +831,12 @@ export default class I3SConverter {
     const propertyTable = getPropertyTable(tileContent, this.options.metadataClass);
     this.createAttributeStorageInfo(tileContent, propertyTable);
 
+    this.conversionDump.attributeMetadataInfo = {
+      attributeStorageInfo: this.attributeMetadataInfo.attributeStorageInfo,
+      fields: this.attributeMetadataInfo.fields,
+      popupInfo: this.attributeMetadataInfo.popupInfo
+    };
+
     const resourcesData = await this._convertResources(
       sourceTile,
       transformationMatrix,
@@ -710,39 +863,30 @@ export default class I3SConverter {
     };
 
     for (const resources of resourcesData || [emptyResources]) {
-      this.layersHasTexture = this.layersHasTexture || Boolean(resources.texture);
-
-      if (this.generateBoundingVolumes && resources.boundingVolumes) {
-        boundingVolumes = resources.boundingVolumes;
-      }
-
-      const lodSelection = convertGeometricErrorToScreenThreshold(sourceTile, boundingVolumes);
-      const maxScreenThresholdSQ = lodSelection.find(
-        (val) => val.metricType === 'maxScreenThresholdSQ'
-      ) || {maxError: 0};
-
-      const nodeInPage = await this._updateNodeInNodePages(
-        maxScreenThresholdSQ,
+      const {node, nodeInPage, nodeData} = await this._generateNodeIndexDocument(
         boundingVolumes,
-        sourceTile,
-        parentNode.inPageId,
-        resources
-      );
-
-      const nodeData = await NodeIndexDocument.createNodeIndexDocument(
+        resources,
         parentNode,
-        boundingVolumes,
-        lodSelection,
-        nodeInPage,
-        resources
+        sourceTile,
+        false
       );
-      const node = await new NodeIndexDocument(nodeInPage.index, this).addData(nodeData);
       nodes.push(node);
 
       if (nodeInPage.mesh) {
         //update a record in a dump file
         if (sourceTile.id) {
-          await this.conversionDump.addNode(sourceTile.id, nodeInPage.index);
+          const dumpMetadata = {
+            boundingVolumes: resources.boundingVolumes,
+            attributesCount: resources.attributes?.length,
+            featureCount: resources.featureCount,
+            geometry: Boolean(resources.geometry),
+            hasUvRegions: resources.hasUvRegions,
+            materialId: nodeInPage.mesh.material.definition,
+            texelCountHint: nodeInPage.mesh.material.texelCountHint,
+            vertexCount: resources.vertexCount
+          };
+          this.conversionDump.setMaterialsDefinitions(this.materialDefinitions);
+          await this.conversionDump.addNode(sourceTile.id, nodeInPage.index, dumpMetadata);
         }
 
         //write resources
@@ -829,9 +973,9 @@ export default class I3SConverter {
     boundingVolumes: BoundingVolumes,
     sourceTile: Tiles3DTileJSONPostprocessed,
     parentId: number,
-    resources: I3SConvertedResources
+    resources: I3SConvertedResources | DumpMetadata
   ): Promise<NodeInPage> {
-    const {meshMaterial, texture, vertexCount, featureCount, geometry, hasUvRegions} = resources;
+    const {vertexCount, featureCount, geometry, hasUvRegions} = resources;
     const nodeInPage: NodeInPage = {
       index: 0,
       lodThreshold: maxScreenThresholdSQ.maxError,
@@ -841,7 +985,13 @@ export default class I3SConverter {
     if (geometry && this.isContentSupported(sourceTile)) {
       nodeInPage.mesh = {
         geometry: {
-          definition: this.findOrCreateGeometryDefinition(Boolean(texture), hasUvRegions),
+          definition: this.findOrCreateGeometryDefinition(
+            Boolean(
+              ('texture' in resources && resources.texture) ||
+                ('texelCountHint' in resources && resources.texelCountHint)
+            ),
+            hasUvRegions
+          ),
           resource: 0
         },
         attribute: {
@@ -853,7 +1003,7 @@ export default class I3SConverter {
       };
     }
 
-    let nodeId = resources.nodeId;
+    let nodeId = 'nodeId' in resources ? resources.nodeId : undefined;
     let node: NodeInPage;
     if (!nodeId) {
       node = await this.nodePages.push(nodeInPage, parentId);
@@ -866,12 +1016,16 @@ export default class I3SConverter {
     }
 
     NodePages.updateAll(node, nodeInPage);
-    if (meshMaterial) {
-      NodePages.updateMaterialByNodeId(node, this._findOrCreateMaterial(meshMaterial));
+    if ('meshMaterial' in resources && resources.meshMaterial) {
+      NodePages.updateMaterialByNodeId(node, this._findOrCreateMaterial(resources.meshMaterial));
+    } else if ('materialId' in resources && resources.materialId !== null) {
+      NodePages.updateMaterialByNodeId(node, resources.materialId);
     }
-    if (texture) {
-      const texelCountHint = texture.image.height * texture.image.width;
+    if ('texture' in resources && resources.texture) {
+      const texelCountHint = resources.texture.image.height * resources.texture.image.width;
       NodePages.updateTexelCountHintByNodeId(node, texelCountHint);
+    } else if ('texelCountHint' in resources && resources.texelCountHint) {
+      NodePages.updateTexelCountHintByNodeId(node, resources.texelCountHint);
     }
     if (vertexCount) {
       this.vertexCounter += vertexCount;
@@ -1178,6 +1332,9 @@ export default class I3SConverter {
       if (!this.layers0!.textureSetDefinitions!.length) {
         this.layers0!.textureSetDefinitions!.push({formats});
         this.layers0!.textureSetDefinitions!.push({formats, atlas: true});
+        if (this.layers0!.textureSetDefinitions) {
+          this.conversionDump.addTexturesDefinitions(this.layers0!.textureSetDefinitions);
+        }
       }
     }
   }

package/src/lib/utils/conversion-dump.ts

@@ -1,6 +1,9 @@
+import {isDeepStrictEqual} from 'util';
 import {DUMP_FILE_SUFFIX} from '../../constants';
-import {removeFile, writeFile} from './file-utils';
+import {isFileExists, openJson, removeFile, writeFile} from './file-utils';
 import {join} from 'path';
+import {BoundingVolumes, I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s';
+import {AttributeMetadataInfoObject} from '../../i3s-converter/helpers/attribute-metadata-info';
 
 export type ConversionDumpOptions = {
   inputUrl: string;
@@ -21,28 +24,53 @@ export type ConversionDumpOptions = {
 type NodeDoneStatus = {
   nodeId: number;
   done: boolean;
-  progress: Record<string, boolean>;
+  progress?: Record<string, boolean>;
+  dumpMetadata?: DumpMetadata;
 };
 
 type TilesConverted = {
   nodes: NodeDoneStatus[];
 };
 
+export type DumpMetadata = {
+  boundingVolumes: BoundingVolumes | null;
+  attributesCount?: number;
+  featureCount: number | null;
+  geometry: boolean;
+  hasUvRegions: boolean;
+  materialId: number | null;
+  texelCountHint?: number;
+  vertexCount: number | null;
+};
+
+export type TextureSetDefinition = {
+  formats: TextureSetDefinitionFormats;
+  atlas?: boolean;
+};
+
 export class ConversionDump {
+  /**Restored/resumed dump indicator */
+  restored: boolean = false;
   /** Conversion options */
   private options?: ConversionDumpOptions;
   /** Tiles conversion progress status map */
   tilesConverted: Record<string, TilesConverted>;
+  /** Textures formats definitions */
+  textureSetDefinitions?: TextureSetDefinition[];
+  /** Attributes Metadata */
+  attributeMetadataInfo?: AttributeMetadataInfoObject;
+  /** Array of materials definitions */
+  materialDefinitions: I3SMaterialDefinition[] = [];
 
   constructor() {
     this.tilesConverted = {};
   }
 
   /**
-   * Create a dump file with convertion options
-   * @param options - converter options
+   * Create a dump with convertion options
+   * @param currentOptions - converter options
    */
-  async createDumpFile(options: ConversionDumpOptions): Promise<void> {
+  async createDump(currentOptions: ConversionDumpOptions): Promise<void> {
     const {
       tilesetName,
       slpk,
@@ -57,7 +85,7 @@ export class ConversionDump {
       mergeMaterials = true,
       metadataClass,
       analyze = false
-    } = options;
+    } = currentOptions;
     this.options = {
       tilesetName,
       slpk,
@@ -74,14 +102,48 @@ export class ConversionDump {
       analyze
     };
 
-    try {
-      await writeFile(
-        options.outputPath,
-        JSON.stringify({options: this.options}),
-        `${options.tilesetName}${DUMP_FILE_SUFFIX}`
+    const dumpFilename = join(
+      this.options.outputPath,
+      this.options.tilesetName,
+      `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+    );
+    if (await isFileExists(dumpFilename)) {
+      const {
+        options,
+        tilesConverted,
+        textureSetDefinitions,
+        attributeMetadataInfo,
+        materialDefinitions
+      } = await openJson(
+        join(this.options.outputPath, this.options.tilesetName),
+        `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
       );
-    } catch (error) {
-      console.log("Can't create dump file", error);
+      if (isDeepStrictEqual(options, JSON.parse(JSON.stringify(this.options)))) {
+        this.tilesConverted = tilesConverted;
+        this.textureSetDefinitions = textureSetDefinitions;
+        this.attributeMetadataInfo = attributeMetadataInfo;
+        this.materialDefinitions = materialDefinitions;
+        this.restored = true;
+        return;
+      }
+    }
+    await this.deleteDumpFile();
+  }
+
+  /**
+   * Reset a dump
+   */
+  reset(): void {
+    this.restored = false;
+    this.tilesConverted = {};
+    if (this.textureSetDefinitions) {
+      delete this.textureSetDefinitions;
+    }
+    if (this.attributeMetadataInfo) {
+      delete this.attributeMetadataInfo;
+    }
+    if (this.materialDefinitions.length > 0) {
+      this.materialDefinitions = [];
     }
   }
 
@@ -92,10 +154,13 @@ export class ConversionDump {
     if (this.options?.outputPath && this.options.tilesetName) {
       try {
         await writeFile(
-          this.options.outputPath,
+          join(this.options.outputPath, this.options.tilesetName),
           JSON.stringify({
             options: this.options,
-            tilesConverted: this.tilesConverted
+            tilesConverted: this.tilesConverted,
+            textureSetDefinitions: this.textureSetDefinitions,
+            attributeMetadataInfo: this.attributeMetadataInfo,
+            materialDefinitions: this.materialDefinitions
           }),
           `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
         );
@@ -109,9 +174,23 @@ export class ConversionDump {
    * Delete a dump file
    */
   async deleteDumpFile(): Promise<void> {
-    if (this.options?.outputPath && this.options.tilesetName) {
+    if (
+      this.options?.outputPath &&
+      this.options.tilesetName &&
+      (await isFileExists(
+        join(
+          this.options.outputPath,
+          this.options.tilesetName,
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        )
+      ))
+    ) {
       await removeFile(
-        join(this.options.outputPath, `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`)
+        join(
+          this.options.outputPath,
+          this.options.tilesetName,
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        )
       );
     }
   }
@@ -139,15 +218,31 @@ export class ConversionDump {
    * @param fileName - source filename
    * @param nodeId - nodeId of the node
    */
-  async addNode(filename: string, nodeId: number) {
+  async addNode(filename: string, nodeId: number, dumpMetadata: DumpMetadata) {
     const {nodes} = this.getRecord(filename) || {nodes: []};
-    nodes.push({nodeId, done: false, progress: {}});
+    nodes.push({nodeId, done: false, progress: {}, dumpMetadata});
     if (nodes.length === 1) {
       this.setRecord(filename, {nodes});
     }
     await this.updateDumpFile();
   }
 
+  /**
+   * Clear dump record got the source filename
+   * @param fileName - source filename
+   */
+  clearDumpRecord(filename: string) {
+    this.setRecord(filename, {nodes: []});
+  }
+
+  /**
+   * Add textures definitions into the dump file
+   * @param textureDefinitions - textures definitions array
+   */
+  addTexturesDefinitions(textureDefinitions: TextureSetDefinition[]) {
+    this.textureSetDefinitions = textureDefinitions;
+  }
+
   /**
    * Update done status object for the writing resources
    * @param fileName - key - source filename
@@ -160,6 +255,9 @@ export class ConversionDump {
       (element) => element.nodeId === nodeId
     );
     if (nodeDump) {
+      if (!nodeDump.progress) {
+        nodeDump.progress = {};
+      }
       nodeDump.progress[resourceType] = value;
       if (!value) {
         nodeDump.done = false;
@@ -181,7 +279,7 @@ export class ConversionDump {
         const {sourceId, resourceType, outputId} = changedRecords[i];
         if (!sourceId || !resourceType || !outputId) continue;
         for (const node of this.tilesConverted[sourceId].nodes) {
-          if (node.nodeId === outputId) {
+          if (node.nodeId === outputId && node.progress) {
             node.progress[resourceType] = true;
 
             let done = false;
@@ -191,7 +289,7 @@ export class ConversionDump {
             }
             node.done = done;
             if (node.done) {
-              node.progress = {};
+              delete node.progress;
             }
             break;
           }
@@ -200,4 +298,28 @@ export class ConversionDump {
     }
     await this.updateDumpFile();
   }
+
+  /**
+   * Check is source file conversion complete
+   * @param filename - source filename
+   * @returns true if source file conversion complete
+   */
+  isFileConversionComplete(filename: string): boolean {
+    let result = true;
+    for (const node of this.tilesConverted[filename]?.nodes || []) {
+      if (!node.done) {
+        result = false;
+        break;
+      }
+    }
+    return result && this.tilesConverted[filename]?.nodes?.length > 0;
+  }
+
+  /**
+   * Set materialDefinitions into a dump
+   * @param materialDefinitions - Array materialDefinitions
+   */
+  setMaterialsDefinitions(materialDefinitions: I3SMaterialDefinition[]): void {
+    this.materialDefinitions = materialDefinitions;
+  }
 }