@loaders.gl/tile-converter 4.1.1 → 4.2.0-alpha.2

package/src/i3s-converter/i3s-converter.ts

@@ -17,7 +17,8 @@ import type {
   BoundingVolumes,
   MaxScreenThresholdSQ,
   NodeInPage,
-  Attribute
+  Attribute,
+  Node3DIndexDocument
 } from '@loaders.gl/i3s';
 import {load, encode, isBrowser} from '@loaders.gl/core';
 import {CesiumIonLoader, Tiles3DLoader} from '@loaders.gl/3d-tiles';
@@ -78,7 +79,7 @@ import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-t
 import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles';
 import {Progress} from './helpers/progress';
 import {composeHashFile, createZip} from '@loaders.gl/zip';
-import {ConversionDump, ConversionDumpOptions} from '../lib/utils/conversion-dump';
+import {ConversionDump, ConversionDumpOptions, DumpMetadata} from '../lib/utils/conversion-dump';
 
 const ION_DEFAULT_TOKEN = process.env?.IonToken;
 const HARDCODED_NODES_PER_PAGE = 64;
@@ -259,9 +260,6 @@ export default class I3SConverter {
       this.nodePages.useWriteFunction(writeFileForSlpk);
     }
 
-    //create a dump file with convertion options
-    await this.conversionDump.createDumpFile(options as ConversionDumpOptions);
-
     try {
       const preloadOptions = await this._fetchPreloadOptions();
       let tilesetUrl = inputUrl;
@@ -415,18 +413,49 @@ export default class I3SConverter {
    */
   private async _createAndSaveTileset(outputPath: string, tilesetName: string): Promise<void> {
     const tilesetPath = join(`${outputPath}`, `${tilesetName}`);
+
+    await this.conversionDump.createDump(this.options as ConversionDumpOptions);
+    if (this.conversionDump.restored && this.options.inquirer) {
+      const result = await this.options.inquirer.prompt([
+        {
+          name: 'resumeConversion',
+          type: 'confirm',
+          message:
+            'Dump file of the previous conversion exists, do you want to resume that conversion?'
+        }
+      ]);
+      if (!result.resumeConversion) {
+        this.conversionDump.reset();
+      }
+    }
+
+    this.layers0Path = join(tilesetPath, 'SceneServer', 'layers', '0');
+
     // Removing the tilesetPath needed to exclude erroneous files after conversion
+    const removePath = this.conversionDump.restored
+      ? join(this.layers0Path, 'nodepages')
+      : tilesetPath;
     try {
-      await removeDir(tilesetPath);
+      await removeDir(removePath);
     } catch (e) {
       // do nothing
     }
 
-    this.layers0Path = join(tilesetPath, 'SceneServer', 'layers', '0');
+    if (this.conversionDump.restored && this.conversionDump.attributeMetadataInfo) {
+      this.attributeMetadataInfo.fromObject(this.conversionDump.attributeMetadataInfo);
+    }
 
     this.materialDefinitions = [];
     this.materialMap = new Map();
 
+    if (this.conversionDump.restored && this.conversionDump.materialDefinitions) {
+      for (let i = 0; i < this.conversionDump.materialDefinitions.length; i++) {
+        const hash = md5(JSON.stringify(this.conversionDump.materialDefinitions[i]));
+        this.materialMap.set(hash, i);
+      }
+      this.materialDefinitions = this.conversionDump.materialDefinitions;
+    }
+
     const sourceRootTile: Tiles3DTileJSONPostprocessed = this.sourceTileset!.root!;
     const sourceBoundingVolume = createBoundingVolume(
       sourceRootTile.boundingVolume,
@@ -471,6 +500,10 @@ export default class I3SConverter {
       this.layers0!.layerType = _3D_OBJECT_LAYER_TYPE;
     }
 
+    if (this.conversionDump.restored && this.conversionDump.textureSetDefinitions) {
+      this.layers0!.textureSetDefinitions = this.conversionDump.textureSetDefinitions;
+    }
+
     this.layers0!.materialDefinitions = this.materialDefinitions;
     // @ts-ignore
     this.layers0.geometryDefinitions = transform(
@@ -607,7 +640,13 @@ export default class I3SConverter {
       transformationMatrix = transformationMatrix.multiplyRight(sourceTile.transform);
     }
     const parentNode = parentNodes[0];
-    const childNodes = await this._createNode(parentNode, sourceTile, transformationMatrix);
+    const restoreResult = await this._restoreNode(parentNode, sourceTile, transformationMatrix);
+    let childNodes;
+    if (restoreResult === null) {
+      childNodes = await this._createNode(parentNode, sourceTile, transformationMatrix);
+    } else {
+      childNodes = restoreResult;
+    }
     await parentNode.addChildren(childNodes);
 
     const newTraversalProps: TraversalConversionProps = {
@@ -651,6 +690,114 @@ export default class I3SConverter {
     }
   }
 
+  /**
+   * Generate NodeIndexDocument
+   * @param boundingVolumes - Bounding volumes
+   * @param resources - converted or dumped node resources data
+   * @param parentNode - 3DNodeIndexDocument of parent node
+   * @param sourceTile - source 3DTile data
+   * @param isDumped - indicator if the node is dumped
+   * @return NodeIndexDocument, nodeInPage and node data
+   */
+  private async _generateNodeIndexDocument(
+    boundingVolumes: BoundingVolumes,
+    resources: I3SConvertedResources | DumpMetadata,
+    parentNode: NodeIndexDocument,
+    sourceTile: Tiles3DTileJSONPostprocessed,
+    isDumped: boolean
+  ): Promise<{node: NodeIndexDocument; nodeInPage: NodeInPage; nodeData: Node3DIndexDocument}> {
+    this.layersHasTexture =
+      this.layersHasTexture ||
+      Boolean(
+        ('texture' in resources && resources.texture) ||
+          ('texelCountHint' in resources && resources.texelCountHint)
+      );
+
+    if (this.generateBoundingVolumes && resources.boundingVolumes) {
+      boundingVolumes = resources.boundingVolumes;
+    }
+
+    const lodSelection = convertGeometricErrorToScreenThreshold(sourceTile, boundingVolumes);
+    const maxScreenThresholdSQ = lodSelection.find(
+      (val) => val.metricType === 'maxScreenThresholdSQ'
+    ) || {maxError: 0};
+
+    if (isDumped) {
+      const draftObb = {
+        center: [],
+        halfSize: [],
+        quaternion: []
+      };
+      await this.nodePages.push({index: 0, obb: draftObb}, parentNode.inPageId);
+    }
+
+    const nodeInPage = await this._updateNodeInNodePages(
+      maxScreenThresholdSQ,
+      boundingVolumes,
+      sourceTile,
+      parentNode.inPageId,
+      resources
+    );
+
+    const nodeData = await NodeIndexDocument.createNodeIndexDocument(
+      parentNode,
+      boundingVolumes,
+      lodSelection,
+      nodeInPage,
+      resources
+    );
+
+    const node = await new NodeIndexDocument(nodeInPage.index, this).addData(nodeData);
+    return {node, nodeInPage, nodeData};
+  }
+
+  /**
+   * Restore 3DNodeIndexDocument from a comversion dump file
+   * @param parentNode - 3DNodeIndexDocument of parent node
+   * @param sourceTile - source 3DTile data
+   * @param transformationMatrix - transformation matrix of the current tile, calculated recursively multiplying
+   *                               transform of all parent tiles and transform of the current tile
+   */
+  private async _restoreNode(
+    parentNode: NodeIndexDocument,
+    sourceTile: Tiles3DTileJSONPostprocessed,
+    transformationMatrix: Matrix4
+  ): Promise<null | NodeIndexDocument[]> {
+    this._checkAddRefinementTypeForTile(sourceTile);
+    await this._updateTilesetOptions();
+    if (
+      this.conversionDump.restored &&
+      sourceTile.id &&
+      this.conversionDump.isFileConversionComplete(sourceTile.id)
+    ) {
+      const sourceBoundingVolume = createBoundingVolume(
+        sourceTile.boundingVolume,
+        transformationMatrix,
+        null
+      );
+      let boundingVolumes = createBoundingVolumes(sourceBoundingVolume, this.geoidHeightModel!);
+      const nodes: NodeIndexDocument[] = [];
+      for (const convertedNode of this.conversionDump.tilesConverted[sourceTile.id].nodes) {
+        const {node} = await this._generateNodeIndexDocument(
+          boundingVolumes,
+          {
+            ...(convertedNode.dumpMetadata as DumpMetadata),
+            nodeId: convertedNode.nodeId
+          } as I3SConvertedResources | DumpMetadata,
+          parentNode,
+          sourceTile,
+          true
+        );
+        nodes.push(node);
+      }
+      return nodes;
+    } else if (this.conversionDump.restored && sourceTile.id) {
+      //clear existing record in a dump
+      this.conversionDump.clearDumpRecord(sourceTile.id);
+    }
+    return null;
+  }
+
   /**
    * Convert tile to one or more I3S nodes
    * @param parentNode - 3DNodeIndexDocument of parent node
@@ -684,6 +831,12 @@ export default class I3SConverter {
     const propertyTable = getPropertyTable(tileContent, this.options.metadataClass);
     this.createAttributeStorageInfo(tileContent, propertyTable);
 
+    this.conversionDump.attributeMetadataInfo = {
+      attributeStorageInfo: this.attributeMetadataInfo.attributeStorageInfo,
+      fields: this.attributeMetadataInfo.fields,
+      popupInfo: this.attributeMetadataInfo.popupInfo
+    };
+
     const resourcesData = await this._convertResources(
       sourceTile,
       transformationMatrix,
@@ -710,39 +863,30 @@ export default class I3SConverter {
     };
 
     for (const resources of resourcesData || [emptyResources]) {
-      this.layersHasTexture = this.layersHasTexture || Boolean(resources.texture);
-
-      if (this.generateBoundingVolumes && resources.boundingVolumes) {
-        boundingVolumes = resources.boundingVolumes;
-      }
-
-      const lodSelection = convertGeometricErrorToScreenThreshold(sourceTile, boundingVolumes);
-      const maxScreenThresholdSQ = lodSelection.find(
-        (val) => val.metricType === 'maxScreenThresholdSQ'
-      ) || {maxError: 0};
-
-      const nodeInPage = await this._updateNodeInNodePages(
-        maxScreenThresholdSQ,
+      const {node, nodeInPage, nodeData} = await this._generateNodeIndexDocument(
         boundingVolumes,
-        sourceTile,
-        parentNode.inPageId,
-        resources
-      );
-
-      const nodeData = await NodeIndexDocument.createNodeIndexDocument(
+        resources,
         parentNode,
-        boundingVolumes,
-        lodSelection,
-        nodeInPage,
-        resources
+        sourceTile,
+        false
       );
-      const node = await new NodeIndexDocument(nodeInPage.index, this).addData(nodeData);
       nodes.push(node);
 
       if (nodeInPage.mesh) {
         //update a record in a dump file
         if (sourceTile.id) {
-          await this.conversionDump.addNode(sourceTile.id, nodeInPage.index);
+          const dumpMetadata = {
+            boundingVolumes: resources.boundingVolumes,
+            attributesCount: resources.attributes?.length,
+            featureCount: resources.featureCount,
+            geometry: Boolean(resources.geometry),
+            hasUvRegions: resources.hasUvRegions,
+            materialId: nodeInPage.mesh.material.definition,
+            texelCountHint: nodeInPage.mesh.material.texelCountHint,
+            vertexCount: resources.vertexCount
+          };
+          this.conversionDump.setMaterialsDefinitions(this.materialDefinitions);
+          await this.conversionDump.addNode(sourceTile.id, nodeInPage.index, dumpMetadata);
         }
 
         //write resources
@@ -829,9 +973,9 @@ export default class I3SConverter {
     boundingVolumes: BoundingVolumes,
     sourceTile: Tiles3DTileJSONPostprocessed,
     parentId: number,
-    resources: I3SConvertedResources
+    resources: I3SConvertedResources | DumpMetadata
   ): Promise<NodeInPage> {
-    const {meshMaterial, texture, vertexCount, featureCount, geometry, hasUvRegions} = resources;
+    const {vertexCount, featureCount, geometry, hasUvRegions} = resources;
     const nodeInPage: NodeInPage = {
       index: 0,
       lodThreshold: maxScreenThresholdSQ.maxError,
@@ -841,7 +985,13 @@ export default class I3SConverter {
     if (geometry && this.isContentSupported(sourceTile)) {
       nodeInPage.mesh = {
         geometry: {
-          definition: this.findOrCreateGeometryDefinition(Boolean(texture), hasUvRegions),
+          definition: this.findOrCreateGeometryDefinition(
+            Boolean(
+              ('texture' in resources && resources.texture) ||
+                ('texelCountHint' in resources && resources.texelCountHint)
+            ),
+            hasUvRegions
+          ),
           resource: 0
         },
         attribute: {
@@ -853,7 +1003,7 @@ export default class I3SConverter {
       };
     }
 
-    let nodeId = resources.nodeId;
+    let nodeId = 'nodeId' in resources ? resources.nodeId : undefined;
     let node: NodeInPage;
     if (!nodeId) {
       node = await this.nodePages.push(nodeInPage, parentId);
@@ -866,12 +1016,16 @@ export default class I3SConverter {
     }
 
     NodePages.updateAll(node, nodeInPage);
-    if (meshMaterial) {
-      NodePages.updateMaterialByNodeId(node, this._findOrCreateMaterial(meshMaterial));
+    if ('meshMaterial' in resources && resources.meshMaterial) {
+      NodePages.updateMaterialByNodeId(node, this._findOrCreateMaterial(resources.meshMaterial));
+    } else if ('materialId' in resources && resources.materialId !== null) {
+      NodePages.updateMaterialByNodeId(node, resources.materialId);
     }
-    if (texture) {
-      const texelCountHint = texture.image.height * texture.image.width;
+    if ('texture' in resources && resources.texture) {
+      const texelCountHint = resources.texture.image.height * resources.texture.image.width;
       NodePages.updateTexelCountHintByNodeId(node, texelCountHint);
+    } else if ('texelCountHint' in resources && resources.texelCountHint) {
+      NodePages.updateTexelCountHintByNodeId(node, resources.texelCountHint);
     }
     if (vertexCount) {
       this.vertexCounter += vertexCount;
@@ -1178,6 +1332,9 @@ export default class I3SConverter {
       if (!this.layers0!.textureSetDefinitions!.length) {
         this.layers0!.textureSetDefinitions!.push({formats});
         this.layers0!.textureSetDefinitions!.push({formats, atlas: true});
+        if (this.layers0!.textureSetDefinitions) {
+          this.conversionDump.addTexturesDefinitions(this.layers0!.textureSetDefinitions);
+        }
       }
     }
   }

package/src/lib/json-schemas/conversion-dump-json-schema.ts

@@ -0,0 +1,285 @@
+export const dumpJsonSchema = {
+  type: 'object',
+  properties: {
+    options: {
+      type: 'object',
+      properties: {
+        inputUrl: {type: 'string'},
+        outputPath: {type: 'string'},
+        tilesetName: {type: 'string'},
+        maxDepth: {type: 'number'},
+        slpk: {type: 'boolean'},
+        egmFilePath: {type: 'string'},
+        token: {type: 'string'},
+        draco: {type: 'boolean'},
+        mergeMaterials: {type: 'boolean'},
+        generateTextures: {type: 'boolean'},
+        generateBoundingVolumes: {type: 'boolean'},
+        metadataClass: {type: 'string'},
+        analyze: {type: 'boolean'}
+      },
+      required: ['inputUrl', 'outputPath', 'tilesetName']
+    },
+    tilesConverted: {
+      type: 'object',
+      patternProperties: {
+        '.*': {
+          type: 'object',
+          properties: {
+            nodes: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  nodeId: {type: ['number', 'string']},
+                  done: {type: 'boolean'},
+                  progress: {type: 'object', patternProperties: {'.*': {type: 'boolean'}}},
+                  dumpMetadata: {
+                    type: 'object',
+                    properties: {
+                      boundingVolumes: {
+                        type: ['object', 'null'],
+                        properties: {
+                          mbs: {
+                            type: 'array',
+                            minItems: 4,
+                            maxItems: 4,
+                            items: {type: 'number'}
+                          },
+                          obb: {
+                            type: 'object',
+                            properties: {
+                              center: {
+                                type: 'array',
+                                minItems: 3,
+                                maxItems: 3,
+                                items: {type: 'number'}
+                              },
+                              halfSize: {
+                                type: 'array',
+                                minItems: 3,
+                                maxItems: 3,
+                                items: {type: 'number'}
+                              },
+                              quaternion: {
+                                type: 'array',
+                                minItems: 4,
+                                maxItems: 4,
+                                items: {type: 'number'}
+                              }
+                            },
+                            required: ['center', 'halfSize', 'quaternion']
+                          }
+                        },
+                        required: ['mbs', 'obb']
+                      },
+                      attributesCount: {type: 'number'},
+                      featureCount: {type: 'number'},
+                      geometry: {type: 'boolean'},
+                      hasUvRegions: {type: 'boolean'},
+                      materialId: {type: 'number'},
+                      texelCountHint: {type: 'number'},
+                      vertexCount: {type: 'number'}
+                    },
+                    required: [
+                      'boundingVolumes',
+                      'featureCount',
+                      'geometry',
+                      'hasUvRegions',
+                      'materialId',
+                      'vertexCount'
+                    ]
+                  }
+                },
+                required: ['nodeId', 'done']
+              }
+            }
+          },
+          required: ['nodes']
+        }
+      }
+    },
+    textureSetDefinitions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          formats: {
+            type: 'array',
+            items: {
+              type: 'object',
+              properties: {
+                name: {type: 'string'},
+                format: {enum: ['jpg', 'png', 'ktx-etc2', 'dds', 'ktx2']}
+              },
+              required: ['name', 'format']
+            }
+          },
+          atlas: {type: 'boolean'}
+        },
+        required: ['formats']
+      }
+    },
+    attributeMetadataInfo: {
+      type: 'object',
+      properties: {
+        attributeStorageInfo: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              key: {type: 'string'},
+              name: {type: 'string'},
+              header: {
+                type: 'array',
+                items: {
+                  type: 'object',
+                  properties: {property: {type: 'string'}, valueType: {type: 'string'}},
+                  required: ['property', 'valueType']
+                }
+              },
+              ordering: {type: 'array', items: {type: 'string'}},
+              attributeValues: {$ref: '#/$defs/AttributeValue'},
+              attributeByteCounts: {$ref: '#/$defs/AttributeValue'},
+              objectIds: {$ref: '#/$defs/AttributeValue'}
+            },
+            required: ['key', 'name', 'header']
+          }
+        },
+        fields: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              name: {type: 'string'},
+              type: {$ref: '#/$defs/ESRIField'},
+              alias: {type: 'string'},
+              domain: {$ref: '#/$defs/Domain'}
+            },
+            required: ['name', 'type']
+          }
+        },
+        popupInfo: {
+          type: 'object',
+          properties: {
+            title: {type: 'string'},
+            description: {type: 'string'},
+            expressionInfos: {type: 'array', items: {}},
+            fieldInfos: {type: 'array', items: {$ref: '#/$defs/FieldInfo'}},
+            mediaInfos: {type: 'array', items: {}},
+            popupElements: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  text: {type: 'string'},
+                  type: {type: 'string'},
+                  fieldInfos: {type: 'array', items: {$ref: '#/$defs/FieldInfo'}}
+                }
+              }
+            }
+          }
+        }
+      },
+      required: ['attributeStorageInfo', 'fields']
+    },
+    materialDefinitions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          pbrMetallicRoughness: {
+            type: 'object',
+            properties: {
+              baseColorFactor: {
+                type: 'array',
+                minItems: 4,
+                maxItems: 4,
+                items: {type: 'number'}
+              },
+              baseColorTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+              metallicFactor: {type: 'number'},
+              roughnessFactor: {type: 'number'},
+              metallicRoughnessTexture: {$ref: '#/$defs/I3SMaterialTexture'}
+            },
+            required: ['metallicFactor', 'roughnessFactor']
+          },
+          normalTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          occlusionTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          emissiveTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          emissiveFactor: {type: 'array', minItems: 3, maxItems: 3, items: {type: 'number'}},
+          alphaMode: {enum: ['opaque', 'mask', 'blend']},
+          alphaCutoff: {type: 'number'},
+          doubleSided: {type: 'boolean'},
+          cullFace: {enum: ['none', 'front', 'back']}
+        },
+        required: ['pbrMetallicRoughness', 'alphaMode']
+      }
+    }
+  },
+  required: ['options', 'tilesConverted'],
+  $defs: {
+    AttributeValue: {
+      type: 'object',
+      properties: {
+        valueType: {type: 'string'},
+        encoding: {type: 'string'},
+        valuesPerElement: {type: 'number'}
+      },
+      required: ['valueType']
+    },
+    ESRIField: {
+      enum: [
+        'esriFieldTypeDate',
+        'esriFieldTypeSingle',
+        'esriFieldTypeDouble',
+        'esriFieldTypeGUID',
+        'esriFieldTypeGlobalID',
+        'esriFieldTypeInteger',
+        'esriFieldTypeOID',
+        'esriFieldTypeSmallInteger',
+        'esriFieldTypeString'
+      ]
+    },
+    Domain: {
+      type: 'object',
+      properties: {
+        type: {type: 'string'},
+        name: {type: 'string'},
+        description: {type: 'string'},
+        fieldType: {type: 'string'},
+        range: {type: 'array', items: {type: 'number'}},
+        codedValues: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {name: {type: 'string'}, code: {type: ['string', 'number']}},
+            required: ['name', 'code']
+          }
+        },
+        mergePolicy: {type: 'string'},
+        splitPolicy: {type: 'string'}
+      },
+      required: ['type', 'name']
+    },
+    FieldInfo: {
+      type: 'object',
+      properties: {
+        fieldName: {type: 'string'},
+        visible: {type: 'boolean'},
+        isEditable: {type: 'boolean'},
+        label: {type: 'string'}
+      },
+      required: ['fieldName', 'visible', 'isEditable', 'label']
+    },
+    I3SMaterialTexture: {
+      type: 'object',
+      properties: {
+        textureSetDefinitionId: {type: 'number'},
+        texCoord: {type: 'number'},
+        factor: {type: 'number'}
+      },
+      required: ['textureSetDefinitionId']
+    }
+  }
+};