@loaders.gl/tile-converter 4.2.0-alpha.1 → 4.2.0-alpha.3

package/src/lib/json-schemas/conversion-dump-json-schema.ts

@@ -0,0 +1,285 @@
+export const dumpJsonSchema = {
+  type: 'object',
+  properties: {
+    options: {
+      type: 'object',
+      properties: {
+        inputUrl: {type: 'string'},
+        outputPath: {type: 'string'},
+        tilesetName: {type: 'string'},
+        maxDepth: {type: 'number'},
+        slpk: {type: 'boolean'},
+        egmFilePath: {type: 'string'},
+        token: {type: 'string'},
+        draco: {type: 'boolean'},
+        mergeMaterials: {type: 'boolean'},
+        generateTextures: {type: 'boolean'},
+        generateBoundingVolumes: {type: 'boolean'},
+        metadataClass: {type: 'string'},
+        analyze: {type: 'boolean'}
+      },
+      required: ['inputUrl', 'outputPath', 'tilesetName']
+    },
+    tilesConverted: {
+      type: 'object',
+      patternProperties: {
+        '.*': {
+          type: 'object',
+          properties: {
+            nodes: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  nodeId: {type: ['number', 'string']},
+                  done: {type: 'boolean'},
+                  progress: {type: 'object', patternProperties: {'.*': {type: 'boolean'}}},
+                  dumpMetadata: {
+                    type: 'object',
+                    properties: {
+                      boundingVolumes: {
+                        type: ['object', 'null'],
+                        properties: {
+                          mbs: {
+                            type: 'array',
+                            minItems: 4,
+                            maxItems: 4,
+                            items: {type: 'number'}
+                          },
+                          obb: {
+                            type: 'object',
+                            properties: {
+                              center: {
+                                type: 'array',
+                                minItems: 3,
+                                maxItems: 3,
+                                items: {type: 'number'}
+                              },
+                              halfSize: {
+                                type: 'array',
+                                minItems: 3,
+                                maxItems: 3,
+                                items: {type: 'number'}
+                              },
+                              quaternion: {
+                                type: 'array',
+                                minItems: 4,
+                                maxItems: 4,
+                                items: {type: 'number'}
+                              }
+                            },
+                            required: ['center', 'halfSize', 'quaternion']
+                          }
+                        },
+                        required: ['mbs', 'obb']
+                      },
+                      attributesCount: {type: 'number'},
+                      featureCount: {type: 'number'},
+                      geometry: {type: 'boolean'},
+                      hasUvRegions: {type: 'boolean'},
+                      materialId: {type: 'number'},
+                      texelCountHint: {type: 'number'},
+                      vertexCount: {type: 'number'}
+                    },
+                    required: [
+                      'boundingVolumes',
+                      'featureCount',
+                      'geometry',
+                      'hasUvRegions',
+                      'materialId',
+                      'vertexCount'
+                    ]
+                  }
+                },
+                required: ['nodeId', 'done']
+              }
+            }
+          },
+          required: ['nodes']
+        }
+      }
+    },
+    textureSetDefinitions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          formats: {
+            type: 'array',
+            items: {
+              type: 'object',
+              properties: {
+                name: {type: 'string'},
+                format: {enum: ['jpg', 'png', 'ktx-etc2', 'dds', 'ktx2']}
+              },
+              required: ['name', 'format']
+            }
+          },
+          atlas: {type: 'boolean'}
+        },
+        required: ['formats']
+      }
+    },
+    attributeMetadataInfo: {
+      type: 'object',
+      properties: {
+        attributeStorageInfo: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              key: {type: 'string'},
+              name: {type: 'string'},
+              header: {
+                type: 'array',
+                items: {
+                  type: 'object',
+                  properties: {property: {type: 'string'}, valueType: {type: 'string'}},
+                  required: ['property', 'valueType']
+                }
+              },
+              ordering: {type: 'array', items: {type: 'string'}},
+              attributeValues: {$ref: '#/$defs/AttributeValue'},
+              attributeByteCounts: {$ref: '#/$defs/AttributeValue'},
+              objectIds: {$ref: '#/$defs/AttributeValue'}
+            },
+            required: ['key', 'name', 'header']
+          }
+        },
+        fields: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              name: {type: 'string'},
+              type: {$ref: '#/$defs/ESRIField'},
+              alias: {type: 'string'},
+              domain: {$ref: '#/$defs/Domain'}
+            },
+            required: ['name', 'type']
+          }
+        },
+        popupInfo: {
+          type: 'object',
+          properties: {
+            title: {type: 'string'},
+            description: {type: 'string'},
+            expressionInfos: {type: 'array', items: {}},
+            fieldInfos: {type: 'array', items: {$ref: '#/$defs/FieldInfo'}},
+            mediaInfos: {type: 'array', items: {}},
+            popupElements: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  text: {type: 'string'},
+                  type: {type: 'string'},
+                  fieldInfos: {type: 'array', items: {$ref: '#/$defs/FieldInfo'}}
+                }
+              }
+            }
+          }
+        }
+      },
+      required: ['attributeStorageInfo', 'fields']
+    },
+    materialDefinitions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          pbrMetallicRoughness: {
+            type: 'object',
+            properties: {
+              baseColorFactor: {
+                type: 'array',
+                minItems: 4,
+                maxItems: 4,
+                items: {type: 'number'}
+              },
+              baseColorTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+              metallicFactor: {type: 'number'},
+              roughnessFactor: {type: 'number'},
+              metallicRoughnessTexture: {$ref: '#/$defs/I3SMaterialTexture'}
+            },
+            required: ['metallicFactor', 'roughnessFactor']
+          },
+          normalTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          occlusionTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          emissiveTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          emissiveFactor: {type: 'array', minItems: 3, maxItems: 3, items: {type: 'number'}},
+          alphaMode: {enum: ['opaque', 'mask', 'blend']},
+          alphaCutoff: {type: 'number'},
+          doubleSided: {type: 'boolean'},
+          cullFace: {enum: ['none', 'front', 'back']}
+        },
+        required: ['pbrMetallicRoughness', 'alphaMode']
+      }
+    }
+  },
+  required: ['options', 'tilesConverted'],
+  $defs: {
+    AttributeValue: {
+      type: 'object',
+      properties: {
+        valueType: {type: 'string'},
+        encoding: {type: 'string'},
+        valuesPerElement: {type: 'number'}
+      },
+      required: ['valueType']
+    },
+    ESRIField: {
+      enum: [
+        'esriFieldTypeDate',
+        'esriFieldTypeSingle',
+        'esriFieldTypeDouble',
+        'esriFieldTypeGUID',
+        'esriFieldTypeGlobalID',
+        'esriFieldTypeInteger',
+        'esriFieldTypeOID',
+        'esriFieldTypeSmallInteger',
+        'esriFieldTypeString'
+      ]
+    },
+    Domain: {
+      type: 'object',
+      properties: {
+        type: {type: 'string'},
+        name: {type: 'string'},
+        description: {type: 'string'},
+        fieldType: {type: 'string'},
+        range: {type: 'array', items: {type: 'number'}},
+        codedValues: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {name: {type: 'string'}, code: {type: ['string', 'number']}},
+            required: ['name', 'code']
+          }
+        },
+        mergePolicy: {type: 'string'},
+        splitPolicy: {type: 'string'}
+      },
+      required: ['type', 'name']
+    },
+    FieldInfo: {
+      type: 'object',
+      properties: {
+        fieldName: {type: 'string'},
+        visible: {type: 'boolean'},
+        isEditable: {type: 'boolean'},
+        label: {type: 'string'}
+      },
+      required: ['fieldName', 'visible', 'isEditable', 'label']
+    },
+    I3SMaterialTexture: {
+      type: 'object',
+      properties: {
+        textureSetDefinitionId: {type: 'number'},
+        texCoord: {type: 'number'},
+        factor: {type: 'number'}
+      },
+      required: ['textureSetDefinitionId']
+    }
+  }
+};

package/src/lib/utils/conversion-dump.ts

@@ -1,9 +1,12 @@
 import {isDeepStrictEqual} from 'util';
 import {DUMP_FILE_SUFFIX} from '../../constants';
-import {isFileExists, openJson, removeFile, writeFile} from './file-utils';
+import {isFileExists, openJson, removeFile, renameFile, writeFile} from './file-utils';
 import {join} from 'path';
 import {BoundingVolumes, I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s';
 import {AttributeMetadataInfoObject} from '../../i3s-converter/helpers/attribute-metadata-info';
+import process from 'process';
+import Ajv from 'ajv';
+import {dumpJsonSchema} from '../json-schemas/conversion-dump-json-schema';
 
 export type ConversionDumpOptions = {
   inputUrl: string;
@@ -22,7 +25,7 @@ export type ConversionDumpOptions = {
 };
 
 type NodeDoneStatus = {
-  nodeId: number;
+  nodeId: number | string;
   done: boolean;
   progress?: Record<string, boolean>;
   dumpMetadata?: DumpMetadata;
@@ -60,7 +63,7 @@ export class ConversionDump {
   /** Attributes Metadata */
   attributeMetadataInfo?: AttributeMetadataInfoObject;
   /** Array of materials definitions */
-  materialDefinitions: I3SMaterialDefinition[] = [];
+  materialDefinitions?: I3SMaterialDefinition[];
 
   constructor() {
     this.tilesConverted = {};
@@ -108,23 +111,34 @@ export class ConversionDump {
       `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
     );
     if (await isFileExists(dumpFilename)) {
-      const {
-        options,
-        tilesConverted,
-        textureSetDefinitions,
-        attributeMetadataInfo,
-        materialDefinitions
-      } = await openJson(
-        join(this.options.outputPath, this.options.tilesetName),
-        `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
-      );
-      if (isDeepStrictEqual(options, JSON.parse(JSON.stringify(this.options)))) {
-        this.tilesConverted = tilesConverted;
-        this.textureSetDefinitions = textureSetDefinitions;
-        this.attributeMetadataInfo = attributeMetadataInfo;
-        this.materialDefinitions = materialDefinitions;
-        this.restored = true;
-        return;
+      try {
+        const dump = await openJson(
+          join(this.options.outputPath, this.options.tilesetName),
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        );
+
+        const {
+          options,
+          tilesConverted,
+          textureSetDefinitions,
+          attributeMetadataInfo,
+          materialDefinitions
+        } = dump;
+
+        const ajv = new Ajv();
+        const dumpJsonValidate = ajv.compile(dumpJsonSchema);
+        const isDumpValid = dumpJsonValidate(dump);
+
+        if (isDumpValid && isDeepStrictEqual(options, JSON.parse(JSON.stringify(this.options)))) {
+          this.tilesConverted = tilesConverted;
+          this.textureSetDefinitions = textureSetDefinitions;
+          this.attributeMetadataInfo = attributeMetadataInfo;
+          this.materialDefinitions = materialDefinitions;
+          this.restored = true;
+          return;
+        }
+      } catch (error) {
+        console.log("Can't open dump file", error);
       }
     }
     await this.deleteDumpFile();
@@ -142,8 +156,8 @@ export class ConversionDump {
     if (this.attributeMetadataInfo) {
       delete this.attributeMetadataInfo;
     }
-    if (this.materialDefinitions.length > 0) {
-      this.materialDefinitions = [];
+    if (this.materialDefinitions) {
+      delete this.materialDefinitions;
     }
   }
 
@@ -153,6 +167,7 @@ export class ConversionDump {
   private async updateDumpFile(): Promise<void> {
     if (this.options?.outputPath && this.options.tilesetName) {
       try {
+        const time = process.hrtime();
         await writeFile(
           join(this.options.outputPath, this.options.tilesetName),
           JSON.stringify({
@@ -162,7 +177,19 @@ export class ConversionDump {
             attributeMetadataInfo: this.attributeMetadataInfo,
             materialDefinitions: this.materialDefinitions
           }),
-          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}.${time[0]}.${time[1]}`
+        );
+        await renameFile(
+          join(
+            this.options.outputPath,
+            this.options.tilesetName,
+            `${this.options.tilesetName}${DUMP_FILE_SUFFIX}.${time[0]}.${time[1]}`
+          ),
+          join(
+            this.options.outputPath,
+            this.options.tilesetName,
+            `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+          )
         );
       } catch (error) {
         console.log("Can't update dump file", error);
@@ -218,9 +245,9 @@ export class ConversionDump {
    * @param fileName - source filename
    * @param nodeId - nodeId of the node
    */
-  async addNode(filename: string, nodeId: number, dumpMetadata: DumpMetadata) {
+  async addNode(filename: string, nodeId: number | string, dumpMetadata?: DumpMetadata) {
     const {nodes} = this.getRecord(filename) || {nodes: []};
-    nodes.push({nodeId, done: false, progress: {}, dumpMetadata});
+    nodes.push({nodeId, done: false, dumpMetadata});
     if (nodes.length === 1) {
       this.setRecord(filename, {nodes});
     }
@@ -250,7 +277,12 @@ export class ConversionDump {
    * @param resourceType - resource type to update status
    * @param value - value
    */
-  updateDoneStatus(filename: string, nodeId: number, resourceType: string, value: boolean) {
+  updateDoneStatus(
+    filename: string,
+    nodeId: number | string,
+    resourceType: string,
+    value: boolean
+  ) {
     const nodeDump = this.tilesConverted[filename]?.nodes.find(
       (element) => element.nodeId === nodeId
     );
@@ -271,7 +303,7 @@ export class ConversionDump {
    * @param writeResults - array of writing resource files results
    */
   async updateConvertedTilesDump(
-    changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[],
+    changedRecords: {outputId?: number | string; sourceId?: string; resourceType?: string}[],
     writeResults: PromiseSettledResult<string | null>[]
   ) {
     for (let i = 0; i < changedRecords.length; i++) {
@@ -299,6 +331,26 @@ export class ConversionDump {
     await this.updateDumpFile();
   }
 
+  /**
+   * Update 3d-tiles-converter dump file
+   * @param filename - source filename
+   * @param nodeId - nodeId
+   * @param done - conversion status
+   */
+  async updateConvertedNodesDumpFile(
+    filename: string,
+    nodeId: number | string,
+    done: boolean
+  ): Promise<void> {
+    const nodeDump = this.tilesConverted[filename]?.nodes.find(
+      (element) => element.nodeId === nodeId
+    );
+    if (nodeDump) {
+      nodeDump.done = done;
+      await this.updateDumpFile();
+    }
+  }
+
   /**
    * Check is source file conversion complete
    * @param filename - source filename

package/src/lib/utils/file-utils.ts

@@ -138,3 +138,16 @@ export function removeFile(path: string) {
 export function getAbsoluteFilePath(filePath: string) {
   return isAbsolute(filePath) ? filePath : join(process.cwd(), filePath);
 }
+
+/**
+ * Rename file with old path by new path
+ * @param oldPath
+ * @param newPath
+ */
+export async function renameFile(oldPath: string, newPath: string): Promise<void> {
+  try {
+    await fs.rename(oldPath, newPath);
+  } catch (err) {
+    console.log("Can't rename file", err);
+  }
+}