@loaders.gl/tile-converter 4.2.0-alpha.1 → 4.2.0-alpha.2

package/src/3d-tiles-converter/helpers/load-i3s.ts

@@ -1,36 +1,45 @@
-import {load} from '@loaders.gl/core';
+import {LoaderWithParser, load} from '@loaders.gl/core';
 import {
   I3STileContent,
   I3STileHeader,
   I3STilesetHeader,
   I3SLoader,
-  I3SLoaderOptions
+  I3SLoaderOptions,
+  parseSLPKArchive
 } from '@loaders.gl/i3s';
+import {FileHandleFile} from '@loaders.gl/loader-utils';
+import {ZipFileSystem} from '@loaders.gl/zip';
+
+export type SLPKUrlParts = {slpkFileName: string; internalFileName: string};
 
 /**
  * Load I3S node content
  * @param sourceTileset - source layer JSON
  * @param sourceTile - source I3S node metadata
  * @param tilesetLoadOptions - load options for Tiles3DLoader
+ * @param slpkFilesystem - loaded instance of ZipFileSystem for local convertion from SLPK file
  * @returns - 3DTiles tile content or null
  */
 export const loadI3SContent = async (
   sourceTileset: I3STilesetHeader | null,
   sourceTile: I3STileHeader,
-  tilesetLoadOptions: I3SLoaderOptions
+  tilesetLoadOptions: I3SLoaderOptions,
+  slpkFilesystem: ZipFileSystem | null
 ): Promise<I3STileContent | null> => {
   if (!sourceTileset || !sourceTile.contentUrl) {
     return null;
   }
 
-  const loadOptions = {
+  const loadOptions: I3SLoaderOptions = {
     ...tilesetLoadOptions,
     i3s: {
       ...tilesetLoadOptions.i3s,
+      // @ts-expect-error
       isTileset: false,
+      // @ts-expect-error
       isTileHeader: false,
       _tileOptions: {
-        attributeUrls: sourceTile.attributeUrls,
+        attributeUrls: sourceTile.attributeUrls || [],
         textureUrl: sourceTile.textureUrl,
         textureFormat: sourceTile.textureFormat,
         textureLoaderOptions: sourceTile.textureLoaderOptions,
@@ -40,13 +49,83 @@ export const loadI3SContent = async (
       },
       _tilesetOptions: {
         store: sourceTileset.store,
+        // @ts-expect-error
         attributeStorageInfo: sourceTileset.attributeStorageInfo,
+        // @ts-expect-error
         fields: sourceTileset.fields
       }
     }
   };
-  const tileContent = await load(sourceTile.contentUrl, I3SLoader, loadOptions);
+  const tileContent = await loadFromArchive(
+    sourceTile.contentUrl,
+    I3SLoader,
+    loadOptions,
+    slpkFilesystem
+  );
 
-  // @ts-expect-error
   return tileContent;
 };
+
+/**
+ * Load local SLPK file to ZipFileSystem instance
+ * @param url - path to SLPK file
+ * @returns instance of ZipFileSystem or null if url is not an SLPK file
+ */
+export async function openSLPK(url: string): Promise<ZipFileSystem | null> {
+  const slpkUrlParts = url.split('.slpk');
+  const {slpkFileName} = getSlpkUrlParts(url) || {};
+  if (slpkFileName) {
+    const slpkFileName = `${slpkUrlParts[0]}.slpk`;
+    const fileProvider = new FileHandleFile(slpkFileName);
+    const archive = await parseSLPKArchive(fileProvider, undefined, slpkFileName);
+    const fileSystem = new ZipFileSystem(archive);
+    return fileSystem;
+  }
+  return null;
+}
+
+/**
+ * Load a resource with load options and .3tz format support
+ * @param url - resource URL
+ * @param loader - loader to parse data (Tiles3DLoader / CesiumIonLoader)
+ * @param loadOptions - i3s loader options
+ * @returns i3s resource
+ */
+export async function loadFromArchive(
+  url: string,
+  loader: LoaderWithParser,
+  loadOptions: I3SLoaderOptions,
+  fileSystem: ZipFileSystem | null
+) {
+  const slpkUrlParts = getSlpkUrlParts(url);
+  if (fileSystem !== null && slpkUrlParts !== null) {
+    const {internalFileName} = slpkUrlParts;
+    const content = await load(internalFileName, loader, {
+      ...loadOptions,
+      fetch: fileSystem.fetch.bind(fileSystem)
+    });
+    return content;
+  }
+  return await load(url, loader, loadOptions);
+}
+
+/**
+ * Extract slpk file path and internal from the url
+ * For example, for `./path/to/file.slpk/nodes/0` it returns
+ * {"slpkFileName": "./path/to/file.slpk", "internalFileName": "/nodes/0" }
+ * @param url full internal file path
+ * @returns object with internal slpk file parts
+ */
+function getSlpkUrlParts(url: string): SLPKUrlParts | null {
+  const slpkUrlParts = url.split('.slpk');
+  let result: SLPKUrlParts | null;
+  // Not '.slpk'. The file will be loaded with global fetch function
+  if (slpkUrlParts.length === 1) {
+    result = null;
+  } else if (slpkUrlParts.length === 2) {
+    result = {slpkFileName: `${slpkUrlParts[0]}.slpk`, internalFileName: slpkUrlParts[1].slice(1)};
+  } else {
+    throw new Error('Unexpected URL format');
+  }
+  return result;
+}

package/src/converter-cli.ts

@@ -210,7 +210,8 @@ async function convert(options: ValidatedTileConversionOptions) {
         outputPath: options.output,
         tilesetName: options.name,
         maxDepth: options.maxDepth,
-        egmFilePath: options.egm
+        egmFilePath: options.egm,
+        inquirer
       });
       break;
     case TILESET_TYPE._3DTILES:

package/src/i3s-converter/helpers/load-3d-tiles.ts

@@ -4,8 +4,17 @@ import type {
   Tiles3DTileJSONPostprocessed,
   Tiles3DTilesetJSONPostprocessed
 } from '@loaders.gl/3d-tiles';
-import {Tiles3DArchiveFileSystem} from '@loaders.gl/3d-tiles';
+import {Tiles3DArchive} from '@loaders.gl/3d-tiles';
 import {LoaderWithParser, load} from '@loaders.gl/core';
+import {FileHandleFile, FileProvider} from '@loaders.gl/loader-utils';
+import {
+  CD_HEADER_SIGNATURE,
+  ZipFileSystem,
+  parseHashTable,
+  parseZipCDFileHeader,
+  parseZipLocalFileHeader,
+  searchFromTheEnd
+} from '@loaders.gl/zip';
 
 /**
  * Load nested 3DTiles tileset. If the sourceTile is not nested tileset - do nothing
@@ -104,7 +113,10 @@ export async function loadFromArchive(
   }
   if (filename) {
     const tz3Path = `${tz3UrlParts[0]}.3tz`;
-    const fileSystem = new Tiles3DArchiveFileSystem(tz3Path);
+    const fileProvider = new FileHandleFile(tz3Path);
+    const hashTable = await loadHashTable(fileProvider);
+    const archive = new Tiles3DArchive(fileProvider, hashTable, tz3Path);
+    const fileSystem = new ZipFileSystem(archive);
     const content = await load(filename, loader, {
       ...loadOptions,
       fetch: fileSystem.fetch.bind(fileSystem)
@@ -123,3 +135,41 @@ export async function loadFromArchive(
 export function isNestedTileset(tile: Tiles3DTileJSONPostprocessed) {
   return tile?.type === 'json' || tile?.type === '3tz';
 }
+
+/**
+ * Load hash file from 3TZ
+ * @param fileProvider - binary reader of 3TZ
+ * @returns hash table of the 3TZ file content or undefined if the hash file is not presented inside
+ */
+async function loadHashTable(
+  fileProvider: FileProvider
+): Promise<undefined | Record<string, bigint>> {
+  let hashTable: undefined | Record<string, bigint>;
+
+  const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE);
+
+  const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
+
+  // '@3dtilesIndex1@' is index file that must be the last in the archive. It allows
+  // to improve load and read performance when the archive contains a very large number
+  // of files.
+  if (cdFileHeader?.fileName === '@3dtilesIndex1@') {
+    const localFileHeader = await parseZipLocalFileHeader(
+      cdFileHeader.localHeaderOffset,
+      fileProvider
+    );
+    if (!localFileHeader) {
+      throw new Error('corrupted 3tz');
+    }
+
+    const fileDataOffset = localFileHeader.fileDataOffset;
+    const hashFile = await fileProvider.slice(
+      fileDataOffset,
+      fileDataOffset + localFileHeader.compressedSize
+    );
+
+    hashTable = parseHashTable(hashFile);
+  }
+
+  return hashTable;
+}

package/src/lib/json-schemas/conversion-dump-json-schema.ts

@@ -0,0 +1,285 @@
+export const dumpJsonSchema = {
+  type: 'object',
+  properties: {
+    options: {
+      type: 'object',
+      properties: {
+        inputUrl: {type: 'string'},
+        outputPath: {type: 'string'},
+        tilesetName: {type: 'string'},
+        maxDepth: {type: 'number'},
+        slpk: {type: 'boolean'},
+        egmFilePath: {type: 'string'},
+        token: {type: 'string'},
+        draco: {type: 'boolean'},
+        mergeMaterials: {type: 'boolean'},
+        generateTextures: {type: 'boolean'},
+        generateBoundingVolumes: {type: 'boolean'},
+        metadataClass: {type: 'string'},
+        analyze: {type: 'boolean'}
+      },
+      required: ['inputUrl', 'outputPath', 'tilesetName']
+    },
+    tilesConverted: {
+      type: 'object',
+      patternProperties: {
+        '.*': {
+          type: 'object',
+          properties: {
+            nodes: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  nodeId: {type: ['number', 'string']},
+                  done: {type: 'boolean'},
+                  progress: {type: 'object', patternProperties: {'.*': {type: 'boolean'}}},
+                  dumpMetadata: {
+                    type: 'object',
+                    properties: {
+                      boundingVolumes: {
+                        type: ['object', 'null'],
+                        properties: {
+                          mbs: {
+                            type: 'array',
+                            minItems: 4,
+                            maxItems: 4,
+                            items: {type: 'number'}
+                          },
+                          obb: {
+                            type: 'object',
+                            properties: {
+                              center: {
+                                type: 'array',
+                                minItems: 3,
+                                maxItems: 3,
+                                items: {type: 'number'}
+                              },
+                              halfSize: {
+                                type: 'array',
+                                minItems: 3,
+                                maxItems: 3,
+                                items: {type: 'number'}
+                              },
+                              quaternion: {
+                                type: 'array',
+                                minItems: 4,
+                                maxItems: 4,
+                                items: {type: 'number'}
+                              }
+                            },
+                            required: ['center', 'halfSize', 'quaternion']
+                          }
+                        },
+                        required: ['mbs', 'obb']
+                      },
+                      attributesCount: {type: 'number'},
+                      featureCount: {type: 'number'},
+                      geometry: {type: 'boolean'},
+                      hasUvRegions: {type: 'boolean'},
+                      materialId: {type: 'number'},
+                      texelCountHint: {type: 'number'},
+                      vertexCount: {type: 'number'}
+                    },
+                    required: [
+                      'boundingVolumes',
+                      'featureCount',
+                      'geometry',
+                      'hasUvRegions',
+                      'materialId',
+                      'vertexCount'
+                    ]
+                  }
+                },
+                required: ['nodeId', 'done']
+              }
+            }
+          },
+          required: ['nodes']
+        }
+      }
+    },
+    textureSetDefinitions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          formats: {
+            type: 'array',
+            items: {
+              type: 'object',
+              properties: {
+                name: {type: 'string'},
+                format: {enum: ['jpg', 'png', 'ktx-etc2', 'dds', 'ktx2']}
+              },
+              required: ['name', 'format']
+            }
+          },
+          atlas: {type: 'boolean'}
+        },
+        required: ['formats']
+      }
+    },
+    attributeMetadataInfo: {
+      type: 'object',
+      properties: {
+        attributeStorageInfo: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              key: {type: 'string'},
+              name: {type: 'string'},
+              header: {
+                type: 'array',
+                items: {
+                  type: 'object',
+                  properties: {property: {type: 'string'}, valueType: {type: 'string'}},
+                  required: ['property', 'valueType']
+                }
+              },
+              ordering: {type: 'array', items: {type: 'string'}},
+              attributeValues: {$ref: '#/$defs/AttributeValue'},
+              attributeByteCounts: {$ref: '#/$defs/AttributeValue'},
+              objectIds: {$ref: '#/$defs/AttributeValue'}
+            },
+            required: ['key', 'name', 'header']
+          }
+        },
+        fields: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              name: {type: 'string'},
+              type: {$ref: '#/$defs/ESRIField'},
+              alias: {type: 'string'},
+              domain: {$ref: '#/$defs/Domain'}
+            },
+            required: ['name', 'type']
+          }
+        },
+        popupInfo: {
+          type: 'object',
+          properties: {
+            title: {type: 'string'},
+            description: {type: 'string'},
+            expressionInfos: {type: 'array', items: {}},
+            fieldInfos: {type: 'array', items: {$ref: '#/$defs/FieldInfo'}},
+            mediaInfos: {type: 'array', items: {}},
+            popupElements: {
+              type: 'array',
+              items: {
+                type: 'object',
+                properties: {
+                  text: {type: 'string'},
+                  type: {type: 'string'},
+                  fieldInfos: {type: 'array', items: {$ref: '#/$defs/FieldInfo'}}
+                }
+              }
+            }
+          }
+        }
+      },
+      required: ['attributeStorageInfo', 'fields']
+    },
+    materialDefinitions: {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          pbrMetallicRoughness: {
+            type: 'object',
+            properties: {
+              baseColorFactor: {
+                type: 'array',
+                minItems: 4,
+                maxItems: 4,
+                items: {type: 'number'}
+              },
+              baseColorTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+              metallicFactor: {type: 'number'},
+              roughnessFactor: {type: 'number'},
+              metallicRoughnessTexture: {$ref: '#/$defs/I3SMaterialTexture'}
+            },
+            required: ['metallicFactor', 'roughnessFactor']
+          },
+          normalTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          occlusionTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          emissiveTexture: {$ref: '#/$defs/I3SMaterialTexture'},
+          emissiveFactor: {type: 'array', minItems: 3, maxItems: 3, items: {type: 'number'}},
+          alphaMode: {enum: ['opaque', 'mask', 'blend']},
+          alphaCutoff: {type: 'number'},
+          doubleSided: {type: 'boolean'},
+          cullFace: {enum: ['none', 'front', 'back']}
+        },
+        required: ['pbrMetallicRoughness', 'alphaMode']
+      }
+    }
+  },
+  required: ['options', 'tilesConverted'],
+  $defs: {
+    AttributeValue: {
+      type: 'object',
+      properties: {
+        valueType: {type: 'string'},
+        encoding: {type: 'string'},
+        valuesPerElement: {type: 'number'}
+      },
+      required: ['valueType']
+    },
+    ESRIField: {
+      enum: [
+        'esriFieldTypeDate',
+        'esriFieldTypeSingle',
+        'esriFieldTypeDouble',
+        'esriFieldTypeGUID',
+        'esriFieldTypeGlobalID',
+        'esriFieldTypeInteger',
+        'esriFieldTypeOID',
+        'esriFieldTypeSmallInteger',
+        'esriFieldTypeString'
+      ]
+    },
+    Domain: {
+      type: 'object',
+      properties: {
+        type: {type: 'string'},
+        name: {type: 'string'},
+        description: {type: 'string'},
+        fieldType: {type: 'string'},
+        range: {type: 'array', items: {type: 'number'}},
+        codedValues: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {name: {type: 'string'}, code: {type: ['string', 'number']}},
+            required: ['name', 'code']
+          }
+        },
+        mergePolicy: {type: 'string'},
+        splitPolicy: {type: 'string'}
+      },
+      required: ['type', 'name']
+    },
+    FieldInfo: {
+      type: 'object',
+      properties: {
+        fieldName: {type: 'string'},
+        visible: {type: 'boolean'},
+        isEditable: {type: 'boolean'},
+        label: {type: 'string'}
+      },
+      required: ['fieldName', 'visible', 'isEditable', 'label']
+    },
+    I3SMaterialTexture: {
+      type: 'object',
+      properties: {
+        textureSetDefinitionId: {type: 'number'},
+        texCoord: {type: 'number'},
+        factor: {type: 'number'}
+      },
+      required: ['textureSetDefinitionId']
+    }
+  }
+};

package/src/lib/utils/conversion-dump.ts

@@ -1,9 +1,12 @@
 import {isDeepStrictEqual} from 'util';
 import {DUMP_FILE_SUFFIX} from '../../constants';
-import {isFileExists, openJson, removeFile, writeFile} from './file-utils';
+import {isFileExists, openJson, removeFile, renameFile, writeFile} from './file-utils';
 import {join} from 'path';
 import {BoundingVolumes, I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s';
 import {AttributeMetadataInfoObject} from '../../i3s-converter/helpers/attribute-metadata-info';
+import process from 'process';
+import Ajv from 'ajv';
+import {dumpJsonSchema} from '../json-schemas/conversion-dump-json-schema';
 
 export type ConversionDumpOptions = {
   inputUrl: string;
@@ -22,7 +25,7 @@ export type ConversionDumpOptions = {
 };
 
 type NodeDoneStatus = {
-  nodeId: number;
+  nodeId: number | string;
   done: boolean;
   progress?: Record<string, boolean>;
   dumpMetadata?: DumpMetadata;
@@ -60,7 +63,7 @@ export class ConversionDump {
   /** Attributes Metadata */
   attributeMetadataInfo?: AttributeMetadataInfoObject;
   /** Array of materials definitions */
-  materialDefinitions: I3SMaterialDefinition[] = [];
+  materialDefinitions?: I3SMaterialDefinition[];
 
   constructor() {
     this.tilesConverted = {};
@@ -108,23 +111,34 @@ export class ConversionDump {
       `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
     );
     if (await isFileExists(dumpFilename)) {
-      const {
-        options,
-        tilesConverted,
-        textureSetDefinitions,
-        attributeMetadataInfo,
-        materialDefinitions
-      } = await openJson(
-        join(this.options.outputPath, this.options.tilesetName),
-        `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
-      );
-      if (isDeepStrictEqual(options, JSON.parse(JSON.stringify(this.options)))) {
-        this.tilesConverted = tilesConverted;
-        this.textureSetDefinitions = textureSetDefinitions;
-        this.attributeMetadataInfo = attributeMetadataInfo;
-        this.materialDefinitions = materialDefinitions;
-        this.restored = true;
-        return;
+      try {
+        const dump = await openJson(
+          join(this.options.outputPath, this.options.tilesetName),
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        );
+
+        const {
+          options,
+          tilesConverted,
+          textureSetDefinitions,
+          attributeMetadataInfo,
+          materialDefinitions
+        } = dump;
+
+        const ajv = new Ajv();
+        const dumpJsonValidate = ajv.compile(dumpJsonSchema);
+        const isDumpValid = dumpJsonValidate(dump);
+
+        if (isDumpValid && isDeepStrictEqual(options, JSON.parse(JSON.stringify(this.options)))) {
+          this.tilesConverted = tilesConverted;
+          this.textureSetDefinitions = textureSetDefinitions;
+          this.attributeMetadataInfo = attributeMetadataInfo;
+          this.materialDefinitions = materialDefinitions;
+          this.restored = true;
+          return;
+        }
+      } catch (error) {
+        console.log("Can't open dump file", error);
       }
     }
     await this.deleteDumpFile();
@@ -142,8 +156,8 @@ export class ConversionDump {
     if (this.attributeMetadataInfo) {
       delete this.attributeMetadataInfo;
     }
-    if (this.materialDefinitions.length > 0) {
-      this.materialDefinitions = [];
+    if (this.materialDefinitions) {
+      delete this.materialDefinitions;
     }
   }
 
@@ -153,6 +167,7 @@ export class ConversionDump {
   private async updateDumpFile(): Promise<void> {
     if (this.options?.outputPath && this.options.tilesetName) {
       try {
+        const time = process.hrtime();
         await writeFile(
           join(this.options.outputPath, this.options.tilesetName),
           JSON.stringify({
@@ -162,7 +177,19 @@ export class ConversionDump {
             attributeMetadataInfo: this.attributeMetadataInfo,
             materialDefinitions: this.materialDefinitions
           }),
-          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}.${time[0]}.${time[1]}`
+        );
+        await renameFile(
+          join(
+            this.options.outputPath,
+            this.options.tilesetName,
+            `${this.options.tilesetName}${DUMP_FILE_SUFFIX}.${time[0]}.${time[1]}`
+          ),
+          join(
+            this.options.outputPath,
+            this.options.tilesetName,
+            `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+          )
         );
       } catch (error) {
         console.log("Can't update dump file", error);
@@ -218,9 +245,9 @@ export class ConversionDump {
    * @param fileName - source filename
    * @param nodeId - nodeId of the node
    */
-  async addNode(filename: string, nodeId: number, dumpMetadata: DumpMetadata) {
+  async addNode(filename: string, nodeId: number | string, dumpMetadata?: DumpMetadata) {
     const {nodes} = this.getRecord(filename) || {nodes: []};
-    nodes.push({nodeId, done: false, progress: {}, dumpMetadata});
+    nodes.push({nodeId, done: false, dumpMetadata});
     if (nodes.length === 1) {
       this.setRecord(filename, {nodes});
     }
@@ -250,7 +277,12 @@ export class ConversionDump {
    * @param resourceType - resource type to update status
    * @param value - value
    */
-  updateDoneStatus(filename: string, nodeId: number, resourceType: string, value: boolean) {
+  updateDoneStatus(
+    filename: string,
+    nodeId: number | string,
+    resourceType: string,
+    value: boolean
+  ) {
     const nodeDump = this.tilesConverted[filename]?.nodes.find(
       (element) => element.nodeId === nodeId
     );
@@ -271,7 +303,7 @@ export class ConversionDump {
    * @param writeResults - array of writing resource files results
    */
   async updateConvertedTilesDump(
-    changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[],
+    changedRecords: {outputId?: number | string; sourceId?: string; resourceType?: string}[],
     writeResults: PromiseSettledResult<string | null>[]
   ) {
     for (let i = 0; i < changedRecords.length; i++) {
@@ -299,6 +331,26 @@ export class ConversionDump {
     await this.updateDumpFile();
   }
 
+  /**
+   * Update 3d-tiles-converter dump file
+   * @param filename - source filename
+   * @param nodeId - nodeId
+   * @param done - conversion status
+   */
+  async updateConvertedNodesDumpFile(
+    filename: string,
+    nodeId: number | string,
+    done: boolean
+  ): Promise<void> {
+    const nodeDump = this.tilesConverted[filename]?.nodes.find(
+      (element) => element.nodeId === nodeId
+    );
+    if (nodeDump) {
+      nodeDump.done = done;
+      await this.updateDumpFile();
+    }
+  }
+
   /**
    * Check is source file conversion complete
    * @param filename - source filename