@loaders.gl/tile-converter 4.1.1 → 4.2.0-alpha.2

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -4,7 +4,7 @@ import type {
   NodeReference,
   I3STilesetHeader
 } from '@loaders.gl/i3s';
-import type {Tiles3DTileJSON} from '@loaders.gl/3d-tiles';
+import type {Tile3DBoundingVolume, Tiles3DTileJSON} from '@loaders.gl/3d-tiles';
 
 import {join} from 'path';
 import process from 'process';
@@ -24,8 +24,10 @@ import {WorkerFarm} from '@loaders.gl/worker-utils';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
 import B3dmConverter, {I3SAttributesData} from './helpers/b3dm-converter';
 import {I3STileHeader} from '@loaders.gl/i3s/src/types';
-import {loadI3SContent} from './helpers/load-i3s';
+import {loadFromArchive, loadI3SContent, openSLPK} from './helpers/load-i3s';
 import {I3SLoaderOptions} from '@loaders.gl/i3s/src/i3s-loader';
+import {ZipFileSystem} from '../../../zip/src';
+import {ConversionDump, ConversionDumpOptions} from '../lib/utils/conversion-dump';
 
 const I3S = 'I3S';
 
@@ -41,6 +43,7 @@ export default class Tiles3DConverter {
   sourceTileset: I3STilesetHeader | null;
   attributeStorageInfo?: AttributeStorageInfo[] | null;
   workerSource: {[key: string]: string} = {};
+  slpkFilesystem: ZipFileSystem | null = null;
   loaderOptions: I3SLoaderOptions = {
     _nodeWorkers: true,
     reuseWorkers: true,
@@ -52,6 +55,7 @@ export default class Tiles3DConverter {
       workerUrl: './modules/i3s/dist/i3s-content-worker-node.js'
     }
   };
+  conversionDump: ConversionDump;
 
   constructor() {
     this.options = {};
@@ -62,6 +66,7 @@ export default class Tiles3DConverter {
     this.sourceTileset = null;
     this.attributeStorageInfo = null;
     this.workerSource = {};
+    this.conversionDump = new ConversionDump();
   }
 
   /**
@@ -79,20 +84,32 @@ export default class Tiles3DConverter {
     tilesetName: string;
     maxDepth?: number;
     egmFilePath: string;
+    inquirer?: Promise<unknown>;
   }): Promise<any> {
     if (isBrowser) {
       console.log(BROWSER_ERROR_MESSAGE);
       return BROWSER_ERROR_MESSAGE;
     }
-    const {inputUrl, outputPath, tilesetName, maxDepth, egmFilePath} = options;
+    const {inputUrl, outputPath, tilesetName, maxDepth, egmFilePath, inquirer} = options;
     this.conversionStartTime = process.hrtime();
-    this.options = {maxDepth};
+    this.options = {maxDepth, inquirer};
 
     console.log('Loading egm file...'); // eslint-disable-line
     this.geoidHeightModel = await load(egmFilePath, PGMLoader);
     console.log('Loading egm file completed!'); // eslint-disable-line
 
-    this.sourceTileset = await load(inputUrl, I3SLoader, this.loaderOptions);
+    this.slpkFilesystem = await openSLPK(inputUrl);
+
+    this.sourceTileset = await loadFromArchive(
+      inputUrl,
+      I3SLoader,
+      {
+        ...this.loaderOptions,
+        // @ts-expect-error `isTileset` can be boolean of 'auto' but TS expects a string
+        i3s: {...this.loaderOptions.i3s, isTileset: true}
+      },
+      this.slpkFilesystem
+    );
 
     if (!this.sourceTileset) {
       return;
@@ -105,11 +122,28 @@ export default class Tiles3DConverter {
 
     this.tilesetPath = join(`${outputPath}`, `${tilesetName}`);
     this.attributeStorageInfo = this.sourceTileset.attributeStorageInfo;
+
+    await this.conversionDump.createDump(options as ConversionDumpOptions);
+    if (this.conversionDump.restored && this.options.inquirer) {
+      const result = await this.options.inquirer.prompt([
+        {
+          name: 'resumeConversion',
+          type: 'confirm',
+          message:
+            'Dump file of the previous conversion exists, do you want to resume that conversion?'
+        }
+      ]);
+      if (!result.resumeConversion) {
+        this.conversionDump.reset();
+      }
+    }
     // Removing the tilesetPath needed to exclude erroneous files after conversion
-    try {
-      await removeDir(this.tilesetPath);
-    } catch (e) {
-      // do nothing
+    if (!this.conversionDump.restored) {
+      try {
+        await removeDir(this.tilesetPath);
+      } catch (e) {
+        // do nothing
+      }
     }
 
     const rootTile: Tiles3DTileJSON = {
@@ -125,8 +159,13 @@ export default class Tiles3DConverter {
 
     const tileset = transform({root: rootTile}, tilesetTemplate());
     await writeFile(this.tilesetPath, JSON.stringify(tileset), 'tileset.json');
+    await this.conversionDump.deleteDumpFile();
 
-    this._finishConversion({slpk: false, outputPath, tilesetName});
+    await this._finishConversion({slpk: false, outputPath, tilesetName});
+
+    if (this.slpkFilesystem) {
+      this.slpkFilesystem.destroy();
+    }
 
     // Clean up worker pools
     const workerFarm = WorkerFarm.getWorkerFarm({});
@@ -148,7 +187,22 @@ export default class Tiles3DConverter {
   ): Promise<void> {
     const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
     if (sourceChild.contentUrl) {
-      const content = await loadI3SContent(this.sourceTileset, sourceChild, this.loaderOptions);
+      if (
+        this.conversionDump.restored &&
+        this.conversionDump.isFileConversionComplete(`${sourceChild.id}.b3dm`) &&
+        (sourceChild.obb || sourceChild.mbs)
+      ) {
+        const {child} = this._createChildAndBoundingVolume(sourceChild);
+        parentNode.children.push(child);
+        await this._addChildren(sourceChild, child, level + 1);
+        return;
+      }
+      const content = await loadI3SContent(
+        this.sourceTileset,
+        sourceChild,
+        this.loaderOptions,
+        this.slpkFilesystem
+      );
 
       if (!content) {
         await this._addChildren(sourceChild, parentNode, level + 1);
@@ -162,33 +216,24 @@ export default class Tiles3DConverter {
         featureAttributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
       }
 
-      if (!sourceChild.obb) {
-        sourceChild.obb = createObbFromMbs(sourceChild.mbs);
-      }
-
-      const boundingVolume = {
-        box: i3sObbTo3dTilesObb(sourceChild.obb, this.geoidHeightModel)
-      };
-      const child: Tiles3DTileJSON = {
-        boundingVolume,
-        geometricError: convertScreenThresholdToGeometricError(sourceChild),
-        children: []
-      };
+      const {child, boundingVolume} = this._createChildAndBoundingVolume(sourceChild);
 
       const i3sAttributesData: I3SAttributesData = {
         tileContent: content,
-        box: boundingVolume.box,
+        box: boundingVolume.box || [],
         textureFormat: sourceChild.textureFormat
       };
 
       const b3dmConverter = new B3dmConverter();
       const b3dm = await b3dmConverter.convert(i3sAttributesData, featureAttributes);
 
-      child.content = {
-        uri: `${sourceChild.id}.b3dm`,
-        boundingVolume
-      };
+      await this.conversionDump.addNode(`${sourceChild.id}.b3dm`, sourceChild.id);
       await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
+      await this.conversionDump.updateConvertedNodesDumpFile(
+        `${sourceChild.id}.b3dm`,
+        sourceChild.id,
+        true
+      );
       parentNode.children.push(child);
 
       await this._addChildren(sourceChild, child, level + 1);
@@ -235,20 +280,48 @@ export default class Tiles3DConverter {
     } else {
       const nodeUrl = this._relativeUrlToFullUrl(parentNode.url, childNodeInfo.href!);
       // load metadata
-      const options = {
+      const options: I3SLoaderOptions = {
         i3s: {
           ...this.loaderOptions,
+          // @ts-expect-error
           isTileHeader: true,
           loadContent: false
         }
       };
 
       console.log(`Node conversion: ${nodeUrl}`); // eslint-disable-line no-console,no-undef
-      header = await load(nodeUrl, I3SLoader, options);
+      header = await loadFromArchive(nodeUrl, I3SLoader, options, this.slpkFilesystem);
     }
     return header;
   }
 
+  /**
+   * Create child and child's boundingVolume for the converted node
+   * @param sourceChild
+   * @returns child and child's boundingVolume
+   */
+  private _createChildAndBoundingVolume(sourceChild: I3STileHeader): {
+    boundingVolume: Tile3DBoundingVolume;
+    child: Tiles3DTileJSON;
+  } {
+    if (!sourceChild.obb) {
+      sourceChild.obb = createObbFromMbs(sourceChild.mbs);
+    }
+    const boundingVolume: Tile3DBoundingVolume = {
+      box: i3sObbTo3dTilesObb(sourceChild.obb, this.geoidHeightModel)
+    };
+    const child: Tiles3DTileJSON = {
+      boundingVolume,
+      geometricError: convertScreenThresholdToGeometricError(sourceChild),
+      children: [],
+      content: {
+        uri: `${sourceChild.id}.b3dm`,
+        boundingVolume
+      }
+    };
+    return {boundingVolume, child};
+  }
+
   /**
    * Make an url of a resource from its relative url having the base url
    * @param baseUrl the base url. A resulting url will be related from this url
@@ -292,7 +365,7 @@ export default class Tiles3DConverter {
         attributeType: this._getAttributeType(attribute)
       };
 
-      promises.push(load(inputUrl, I3SAttributeLoader, options));
+      promises.push(loadFromArchive(inputUrl, I3SAttributeLoader, options, this.slpkFilesystem));
     }
     const attributesList = await Promise.all(promises);
     this._replaceNestedArrays(attributesList);

package/src/3d-tiles-converter/helpers/load-i3s.ts

@@ -1,36 +1,45 @@
-import {load} from '@loaders.gl/core';
+import {LoaderWithParser, load} from '@loaders.gl/core';
 import {
   I3STileContent,
   I3STileHeader,
   I3STilesetHeader,
   I3SLoader,
-  I3SLoaderOptions
+  I3SLoaderOptions,
+  parseSLPKArchive
 } from '@loaders.gl/i3s';
+import {FileHandleFile} from '@loaders.gl/loader-utils';
+import {ZipFileSystem} from '@loaders.gl/zip';
+
+export type SLPKUrlParts = {slpkFileName: string; internalFileName: string};
 
 /**
  * Load I3S node content
  * @param sourceTileset - source layer JSON
  * @param sourceTile - source I3S node metadata
  * @param tilesetLoadOptions - load options for Tiles3DLoader
+ * @param slpkFilesystem - loaded instance of ZipFileSystem for local convertion from SLPK file
  * @returns - 3DTiles tile content or null
  */
 export const loadI3SContent = async (
   sourceTileset: I3STilesetHeader | null,
   sourceTile: I3STileHeader,
-  tilesetLoadOptions: I3SLoaderOptions
+  tilesetLoadOptions: I3SLoaderOptions,
+  slpkFilesystem: ZipFileSystem | null
 ): Promise<I3STileContent | null> => {
   if (!sourceTileset || !sourceTile.contentUrl) {
     return null;
   }
 
-  const loadOptions = {
+  const loadOptions: I3SLoaderOptions = {
     ...tilesetLoadOptions,
     i3s: {
       ...tilesetLoadOptions.i3s,
+      // @ts-expect-error
       isTileset: false,
+      // @ts-expect-error
       isTileHeader: false,
       _tileOptions: {
-        attributeUrls: sourceTile.attributeUrls,
+        attributeUrls: sourceTile.attributeUrls || [],
         textureUrl: sourceTile.textureUrl,
         textureFormat: sourceTile.textureFormat,
         textureLoaderOptions: sourceTile.textureLoaderOptions,
@@ -40,13 +49,83 @@ export const loadI3SContent = async (
       },
       _tilesetOptions: {
         store: sourceTileset.store,
+        // @ts-expect-error
         attributeStorageInfo: sourceTileset.attributeStorageInfo,
+        // @ts-expect-error
         fields: sourceTileset.fields
       }
     }
   };
-  const tileContent = await load(sourceTile.contentUrl, I3SLoader, loadOptions);
+  const tileContent = await loadFromArchive(
+    sourceTile.contentUrl,
+    I3SLoader,
+    loadOptions,
+    slpkFilesystem
+  );
 
-  // @ts-expect-error
   return tileContent;
 };
+
+/**
+ * Load local SLPK file to ZipFileSystem instance
+ * @param url - path to SLPK file
+ * @returns instance of ZipFileSystem or null if url is not an SLPK file
+ */
+export async function openSLPK(url: string): Promise<ZipFileSystem | null> {
+  const slpkUrlParts = url.split('.slpk');
+  const {slpkFileName} = getSlpkUrlParts(url) || {};
+  if (slpkFileName) {
+    const slpkFileName = `${slpkUrlParts[0]}.slpk`;
+    const fileProvider = new FileHandleFile(slpkFileName);
+    const archive = await parseSLPKArchive(fileProvider, undefined, slpkFileName);
+    const fileSystem = new ZipFileSystem(archive);
+    return fileSystem;
+  }
+  return null;
+}
+
+/**
+ * Load a resource with load options and .3tz format support
+ * @param url - resource URL
+ * @param loader - loader to parse data (Tiles3DLoader / CesiumIonLoader)
+ * @param loadOptions - i3s loader options
+ * @returns i3s resource
+ */
+export async function loadFromArchive(
+  url: string,
+  loader: LoaderWithParser,
+  loadOptions: I3SLoaderOptions,
+  fileSystem: ZipFileSystem | null
+) {
+  const slpkUrlParts = getSlpkUrlParts(url);
+  if (fileSystem !== null && slpkUrlParts !== null) {
+    const {internalFileName} = slpkUrlParts;
+    const content = await load(internalFileName, loader, {
+      ...loadOptions,
+      fetch: fileSystem.fetch.bind(fileSystem)
+    });
+    return content;
+  }
+  return await load(url, loader, loadOptions);
+}
+
+/**
+ * Extract slpk file path and internal from the url
+ * For example, for `./path/to/file.slpk/nodes/0` it returns
+ * {"slpkFileName": "./path/to/file.slpk", "internalFileName": "/nodes/0" }
+ * @param url full internal file path
+ * @returns object with internal slpk file parts
+ */
+function getSlpkUrlParts(url: string): SLPKUrlParts | null {
+  const slpkUrlParts = url.split('.slpk');
+  let result: SLPKUrlParts | null;
+  // Not '.slpk'. The file will be loaded with global fetch function
+  if (slpkUrlParts.length === 1) {
+    result = null;
+  } else if (slpkUrlParts.length === 2) {
+    result = {slpkFileName: `${slpkUrlParts[0]}.slpk`, internalFileName: slpkUrlParts[1].slice(1)};
+  } else {
+    throw new Error('Unexpected URL format');
+  }
+  return result;
+}

package/src/converter-cli.ts

@@ -210,7 +210,8 @@ async function convert(options: ValidatedTileConversionOptions) {
         outputPath: options.output,
         tilesetName: options.name,
         maxDepth: options.maxDepth,
-        egmFilePath: options.egm
+        egmFilePath: options.egm,
+        inquirer
       });
       break;
     case TILESET_TYPE._3DTILES:

package/src/i3s-converter/helpers/attribute-metadata-info.ts

@@ -9,6 +9,12 @@ import type {
 
 import {AttributeType} from '../types';
 
+export type AttributeMetadataInfoObject = {
+  attributeStorageInfo: AttributeStorageInfo[];
+  fields: Field[];
+  popupInfo: PopupInfo | undefined;
+};
+
 export class AttributeMetadataInfo {
   private _attributeStorageInfo: AttributeStorageInfo[];
   private _fields: Field[];
@@ -95,6 +101,16 @@ export class AttributeMetadataInfo {
     }
   }
 
+  /**
+   * Set AttributeMetadataInfo from object
+   * @param object - object with AttributeMetadataInfo props
+   */
+  fromObject(object: AttributeMetadataInfoObject) {
+    this._attributeStorageInfo = object.attributeStorageInfo;
+    this._fields = object.fields;
+    this._popupInfo = object.popupInfo;
+  }
+
   /**
    * Generates storage attribute for map segmentation.
    * @param attributeIndex - order index of attribute (f_0, f_1 ...).

package/src/i3s-converter/helpers/load-3d-tiles.ts

@@ -4,8 +4,17 @@ import type {
   Tiles3DTileJSONPostprocessed,
   Tiles3DTilesetJSONPostprocessed
 } from '@loaders.gl/3d-tiles';
-import {Tiles3DArchiveFileSystem} from '@loaders.gl/3d-tiles';
+import {Tiles3DArchive} from '@loaders.gl/3d-tiles';
 import {LoaderWithParser, load} from '@loaders.gl/core';
+import {FileHandleFile, FileProvider} from '@loaders.gl/loader-utils';
+import {
+  CD_HEADER_SIGNATURE,
+  ZipFileSystem,
+  parseHashTable,
+  parseZipCDFileHeader,
+  parseZipLocalFileHeader,
+  searchFromTheEnd
+} from '@loaders.gl/zip';
 
 /**
  * Load nested 3DTiles tileset. If the sourceTile is not nested tileset - do nothing
@@ -104,7 +113,10 @@ export async function loadFromArchive(
   }
   if (filename) {
     const tz3Path = `${tz3UrlParts[0]}.3tz`;
-    const fileSystem = new Tiles3DArchiveFileSystem(tz3Path);
+    const fileProvider = new FileHandleFile(tz3Path);
+    const hashTable = await loadHashTable(fileProvider);
+    const archive = new Tiles3DArchive(fileProvider, hashTable, tz3Path);
+    const fileSystem = new ZipFileSystem(archive);
     const content = await load(filename, loader, {
       ...loadOptions,
       fetch: fileSystem.fetch.bind(fileSystem)
@@ -123,3 +135,41 @@ export async function loadFromArchive(
 export function isNestedTileset(tile: Tiles3DTileJSONPostprocessed) {
   return tile?.type === 'json' || tile?.type === '3tz';
 }
+
+/**
+ * Load hash file from 3TZ
+ * @param fileProvider - binary reader of 3TZ
+ * @returns hash table of the 3TZ file content or undefined if the hash file is not presented inside
+ */
+async function loadHashTable(
+  fileProvider: FileProvider
+): Promise<undefined | Record<string, bigint>> {
+  let hashTable: undefined | Record<string, bigint>;
+
+  const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE);
+
+  const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
+
+  // '@3dtilesIndex1@' is index file that must be the last in the archive. It allows
+  // to improve load and read performance when the archive contains a very large number
+  // of files.
+  if (cdFileHeader?.fileName === '@3dtilesIndex1@') {
+    const localFileHeader = await parseZipLocalFileHeader(
+      cdFileHeader.localHeaderOffset,
+      fileProvider
+    );
+    if (!localFileHeader) {
+      throw new Error('corrupted 3tz');
+    }
+
+    const fileDataOffset = localFileHeader.fileDataOffset;
+    const hashFile = await fileProvider.slice(
+      fileDataOffset,
+      fileDataOffset + localFileHeader.compressedSize
+    );
+
+    hashTable = parseHashTable(hashFile);
+  }
+
+  return hashTable;
+}

package/src/i3s-converter/helpers/node-index-document.ts

@@ -12,6 +12,7 @@ import {openJson, writeFile, writeFileForSlpk} from '../../lib/utils/file-utils'
 import I3SConverter from '../i3s-converter';
 import {NODE as nodeTemplate} from '../json-templates/node';
 import {I3SConvertedResources} from '../types';
+import {DumpMetadata} from '../../lib/utils/conversion-dump';
 
 /**
  * Wrapper for https://github.com/Esri/i3s-spec/blob/master/docs/1.7/3DNodeIndexDocument.cmn.md data
@@ -287,9 +288,8 @@ export class NodeIndexDocument {
     boundingVolumes: BoundingVolumes,
     lodSelection: LodSelection[],
     nodeInPage: NodeInPage,
-    resources: I3SConvertedResources
+    resources: I3SConvertedResources | DumpMetadata
   ): Promise<Node3DIndexDocument> {
-    const {texture, attributes} = resources;
     const nodeId = nodeInPage.index!;
     const parentNodeData = await parentNode.load();
     const nodeData = {
@@ -313,19 +313,29 @@ export class NodeIndexDocument {
       node.geometryData = [{href: './geometries/0'}];
       node.sharedResource = {href: './shared'};
 
-      if (texture) {
+      if (
+        ('texture' in resources && resources.texture) ||
+        ('texelCountHint' in resources && resources.texelCountHint)
+      ) {
         node.textureData = [{href: './textures/0'}, {href: './textures/1'}];
       }
 
       if (
-        attributes &&
-        attributes.length &&
-        parentNode.converter.layers0?.attributeStorageInfo?.length
+        ('attributes' in resources &&
+          resources.attributes &&
+          resources.attributes.length &&
+          parentNode.converter.layers0?.attributeStorageInfo?.length) ||
+        ('attributesCount' in resources &&
+          resources.attributesCount &&
+          parentNode.converter.layers0?.attributeStorageInfo?.length)
       ) {
+        const attributesLength =
+          ('attributes' in resources ? resources.attributes?.length : resources.attributesCount) ||
+          0;
         node.attributeData = [];
         const minimumLength =
-          attributes.length < parentNode.converter.layers0.attributeStorageInfo.length
-            ? attributes.length
+          attributesLength < parentNode.converter.layers0.attributeStorageInfo.length
+            ? attributesLength
             : parentNode.converter.layers0.attributeStorageInfo.length;
         for (let index = 0; index < minimumLength; index++) {
           const folderName = parentNode.converter.layers0.attributeStorageInfo[index].key;