@loaders.gl/tile-converter 4.2.0-alpha.2 → 4.2.0-alpha.3

package/dist/index.cjs

@@ -4676,7 +4676,6 @@ var I3SConverter = class {
     } else {
       console.log("Feature metadata classes have not been found");
     }
-    console.log(`------------------------------------------------`);
     if (!meshTopologyTypes.has("TRIANGLES" /* TRIANGLES */) && !meshTopologyTypes.has("TRIANGLE_STRIP" /* TRIANGLE_STRIP */)) {
       console.log(
         "The tileset is of unsupported mesh topology types. The conversion will be interrupted."
@@ -4684,6 +4683,7 @@ var I3SConverter = class {
       console.log(`------------------------------------------------`);
       return false;
     }
+    console.log(`------------------------------------------------`);
     return true;
   }
   /**
@@ -6191,6 +6191,20 @@ function getSlpkUrlParts(url) {
   }
   return result;
 }
+async function getNodeCount(fileSystem) {
+  if (!(fileSystem == null ? void 0 : fileSystem.fileProvider)) {
+    return 0;
+  }
+  let count = 0;
+  const filesIterator = (0, import_zip3.makeZipCDHeaderIterator)(fileSystem.fileProvider);
+  for await (const file of filesIterator) {
+    const filename = file.fileName;
+    if (filename.indexOf("3dNodeIndexDocument.json.gz") >= 0) {
+      count++;
+    }
+  }
+  return count;
+}
 
 // src/3d-tiles-converter/3d-tiles-converter.ts
 var I3S = "I3S";
@@ -6218,6 +6232,7 @@ var Tiles3DConverter = class {
     this.attributeStorageInfo = null;
     this.workerSource = {};
     this.conversionDump = new ConversionDump();
+    this.progress = new Progress();
   }
   /**
    * Convert i3s format data to 3dTiles
@@ -6234,13 +6249,21 @@ var Tiles3DConverter = class {
       console.log(BROWSER_ERROR_MESSAGE);
       return BROWSER_ERROR_MESSAGE;
     }
-    const { inputUrl, outputPath, tilesetName, maxDepth, egmFilePath, inquirer } = options;
+    const { inputUrl, outputPath, tilesetName, maxDepth, egmFilePath, inquirer, analyze } = options;
     this.conversionStartTime = import_process5.default.hrtime();
     this.options = { maxDepth, inquirer };
     console.log("Loading egm file...");
     this.geoidHeightModel = await (0, import_core15.load)(egmFilePath, PGMLoader);
     console.log("Loading egm file completed!");
     this.slpkFilesystem = await openSLPK(inputUrl);
+    let preprocessResult = true;
+    if (analyze || this.slpkFilesystem) {
+      preprocessResult = await this.preprocessConversion();
+      if (!preprocessResult || analyze) {
+        return;
+      }
+    }
+    this.progress.startMonitoring();
     this.sourceTileset = await loadFromArchive2(
       inputUrl,
       import_i3s2.I3SLoader,
@@ -6291,6 +6314,7 @@ var Tiles3DConverter = class {
     const tileset = (0, import_json_map_transform10.default)({ root: rootTile }, TILESET());
     await writeFile(this.tilesetPath, JSON.stringify(tileset), "tileset.json");
     await this.conversionDump.deleteDumpFile();
+    this.progress.stopMonitoring();
     await this._finishConversion({ slpk: false, outputPath, tilesetName });
     if (this.slpkFilesystem) {
       this.slpkFilesystem.destroy();
@@ -6298,6 +6322,29 @@ var Tiles3DConverter = class {
     const workerFarm = import_worker_utils2.WorkerFarm.getWorkerFarm({});
     workerFarm.destroy();
   }
+  /**
+   * Preprocess stage of the tile converter. Calculate number of nodes
+   * @returns true - the conversion is possible, false - the tileset's content is not supported
+   */
+  async preprocessConversion() {
+    console.log(`Analyze source layer`);
+    const nodesCount = await getNodeCount(this.slpkFilesystem);
+    this.progress.stepsTotal = nodesCount;
+    console.log(`------------------------------------------------`);
+    console.log(`Preprocess results:`);
+    if (this.slpkFilesystem) {
+      console.log(`Node count: ${nodesCount}`);
+      if (nodesCount === 0) {
+        console.log("Node count is 0. The conversion will be interrupted.");
+        console.log(`------------------------------------------------`);
+        return false;
+      }
+    } else {
+      console.log(`Node count cannot be calculated for the remote dataset`);
+    }
+    console.log(`------------------------------------------------`);
+    return true;
+  }
   /**
    * Convert particular I3S Node
    * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
@@ -6306,6 +6353,7 @@ var Tiles3DConverter = class {
    * @param childNodeInfo child node to convert
    */
   async convertChildNode(parentSourceNode, parentNode, level, childNodeInfo) {
+    let nextParentNode = parentNode;
     const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
     if (sourceChild.contentUrl) {
       if (this.conversionDump.restored && this.conversionDump.isFileConversionComplete(`${sourceChild.id}.b3dm`) && (sourceChild.obb || sourceChild.mbs)) {
@@ -6345,10 +6393,18 @@ var Tiles3DConverter = class {
         true
       );
       parentNode.children.push(child);
-      await this._addChildren(sourceChild, child, level + 1);
-    } else {
-      await this._addChildren(sourceChild, parentNode, level + 1);
-    }
+      nextParentNode = child;
+    }
+    this.progress.stepsDone += 1;
+    let timeRemainingString = "Calculating time left...";
+    const timeRemaining = this.progress.getTimeRemainingString();
+    if (timeRemaining) {
+      timeRemainingString = `${timeRemaining} left`;
+    }
+    const percentString = this.progress.getPercentString();
+    const progressString = percentString ? ` ${percentString}%, ${timeRemainingString}` : "";
+    console.log(`[converted${progressString}]: ${childNodeInfo.id}`);
+    await this._addChildren(sourceChild, nextParentNode, level + 1);
   }
   /**
    * The recursive function of traversal of a nodes tree

package/dist/pgm-loader.js

@@ -1,5 +1,5 @@
 import { Geoid, parsePGM } from '@math.gl/geoid';
-const VERSION = typeof "4.2.0-alpha.2" !== 'undefined' ? "4.2.0-alpha.2" : 'latest';
+const VERSION = typeof "4.2.0-alpha.3" !== 'undefined' ? "4.2.0-alpha.3" : 'latest';
 export { Geoid };
 export const PGMLoader = {
   name: 'PGM - Netpbm grayscale image format',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/tile-converter",
-  "version": "4.2.0-alpha.2",
+  "version": "4.2.0-alpha.3",
   "description": "Converter",
   "license": "MIT",
   "type": "module",
@@ -52,18 +52,18 @@
     "build-i3s-server-bundle": "esbuild src/i3s-server/bin/www.ts --outfile=dist/i3s-server/bin/i3s-server.min.cjs --platform=node --target=esnext,node14 --minify --bundle --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/3d-tiles": "4.2.0-alpha.2",
-    "@loaders.gl/crypto": "4.2.0-alpha.2",
-    "@loaders.gl/draco": "4.2.0-alpha.2",
-    "@loaders.gl/gltf": "4.2.0-alpha.2",
-    "@loaders.gl/i3s": "4.2.0-alpha.2",
-    "@loaders.gl/images": "4.2.0-alpha.2",
-    "@loaders.gl/loader-utils": "4.2.0-alpha.2",
-    "@loaders.gl/polyfills": "4.2.0-alpha.2",
-    "@loaders.gl/textures": "4.2.0-alpha.2",
-    "@loaders.gl/tiles": "4.2.0-alpha.2",
-    "@loaders.gl/worker-utils": "4.2.0-alpha.2",
-    "@loaders.gl/zip": "4.2.0-alpha.2",
+    "@loaders.gl/3d-tiles": "4.2.0-alpha.3",
+    "@loaders.gl/crypto": "4.2.0-alpha.3",
+    "@loaders.gl/draco": "4.2.0-alpha.3",
+    "@loaders.gl/gltf": "4.2.0-alpha.3",
+    "@loaders.gl/i3s": "4.2.0-alpha.3",
+    "@loaders.gl/images": "4.2.0-alpha.3",
+    "@loaders.gl/loader-utils": "4.2.0-alpha.3",
+    "@loaders.gl/polyfills": "4.2.0-alpha.3",
+    "@loaders.gl/textures": "4.2.0-alpha.3",
+    "@loaders.gl/tiles": "4.2.0-alpha.3",
+    "@loaders.gl/worker-utils": "4.2.0-alpha.3",
+    "@loaders.gl/zip": "4.2.0-alpha.3",
     "@math.gl/core": "^4.0.0",
     "@math.gl/culling": "^4.0.0",
     "@math.gl/geoid": "^4.0.0",
@@ -88,7 +88,7 @@
     "join-images": "^1.1.3",
     "sharp": "^0.31.3"
   },
-  "gitHead": "d66a6a4626ea84c5f2cad5fa5cf7ebb6943c57c8",
+  "gitHead": "cf18e05801314a62c9fc4c2a85e8888a8a852083",
   "devDependencies": {
     "@types/express": "^4.17.17",
     "@types/node": "^20.4.2"

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -24,10 +24,11 @@ import {WorkerFarm} from '@loaders.gl/worker-utils';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
 import B3dmConverter, {I3SAttributesData} from './helpers/b3dm-converter';
 import {I3STileHeader} from '@loaders.gl/i3s/src/types';
-import {loadFromArchive, loadI3SContent, openSLPK} from './helpers/load-i3s';
+import {getNodeCount, loadFromArchive, loadI3SContent, openSLPK} from './helpers/load-i3s';
 import {I3SLoaderOptions} from '@loaders.gl/i3s/src/i3s-loader';
 import {ZipFileSystem} from '../../../zip/src';
 import {ConversionDump, ConversionDumpOptions} from '../lib/utils/conversion-dump';
+import {Progress} from '../i3s-converter/helpers/progress';
 
 const I3S = 'I3S';
 
@@ -56,6 +57,7 @@ export default class Tiles3DConverter {
     }
   };
   conversionDump: ConversionDump;
+  progress: Progress;
 
   constructor() {
     this.options = {};
@@ -67,6 +69,7 @@ export default class Tiles3DConverter {
     this.attributeStorageInfo = null;
     this.workerSource = {};
     this.conversionDump = new ConversionDump();
+    this.progress = new Progress();
   }
 
   /**
@@ -85,12 +88,13 @@ export default class Tiles3DConverter {
     maxDepth?: number;
     egmFilePath: string;
     inquirer?: Promise<unknown>;
+    analyze?: boolean;
   }): Promise<any> {
     if (isBrowser) {
       console.log(BROWSER_ERROR_MESSAGE);
       return BROWSER_ERROR_MESSAGE;
     }
-    const {inputUrl, outputPath, tilesetName, maxDepth, egmFilePath, inquirer} = options;
+    const {inputUrl, outputPath, tilesetName, maxDepth, egmFilePath, inquirer, analyze} = options;
     this.conversionStartTime = process.hrtime();
     this.options = {maxDepth, inquirer};
 
@@ -100,6 +104,16 @@ export default class Tiles3DConverter {
 
     this.slpkFilesystem = await openSLPK(inputUrl);
 
+    let preprocessResult = true;
+    if (analyze || this.slpkFilesystem) {
+      preprocessResult = await this.preprocessConversion();
+      if (!preprocessResult || analyze) {
+        return;
+      }
+    }
+
+    this.progress.startMonitoring();
+
     this.sourceTileset = await loadFromArchive(
       inputUrl,
       I3SLoader,
@@ -161,6 +175,8 @@ export default class Tiles3DConverter {
     await writeFile(this.tilesetPath, JSON.stringify(tileset), 'tileset.json');
     await this.conversionDump.deleteDumpFile();
 
+    this.progress.stopMonitoring();
+
     await this._finishConversion({slpk: false, outputPath, tilesetName});
 
     if (this.slpkFilesystem) {
@@ -172,6 +188,32 @@ export default class Tiles3DConverter {
     workerFarm.destroy();
   }
 
+  /**
+   * Preprocess stage of the tile converter. Calculate number of nodes
+   * @returns true - the conversion is possible, false - the tileset's content is not supported
+   */
+  private async preprocessConversion(): Promise<boolean> {
+    console.log(`Analyze source layer`);
+    const nodesCount = await getNodeCount(this.slpkFilesystem);
+    this.progress.stepsTotal = nodesCount;
+
+    console.log(`------------------------------------------------`);
+    console.log(`Preprocess results:`);
+    if (this.slpkFilesystem) {
+      console.log(`Node count: ${nodesCount}`);
+      if (nodesCount === 0) {
+        console.log('Node count is 0. The conversion will be interrupted.');
+        console.log(`------------------------------------------------`);
+        return false;
+      }
+    } else {
+      console.log(`Node count cannot be calculated for the remote dataset`);
+    }
+
+    console.log(`------------------------------------------------`);
+    return true;
+  }
+
   /**
    * Convert particular I3S Node
    * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
@@ -185,6 +227,7 @@ export default class Tiles3DConverter {
     level: number,
     childNodeInfo: NodeReference
   ): Promise<void> {
+    let nextParentNode = parentNode;
     const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
     if (sourceChild.contentUrl) {
       if (
@@ -235,11 +278,20 @@ export default class Tiles3DConverter {
         true
       );
       parentNode.children.push(child);
+      nextParentNode = child;
+    }
 
-      await this._addChildren(sourceChild, child, level + 1);
-    } else {
-      await this._addChildren(sourceChild, parentNode, level + 1);
+    this.progress.stepsDone += 1;
+    let timeRemainingString = 'Calculating time left...';
+    const timeRemaining = this.progress.getTimeRemainingString();
+    if (timeRemaining) {
+      timeRemainingString = `${timeRemaining} left`;
     }
+    const percentString = this.progress.getPercentString();
+    const progressString = percentString ? ` ${percentString}%, ${timeRemainingString}` : '';
+    console.log(`[converted${progressString}]: ${childNodeInfo.id}`); // eslint-disable-line
+
+    await this._addChildren(sourceChild, nextParentNode, level + 1);
   }
 
   /**

package/src/3d-tiles-converter/helpers/load-i3s.ts

@@ -8,7 +8,7 @@ import {
   parseSLPKArchive
 } from '@loaders.gl/i3s';
 import {FileHandleFile} from '@loaders.gl/loader-utils';
-import {ZipFileSystem} from '@loaders.gl/zip';
+import {ZipFileSystem, makeZipCDHeaderIterator} from '@loaders.gl/zip';
 
 export type SLPKUrlParts = {slpkFileName: string; internalFileName: string};
 
@@ -129,3 +129,23 @@ function getSlpkUrlParts(url: string): SLPKUrlParts | null {
   }
   return result;
 }
+
+/**
+ * Get nodes count inside SLPK
+ * @param fileSystem - file system of SLPK
+ * @returns number of nodes
+ */
+export async function getNodeCount(fileSystem: ZipFileSystem | null): Promise<number> {
+  if (!fileSystem?.fileProvider) {
+    return 0;
+  }
+  let count = 0;
+  const filesIterator = makeZipCDHeaderIterator(fileSystem.fileProvider);
+  for await (const file of filesIterator) {
+    const filename = file.fileName;
+    if (filename.indexOf('3dNodeIndexDocument.json.gz') >= 0) {
+      count++;
+    }
+  }
+  return count;
+}

package/src/converter-cli.ts

@@ -59,6 +59,8 @@ type TileConversionOptions = {
   metadataClass?: string;
   /** With this options the tileset content will be analyzed without conversion */
   analyze?: boolean;
+  /** Skip all prompts that stop conversion and wait for a user input */
+  quiet?: boolean;
 };
 
 /* During validation we check that particular options are defined so they can't be undefined */
@@ -100,36 +102,40 @@ async function main() {
   if (options.addHash) {
     const validatedOptions = validateOptions(options, true);
     let finalPath = validatedOptions.tileset;
-    if (validatedOptions.output === 'data') {
-      const nameWithoutExt = validatedOptions.tileset.substring(
-        0,
-        validatedOptions.tileset.length - 5
-      );
 
-      const result = await inquirer.prompt<{isNewFileRequired: boolean}>([
-        {
-          name: 'isNewFileRequired',
-          type: 'list',
-          message: 'What would you like to do?',
-          choices: [
-            {
-              name: 'Add hash file to the current SLPK file',
-              value: false
-            },
-            {
-              name: `Create a new file ${nameWithoutExt}-hash.slpk with hash file inside`,
-              value: true
-            }
-          ]
-        }
-      ]);
+    if (!options.quiet) {
+      if (validatedOptions.output === 'data') {
+        const nameWithoutExt = validatedOptions.tileset.substring(
+          0,
+          validatedOptions.tileset.length - 5
+        );
+
+        const result = await inquirer.prompt<{isNewFileRequired: boolean}>([
+          {
+            name: 'isNewFileRequired',
+            type: 'list',
+            message: 'What would you like to do?',
+            choices: [
+              {
+                name: 'Add hash file to the current SLPK file',
+                value: false
+              },
+              {
+                name: `Create a new file ${nameWithoutExt}-hash.slpk with hash file inside`,
+                value: true
+              }
+            ]
+          }
+        ]);
 
-      if (result.isNewFileRequired) {
-        finalPath = `${nameWithoutExt}-hash.slpk`;
+        if (result.isNewFileRequired) {
+          finalPath = `${nameWithoutExt}-hash.slpk`;
+        }
+      } else {
+        finalPath = validatedOptions.output;
       }
-    } else {
-      finalPath = validatedOptions.output;
     }
+
     if (finalPath !== validatedOptions.tileset) {
       await copyFile(validatedOptions.tileset, finalPath);
     }
@@ -190,6 +196,9 @@ function printHelp(): void {
     '--metadata-class [One of the list of feature metadata classes, detected by converter on "analyze" stage, default: not set]'
   );
   console.log('--validate [Enable validation]');
+  console.log(
+    '--quiet [Skip all prompts that stop conversion and wait for a user input: default: false]'
+  );
   process.exit(0); // eslint-disable-line
 }
 
@@ -211,7 +220,8 @@ async function convert(options: ValidatedTileConversionOptions) {
         tilesetName: options.name,
         maxDepth: options.maxDepth,
         egmFilePath: options.egm,
-        inquirer
+        analyze: options.analyze,
+        inquirer: options.quiet ? undefined : inquirer
       });
       break;
     case TILESET_TYPE._3DTILES:
@@ -233,7 +243,7 @@ async function convert(options: ValidatedTileConversionOptions) {
         instantNodeWriting: options.instantNodeWriting,
         metadataClass: options.metadataClass,
         analyze: options.analyze,
-        inquirer
+        inquirer: options.quiet ? undefined : inquirer
       });
       break;
     default:
@@ -308,7 +318,8 @@ function parseOptions(args: string[]): TileConversionOptions {
     generateBoundingVolumes: false,
     validate: false,
     slpk: false,
-    addHash: false
+    addHash: false,
+    quiet: false
   };
 
   // eslint-disable-next-line complexity
@@ -369,6 +380,9 @@ function parseOptions(args: string[]): TileConversionOptions {
         case '--analyze':
           opts.analyze = getBooleanValue(index, args);
           break;
+        case '--quiet':
+          opts.quiet = getBooleanValue(index, args);
+          break;
         case '--metadata-class':
           opts.metadataClass = getStringValue(index, args);
           break;

package/src/i3s-converter/i3s-converter.ts

@@ -322,7 +322,6 @@ export default class I3SConverter {
       console.log('Feature metadata classes have not been found');
     }
 
-    console.log(`------------------------------------------------`);
     if (
       !meshTopologyTypes.has(GLTFPrimitiveModeString.TRIANGLES) &&
       !meshTopologyTypes.has(GLTFPrimitiveModeString.TRIANGLE_STRIP)
@@ -334,6 +333,7 @@ export default class I3SConverter {
       return false;
     }
 
+    console.log(`------------------------------------------------`);
     return true;
   }