@loaders.gl/tile-converter 4.0.2 → 4.0.3

package/src/i3s-converter/helpers/progress.ts

@@ -0,0 +1,166 @@
+import process from 'process';
+import {timeConverter} from '../../lib/utils/statistic-utills';
+
+/** Defines a threshold that is used to check if the process velocity can be consifered trust. */
+const THRESHOLD_DEFAULT = 0.2;
+
+/**
+ * Implements methods to keep track on the progress of a long process.
+ */
+export class Progress {
+  /** Total amount of work, e.g. number of files to save or number of bytes to send */
+  private _stepsTotal: number = 0;
+  /** Amount of work already done */
+  private _stepsDone: number = 0;
+  /** Time in milli-seconds when the process started */
+  private startTime: number = 0;
+  /** Time in milli-seconds when the process stopped */
+  private stopTime: number = 0;
+  /** Time in milli-seconds when stepsDone was updated */
+  private timeOfUpdatingStepsDone: number = 0;
+  /** Time in milli-seconds spent for performing one step*/
+  private milliSecForOneStep: number = 0;
+  private trust: boolean = false;
+  /**
+   * The number of digits to appear after decimal point in the string representation of the count of steps already done.
+   * It's calculated based on the total count of steps.
+   */
+  private numberOfDigitsInPercentage: number = 0;
+  /** Defines a threshold that is used to check if the process velocity can be consifered trust. */
+  private threshold: number;
+  /** Function that is used to get the time stamp */
+  private getTime: () => bigint;
+
+  constructor(options: {threshold?: number; getTime?: () => bigint} = {}) {
+    this.getTime = options.getTime || process.hrtime.bigint;
+    this.threshold = options.threshold || THRESHOLD_DEFAULT;
+  }
+
+  /** Total amount of work, e.g. number of files to save or number of bytes to send */
+  get stepsTotal() {
+    return this._stepsTotal;
+  }
+
+  set stepsTotal(stepsTotal) {
+    this._stepsTotal = stepsTotal;
+    this.numberOfDigitsInPercentage =
+      this.stepsTotal > 100 ? Math.ceil(Math.log10(this.stepsTotal)) - 2 : 0;
+  }
+
+  /** Amount of work already done */
+  get stepsDone() {
+    return this._stepsDone;
+  }
+
+  set stepsDone(stepsDone) {
+    this._stepsDone = stepsDone;
+    this.timeOfUpdatingStepsDone = this.getCurrentTimeInMilliSeconds();
+    if (this._stepsDone) {
+      const diff = this.timeOfUpdatingStepsDone - this.startTime;
+      const milliSecForOneStep = diff / this._stepsDone;
+
+      this.trust = this.isVelocityTrust(milliSecForOneStep, this.milliSecForOneStep);
+      this.milliSecForOneStep = milliSecForOneStep;
+    }
+  }
+
+  /**
+   * Saves the current time as we start monitoring the process.
+   */
+  startMonitoring() {
+    this.startTime = this.getCurrentTimeInMilliSeconds();
+    this.milliSecForOneStep = 0;
+    this.trust = false;
+    this.timeOfUpdatingStepsDone = 0;
+    this.stopTime = 0;
+    this.stepsDone = 0;
+  }
+
+  /**
+   * Saves the current time as we stop monitoring the process.
+   */
+  stopMonitoring() {
+    this.stopTime = this.getCurrentTimeInMilliSeconds();
+  }
+
+  /**
+   * Gets percentage of the work already done.
+   * @returns percentage of the work already done.
+   */
+  getPercent(): number | null {
+    if (!this._stepsTotal) {
+      return null;
+    }
+    const percent = (this._stepsDone / this._stepsTotal) * 100.0;
+    return percent;
+  }
+
+  /**
+   * Gets string representation of percentage of the work already done.
+   * @returns string representation of percentage or an empty string if the percetage value cannot be calculated.
+   */
+  getPercentString() {
+    const percent = this.getPercent();
+    return percent !== null ? percent.toFixed(this.numberOfDigitsInPercentage) : '';
+  }
+
+  /**
+   * Gets the time elapsed since the monitoring started
+   * @returns Number of milliseconds elapsed
+   */
+  getTimeCurrentlyElapsed(): number {
+    const currentTime = this.stopTime ? this.stopTime : this.getCurrentTimeInMilliSeconds();
+    const diff = currentTime - this.startTime;
+    return diff;
+  }
+
+  /**
+   * Gets the time remaining (expected at the moment of updating 'stepsDone') to complete the work.
+   * @returns Number of milliseconds remaining
+   */
+  getTimeRemaining(): {timeRemaining: number; trust: boolean} | null {
+    if (!this._stepsDone || !this.startTime) {
+      return null;
+    }
+
+    const timeRemainingInMilliSeconds =
+      (this._stepsTotal - this._stepsDone) * this.milliSecForOneStep;
+    return {timeRemaining: timeRemainingInMilliSeconds, trust: this.trust};
+  }
+
+  /**
+   * Gets the string representation of the time remaining (expected at the moment of updating 'stepsDone') to complete the work.
+   * @returns string representation of the time remaining.
+   * It's an empty string if the time cannot be pedicted or it's still being calculated.
+   */
+  getTimeRemainingString(): string {
+    const timeRemainingObject = this.getTimeRemaining();
+    return timeRemainingObject?.trust ? timeConverter(timeRemainingObject.timeRemaining) : '';
+  }
+
+  /**
+   * Check if the computed velociy of the process can be considered trust.
+   * At the beginning of the process the number of samples collected ('time necessary to perform one step' averaged) is too small,
+   * which results in huge deviation of the cumputed velocity of the process.
+   * It makes sense to perform the check before reporting the time remainig so the end user is not confused.
+   * @param current - current value
+   * @param previous - previous value
+   * @returns true if the computed velociy can be considered trust, or false otherwise
+   */
+  private isVelocityTrust(current: number, previous: number): boolean {
+    if (previous) {
+      const dev = Math.abs((current - previous) / previous);
+      return dev < this.threshold;
+    }
+    return false;
+  }
+
+  /**
+   * Gets current time in milliseconds.
+   * @returns current time in milliseconds.
+   */
+  private getCurrentTimeInMilliSeconds(): number {
+    // process.hrtime.bigint() returns the time in nanoseconds. We need the time in milliseconds.
+    return Number(this.getTime() / BigInt(1e6));
+  }
+}

package/src/i3s-converter/i3s-converter.ts

@@ -79,6 +79,7 @@ import {BoundingSphere, OrientedBoundingBox} from '@math.gl/culling';
 import {createBoundingVolume} from '@loaders.gl/tiles';
 import {TraversalConversionProps, traverseDatasetWith} from './helpers/tileset-traversal';
 import {analyzeTileContent, mergePreprocessData} from './helpers/preprocess-3d-tiles';
+import {Progress} from './helpers/progress';
 
 const ION_DEFAULT_TOKEN = process.env?.IonToken;
 const HARDCODED_NODES_PER_PAGE = 64;
@@ -87,6 +88,7 @@ const _3D_OBJECT_LAYER_TYPE = '3DObject';
 const REFRESH_TOKEN_TIMEOUT = 1800; // 30 minutes in seconds
 const CESIUM_DATASET_PREFIX = 'https://';
 // const FS_FILE_TOO_LARGE = 'ERR_FS_FILE_TOO_LARGE';
+const PROGRESS_PHASE1_COUNT = 'phase1-count';
 
 /**
  * Converter from 3d-tiles tileset to i3s layer
@@ -138,15 +140,7 @@ export default class I3SConverter {
     meshTopologyTypes: new Set(),
     metadataClasses: new Set()
   };
-  /** Total count of tiles in tileset */
-  tileCountTotal: number = 0;
-  /** Count of tiles already converted plus one (refers to the tile currently being converted) */
-  tileCountCurrentlyConverting: number = 0;
-  /**
-   * The number of digits to appear after decimal point in the string representation of the tile count.
-   * It's calculated based on the total count of tiles.
-   */
-  numberOfDigitsInPercentage: number = 0;
+  progresses: Record<string, Progress> = {};
 
   constructor() {
     this.attributeMetadataInfo = new AttributeMetadataInfo();
@@ -244,6 +238,7 @@ export default class I3SConverter {
       inquirer,
       metadataClass
     };
+    this.progresses[PROGRESS_PHASE1_COUNT] = new Progress();
     this.compressList = (this.options.instantNodeWriting && []) || null;
     this.validate = Boolean(validate);
     this.Loader = inputUrl.indexOf(CESIUM_DATASET_PREFIX) !== -1 ? CesiumIonLoader : Tiles3DLoader;
@@ -310,12 +305,9 @@ export default class I3SConverter {
     );
     const {meshTopologyTypes, metadataClasses} = this.preprocessData;
 
-    this.numberOfDigitsInPercentage =
-      this.tileCountTotal > 100 ? Math.ceil(Math.log10(this.tileCountTotal)) - 2 : 0;
-
     console.log(`------------------------------------------------`);
     console.log(`Preprocess results:`);
-    console.log(`Tile count: ${this.tileCountTotal}`);
+    console.log(`Tile count: ${this.progresses[PROGRESS_PHASE1_COUNT].stepsTotal}`);
     console.log(`glTF mesh topology types: ${Array.from(meshTopologyTypes).join(', ')}`);
 
     if (metadataClasses.size) {
@@ -357,7 +349,7 @@ export default class I3SConverter {
       return null;
     }
     if (sourceTile.id) {
-      this.tileCountTotal++;
+      this.progresses[PROGRESS_PHASE1_COUNT].stepsTotal += 1;
       console.log(`[analyze]: ${sourceTile.id}`); // eslint-disable-line
     }
 
@@ -374,7 +366,6 @@ export default class I3SConverter {
     }
     const tilePreprocessData = await analyzeTileContent(tileContent);
     mergePreprocessData(this.preprocessData, tilePreprocessData);
-
     return null;
   }
 
@@ -451,7 +442,7 @@ export default class I3SConverter {
       obb: boundingVolumes.obb,
       children: []
     });
-
+    this.progresses[PROGRESS_PHASE1_COUNT].startMonitoring();
     const rootNode = await NodeIndexDocument.createRootNode(boundingVolumes, this);
     await traverseDatasetWith<TraversalConversionProps>(
       sourceRootTile,
@@ -463,6 +454,7 @@ export default class I3SConverter {
       this.finalizeTile.bind(this),
       this.options.maxDepth
     );
+    this.progresses[PROGRESS_PHASE1_COUNT].stopMonitoring();
 
     this.layers0!.attributeStorageInfo = this.attributeMetadataInfo.attributeStorageInfo;
     this.layers0!.fields = this.attributeMetadataInfo.fields;
@@ -618,16 +610,8 @@ export default class I3SConverter {
       return traversalProps;
     }
     if (sourceTile.id) {
-      // Print the tile number that is currently being converted.
-      this.tileCountCurrentlyConverting++;
-      let percentString = '';
-      if (this.tileCountTotal) {
-        const percent = (this.tileCountCurrentlyConverting / this.tileCountTotal) * 100;
-        percentString = ' ' + percent.toFixed(this.numberOfDigitsInPercentage);
-      }
-      console.log(`[convert${percentString}%]: ${sourceTile.id}`); // eslint-disable-line
+      console.log(`[convert]: ${sourceTile.id}`); // eslint-disable-line
     }
-
     const {parentNodes, transform} = traversalProps;
     let transformationMatrix: Matrix4 = transform.clone();
     if (sourceTile.transform) {
@@ -641,6 +625,20 @@ export default class I3SConverter {
       transform: transformationMatrix,
       parentNodes: childNodes
     };
+
+    if (sourceTile.id) {
+      this.progresses[PROGRESS_PHASE1_COUNT].stepsDone += 1;
+
+      let timeRemainingString = 'Calculating time left...';
+      const timeRemainingStringBasedOnCount =
+        this.progresses[PROGRESS_PHASE1_COUNT].getTimeRemainingString();
+      if (timeRemainingStringBasedOnCount) {
+        timeRemainingString = `${timeRemainingStringBasedOnCount} left`;
+      }
+
+      let percentString = this.progresses[PROGRESS_PHASE1_COUNT].getPercentString();
+      console.log(`[converted ${percentString}%, ${timeRemainingString}]: ${sourceTile.id}`); // eslint-disable-line
+    }
     return newTraversalProps;
   }
 

package/src/lib/utils/statistic-utills.ts

@@ -2,15 +2,33 @@ import {join} from 'path';
 import {promises as fs} from 'fs';
 import {getAbsoluteFilePath} from './file-utils';
 
-export function timeConverter(time) {
-  const nanoSecondsInMillisecond = 1e6;
-  let timeInSeconds = time[0];
+/**
+ * Converts time value to string.
+ * @param time - high-resolution real time in a [seconds, nanoseconds] tuple Array, or a value on milliseconds.
+ * @returns string representation of the time
+ */
+export function timeConverter(time: number | [number, number]): string {
+  if (typeof time === 'number') {
+    // time - real time in milli-seconds
+    const milliSecondsInSecond = 1e3;
+    const timeInSeconds = Math.floor(time / milliSecondsInSecond);
+    const milliseconds = time - timeInSeconds * milliSecondsInSecond;
+    return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);
+  } else {
+    // time - high-resolution real time in a [seconds, nanoseconds] tuple Array
+    const nanoSecondsInMillisecond = 1e6;
+    const timeInSeconds = time[0];
+    const milliseconds = time[1] / nanoSecondsInMillisecond;
+    return timeConverterFromSecondsAndMilliseconds(timeInSeconds, milliseconds);
+  }
+}
+
+function timeConverterFromSecondsAndMilliseconds(timeInSeconds: number, milliseconds: number) {
   const hours = Math.floor(timeInSeconds / 3600);
   timeInSeconds = timeInSeconds - hours * 3600;
   const minutes = Math.floor(timeInSeconds / 60);
   timeInSeconds = timeInSeconds - minutes * 60;
   const seconds = Math.floor(timeInSeconds);
-  const milliseconds = time[1] / nanoSecondsInMillisecond;
   let result = '';
 
   if (hours) {