@loaders.gl/tile-converter 4.1.0-alpha.1 → 4.1.0-alpha.10

package/src/lib/utils/conversion-dump.ts

@@ -0,0 +1,198 @@
+import {DUMP_FILE_SUFFIX} from '../../constants';
+import {removeFile, writeFile} from './file-utils';
+import {join} from 'path';
+
+export type ConversionDumpOptions = {
+  inputUrl: string;
+  outputPath: string;
+  tilesetName: string;
+  maxDepth: number;
+  slpk: boolean;
+  egmFilePath: string;
+  token: string;
+  draco: boolean;
+  mergeMaterials: boolean;
+  generateTextures: boolean;
+  generateBoundingVolumes: boolean;
+  metadataClass: string;
+  analyze: boolean;
+};
+
+type NodeDoneStatus = {
+  nodeId: number;
+  done: Record<string, boolean> | boolean;
+};
+
+type TilesConverted = {
+  nodes: NodeDoneStatus[];
+};
+
+export class ConversionDump {
+  /** Conversion options */
+  private options?: ConversionDumpOptions;
+  /** Tiles conversion progress status map */
+  tilesConverted: Record<string, TilesConverted>;
+
+  constructor() {
+    this.tilesConverted = {};
+  }
+
+  /**
+   * Create a dump file with convertion options
+   * @param options - converter options
+   */
+  async createDumpFile(options: ConversionDumpOptions): Promise<void> {
+    const {
+      tilesetName,
+      slpk,
+      egmFilePath,
+      inputUrl,
+      outputPath,
+      draco = true,
+      maxDepth,
+      token,
+      generateTextures,
+      generateBoundingVolumes,
+      mergeMaterials = true,
+      metadataClass,
+      analyze = false
+    } = options;
+    this.options = {
+      tilesetName,
+      slpk,
+      egmFilePath,
+      inputUrl,
+      outputPath,
+      draco,
+      maxDepth,
+      token,
+      generateTextures,
+      generateBoundingVolumes,
+      mergeMaterials,
+      metadataClass,
+      analyze
+    };
+
+    try {
+      await writeFile(
+        options.outputPath,
+        JSON.stringify({options: this.options}),
+        `${options.tilesetName}${DUMP_FILE_SUFFIX}`
+      );
+    } catch (error) {
+      console.log("Can't create dump file", error);
+    }
+  }
+
+  /**
+   * Update conversion status in the dump file
+   */
+  private async updateDumpFile(): Promise<void> {
+    if (this.options?.outputPath && this.options.tilesetName) {
+      try {
+        await writeFile(
+          this.options.outputPath,
+          JSON.stringify({
+            options: this.options,
+            tilesConverted: this.tilesConverted
+          }),
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        );
+      } catch (error) {
+        console.log("Can't update dump file", error);
+      }
+    }
+  }
+
+  /**
+   * Delete a dump file
+   */
+  async deleteDumpFile(): Promise<void> {
+    if (this.options?.outputPath && this.options.tilesetName) {
+      await removeFile(
+        join(this.options.outputPath, `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`)
+      );
+    }
+  }
+
+  /**
+   * Get record from the tilesConverted Map
+   * @param fileName - source filename
+   * @returns existing object from the tilesConverted Map
+   */
+  private getRecord(fileName: string) {
+    return this.tilesConverted[fileName];
+  }
+
+  /**
+   * Set a record for the dump file
+   * @param fileName - key - source filename
+   * @param object - value
+   */
+  private setRecord(fileName: string, object: any) {
+    this.tilesConverted[fileName] = object;
+  }
+
+  /**
+   * Add a node into the dump file for the source file record
+   * @param fileName - source filename
+   * @param nodeId - nodeId of the node
+   */
+  async addNode(filename: string, nodeId: number) {
+    const {nodes} = this.getRecord(filename) || {nodes: []};
+    nodes.push({nodeId, done: {}});
+    if (nodes.length === 1) {
+      this.setRecord(filename, {nodes});
+    }
+    await this.updateDumpFile();
+  }
+
+  /**
+   * Update done status object for the writing resources
+   * @param fileName - key - source filename
+   * @param nodeId - nodeId for the source filename
+   * @param resourceType - resource type to update status
+   * @param value - value
+   */
+  updateDoneStatus(filename: string, nodeId: number, resourceType: string, value: boolean) {
+    const nodeDump = this.tilesConverted[filename]?.nodes.find(
+      (element) => element.nodeId === nodeId
+    );
+    if (nodeDump) {
+      nodeDump.done[resourceType] = value;
+    }
+  }
+
+  /**
+   * Update dump file according to writing results
+   * @param changedRecords - array of parameters ids for the written resources
+   * @param writeResults - array of writing resource files results
+   */
+  async updateConvertedTilesDump(
+    changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[],
+    writeResults: PromiseSettledResult<string | null>[]
+  ) {
+    for (let i = 0; i < changedRecords.length; i++) {
+      if (changedRecords[i] && 'value' in writeResults[i]) {
+        const {sourceId, resourceType, outputId} = changedRecords[i];
+        if (!sourceId || !resourceType || !outputId) continue;
+        for (const node of this.tilesConverted[sourceId].nodes) {
+          if (typeof node.done !== 'boolean' && node.nodeId === outputId) {
+            node.done[resourceType] = true;
+          }
+          if (typeof node.done !== 'boolean') {
+            let done = false;
+            for (const key in node.done) {
+              done = node.done[key];
+              if (!done) break;
+            }
+            if (done) {
+              node.done = true;
+            }
+          }
+        }
+      }
+    }
+    await this.updateDumpFile();
+  }
+}

package/src/lib/utils/write-queue.ts

@@ -1,11 +1,15 @@
 import {Queue} from './queue';
 import process from 'process';
+import {ConversionDump} from './conversion-dump';
 
 /** Memory limit size is based on testing */
 const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
 
 export type WriteQueueItem = {
   archiveKey?: string;
+  sourceId?: string;
+  outputId?: number;
+  resourceType?: string;
   /**
    * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);
    * Arguments for this call are specified in writeQueue.enqueue call like this:
@@ -27,13 +31,19 @@ export type WriteQueueItem = {
 
 export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
   private intervalId?: NodeJS.Timeout;
+  private conversionDump: ConversionDump;
   public writePromise: Promise<void> | null = null;
   public fileMap: {[key: string]: string} = {};
   public listeningInterval: number;
   public writeConcurrency: number;
 
-  constructor(listeningInterval: number = 2000, writeConcurrency: number = 400) {
+  constructor(
+    conversionDump: ConversionDump,
+    listeningInterval: number = 2000,
+    writeConcurrency: number = 400
+  ) {
     super();
+    this.conversionDump = conversionDump;
     this.listeningInterval = listeningInterval;
     this.writeConcurrency = writeConcurrency;
   }
@@ -81,18 +91,21 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
     while (this.length) {
       const promises: Promise<string | null>[] = [];
       const archiveKeys: (string | undefined)[] = [];
+      const changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[] = [];
       for (let i = 0; i < this.writeConcurrency; i++) {
         const item = this.dequeue();
         if (!item) {
           break;
         }
-        const {archiveKey, writePromise} = item as WriteQueueItem;
+        const {archiveKey, sourceId, outputId, resourceType, writePromise} = item as WriteQueueItem;
         archiveKeys.push(archiveKey);
+        changedRecords.push({sourceId, outputId, resourceType});
         const promise = writePromise();
         promises.push(promise);
       }
       const writeResults = await Promise.allSettled(promises);
       this.updateFileMap(archiveKeys, writeResults);
+      await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults);
     }
   }
 

package/dist/lib/utils/statistic-utills.d.js

@@ -1,2 +0,0 @@
-
-//# sourceMappingURL=statistic-utills.d.js.map

package/dist/lib/utils/statistic-utills.d.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"statistic-utills.d.js","names":[],"sources":["../../../src/lib/utils/statistic-utills.d.ts"],"sourcesContent":["/**\n * Do milliseconds time conversion to readable time string.\n * @param tile - 3d-tiles tile Object\n * @param coordinates - node converted coordinates\n * @returns String which characterizes conversion time period\n */\nexport function timeConverter(time: [number, number]): String;\n\n/**\n * Calculate files sizes after conversion.\n * @param params - Object with params of conversion.\n * @returns Promise with generated files size in bytes.\n */\nexport function calculateFilesSize(params: {\n  slpk: boolean;\n  outputPath: string;\n  tilesetName: string;\n}): Number;\n\n/**\n * Reqursivelly calculate files sizes in directory.\n * @param dirPath - Directory path.\n * @returns Promise with files size in directory.\n */\nexport function getTotalFilesSize(dirPath: string): Number;\n"],"mappings":""}