@loaders.gl/tile-converter 4.1.0-alpha.9 → 4.2.0-alpha.1

package/src/i3s-converter/types.ts

@@ -241,3 +241,11 @@ export const AttributeType = {
   /** Integer data type name for feature attributes */
   SHORT_INT_TYPE: 'Int32'
 } as const;
+
+export enum ResourceType {
+  ATTRIBUTES = 'ATTRIBUTES',
+  DRACO_GEOMETRY = 'DRACO_GEOMETRY',
+  GEOMETRY = 'GEOMETRY',
+  SHARED = 'SHARED',
+  TEXTURE = 'TEXTURE'
+}

package/src/lib/utils/compress-util.ts

@@ -1,13 +1,5 @@
 import {createGzip} from 'zlib';
-import {join} from 'path';
-import {promises as fs, createReadStream, createWriteStream} from 'fs';
-import archiver from 'archiver';
-import {removeFile} from './file-utils';
-import {ChildProcessProxy} from '@loaders.gl/worker-utils';
-import JSZip from 'jszip';
-import {MD5Hash} from '@loaders.gl/crypto';
-import crypt from 'crypt';
-import {getAbsoluteFilePath} from './file-utils';
+import {createReadStream, createWriteStream} from 'fs';
 
 /**
  * Compress file to gzip file
@@ -33,258 +25,3 @@ export function compressFileWithGzip(pathFile: string): Promise<string> {
     input.pipe(gzip).pipe(output);
   });
 }
-
-/**
- * Compress files from map into slpk file
- *
- * @param fileMap - map with file paths (key: output path, value: input path)
- * @param outputFile - output slpk file
- * @param level - compression level
- */
-export async function compressFilesWithZip(
-  fileMap: {[key: string]: string},
-  outputFile: string,
-  level: number = 0
-) {
-  // Before creating a new file, we need to delete the old file
-  try {
-    await removeFile(outputFile);
-  } catch (e) {
-    // Do nothing if old file doesn't exist
-  }
-
-  const output = createWriteStream(outputFile);
-  const archive = archiver('zip', {
-    zlib: {level} // Sets the compression level.
-  });
-
-  return new Promise(async (resolve, reject) => {
-    // listen for all archive data to be writte
-    // 'close' event is fired only when a file descriptor is involved
-    output.on('close', function () {
-      console.log(`${outputFile} saved.`); // eslint-disable-line no-undef,no-console
-      console.log(`${archive.pointer()} total bytes`); // eslint-disable-line no-undef,no-console
-      resolve(null);
-    });
-
-    // This event is fired when the data source is drained no matter what was the data source.
-    // It is not part of this library but rather from the NodeJS Stream API.
-    // @see: https://nodejs.org/api/stream.html#stream_event_end
-    output.on('end', function () {
-      console.log('Data has been drained'); // eslint-disable-line no-undef,no-console
-      resolve(null);
-    });
-
-    // good practice to catch warnings (ie stat failures and other non-blocking errors)
-    archive.on('warning', function (err) {
-      console.log(err); // eslint-disable-line no-undef,no-console
-      reject(err);
-    });
-
-    // good practice to catch this error explicitly
-    archive.on('error', function (err) {
-      reject(err);
-    });
-
-    // pipe archive data to the file
-    archive.pipe(output);
-
-    for (const subFileName in fileMap) {
-      const subFileData = fileMap[subFileName];
-      await appendFileToArchive(archive, subFileName, subFileData);
-    }
-
-    // finalize the archive (ie we are done appending files but streams have to finish yet)
-    archive.finalize();
-  });
-}
-
-/**
- * Compress files using external tool 'zip'/'7z'
- *
- * @param inputFolder - folder to archive - for cwd option
- * @param outputFile - output slpk file
- * @param level - compression level
- * @param inputFiles - input files path to pass to the executable as option
- * @param sevenZipExe - path to 7z.exe executable
- */
-export async function compressWithChildProcess(
-  inputFolder: string,
-  outputFile: string,
-  level: number,
-  inputFiles: string,
-  sevenZipExe: string
-) {
-  // eslint-disable-next-line no-undef
-  if (process.platform === 'win32') {
-    await compressWithChildProcessWindows(inputFolder, outputFile, level, inputFiles, sevenZipExe);
-  } else {
-    await compressWithChildProcessUnix(inputFolder, outputFile, level, inputFiles);
-  }
-}
-
-/**
- * Compress files using external linux tool 'zip'
- *
- * @param inputFolder - folder to archive - for cwd option
- * @param outputFile - output slpk file
- * @param level - compression level
- * @param inputFiles - input files path to pass to the executable as option
- */
-async function compressWithChildProcessUnix(
-  inputFolder: string,
-  outputFile: string,
-  level: number = 0,
-  inputFiles: string = '.'
-) {
-  const fullOutputFile = getAbsoluteFilePath(outputFile);
-  const args = [`-${level}`, '-r', fullOutputFile, inputFiles];
-  const childProcess = new ChildProcessProxy();
-  await childProcess.start({
-    command: 'zip',
-    arguments: args,
-    spawn: {
-      cwd: inputFolder
-    },
-    wait: 0
-  });
-}
-
-/**
- * Compress files using windows external tool '7z'
- *
- * @param inputFolder - folder to archive - for cwd option
- * @param outputFile - output slpk file
- * @param level - compression level
- * @param inputFiles - input files path to pass to the executable as option
- * @param sevenZipExe - path to 7z.exe executable
- */
-async function compressWithChildProcessWindows(
-  inputFolder: string,
-  outputFile: string,
-  level: number = 0,
-  inputFiles: string = join('.', '*'),
-  sevenZipExe: string
-) {
-  // Workaround for @listfile issue. In 7z.exe @-leading files are handled as listfiles
-  // https://sevenzip.osdn.jp/chm/cmdline/syntax.htm
-  if (inputFiles[0] === '@') {
-    inputFiles = `*${inputFiles.substr(1)}`;
-  }
-
-  const fullOutputFile = getAbsoluteFilePath(outputFile);
-  const args = ['a', '-tzip', `-mx=${level}`, fullOutputFile, inputFiles];
-  const childProcess = new ChildProcessProxy();
-  await childProcess.start({
-    command: sevenZipExe,
-    arguments: args,
-    spawn: {
-      cwd: `${inputFolder}`
-    },
-    wait: 0
-  });
-}
-
-/**
- * Generate hash file from zip archive
- * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/slpk_hashtable.cmn.md
- *
- * @param inputZipFile
- * @param outputFile
- */
-export async function generateHash128FromZip(inputZipFile: string, outputFile: string) {
-  const input = await fs.readFile(inputZipFile);
-  const zip = await JSZip.loadAsync(input);
-  const hashTable: {key: string; value: string}[] = [];
-  const zipFiles = zip.files;
-  for (const relativePath in zipFiles) {
-    const zipEntry = zipFiles[relativePath];
-    // Had to use a workaround because the correct string is getting the wrong data
-    // const content = await zipEntry.async('nodebuffer');
-    // _data isn't described in the interface, so lint thought it was wrong
-    const _data = '_data';
-    const content = zipEntry[_data].compressedContent;
-    if (zipEntry.dir) continue; // eslint-disable-line no-continue
-    // eslint-disable-next-line no-undef
-    const hash = await new MD5Hash().hash(Buffer.from(relativePath.toLowerCase()), 'base64');
-    // eslint-disable-next-line no-undef
-    hashTable.push({key: atob(hash), value: content.byteOffset});
-  }
-
-  hashTable.sort((prev, next) => {
-    if (prev.key === next.key) {
-      return prev.value < next.value ? -1 : 1;
-    }
-    return prev.key < next.key ? -1 : 1;
-  });
-
-  const output = createWriteStream(outputFile);
-  return new Promise((resolve, reject) => {
-    output.on('close', function () {
-      console.log(`${outputFile} generated and saved`); // eslint-disable-line
-      resolve(null);
-    });
-    output.on('error', function (err) {
-      console.log(err); // eslint-disable-line
-      reject(err);
-    });
-    for (const key in hashTable) {
-      const item = hashTable[key];
-      const value = longToByteArray(item.value);
-      // TODO: perhaps you need to wait for the 'drain' event if the write returns 'false'
-      // eslint-disable-next-line no-undef
-      output.write(Buffer.from(crypt.hexToBytes(item.key).concat(value)));
-    }
-    output.close();
-  });
-}
-
-/**
- * Encode 64 bit value to byte array
- *
- * @param long - stringified number
- * @returns
- */
-function longToByteArray(long: string): number[] {
-  const buffer = new ArrayBuffer(8); // JS numbers are 8 bytes long, or 64 bits
-  const longNum = new Float64Array(buffer); // so equivalent to Float64
-  longNum[0] = parseInt(long);
-  return Array.from(new Uint8Array(buffer)).reverse(); // reverse to get little endian
-}
-
-/**
- * Add file to zip archive
- *
- * @param inputFile
- * @param fileName
- * @param zipFile
- * @param sevenZipExe
- */
-export async function addFileToZip(
-  inputFolder: string,
-  fileName: string,
-  zipFile: string,
-  sevenZipExe: string
-) {
-  await compressWithChildProcess(inputFolder, zipFile, 0, fileName, sevenZipExe);
-  console.log(`${fileName} added to ${zipFile}.`); // eslint-disable-line
-}
-
-/**
- *
- * @param archive zip archive instance
- * @param subFileName file path inside archive
- * @param subFileData source file path
- * @returns
- */
-function appendFileToArchive(archive: any, subFileName: string, subFileData: string) {
-  return new Promise((resolve) => {
-    const fileStream = createReadStream(subFileData);
-    console.log(`Compression start: ${subFileName}`); // eslint-disable-line no-undef,no-console
-    fileStream.on('close', () => {
-      console.log(`Compression finish: ${subFileName}`); // eslint-disable-line no-undef,no-console
-      resolve(null);
-    });
-    archive.append(fileStream, {name: subFileName});
-  });
-}

package/src/lib/utils/conversion-dump.ts

@@ -0,0 +1,325 @@
+import {isDeepStrictEqual} from 'util';
+import {DUMP_FILE_SUFFIX} from '../../constants';
+import {isFileExists, openJson, removeFile, writeFile} from './file-utils';
+import {join} from 'path';
+import {BoundingVolumes, I3SMaterialDefinition, TextureSetDefinitionFormats} from '@loaders.gl/i3s';
+import {AttributeMetadataInfoObject} from '../../i3s-converter/helpers/attribute-metadata-info';
+
+export type ConversionDumpOptions = {
+  inputUrl: string;
+  outputPath: string;
+  tilesetName: string;
+  maxDepth: number;
+  slpk: boolean;
+  egmFilePath: string;
+  token: string;
+  draco: boolean;
+  mergeMaterials: boolean;
+  generateTextures: boolean;
+  generateBoundingVolumes: boolean;
+  metadataClass: string;
+  analyze: boolean;
+};
+
+type NodeDoneStatus = {
+  nodeId: number;
+  done: boolean;
+  progress?: Record<string, boolean>;
+  dumpMetadata?: DumpMetadata;
+};
+
+type TilesConverted = {
+  nodes: NodeDoneStatus[];
+};
+
+export type DumpMetadata = {
+  boundingVolumes: BoundingVolumes | null;
+  attributesCount?: number;
+  featureCount: number | null;
+  geometry: boolean;
+  hasUvRegions: boolean;
+  materialId: number | null;
+  texelCountHint?: number;
+  vertexCount: number | null;
+};
+
+export type TextureSetDefinition = {
+  formats: TextureSetDefinitionFormats;
+  atlas?: boolean;
+};
+
+export class ConversionDump {
+  /**Restored/resumed dump indicator */
+  restored: boolean = false;
+  /** Conversion options */
+  private options?: ConversionDumpOptions;
+  /** Tiles conversion progress status map */
+  tilesConverted: Record<string, TilesConverted>;
+  /** Textures formats definitions */
+  textureSetDefinitions?: TextureSetDefinition[];
+  /** Attributes Metadata */
+  attributeMetadataInfo?: AttributeMetadataInfoObject;
+  /** Array of materials definitions */
+  materialDefinitions: I3SMaterialDefinition[] = [];
+
+  constructor() {
+    this.tilesConverted = {};
+  }
+
+  /**
+   * Create a dump with convertion options
+   * @param currentOptions - converter options
+   */
+  async createDump(currentOptions: ConversionDumpOptions): Promise<void> {
+    const {
+      tilesetName,
+      slpk,
+      egmFilePath,
+      inputUrl,
+      outputPath,
+      draco = true,
+      maxDepth,
+      token,
+      generateTextures,
+      generateBoundingVolumes,
+      mergeMaterials = true,
+      metadataClass,
+      analyze = false
+    } = currentOptions;
+    this.options = {
+      tilesetName,
+      slpk,
+      egmFilePath,
+      inputUrl,
+      outputPath,
+      draco,
+      maxDepth,
+      token,
+      generateTextures,
+      generateBoundingVolumes,
+      mergeMaterials,
+      metadataClass,
+      analyze
+    };
+
+    const dumpFilename = join(
+      this.options.outputPath,
+      this.options.tilesetName,
+      `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+    );
+    if (await isFileExists(dumpFilename)) {
+      const {
+        options,
+        tilesConverted,
+        textureSetDefinitions,
+        attributeMetadataInfo,
+        materialDefinitions
+      } = await openJson(
+        join(this.options.outputPath, this.options.tilesetName),
+        `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+      );
+      if (isDeepStrictEqual(options, JSON.parse(JSON.stringify(this.options)))) {
+        this.tilesConverted = tilesConverted;
+        this.textureSetDefinitions = textureSetDefinitions;
+        this.attributeMetadataInfo = attributeMetadataInfo;
+        this.materialDefinitions = materialDefinitions;
+        this.restored = true;
+        return;
+      }
+    }
+    await this.deleteDumpFile();
+  }
+
+  /**
+   * Reset a dump
+   */
+  reset(): void {
+    this.restored = false;
+    this.tilesConverted = {};
+    if (this.textureSetDefinitions) {
+      delete this.textureSetDefinitions;
+    }
+    if (this.attributeMetadataInfo) {
+      delete this.attributeMetadataInfo;
+    }
+    if (this.materialDefinitions.length > 0) {
+      this.materialDefinitions = [];
+    }
+  }
+
+  /**
+   * Update conversion status in the dump file
+   */
+  private async updateDumpFile(): Promise<void> {
+    if (this.options?.outputPath && this.options.tilesetName) {
+      try {
+        await writeFile(
+          join(this.options.outputPath, this.options.tilesetName),
+          JSON.stringify({
+            options: this.options,
+            tilesConverted: this.tilesConverted,
+            textureSetDefinitions: this.textureSetDefinitions,
+            attributeMetadataInfo: this.attributeMetadataInfo,
+            materialDefinitions: this.materialDefinitions
+          }),
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        );
+      } catch (error) {
+        console.log("Can't update dump file", error);
+      }
+    }
+  }
+
+  /**
+   * Delete a dump file
+   */
+  async deleteDumpFile(): Promise<void> {
+    if (
+      this.options?.outputPath &&
+      this.options.tilesetName &&
+      (await isFileExists(
+        join(
+          this.options.outputPath,
+          this.options.tilesetName,
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        )
+      ))
+    ) {
+      await removeFile(
+        join(
+          this.options.outputPath,
+          this.options.tilesetName,
+          `${this.options.tilesetName}${DUMP_FILE_SUFFIX}`
+        )
+      );
+    }
+  }
+
+  /**
+   * Get record from the tilesConverted Map
+   * @param fileName - source filename
+   * @returns existing object from the tilesConverted Map
+   */
+  private getRecord(fileName: string) {
+    return this.tilesConverted[fileName];
+  }
+
+  /**
+   * Set a record for the dump file
+   * @param fileName - key - source filename
+   * @param object - value
+   */
+  private setRecord(fileName: string, object: any) {
+    this.tilesConverted[fileName] = object;
+  }
+
+  /**
+   * Add a node into the dump file for the source file record
+   * @param fileName - source filename
+   * @param nodeId - nodeId of the node
+   */
+  async addNode(filename: string, nodeId: number, dumpMetadata: DumpMetadata) {
+    const {nodes} = this.getRecord(filename) || {nodes: []};
+    nodes.push({nodeId, done: false, progress: {}, dumpMetadata});
+    if (nodes.length === 1) {
+      this.setRecord(filename, {nodes});
+    }
+    await this.updateDumpFile();
+  }
+
+  /**
+   * Clear dump record got the source filename
+   * @param fileName - source filename
+   */
+  clearDumpRecord(filename: string) {
+    this.setRecord(filename, {nodes: []});
+  }
+
+  /**
+   * Add textures definitions into the dump file
+   * @param textureDefinitions - textures definitions array
+   */
+  addTexturesDefinitions(textureDefinitions: TextureSetDefinition[]) {
+    this.textureSetDefinitions = textureDefinitions;
+  }
+
+  /**
+   * Update done status object for the writing resources
+   * @param fileName - key - source filename
+   * @param nodeId - nodeId for the source filename
+   * @param resourceType - resource type to update status
+   * @param value - value
+   */
+  updateDoneStatus(filename: string, nodeId: number, resourceType: string, value: boolean) {
+    const nodeDump = this.tilesConverted[filename]?.nodes.find(
+      (element) => element.nodeId === nodeId
+    );
+    if (nodeDump) {
+      if (!nodeDump.progress) {
+        nodeDump.progress = {};
+      }
+      nodeDump.progress[resourceType] = value;
+      if (!value) {
+        nodeDump.done = false;
+      }
+    }
+  }
+
+  /**
+   * Update dump file according to writing results
+   * @param changedRecords - array of parameters ids for the written resources
+   * @param writeResults - array of writing resource files results
+   */
+  async updateConvertedTilesDump(
+    changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[],
+    writeResults: PromiseSettledResult<string | null>[]
+  ) {
+    for (let i = 0; i < changedRecords.length; i++) {
+      if (changedRecords[i] && 'value' in writeResults[i]) {
+        const {sourceId, resourceType, outputId} = changedRecords[i];
+        if (!sourceId || !resourceType || !outputId) continue;
+        for (const node of this.tilesConverted[sourceId].nodes) {
+          if (node.nodeId === outputId && node.progress) {
+            node.progress[resourceType] = true;
+
+            let done = false;
+            for (const key in node.progress) {
+              done = node.progress[key];
+              if (!done) break;
+            }
+            node.done = done;
+            if (node.done) {
+              delete node.progress;
+            }
+            break;
+          }
+        }
+      }
+    }
+    await this.updateDumpFile();
+  }
+
+  /**
+   * Check is source file conversion complete
+   * @param filename - source filename
+   * @returns true if source file conversion complete
+   */
+  isFileConversionComplete(filename: string): boolean {
+    let result = true;
+    for (const node of this.tilesConverted[filename]?.nodes || []) {
+      if (!node.done) {
+        result = false;
+        break;
+      }
+    }
+    return result && this.tilesConverted[filename]?.nodes?.length > 0;
+  }
+
+  /**
+   * Set materialDefinitions into a dump
+   * @param materialDefinitions - Array materialDefinitions
+   */
+  setMaterialsDefinitions(materialDefinitions: I3SMaterialDefinition[]): void {
+    this.materialDefinitions = materialDefinitions;
+  }
+}

package/src/lib/utils/statistic-utills.ts

@@ -44,7 +44,7 @@ function timeConverterFromSecondsAndMilliseconds(timeInSeconds: number, millisec
   }
 
   if (!result) {
-    result += `${milliseconds}ms`;
+    result += `${Math.floor(milliseconds)}ms`;
   }
 
   return result;

package/src/lib/utils/write-queue.ts

@@ -1,11 +1,15 @@
 import {Queue} from './queue';
 import process from 'process';
+import {ConversionDump} from './conversion-dump';
 
 /** Memory limit size is based on testing */
 const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
 
 export type WriteQueueItem = {
   archiveKey?: string;
+  sourceId?: string;
+  outputId?: number;
+  resourceType?: string;
   /**
    * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);
    * Arguments for this call are specified in writeQueue.enqueue call like this:
@@ -27,13 +31,19 @@ export type WriteQueueItem = {
 
 export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
   private intervalId?: NodeJS.Timeout;
+  private conversionDump: ConversionDump;
   public writePromise: Promise<void> | null = null;
   public fileMap: {[key: string]: string} = {};
   public listeningInterval: number;
   public writeConcurrency: number;
 
-  constructor(listeningInterval: number = 2000, writeConcurrency: number = 400) {
+  constructor(
+    conversionDump: ConversionDump,
+    listeningInterval: number = 2000,
+    writeConcurrency: number = 400
+  ) {
     super();
+    this.conversionDump = conversionDump;
     this.listeningInterval = listeningInterval;
     this.writeConcurrency = writeConcurrency;
   }
@@ -81,18 +91,21 @@ export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
     while (this.length) {
       const promises: Promise<string | null>[] = [];
       const archiveKeys: (string | undefined)[] = [];
+      const changedRecords: {outputId?: number; sourceId?: string; resourceType?: string}[] = [];
       for (let i = 0; i < this.writeConcurrency; i++) {
         const item = this.dequeue();
         if (!item) {
           break;
         }
-        const {archiveKey, writePromise} = item as WriteQueueItem;
+        const {archiveKey, sourceId, outputId, resourceType, writePromise} = item as WriteQueueItem;
         archiveKeys.push(archiveKey);
+        changedRecords.push({sourceId, outputId, resourceType});
         const promise = writePromise();
         promises.push(promise);
       }
       const writeResults = await Promise.allSettled(promises);
       this.updateFileMap(archiveKeys, writeResults);
+      await this.conversionDump.updateConvertedTilesDump(changedRecords, writeResults);
     }
   }