@loaders.gl/tile-converter 3.2.0-alpha.1 → 3.2.0-alpha.2

package/src/lib/utils/{compress-util.js → compress-util.ts}

@@ -9,7 +9,13 @@ import {MD5Hash} from '@loaders.gl/crypto';
 import crypt from 'crypt';
 import {getAbsoluteFilePath} from './file-utils';
 
-export function compressFileWithGzip(pathFile) {
+/**
+ * Compress file to gzip file
+ *
+ * @param pathFile - the path to the file
+ * @return the path to the gzip file
+ */
+export function compressFileWithGzip(pathFile: string): Promise<string> {
   const compressedPathFile = `${pathFile}.gz`;
   const gzip = createGzip();
   const input = createReadStream(pathFile);
@@ -28,7 +34,18 @@ export function compressFileWithGzip(pathFile) {
   });
 }
 
-export async function compressFilesWithZip(fileMap, outputFile, level = 0) {
+/**
+ * Compress files from map into slpk file
+ *
+ * @param fileMap - map with file paths (key: output path, value: input path)
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ */
+export async function compressFilesWithZip(
+  fileMap: {[key: string]: string},
+  outputFile: string,
+  level: number = 0
+) {
   // Before creating a new file, we need to delete the old file
   try {
     await removeFile(outputFile);
@@ -82,16 +99,44 @@ export async function compressFilesWithZip(fileMap, outputFile, level = 0) {
   });
 }
 
-export async function compressWithChildProcess() {
+/**
+ * Compress files using external tool 'zip'/'7z'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ * @param sevenZipExe - path to 7z.exe executable
+ */
+export async function compressWithChildProcess(
+  inputFolder: string,
+  outputFile: string,
+  level: number,
+  inputFiles: string,
+  sevenZipExe: string
+) {
   // eslint-disable-next-line no-undef
   if (process.platform === 'win32') {
-    await compressWithChildProcessWindows(...arguments);
+    await compressWithChildProcessWindows(inputFolder, outputFile, level, inputFiles, sevenZipExe);
   } else {
-    await compressWithChildProcessUnix(...arguments);
+    await compressWithChildProcessUnix(inputFolder, outputFile, level, inputFiles);
   }
 }
 
-async function compressWithChildProcessUnix(inputFolder, outputFile, level = 0, inputFiles = '.') {
+/**
+ * Compress files using external linux tool 'zip'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ */
+async function compressWithChildProcessUnix(
+  inputFolder: string,
+  outputFile: string,
+  level: number = 0,
+  inputFiles: string = '.'
+) {
   const fullOutputFile = getAbsoluteFilePath(outputFile);
   const args = [`-${level}`, '-r', fullOutputFile, inputFiles];
   const childProcess = new ChildProcessProxy();
@@ -105,12 +150,21 @@ async function compressWithChildProcessUnix(inputFolder, outputFile, level = 0,
   });
 }
 
+/**
+ * Compress files using windows external tool '7z'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ * @param sevenZipExe - path to 7z.exe executable
+ */
 async function compressWithChildProcessWindows(
-  inputFolder,
-  outputFile,
-  level = 0,
-  inputFiles = join('.', '*'),
-  sevenZipExe
+  inputFolder: string,
+  outputFile: string,
+  level: number = 0,
+  inputFiles: string = join('.', '*'),
+  sevenZipExe: string
 ) {
   // Workaround for @listfile issue. In 7z.exe @-leading files are handled as listfiles
   // https://sevenzip.osdn.jp/chm/cmdline/syntax.htm
@@ -131,10 +185,17 @@ async function compressWithChildProcessWindows(
   });
 }
 
-export async function generateHash128FromZip(inputZipFile, outputFile) {
+/**
+ * Generate hash file from zip archive
+ * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/slpk_hashtable.cmn.md
+ *
+ * @param inputZipFile
+ * @param outputFile
+ */
+export async function generateHash128FromZip(inputZipFile: string, outputFile: string) {
   const input = await fs.readFile(inputZipFile);
   const zip = await JSZip.loadAsync(input);
-  const hashTable = [];
+  const hashTable: {key: string; value: string}[] = [];
   const zipFiles = zip.files;
   for (const relativePath in zipFiles) {
     const zipEntry = zipFiles[relativePath];
@@ -169,7 +230,7 @@ export async function generateHash128FromZip(inputZipFile, outputFile) {
     });
     for (const key in hashTable) {
       const item = hashTable[key];
-      const value = _longToByteArray(item.value);
+      const value = longToByteArray(item.value);
       // TODO: perhaps you need to wait for the 'drain' event if the write returns 'false'
       // eslint-disable-next-line no-undef
       output.write(Buffer.from(crypt.hexToBytes(item.key).concat(value)));
@@ -178,19 +239,45 @@ export async function generateHash128FromZip(inputZipFile, outputFile) {
   });
 }
 
-function _longToByteArray(long) {
+/**
+ * Encode 64 bit value to byte array
+ *
+ * @param long - stringified number
+ * @returns
+ */
+function longToByteArray(long: string): number[] {
   const buffer = new ArrayBuffer(8); // JS numbers are 8 bytes long, or 64 bits
   const longNum = new Float64Array(buffer); // so equivalent to Float64
-  longNum[0] = long;
+  longNum[0] = parseInt(long);
   return Array.from(new Uint8Array(buffer)).reverse(); // reverse to get little endian
 }
 
-export async function addFileToZip(inputFolder, fileName, zipFile, sevenZipExe) {
+/**
+ * Add file to zip archive
+ *
+ * @param inputFile
+ * @param fileName
+ * @param zipFile
+ * @param sevenZipExe
+ */
+export async function addFileToZip(
+  inputFolder: string,
+  fileName: string,
+  zipFile: string,
+  sevenZipExe: string
+) {
   await compressWithChildProcess(inputFolder, zipFile, 0, fileName, sevenZipExe);
   console.log(`${fileName} added to ${zipFile}.`); // eslint-disable-line
 }
 
-function appendFileToArchive(archive, subFileName, subFileData) {
+/**
+ *
+ * @param archive zip archive instance
+ * @param subFileName file path inside archive
+ * @param subFileData source file path
+ * @returns
+ */
+function appendFileToArchive(archive: any, subFileName: string, subFileData: string) {
   return new Promise((resolve) => {
     const fileStream = createReadStream(subFileData);
     console.log(`Compression start: ${subFileName}`); // eslint-disable-line no-undef,no-console

package/src/lib/utils/file-utils.ts

@@ -0,0 +1,84 @@
+import {promises as fs} from 'fs';
+import {isAbsolute, join} from 'path';
+import {compressFileWithGzip} from './compress-util';
+
+/**
+ * Write a file with data and name fileName to path
+ *
+ * @param path - output path
+ * @param data - file content
+ * @param fileName - name of output file (default: index.json)
+ */
+export async function writeFile(
+  path: string,
+  data: string | Uint8Array | ArrayBuffer | Promise<ArrayBuffer>,
+  fileName: string = 'index.json'
+): Promise<string> {
+  let toWriteData: string | Uint8Array;
+  if (data instanceof Promise) {
+    toWriteData = new Uint8Array(await data);
+  } else if (data instanceof ArrayBuffer) {
+    toWriteData = new Uint8Array(data as ArrayBuffer);
+  } else {
+    toWriteData = data;
+  }
+  await fs.mkdir(path, {recursive: true});
+  const pathFile = join(path, fileName);
+  try {
+    await fs.writeFile(pathFile, toWriteData);
+  } catch (err) {
+    throw err;
+  }
+  console.log(`${pathFile} saved.`); // eslint-disable-line
+  return pathFile;
+}
+
+/**
+ * Write a file with data and name fileName to path - specific one for further packaging into slpk
+ *
+ * @param path - output path
+ * @param data - file content
+ * @param fileName - name of output file (default: index.json)
+ * @param compress - if need to compress file with gzip (default: true)
+ */
+export async function writeFileForSlpk(
+  path: string,
+  data: string | Uint8Array | ArrayBuffer | Promise<ArrayBuffer>,
+  fileName: string = 'index.json',
+  compress: boolean = true
+): Promise<string> {
+  const pathFile = await writeFile(path, data, fileName);
+  if (compress) {
+    const pathGzFile = await compressFileWithGzip(pathFile);
+    // After compression, we don't need an uncompressed file
+    await removeFile(pathFile);
+    return pathGzFile;
+  }
+  return pathFile;
+}
+
+/**
+ * Remove dir with path
+ *
+ * @param path
+ */
+export function removeDir(path: string) {
+  return fs.rmdir(path, {recursive: true});
+}
+
+/**
+ * Remove file with path
+ *
+ * @param path
+ */
+export function removeFile(path: string) {
+  return fs.unlink(path);
+}
+
+/**
+ * Generates absolute file path
+ * @param filePath
+ */
+export function getAbsoluteFilePath(filePath: string) {
+  return isAbsolute(filePath) ? filePath : join(process.cwd(), filePath); // eslint-disable-line no-undef
+}

package/src/lib/utils/{lod-conversion-utils.js → lod-conversion-utils.ts}

@@ -1,9 +1,11 @@
+import {BoundingVolumes} from '@loaders.gl/i3s';
+import {Tile3D} from '@loaders.gl/tiles';
+
 // https://cesium.com/docs/cesiumjs-ref-doc/Cesium3DTileset.html
 const DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR = 16;
 /**
  * Do conversion from geometric error to screen threshold
- * @param {Object} tile
- * @param {Object} coordinates
+ * 
  * In 3DTiles we have HLOD logic and parent tile also has bigger lodMetric value then its children.
  * In I3s we have reverse logic related to maxError. Parent has lower maxError than its child.
  * In nodes where are no children tile.lodMetricValue is 0. This is because of logic of HLOD in 3DTiles
@@ -13,9 +15,24 @@ const DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR = 16;
  * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/lodSelection.cmn.md
  * To avoid infinity values when we do calculations of maxError we shold replace 0 with value which allows us
  * to make child maxError bigger than his parent maxError.
+ * 
+ * @param tile - 3d-tiles tile Object
+ * @param coordinates - node converted coordinates
+ * @returns An array of LOD metrics in format compatible with i3s 3DNodeIndexDocument.lodSelection
+ * @example 
+ *  [
+        {
+            "metricType": "maxScreenThresholdSQ",
+            "maxError": 870638.071285568
+        },
+        {
+            "metricType": "maxScreenThreshold",
+            "maxError": 1052.8679031638949
+        }
+    ]
  */
-export function convertGeometricErrorToScreenThreshold(tile, coordinates) {
-  const lodSelection = [];
+export function convertGeometricErrorToScreenThreshold(tile: Tile3D, coordinates: BoundingVolumes) {
+  const lodSelection: {metricType: string; maxError: number}[] = [];
   const boundingVolume = tile.boundingVolume;
   const lodMetricValue = tile.lodMetricValue || 0.1;
   const maxScreenThreshold = {
@@ -38,7 +55,12 @@ export function convertGeometricErrorToScreenThreshold(tile, coordinates) {
   return lodSelection;
 }
 
-export function convertScreenThresholdToGeometricError(node) {
+/**
+ * Convert LOD metric from "Screen Threshold" to "Screen Space Error"
+ * @param node - i3s node data
+ * @returns lod metric in 3d-tiles format
+ */
+export function convertScreenThresholdToGeometricError(node: Tile3D): number {
   const metricData = node.header.lodSelection.maxScreenThreshold || {};
   let maxError = metricData.maxError;
   if (!maxError) {

package/src/lib/utils/queue.ts

@@ -0,0 +1,17 @@
+export class Queue<T> extends Array<T> {
+  enqueue(val: T) {
+    this.push(val);
+  }
+
+  dequeue(): T | undefined {
+    return this.shift();
+  }
+
+  peek(): T {
+    return this[0];
+  }
+
+  isEmpty() {
+    return this.length === 0;
+  }
+}

package/src/lib/utils/write-queue.ts

@@ -0,0 +1,75 @@
+import {Queue} from './queue';
+
+export type WriteQueueItem = {
+  archiveKey?: string;
+  writePromise: Promise<string>;
+};
+
+export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
+  private intervalId?: NodeJS.Timeout;
+  public writePromise: Promise<void> | null = null;
+  public fileMap: {[key: string]: string} = {};
+  public listeningInterval: number;
+  public writeConcurrency: number;
+
+  constructor(listeningInterval: number = 2000, writeConcurrency: number = 400) {
+    super();
+    this.listeningInterval = listeningInterval;
+    this.writeConcurrency = writeConcurrency;
+  }
+
+  startListening() {
+    this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
+  }
+
+  stopListening() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+    }
+  }
+
+  async startWrite(): Promise<void> {
+    if (this.writePromise) {
+      await this.writePromise;
+      this.writePromise = null;
+      return;
+    }
+    this.writePromise = this.doWrite();
+    await this.writePromise;
+    this.writePromise = null;
+  }
+
+  async finalize(): Promise<void> {
+    this.stopListening();
+    await this.startWrite();
+  }
+
+  private async doWrite(): Promise<void> {
+    while (this.length) {
+      const promises: Promise<string>[] = [];
+      const archiveKeys: (string | undefined)[] = [];
+      for (let i = 0; i < this.writeConcurrency; i++) {
+        const item = this.dequeue();
+        if (!item) {
+          break;
+        }
+        const {archiveKey, writePromise} = item as WriteQueueItem;
+        archiveKeys.push(archiveKey);
+        promises.push(writePromise);
+      }
+      const writeResults = await Promise.all(promises);
+      this.updateFileMap(archiveKeys, writeResults);
+    }
+    this.writePromise = null;
+  }
+
+  private updateFileMap(archiveKeys: (string | undefined)[], writeResults: string[]) {
+    for (let i = 0; i < archiveKeys.length; i++) {
+      const archiveKey = archiveKeys[i];
+      if (!archiveKey) {
+        continue;
+      }
+      this.fileMap[archiveKey] = writeResults[i];
+    }
+  }
+}

package/src/workers/3d-tiles-attributes-worker.ts

@@ -0,0 +1,6 @@
+import {createWorker} from '@loaders.gl/worker-utils';
+import B3dmConverter from '../3d-tiles-converter/helpers/b3dm-converter';
+
+const b3dmConverter = new B3dmConverter();
+
+createWorker(async (data, options = {}) => b3dmConverter.convert(data, options.attributes));

package/src/workers/i3s-attributes-worker.ts

@@ -0,0 +1,6 @@
+import {createWorker} from '@loaders.gl/worker-utils';
+import {convertAttributes} from '../i3s-converter/helpers/geometry-converter';
+
+createWorker(
+  async (data, options = {}) => await convertAttributes(data, options.useCartesianPositions)
+);

package/dist/es5/i3s-converter/helpers/geometry-converter.d.ts

@@ -1,44 +0,0 @@
-import {Vector3, Matrix4} from '@math.gl/core';
-import {GeoidHeightModel} from '../../lib/geoid-height-model';
-/**
- * Convert binary data from b3dm file to i3s resources
- *
- * @param tileContent - 3d tile content
- * @param options - converter options
- * @returns A promise that resolves to object with `geometry`, compressedGeometry`, `texture` and `sharedResources` appropriate
- *  for use  I3S tiles.
- */
-export default function convertB3dmToI3sGeometry(
-  tileContent: {
-    batchTableJson: {
-      id: [];
-    };
-    cartographicOrigin: Vector3;
-    cartesianModelMatrix: Matrix4;
-    gltf: {
-      scene: {
-        nodes: [];
-      };
-      images: [];
-      materials: [];
-    };
-  },
-  nodeId: number,
-  featuresHashArray: any,
-  attributeStorageInfo: any,
-  draco: boolean,
-  generateBoundingVolumes: boolean,
-  geoidHeightModel: GeoidHeightModel
-): Promise<
-  {
-    geometry: ArrayBuffer;
-    compressedGeometry: ArrayBuffer;
-    texture: any;
-    sharedResources: any;
-    meshMaterial: any;
-    vertexCount: number;
-    attributes: any;
-    featureCount: number;
-    boundingVolumes: any;
-  }[]
->;

package/dist/es5/lib/utils/compress-util.d.ts

@@ -1,53 +0,0 @@
-/**
- * Compress file to gzip file
- *
- * @param pathFile - the path to the file
- * @return the path to the gzip file
- */
-export function compressFileWithGzip(pathFile: string): Promise<string>;
-
-/**
- * Compress files from map into slpk file
- *
- * @param fileMap - map with file paths (key: output path, value: input path)
- * @param outputFile - output slpk file
- * @param level - compression level
- */
-export function compressFilesWithZip(
-  fileMap: object,
-  outputFile: string,
-  level: number
-): Promise<void>;
-
-/**
- * Compress files using external tool 'zip'
- *
- * @param inputFolder - folder to archive
- * @param outputFile - output slpk file
- * @param level - compression level
- */
-export function compressWithChildProcess(
-  inputFolder: string,
-  outputFile: string,
-  level: number,
-  sevenZipExe: string
-): Promise<void>;
-
-/**
- * Generate hash file from zip archive
- * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/slpk_hashtable.cmn.md
- *
- * @param inputZipFile
- * @param outputFile
- */
-export function generateHash128FromZip(inputZipFile, outputFile): Promise<void>;
-
-/**
- * Add file to zip archive
- *
- * @param inputFile
- * @param fileName
- * @param zipFile
- * @param sevenZipExe
- */
-export function addFileToZip(inputFile, fileName, zipFile, sevenZipExe);

package/dist/es5/lib/utils/file-utils.d.ts

@@ -1,43 +0,0 @@
-/**
- * Write a file with data and name fileName to path
- *
- * @param path - output path
- * @param data - file content
- * @param fileName - name of output file (default: index.json)
- */
-export function writeFile(path: string, data: string | Uint8Array | ArrayBuffer, fileName?: string);
-
-/**
- * Write a file with data and name fileName to path - specific one for further packaging into slpk
- *
- * @param path - output path
- * @param data - file content
- * @param fileName - name of output file (default: index.json)
- * @param compress - if need to compress file with gzip (default: true)
- */
-export function writeFileForSlpk(
-  path: string,
-  data: string | Uint8Array | ArrayBuffer,
-  fileName?: string,
-  compress?: boolean
-);
-
-/**
- * Remove dir with path
- *
- * @param path
- */
-export function removeDir(path: string): Promise<void>;
-
-/**
- * Remove file with path
- *
- * @param path
- */
-export function removeFile(path: string): Promise<void>;
-
-/**
- * Generates absolute file path
- * @param filePath
- */
-export function getAbsoluteFilePath(filePath: string): string;

package/dist/es5/lib/utils/lod-conversion-utils.d.ts

@@ -1,32 +0,0 @@
-/**
- * Convert LOD metric from "Screen Space Error" to "Screen Threshold"
- * 
- * @param tile - 3d-tiles tile Object
- * @param coordinates - node converted coordinates
- * @returns An array of LOD metrics in format compatible with i3s 3DNodeIndexDocument.lodSelection
- * @example 
- *  [
-        {
-            "metricType": "maxScreenThresholdSQ",
-            "maxError": 870638.071285568
-        },
-        {
-            "metricType": "maxScreenThreshold",
-            "maxError": 1052.8679031638949
-        }
-    ]
- */
-export function convertGeometricErrorToScreenThreshold(
-  tile: Object,
-  coordinates: {
-    mbs?: number[];
-  }
-): Array<{metricType: string; maxError: number}>;
-
-/**
- * Convert LOD metric from "Screen Threshold" to "Screen Space Error"
- * @param node - i3s node data
- * @returns lod metric in 3d-tiles format
- * @todo implement this function
- */
-export function convertScreenThresholdToGeometricError(node: Object): number;

package/dist/esm/i3s-converter/helpers/geometry-converter.d.ts

@@ -1,44 +0,0 @@
-import {Vector3, Matrix4} from '@math.gl/core';
-import {GeoidHeightModel} from '../../lib/geoid-height-model';
-/**
- * Convert binary data from b3dm file to i3s resources
- *
- * @param tileContent - 3d tile content
- * @param options - converter options
- * @returns A promise that resolves to object with `geometry`, compressedGeometry`, `texture` and `sharedResources` appropriate
- *  for use  I3S tiles.
- */
-export default function convertB3dmToI3sGeometry(
-  tileContent: {
-    batchTableJson: {
-      id: [];
-    };
-    cartographicOrigin: Vector3;
-    cartesianModelMatrix: Matrix4;
-    gltf: {
-      scene: {
-        nodes: [];
-      };
-      images: [];
-      materials: [];
-    };
-  },
-  nodeId: number,
-  featuresHashArray: any,
-  attributeStorageInfo: any,
-  draco: boolean,
-  generateBoundingVolumes: boolean,
-  geoidHeightModel: GeoidHeightModel
-): Promise<
-  {
-    geometry: ArrayBuffer;
-    compressedGeometry: ArrayBuffer;
-    texture: any;
-    sharedResources: any;
-    meshMaterial: any;
-    vertexCount: number;
-    attributes: any;
-    featureCount: number;
-    boundingVolumes: any;
-  }[]
->;