@loaders.gl/tile-converter 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/i3s-server/routes/slpk-router.js

@@ -0,0 +1,33 @@
+const express = require('express');
+const {getFileByUrl} = require('../controllers/slpk-controller');
+const createSceneServer = require('../utils/create-scene-server');
+
+const sceneServerRouter = express.Router();
+sceneServerRouter.get('*', async function (req, res, next) {
+  const file = await getFileByUrl('/');
+  if (file) {
+    const layer = JSON.parse(file.toString());
+    const sceneServerResponse = createSceneServer(layer.name, layer);
+    res.send(sceneServerResponse);
+  } else {
+    res.status(404);
+    res.send('File not found');
+  }
+});
+
+const router = express.Router();
+router.get('*', async function (req, res, next) {
+  console.log(req.path);
+  const file = await getFileByUrl(req.path);
+  if (file) {
+    res.send(file);
+  } else {
+    res.status(404);
+    res.send('File not found');
+  }
+});
+
+module.exports = {
+  sceneServerRouter,
+  router
+};

package/src/i3s-server/utils/create-scene-server.js

@@ -0,0 +1,15 @@
+const {v4: uuidv4} = require('uuid');
+
+const createSceneServer = (name, layer) => {
+  return {
+    serviceItemId: uuidv4().replace(/-/gi, ''),
+    serviceName: name,
+    name,
+    currentVersion: '10.7',
+    serviceVersion: '1.8',
+    supportedBindings: ['REST'],
+    layers: [layer]
+  };
+};
+
+module.exports = createSceneServer;

package/src/index.ts

@@ -1,6 +1,2 @@
 export {default as I3SConverter} from './i3s-converter/i3s-converter';
-export {default as NodePages} from './i3s-converter/helpers/node-pages';
-
 export {default as Tiles3DConverter} from './3d-tiles-converter/3d-tiles-converter';
-
-export {DepsInstaller} from './deps-installer/deps-installer';

package/src/lib/utils/cli-utils.ts

@@ -0,0 +1,78 @@
+/**
+ * Get string option value from cli arguments
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+export function getStringValue(index: number, args: string[]): string {
+  if (index + 1 >= args.length) {
+    return '';
+  }
+  const value = args[index + 1];
+  if (value.indexOf('--') === 0) {
+    return '';
+  }
+  return value;
+}
+
+/**
+ * Modyfy URL path to be compatible with fetch
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+export function getURLValue(index: number, args: string[]): string {
+  const value = getStringValue(index, args);
+  console.log(`Input tileset value: ${value}`);
+  console.log(`Modified tileset value: ${value.replace(/\\/g, '/')}`);
+  return value.replace(/\\/g, '/');
+}
+
+export function validateOptionsWithEqual(args: string[]): string[] {
+  return args.reduce((acc: string[], curr) => {
+    const equalSignIndex = curr.indexOf('=');
+    const beforeEqual = curr.slice(0, equalSignIndex);
+    const afterEqual = curr.slice(equalSignIndex + 1, curr.length);
+    const condition = curr.includes('=') && curr.startsWith('--') && afterEqual;
+    if (condition) {
+      return acc.concat(beforeEqual, afterEqual);
+    }
+    return acc.concat(curr);
+  }, []);
+}
+
+/**
+ * Get integer option value from cli arguments
+ * @param index - option's name index in the argument's array
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - number value of the option
+ */
+export function getIntegerValue(index: number, args: string[]): number {
+  const stringValue: string = getStringValue(index, args);
+  const result: number = Number.parseInt(stringValue);
+  if (isFinite(result)) {
+    return result;
+  }
+  return NaN;
+}
+
+/**
+ * Get boolean option value from cli arguments
+ * @param index - option's name index in the argument's array
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - boolean value of the option
+ */
+export function getBooleanValue(index: number, args: string[]): boolean {
+  const stringValue: string = getStringValue(index, args).toLowerCase().trim();
+  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
+    return false;
+  }
+  if (!stringValue || stringValue === 'true') {
+    return true;
+  }
+  return false;
+}

package/src/lib/utils/{compress-util.js → compress-util.ts}

@@ -9,7 +9,13 @@ import {MD5Hash} from '@loaders.gl/crypto';
 import crypt from 'crypt';
 import {getAbsoluteFilePath} from './file-utils';
 
-export function compressFileWithGzip(pathFile) {
+/**
+ * Compress file to gzip file
+ *
+ * @param pathFile - the path to the file
+ * @return the path to the gzip file
+ */
+export function compressFileWithGzip(pathFile: string): Promise<string> {
   const compressedPathFile = `${pathFile}.gz`;
   const gzip = createGzip();
   const input = createReadStream(pathFile);
@@ -28,7 +34,18 @@ export function compressFileWithGzip(pathFile) {
   });
 }
 
-export async function compressFilesWithZip(fileMap, outputFile, level = 0) {
+/**
+ * Compress files from map into slpk file
+ *
+ * @param fileMap - map with file paths (key: output path, value: input path)
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ */
+export async function compressFilesWithZip(
+  fileMap: {[key: string]: string},
+  outputFile: string,
+  level: number = 0
+) {
   // Before creating a new file, we need to delete the old file
   try {
     await removeFile(outputFile);
@@ -82,16 +99,44 @@ export async function compressFilesWithZip(fileMap, outputFile, level = 0) {
   });
 }
 
-export async function compressWithChildProcess() {
+/**
+ * Compress files using external tool 'zip'/'7z'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ * @param sevenZipExe - path to 7z.exe executable
+ */
+export async function compressWithChildProcess(
+  inputFolder: string,
+  outputFile: string,
+  level: number,
+  inputFiles: string,
+  sevenZipExe: string
+) {
   // eslint-disable-next-line no-undef
   if (process.platform === 'win32') {
-    await compressWithChildProcessWindows(...arguments);
+    await compressWithChildProcessWindows(inputFolder, outputFile, level, inputFiles, sevenZipExe);
   } else {
-    await compressWithChildProcessUnix(...arguments);
+    await compressWithChildProcessUnix(inputFolder, outputFile, level, inputFiles);
   }
 }
 
-async function compressWithChildProcessUnix(inputFolder, outputFile, level = 0, inputFiles = '.') {
+/**
+ * Compress files using external linux tool 'zip'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ */
+async function compressWithChildProcessUnix(
+  inputFolder: string,
+  outputFile: string,
+  level: number = 0,
+  inputFiles: string = '.'
+) {
   const fullOutputFile = getAbsoluteFilePath(outputFile);
   const args = [`-${level}`, '-r', fullOutputFile, inputFiles];
   const childProcess = new ChildProcessProxy();
@@ -105,12 +150,21 @@ async function compressWithChildProcessUnix(inputFolder, outputFile, level = 0,
   });
 }
 
+/**
+ * Compress files using windows external tool '7z'
+ *
+ * @param inputFolder - folder to archive - for cwd option
+ * @param outputFile - output slpk file
+ * @param level - compression level
+ * @param inputFiles - input files path to pass to the executable as option
+ * @param sevenZipExe - path to 7z.exe executable
+ */
 async function compressWithChildProcessWindows(
-  inputFolder,
-  outputFile,
-  level = 0,
-  inputFiles = join('.', '*'),
-  sevenZipExe
+  inputFolder: string,
+  outputFile: string,
+  level: number = 0,
+  inputFiles: string = join('.', '*'),
+  sevenZipExe: string
 ) {
   // Workaround for @listfile issue. In 7z.exe @-leading files are handled as listfiles
   // https://sevenzip.osdn.jp/chm/cmdline/syntax.htm
@@ -131,10 +185,17 @@ async function compressWithChildProcessWindows(
   });
 }
 
-export async function generateHash128FromZip(inputZipFile, outputFile) {
+/**
+ * Generate hash file from zip archive
+ * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/slpk_hashtable.cmn.md
+ *
+ * @param inputZipFile
+ * @param outputFile
+ */
+export async function generateHash128FromZip(inputZipFile: string, outputFile: string) {
   const input = await fs.readFile(inputZipFile);
   const zip = await JSZip.loadAsync(input);
-  const hashTable = [];
+  const hashTable: {key: string; value: string}[] = [];
   const zipFiles = zip.files;
   for (const relativePath in zipFiles) {
     const zipEntry = zipFiles[relativePath];
@@ -169,7 +230,7 @@ export async function generateHash128FromZip(inputZipFile, outputFile) {
     });
     for (const key in hashTable) {
       const item = hashTable[key];
-      const value = _longToByteArray(item.value);
+      const value = longToByteArray(item.value);
       // TODO: perhaps you need to wait for the 'drain' event if the write returns 'false'
       // eslint-disable-next-line no-undef
       output.write(Buffer.from(crypt.hexToBytes(item.key).concat(value)));
@@ -178,19 +239,45 @@ export async function generateHash128FromZip(inputZipFile, outputFile) {
   });
 }
 
-function _longToByteArray(long) {
+/**
+ * Encode 64 bit value to byte array
+ *
+ * @param long - stringified number
+ * @returns
+ */
+function longToByteArray(long: string): number[] {
   const buffer = new ArrayBuffer(8); // JS numbers are 8 bytes long, or 64 bits
   const longNum = new Float64Array(buffer); // so equivalent to Float64
-  longNum[0] = long;
+  longNum[0] = parseInt(long);
   return Array.from(new Uint8Array(buffer)).reverse(); // reverse to get little endian
 }
 
-export async function addFileToZip(inputFolder, fileName, zipFile, sevenZipExe) {
+/**
+ * Add file to zip archive
+ *
+ * @param inputFile
+ * @param fileName
+ * @param zipFile
+ * @param sevenZipExe
+ */
+export async function addFileToZip(
+  inputFolder: string,
+  fileName: string,
+  zipFile: string,
+  sevenZipExe: string
+) {
   await compressWithChildProcess(inputFolder, zipFile, 0, fileName, sevenZipExe);
   console.log(`${fileName} added to ${zipFile}.`); // eslint-disable-line
 }
 
-function appendFileToArchive(archive, subFileName, subFileData) {
+/**
+ *
+ * @param archive zip archive instance
+ * @param subFileName file path inside archive
+ * @param subFileData source file path
+ * @returns
+ */
+function appendFileToArchive(archive: any, subFileName: string, subFileData: string) {
   return new Promise((resolve) => {
     const fileStream = createReadStream(subFileData);
     console.log(`Compression start: ${subFileName}`); // eslint-disable-line no-undef,no-console

package/src/lib/utils/file-utils.ts

@@ -0,0 +1,140 @@
+import {load} from '@loaders.gl/core';
+import {JSONLoader} from '@loaders.gl/loader-utils';
+import {promises as fs} from 'fs';
+import {isAbsolute, join} from 'path';
+import {compressFileWithGzip} from './compress-util';
+
+/**
+ * Write a file with data and name fileName to path
+ *
+ * @param path - output path
+ * @param data - file content
+ * @param fileName - name of output file (default: index.json)
+ */
+export async function writeFile(
+  path: string,
+  data: string | Uint8Array | ArrayBuffer | Promise<ArrayBuffer>,
+  fileName: string = 'index.json'
+): Promise<string> {
+  let toWriteData: string | Uint8Array;
+  if (data instanceof Promise) {
+    toWriteData = new Uint8Array(await data);
+  } else if (data instanceof ArrayBuffer) {
+    toWriteData = new Uint8Array(data as ArrayBuffer);
+  } else {
+    toWriteData = data;
+  }
+  await fs.mkdir(path, {recursive: true});
+  const pathFile = join(path, fileName);
+  try {
+    await fs.writeFile(pathFile, toWriteData);
+  } catch (err) {
+    throw err;
+  }
+  console.log(`${pathFile} saved.`); // eslint-disable-line
+  return pathFile;
+}
+
+/**
+ * Write a file with data and name fileName to path - specific one for further packaging into slpk
+ *
+ * @param path - output path
+ * @param data - file content
+ * @param fileName - name of output file (default: index.json)
+ * @param compress - if need to compress file with gzip (default: true)
+ * @param compressList - if set - the file should be added to this list and compressed in the end of conversion
+ */
+export async function writeFileForSlpk(
+  path: string,
+  data: string | Uint8Array | ArrayBuffer | Promise<ArrayBuffer>,
+  fileName: string = 'index.json',
+  compress: boolean = true,
+  compressList?: string[] | null
+): Promise<string | null> {
+  const pathFile = await writeFile(path, data, fileName);
+  if (compress) {
+    if (compressList) {
+      if (!compressList.includes(pathFile)) {
+        compressList.push(pathFile);
+        return `${pathFile}.gz`;
+      } else {
+        return null;
+      }
+    } else {
+      const pathGzFile = await compressFileWithGzip(pathFile);
+      // After compression, we don't need an uncompressed file
+      await removeFile(pathFile);
+      return pathGzFile;
+    }
+  }
+  return pathFile;
+}
+
+/**
+ * Open json file
+ * @param path - path to the file
+ * @param fileName - file name
+ * @returns object
+ */
+export async function openJson(path: string, fileName: string): Promise<{[key: string]: any}> {
+  return new Promise((resolve, reject) => {
+    let count = 0;
+    console.log(`load ${path}/${fileName}.`); // eslint-disable-line
+    const intervalId = setInterval(() => {
+      const pathFile = join(path, fileName);
+      load(pathFile, JSONLoader)
+        .then((result) => {
+          clearInterval(intervalId);
+          resolve(result);
+        })
+        .catch(() => {
+          count++;
+          if (count > 100) {
+            clearInterval(intervalId);
+            reject(new Error(`Cannon load ${path}/${fileName}.`));
+          }
+        });
+    }, 200);
+  });
+}
+
+/**
+ * Check if the file exists
+ * @param fileName - full name of file
+ * @returns true if file exists, otherwise - false
+ */
+export async function isFileExists(fileName: string): Promise<boolean> {
+  try {
+    await fs.stat(fileName);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Remove dir with path
+ *
+ * @param path
+ */
+export function removeDir(path: string) {
+  // (node:35607) [DEP0147] DeprecationWarning: In future versions of Node.js, fs.rmdir(path, { recursive: true }) will be removed. Use fs.rm(path, { recursive: true }) instead
+  return fs.rm(path, {recursive: true});
+}
+
+/**
+ * Remove file with path
+ *
+ * @param path
+ */
+export function removeFile(path: string) {
+  return fs.unlink(path);
+}
+
+/**
+ * Generates absolute file path
+ * @param filePath
+ */
+export function getAbsoluteFilePath(filePath: string) {
+  return isAbsolute(filePath) ? filePath : join(process.cwd(), filePath);
+}

package/src/lib/utils/geometry-utils.ts

@@ -0,0 +1,14 @@
+/**
+ * luma.gl can not work without indices now:
+ * https://github.com/visgl/luma.gl/blob/d8cad75b9f8ca3e578cf078ed9d19e619c2ddbc9/modules/experimental/src/gltf/gltf-instantiator.js#L115
+ * This method generates syntetic indices array: [0, 1, 2, 3, .... , vertexCount-1]
+ * @param {number} vertexCount - vertex count in the geometry
+ * @returns {Uint32Array} indices array.
+ */
+export const generateSyntheticIndices = (vertexCount: number): Uint32Array => {
+  const result = new Uint32Array(vertexCount);
+  for (let index = 0; index < vertexCount; index++) {
+    result[index] = index;
+  }
+  return result;
+};

package/src/lib/utils/{lod-conversion-utils.js → lod-conversion-utils.ts}

@@ -1,9 +1,12 @@
+import {Tiles3DTileJSONPostprocessed} from '@loaders.gl/3d-tiles';
+import {BoundingVolumes} from '@loaders.gl/i3s';
+import {Tile3D} from '@loaders.gl/tiles';
+
 // https://cesium.com/docs/cesiumjs-ref-doc/Cesium3DTileset.html
 const DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR = 16;
 /**
  * Do conversion from geometric error to screen threshold
- * @param {Object} tile
- * @param {Object} coordinates
+ * 
  * In 3DTiles we have HLOD logic and parent tile also has bigger lodMetric value then its children.
  * In I3s we have reverse logic related to maxError. Parent has lower maxError than its child.
  * In nodes where are no children tile.lodMetricValue is 0. This is because of logic of HLOD in 3DTiles
@@ -13,9 +16,27 @@ const DEFAULT_MAXIMUM_SCREEN_SPACE_ERROR = 16;
  * https://github.com/Esri/i3s-spec/blob/master/docs/1.7/lodSelection.cmn.md
  * To avoid infinity values when we do calculations of maxError we shold replace 0 with value which allows us
  * to make child maxError bigger than his parent maxError.
+ * 
+ * @param tile - 3d-tiles tile JSON
+ * @param coordinates - node converted coordinates
+ * @returns An array of LOD metrics in format compatible with i3s 3DNodeIndexDocument.lodSelection
+ * @example 
+ *  [
+        {
+            "metricType": "maxScreenThresholdSQ",
+            "maxError": 870638.071285568
+        },
+        {
+            "metricType": "maxScreenThreshold",
+            "maxError": 1052.8679031638949
+        }
+    ]
  */
-export function convertGeometricErrorToScreenThreshold(tile, coordinates) {
-  const lodSelection = [];
+export function convertGeometricErrorToScreenThreshold(
+  tile: Tiles3DTileJSONPostprocessed,
+  coordinates: BoundingVolumes
+) {
+  const lodSelection: {metricType: string; maxError: number}[] = [];
   const boundingVolume = tile.boundingVolume;
   const lodMetricValue = tile.lodMetricValue || 0.1;
   const maxScreenThreshold = {
@@ -38,7 +59,12 @@ export function convertGeometricErrorToScreenThreshold(tile, coordinates) {
   return lodSelection;
 }
 
-export function convertScreenThresholdToGeometricError(node) {
+/**
+ * Convert LOD metric from "Screen Threshold" to "Screen Space Error"
+ * @param node - i3s node data
+ * @returns lod metric in 3d-tiles format
+ */
+export function convertScreenThresholdToGeometricError(node: Tile3D): number {
   const metricData = node.header.lodSelection.maxScreenThreshold || {};
   let maxError = metricData.maxError;
   if (!maxError) {

package/src/lib/utils/queue.ts

@@ -0,0 +1,17 @@
+export class Queue<T> extends Array<T> {
+  enqueue(val: T) {
+    this.push(val);
+  }
+
+  dequeue(): T | undefined {
+    return this.shift();
+  }
+
+  peek(): T {
+    return this[0];
+  }
+
+  isEmpty() {
+    return this.length === 0;
+  }
+}

package/src/lib/utils/write-queue.ts

@@ -0,0 +1,110 @@
+import {Queue} from './queue';
+import process from 'process';
+
+/** Memory limit size is based on testing */
+const MEMORY_LIMIT = 4 * 1024 * 1024 * 1024; // 4GB
+
+export type WriteQueueItem = {
+  archiveKey?: string;
+  /**
+   * writePromise() returns a Promise that will be awaited in Promise.allSettled(promises);
+   * Arguments for this call are specified in writeQueue.enqueue call like this:
+   * await writeQueue.enqueue({
+   *     archiveKey: `nodePages/xxx.json.gz`,
+   *     writePromise: () => writeFileForSlpk(slpkPath, data, `xxx.json`)
+   * });
+   * Note, a function like writeFileForSlpk should NOT be called when initializing the object for enqueue().
+   * If he function is called, the promise will be created
+   * and the function will allocate resources (file descriptors) for file writing.
+   * It will be done for ALL items in the queue, which is not supposed to happen.
+   * That's why the function should be passed as
+   *   writePromise: () => writeFileForSlpk(slpkPath, content, `xxx.json`)
+   * instead of
+   *  writePromise: writeFileForSlpk(slpkPath, content, `xxx.json`) // INCORRECT !
+   */
+  writePromise: () => Promise<string | null>;
+};
+
+export default class WriteQueue<T extends WriteQueueItem> extends Queue<T> {
+  private intervalId?: NodeJS.Timeout;
+  public writePromise: Promise<void> | null = null;
+  public fileMap: {[key: string]: string} = {};
+  public listeningInterval: number;
+  public writeConcurrency: number;
+
+  constructor(listeningInterval: number = 2000, writeConcurrency: number = 400) {
+    super();
+    this.listeningInterval = listeningInterval;
+    this.writeConcurrency = writeConcurrency;
+  }
+
+  async enqueue(val: T, writeImmediately: boolean = false) {
+    if (writeImmediately) {
+      const {archiveKey, writePromise} = val as WriteQueueItem;
+      const result = await writePromise();
+      if (archiveKey && result) {
+        this.fileMap[archiveKey] = result;
+      }
+    } else {
+      super.enqueue(val);
+      /** https://nodejs.org/docs/latest-v14.x/api/process.html#process_process_memoryusage */
+      if (process.memoryUsage().rss > MEMORY_LIMIT) {
+        await this.startWrite();
+      }
+    }
+  }
+
+  startListening() {
+    this.intervalId = setInterval(this.startWrite.bind(this), this.listeningInterval);
+  }
+
+  stopListening() {
+    if (this.intervalId) {
+      clearInterval(this.intervalId);
+    }
+  }
+
+  async startWrite(): Promise<void> {
+    if (!this.writePromise) {
+      this.writePromise = this.doWrite();
+    }
+    await this.writePromise;
+    this.writePromise = null;
+  }
+
+  async finalize(): Promise<void> {
+    this.stopListening();
+    await this.startWrite();
+  }
+
+  private async doWrite(): Promise<void> {
+    while (this.length) {
+      const promises: Promise<string | null>[] = [];
+      const archiveKeys: (string | undefined)[] = [];
+      for (let i = 0; i < this.writeConcurrency; i++) {
+        const item = this.dequeue();
+        if (!item) {
+          break;
+        }
+        const {archiveKey, writePromise} = item as WriteQueueItem;
+        archiveKeys.push(archiveKey);
+        const promise = writePromise();
+        promises.push(promise);
+      }
+      const writeResults = await Promise.allSettled(promises);
+      this.updateFileMap(archiveKeys, writeResults);
+    }
+  }
+
+  private updateFileMap(
+    archiveKeys: (string | undefined)[],
+    writeResults: PromiseSettledResult<string | null>[]
+  ) {
+    for (let i = 0; i < archiveKeys.length; i++) {
+      const archiveKey = archiveKeys[i];
+      if (archiveKey && 'value' in writeResults[i]) {
+        this.fileMap[archiveKey] = (writeResults[i] as PromiseFulfilledResult<string>).value;
+      }
+    }
+  }
+}

package/src/pgm-loader.ts

@@ -1,11 +1,10 @@
 import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import {parsePGM} from '@math.gl/geoid';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-import {parsePgm} from './lib/pgm-parser';
-
 /**
  * Loader for PGM - Netpbm grayscale image format
  */
@@ -15,7 +14,8 @@ export const PGMLoader: LoaderWithParser = {
   module: 'tile-converter',
   version: VERSION,
   mimeTypes: ['image/x-portable-graymap'],
-  parse: (arrayBuffer, options) => parsePgm(new Uint8Array(arrayBuffer), options),
+  // @ts-expect-error LoaderOptions does not have cubic parameter
+  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options),
   extensions: ['pgm'],
   options: {
     // TODO - use pgm namespace