@loaders.gl/tile-converter 4.0.0-alpha.4 → 4.0.0-alpha.6

package/src/3d-tiles-converter/helpers/texture-atlas.ts

@@ -13,7 +13,7 @@ export function convertTextureAtlas(texCoords: Float32Array, uvRegions: Uint16Ar
     const uv = texCoords.subarray(index, index + 2);
     const regions = normalisedRegions.slice(index * 2, index * 2 + 4);
     // fract(texCoords)
-    const fractatedUV = fract(uv);
+    const fractatedUV = fract([uv[0], uv[1]]);
     // (uvRegions.zw - uvRegions.xy)
     const subtracted = [regions[2] - regions[0], regions[3] - regions[1]];
     // fract(texCoords) * (uvRegions.zw - uvRegions.xy)
@@ -35,7 +35,7 @@ export function convertTextureAtlas(texCoords: Float32Array, uvRegions: Uint16Ar
  * Do fractation of UV array.
  * @param uv
  */
-function fract(uv: number[2]): number[2] {
+function fract(uv: [number, number]): [number, number] {
   return [uv[0] - Math.floor(uv[0]), uv[1] - Math.floor(uv[1])];
 }
 
@@ -43,9 +43,9 @@ function fract(uv: number[2]): number[2] {
  * Normalize uvRegions by dividing by the maximum Uint16 value
  * @param regions
  */
-function normalizeRegions(regions: Uint16Array): number[4] {
+function normalizeRegions(regions: Uint16Array): number[] {
   const MAX_UINT_16_VALUE = 65535;
-  const normalizedRegions = [];
+  const normalizedRegions: number[] = [];
 
   for (let index = 0; index < regions.length; index++) {
     normalizedRegions[index] = regions[index] / MAX_UINT_16_VALUE;

package/src/3d-tiles-converter/json-templates/{tileset.js → tileset.ts}

@@ -1,13 +1,13 @@
 import transform from 'json-map-transform';
 
-const ASSET = {
+const ASSET = () => ({
   version: {
     path: 'version',
     default: '1.0'
   }
-};
+});
 
-const TILE = {
+const TILE = () => ({
   boundingVolume: {
     path: 'boundingVolume'
   },
@@ -19,14 +19,14 @@ const TILE = {
   },
   children: {
     path: 'children',
-    transform: (val) => val.map((tile) => transform(tile, TILE))
+    transform: (val) => val.map((tile) => transform(tile, TILE()))
   }
-};
+});
 
-export const TILESET = {
+export const TILESET = () => ({
   asset: {
     path: 'asset',
-    transform: (val) => transform(val, ASSET)
+    transform: (val) => transform(val, ASSET())
   },
   geometricError: {
     path: 'root',
@@ -34,6 +34,6 @@ export const TILESET = {
   },
   root: {
     path: 'root',
-    transform: (val) => transform(val, TILE)
+    transform: (val) => transform(val, TILE())
   }
-};
+});

package/src/constants.ts

@@ -0,0 +1,2 @@
+export const BROWSER_ERROR_MESSAGE =
+  'Tile converter does not work in browser, only in node js environment';

package/src/converter-cli.ts

@@ -0,0 +1,370 @@
+/* eslint-disable no-console */
+import '@loaders.gl/polyfills';
+import {join} from 'path';
+import {I3SConverter, Tiles3DConverter} from '@loaders.gl/tile-converter';
+import {DepsInstaller} from './deps-installer/deps-installer';
+
+type TileConversionOptions = {
+  /** "I3S" - for I3S to 3DTiles conversion, "3DTILES" for 3DTiles to I3S conversion */
+  inputType?: string;
+  /** "tileset.json" file (3DTiles) / "http://..../SceneServer/layers/0" resource (I3S) */
+  tileset?: string;
+  /** Tileset name. This option is used for naming in resulting json resouces and for resulting path/*.slpk file naming */
+  name?: string;
+  /** Output folder. This folder will be created by converter if doesn't exist. It is relative to the converter path.
+   * Default: "data" folder */
+  output: string;
+  /** Keep created 3DNodeIndexDocument files on disk instead of memory. This option reduce memory usage but decelerates conversion speed */
+  instantNodeWriting: boolean;
+  /** Try to merge similar materials to be able to merge meshes into one node (I3S to 3DTiles conversion only) */
+  mergeMaterials: boolean;
+  /** 3DTiles->I3S only. location of 7z.exe archiver to create slpk on Windows OS, default: "C:\Program Files\7-Zip\7z.exe" */
+  sevenZipExe: string;
+  /** location of the Earth Gravity Model (*.pgm) file to convert heights from ellipsoidal to gravity-related format,
+   * default: "./deps/egm2008-5.pgm". A model file can be loaded from GeographicLib
+   * https://geographiclib.sourceforge.io/html/geoid.html */
+  egm: string;
+  /** 3DTile->I3S only. Token for Cesium ION tileset authentication. */
+  token?: string;
+  /** 3DTiles->I3S only. Enable draco compression for geometry. Default: true */
+  draco: boolean;
+  /** Run the script for installing dependencies. Run this options separate from others. Now "*.pgm" file installation is
+   * implemented */
+  installDependencies: boolean;
+  /** 3DTile->I3S only. Enable KTX2 textures generation if only one of (JPG, PNG) texture is provided or generate JPG texture
+   * if only KTX2 is provided */
+  generateTextures: boolean;
+  /** 3DTile->I3S only. Will generate obb and mbs bounding volumes from geometry */
+  generateBoundingVolumes: boolean;
+  /** Validate the dataset during conversion. Validation messages will be posted in the console output */
+  validate: boolean;
+  /** Maximal depth of the hierarchical tiles tree traversal, default: infinite */
+  maxDepth?: number;
+  /** 3DTiles->I3S only. Whether the converter generates *.slpk (Scene Layer Package) I3S output file */
+  slpk: boolean;
+};
+
+/* During validation we check that particular options are defined so they can't be undefined */
+type ValidatedTileConversionOptions = TileConversionOptions & {
+  /** "I3S" - for I3S to 3DTiles conversion, "3DTILES" for 3DTiles to I3S conversion */
+  inputType: string;
+  /** "tileset.json" file (3DTiles) / "http://..../SceneServer/layers/0" resource (I3S) */
+  tileset: string;
+  /** Tileset name. This option is used for naming in resulting json resouces and for resulting path/*.slpk file naming */
+  name: string;
+};
+
+const TILESET_TYPE = {
+  I3S: 'I3S',
+  _3DTILES: '3DTILES'
+};
+
+/**
+ * CLI entry
+ * @returns
+ */
+async function main() {
+  const [, , ...args] = process.argv;
+
+  if (args.length === 0) {
+    printHelp();
+  }
+
+  const validatedOptionsArr = validateOptionsWithEqual(args);
+
+  const options: TileConversionOptions = parseOptions(validatedOptionsArr);
+
+  if (options.installDependencies) {
+    const depthInstaller = new DepsInstaller();
+    depthInstaller.install('deps');
+    return;
+  }
+
+  const validatedOptions: ValidatedTileConversionOptions = validateOptions(options);
+
+  await convert(validatedOptions);
+}
+
+main().catch((error) => {
+  console.log(error);
+  process.exit(1); // eslint-disable-line
+});
+
+/**
+ * Output for `npx tile-converter --help`
+ */
+function printHelp(): void {
+  console.log('cli: converter 3dTiles to I3S or I3S to 3dTiles...');
+  console.log(
+    '--install-dependencies [Run the script for installing dependencies. Run this options separate from others. Now "*.pgm" file installation is implemented]'
+  );
+  console.log(
+    '--max-depth [Maximal depth of hierarchical tiles tree traversal, default: infinite]'
+  );
+  console.log('--name [Tileset name]');
+  console.log('--output [Output folder, default: "data" folder]');
+  console.log(
+    '--instant-node-writing [Keep created 3DNodeIndexDocument files on disk instead of memory. This option reduce memory usage but decelerates conversion speed]'
+  );
+  console.log(
+    '--split-nodes [Prevent to merge similar materials that could lead to incorrect visualization (I3S to 3DTiles conversion only)]'
+  );
+  console.log('--slpk [Generate slpk (Scene Layer Packages) I3S output file]');
+  console.log(
+    '--tileset [tileset.json file (3DTiles) / http://..../SceneServer/layers/0 resource (I3S)]'
+  );
+  console.log('--input-type [tileset input type: I3S or 3DTILES]');
+  console.log(
+    '--7zExe [location of 7z.exe archiver to create slpk on Windows, default: "C:\\Program Files\\7-Zip\\7z.exe"]'
+  );
+  console.log(
+    '--egm [location of Earth Gravity Model *.pgm file to convert heights from ellipsoidal to gravity-related format. A model file can be loaded from GeographicLib https://geographiclib.sourceforge.io/html/geoid.html], default: "./deps/egm2008-5.zip"'
+  );
+  console.log('--token [Token for Cesium ION tilesets authentication]');
+  console.log('--no-draco [Disable draco compression for geometry]');
+  console.log(
+    '--generate-textures [Enable KTX2 textures generation if only one of (JPG, PNG) texture is provided or generate JPG texture if only KTX2 is provided]'
+  );
+  console.log(
+    '--generate-bounding-volumes [Will generate obb and mbs bounding volumes from geometry]'
+  );
+  console.log('--validate [Enable validation]');
+  process.exit(0); // eslint-disable-line
+}
+
+/**
+ * Run conversion process
+ * @param options validated tile-converter options
+ */
+async function convert(options: ValidatedTileConversionOptions) {
+  console.log(`------------------------------------------------`); // eslint-disable-line
+  console.log(`Starting conversion of ${options.inputType}`); // eslint-disable-line
+  console.log(`------------------------------------------------`); // eslint-disable-line
+  const inputType = options.inputType.toUpperCase();
+  switch (inputType) {
+    case TILESET_TYPE.I3S:
+      const tiles3DConverter = new Tiles3DConverter();
+      tiles3DConverter.convert({
+        inputUrl: options.tileset,
+        outputPath: options.output,
+        tilesetName: options.name,
+        maxDepth: options.maxDepth,
+        egmFilePath: options.egm
+      });
+      break;
+    case TILESET_TYPE._3DTILES:
+      const converter = new I3SConverter();
+      await converter.convert({
+        inputUrl: options.tileset,
+        outputPath: options.output,
+        tilesetName: options.name,
+        maxDepth: options.maxDepth,
+        slpk: options.slpk,
+        sevenZipExe: options.sevenZipExe,
+        egmFilePath: options.egm,
+        token: options.token,
+        draco: options.draco,
+        mergeMaterials: options.mergeMaterials,
+        generateTextures: options.generateTextures,
+        generateBoundingVolumes: options.generateBoundingVolumes,
+        validate: options.validate,
+        instantNodeWriting: options.instantNodeWriting
+      });
+      break;
+    default:
+      printHelp();
+  }
+}
+
+// OPTIONS
+/**
+ * Validate input options of the CLI command
+ * @param options - input options of the CLI command
+ * @returns validated options
+ */
+function validateOptions(options: TileConversionOptions): ValidatedTileConversionOptions {
+  const mandatoryOptionsWithExceptions: {
+    [key: string]: () => void;
+  } = {
+    name: () => console.log('Missed: --name [Tileset name]'),
+    output: () => console.log('Missed: --output [Output path name]'),
+    sevenZipExe: () => console.log('Missed: --7zExe [7z archiver executable path]'),
+    egm: () => console.log('Missed: --egm [*.pgm earth gravity model file path]'),
+    tileset: () => console.log('Missed: --tileset [tileset.json file]'),
+    inputType: () =>
+      console.log('Missed/Incorrect: --input-type [tileset input type: I3S or 3DTILES]')
+  };
+  const exceptions: (() => void)[] = [];
+  for (const mandatoryOption in mandatoryOptionsWithExceptions) {
+    const optionValue = options[mandatoryOption];
+    const isWrongInputType =
+      Boolean(optionValue) &&
+      mandatoryOption === 'inputType' &&
+      !Object.values(TILESET_TYPE).includes(optionValue.toUpperCase());
+
+    if (!optionValue || isWrongInputType) {
+      exceptions.push(mandatoryOptionsWithExceptions[mandatoryOption]);
+    }
+  }
+  if (exceptions.length) {
+    exceptions.forEach((exeption) => exeption());
+    process.exit(1);
+  }
+  return <ValidatedTileConversionOptions>options;
+}
+
+function validateOptionsWithEqual(args: string[]): string[] {
+  return args.reduce((acc: string[], curr) => {
+    const equalSignIndex = curr.indexOf('=');
+    const beforeEqual = curr.slice(0, equalSignIndex);
+    const afterEqual = curr.slice(equalSignIndex + 1, curr.length);
+    const condition = curr.includes('=') && curr.startsWith('--') && afterEqual;
+    if (condition) {
+      return acc.concat(beforeEqual, afterEqual);
+    }
+    return acc.concat(curr);
+  }, []);
+}
+
+/**
+ * Parse option from the cli arguments array
+ * @param args
+ * @returns
+ */
+function parseOptions(args: string[]): TileConversionOptions {
+  const opts: TileConversionOptions = {
+    output: 'data',
+    instantNodeWriting: false,
+    mergeMaterials: true,
+    sevenZipExe: 'C:\\Program Files\\7-Zip\\7z.exe',
+    egm: join(process.cwd(), 'deps', 'egm2008-5.pgm'),
+    draco: true,
+    installDependencies: false,
+    generateTextures: false,
+    generateBoundingVolumes: false,
+    validate: false,
+    slpk: false
+  };
+
+  // eslint-disable-next-line complexity
+  args.forEach((arg, index) => {
+    if (arg.indexOf('--') === 0) {
+      switch (arg) {
+        case '--input-type':
+          opts.inputType = getStringValue(index, args);
+          break;
+        case '--tileset':
+          opts.tileset = getURLValue(index, args);
+          break;
+        case '--name':
+          opts.name = getStringValue(index, args);
+          break;
+        case '--output':
+          opts.output = getStringValue(index, args);
+          break;
+        case '--instant-node-writing':
+          opts.instantNodeWriting = getBooleanValue(index, args);
+          break;
+        case '--split-nodes':
+          opts.mergeMaterials = getBooleanValue(index, args);
+          break;
+        case '--max-depth':
+          opts.maxDepth = getIntegerValue(index, args);
+          break;
+        case '--slpk':
+          opts.slpk = getBooleanValue(index, args);
+          break;
+        case '--7zExe':
+          opts.sevenZipExe = getStringValue(index, args);
+          break;
+        case '--egm':
+          opts.egm = getStringValue(index, args);
+          break;
+        case '--token':
+          opts.token = getStringValue(index, args);
+          break;
+        case '--no-draco':
+          opts.draco = getBooleanValue(index, args);
+          break;
+        case '--validate':
+          opts.validate = getBooleanValue(index, args);
+          break;
+        case '--install-dependencies':
+          opts.installDependencies = getBooleanValue(index, args);
+          break;
+        case '--generate-textures':
+          opts.generateTextures = getBooleanValue(index, args);
+          break;
+        case '--generate-bounding-volumes':
+          opts.generateBoundingVolumes = getBooleanValue(index, args);
+          break;
+        case '--help':
+          printHelp();
+          break;
+        default:
+          console.warn(`Unknown option ${arg}`);
+          process.exit(0); // eslint-disable-line
+      }
+    }
+  });
+  return opts;
+}
+
+/**
+ * Get string option value from cli arguments
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+function getStringValue(index: number, args: string[]): string {
+  if (index + 1 >= args.length) {
+    return '';
+  }
+  const value = args[index + 1];
+  if (value.indexOf('--') === 0) {
+    return '';
+  }
+  return value;
+}
+
+/**
+ * Modyfy URL path to be compatible with fetch
+ * @param index - option's name index in the argument's array.
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - string value of the option
+ */
+function getURLValue(index: number, args: string[]): string {
+  const value = getStringValue(index, args);
+  console.log(`Input tileset value: ${value}`);
+  console.log(`Modified tileset value: ${value.replace(/\\/g, '/')}`);
+  return value.replace(/\\/g, '/');
+}
+
+/**
+ * Get integer option value from cli arguments
+ * @param index - option's name index in the argument's array
+ *                The value of the option should be next to name of the option.
+ * @param args - cli arguments array
+ * @returns - number value of the option
+ */
+function getIntegerValue(index: number, args: string[]): number {
+  const stringValue: string = getStringValue(index, args);
+  const result: number = Number.parseInt(stringValue);
+  if (isFinite(result)) {
+    return result;
+  }
+  return NaN;
+}
+
+function getBooleanValue(index: number, args: string[]): boolean {
+  const stringValue: string = getStringValue(index, args).toLowerCase().trim();
+  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
+    return false;
+  }
+  if (!stringValue || stringValue === 'true') {
+    return true;
+  }
+  return false;
+}

package/src/deps-installer/deps-installer.ts

@@ -0,0 +1,71 @@
+import {load, fetchFile} from '@loaders.gl/core';
+import {ZipLoader} from '@loaders.gl/zip';
+import {writeFile} from '../lib/utils/file-utils';
+import {join} from 'path';
+import {ChildProcessProxy} from '@loaders.gl/worker-utils';
+
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
+
+const PGM_LINK = 'https://raw.githubusercontent.com/visgl/deck.gl-data/master/egm/egm2008-5.zip';
+
+/**
+ * Install external dependencies for converter:
+ * * PGM file (implemented);
+ * * Draco library (not implemented);
+ * * 7z archiver (not implemented);
+ */
+export class DepsInstaller {
+  /**
+   * Run instalation
+   * @param path destination folder
+   * @param workersPath destination folder for workers.
+   *    This path is '' by default and is not used by tile-converter.
+   *    It is used in tests to prevent rewriting actual workers during tests running
+   */
+  async install(path: string = '', workersPath: string = ''): Promise<void> {
+    console.log('Installing "EGM2008-5" model...'); // eslint-disable-line no-console
+    const fileMap = await load(PGM_LINK, ZipLoader, {});
+
+    let depsPath = process.cwd();
+    if (path) {
+      depsPath = join(depsPath, path);
+    }
+
+    await writeFile(depsPath, new Uint8Array(fileMap['geoids/egm2008-5.pgm']), 'egm2008-5.pgm');
+
+    console.log('Installing "I3S Content Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('i3s', 'i3s-content-nodejs-worker.js', workersPath);
+
+    console.log('Installing "Draco Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('draco', 'draco-nodejs-worker.js', workersPath);
+
+    console.log('Installing "Basis Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('textures', 'basis-nodejs-worker.js', workersPath);
+
+    console.log('Installing "join-images" npm package');
+    const childProcess = new ChildProcessProxy();
+    await childProcess.start({
+      command: 'npm',
+      // `npm install sharp join-images` works unstable. It fails because installed `sharp` version
+      // may be different from the version required by `join-images`. Pointing to specific versions
+      // resolve this issue
+      arguments: ['install', 'sharp@^0.30.4', 'join-images@^1.1.3'],
+      wait: 0
+    });
+
+    console.log('All dependencies were installed succesfully.'); // eslint-disable-line no-console
+  }
+
+  private async installWorker(module: string, name: string, extraPath: string) {
+    const fileResponse = await fetchFile(
+      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${name}`
+    );
+    const fileData = await fileResponse.arrayBuffer();
+    if (!fileData) {
+      return;
+    }
+    const path = join(process.cwd(), extraPath, 'modules', module, 'dist');
+    await writeFile(path, fileData, name);
+  }
+}

package/src/i3s-attributes-worker.ts

@@ -0,0 +1,50 @@
+import type {WorkerObject} from '@loaders.gl/worker-utils';
+import type {ConvertedAttributes} from './i3s-converter/types';
+import type {Matrix4, Vector3} from '@math.gl/core';
+import type {GLTFImagePostprocessed, GLTFNodePostprocessed} from '@loaders.gl/gltf';
+
+import {processOnWorker} from '@loaders.gl/worker-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type I3SAttributesWorkerOptions = {
+  _nodeWorkers: boolean;
+  reuseWorkers: boolean;
+  useCartesianPositions: boolean;
+  source: string;
+};
+
+export type B3DMAttributesData = {
+  gltfMaterials?: {id: string}[];
+  nodes: GLTFNodePostprocessed[];
+  images: GLTFImagePostprocessed[];
+  cartographicOrigin: Vector3;
+  cartesianModelMatrix: Matrix4;
+};
+
+/**
+ * I3S Attributes Worker to handle B3DM object
+ */
+export const I3SAttributesWorker = {
+  id: 'i3s-attributes',
+  name: 'I3S Attributes Worker',
+  module: 'tile-converter',
+  version: VERSION,
+  options: {
+    useCartesianPositions: false
+  }
+};
+
+/**
+ * Performs I3S attributes transformation
+ */
+export function transformI3SAttributesOnWorker(
+  attributesData: B3DMAttributesData,
+  options: I3SAttributesWorkerOptions
+): Promise<Map<string, ConvertedAttributes>> {
+  return processOnWorker(I3SAttributesWorker, attributesData, options);
+}
+
+export const _typecheckI3SAttributesWorker: WorkerObject = I3SAttributesWorker;