@loaders.gl/tile-converter 4.0.0-alpha.8 → 4.0.0-beta.1

package/src/converter-cli.ts

@@ -1,8 +1,16 @@
 /* eslint-disable no-console */
 import '@loaders.gl/polyfills';
 import {join} from 'path';
+import inquirer from 'inquirer';
 import {I3SConverter, Tiles3DConverter} from '@loaders.gl/tile-converter';
 import {DepsInstaller} from './deps-installer/deps-installer';
+import {
+  getBooleanValue,
+  getIntegerValue,
+  getStringValue,
+  getURLValue,
+  validateOptionsWithEqual
+} from './lib/utils/cli-utils';
 
 type TileConversionOptions = {
   /** "I3S" - for I3S to 3DTiles conversion, "3DTILES" for 3DTiles to I3S conversion */
@@ -42,6 +50,8 @@ type TileConversionOptions = {
   maxDepth?: number;
   /** 3DTiles->I3S only. Whether the converter generates *.slpk (Scene Layer Package) I3S output file */
   slpk: boolean;
+  /** Feature metadata class from EXT_FEATURE_METADATA or EXT_STRUCTURAL_METADATA extensions  */
+  metadataClass?: string;
 };
 
 /* During validation we check that particular options are defined so they can't be undefined */
@@ -125,8 +135,9 @@ function printHelp(): void {
   console.log(
     '--generate-textures [Enable KTX2 textures generation if only one of (JPG, PNG) texture is provided or generate JPG texture if only KTX2 is provided]'
   );
+  console.log('--generate-bounding-volumes [Generate obb and mbs bounding volumes from geometry]');
   console.log(
-    '--generate-bounding-volumes [Will generate obb and mbs bounding volumes from geometry]'
+    '--metadata-class [One of the list of feature metadata classes, detected by converter on "analyze" stage, default: not set]'
   );
   console.log('--validate [Enable validation]');
   process.exit(0); // eslint-disable-line
@@ -168,7 +179,8 @@ async function convert(options: ValidatedTileConversionOptions) {
         generateTextures: options.generateTextures,
         generateBoundingVolumes: options.generateBoundingVolumes,
         validate: options.validate,
-        instantNodeWriting: options.instantNodeWriting
+        instantNodeWriting: options.instantNodeWriting,
+        inquirer
       });
       break;
     default:
@@ -213,19 +225,6 @@ function validateOptions(options: TileConversionOptions): ValidatedTileConversio
   return <ValidatedTileConversionOptions>options;
 }
 
-function validateOptionsWithEqual(args: string[]): string[] {
-  return args.reduce((acc: string[], curr) => {
-    const equalSignIndex = curr.indexOf('=');
-    const beforeEqual = curr.slice(0, equalSignIndex);
-    const afterEqual = curr.slice(equalSignIndex + 1, curr.length);
-    const condition = curr.includes('=') && curr.startsWith('--') && afterEqual;
-    if (condition) {
-      return acc.concat(beforeEqual, afterEqual);
-    }
-    return acc.concat(curr);
-  }, []);
-}
-
 /**
  * Parse option from the cli arguments array
  * @param args
@@ -298,6 +297,8 @@ function parseOptions(args: string[]): TileConversionOptions {
         case '--generate-bounding-volumes':
           opts.generateBoundingVolumes = getBooleanValue(index, args);
           break;
+        case '--metadata-class':
+          opts.metadataClass = getStringValue(index, args);
         case '--help':
           printHelp();
           break;
@@ -309,62 +310,3 @@ function parseOptions(args: string[]): TileConversionOptions {
   });
   return opts;
 }
-
-/**
- * Get string option value from cli arguments
- * @param index - option's name index in the argument's array.
- *                The value of the option should be next to name of the option.
- * @param args - cli arguments array
- * @returns - string value of the option
- */
-function getStringValue(index: number, args: string[]): string {
-  if (index + 1 >= args.length) {
-    return '';
-  }
-  const value = args[index + 1];
-  if (value.indexOf('--') === 0) {
-    return '';
-  }
-  return value;
-}
-
-/**
- * Modyfy URL path to be compatible with fetch
- * @param index - option's name index in the argument's array.
- *                The value of the option should be next to name of the option.
- * @param args - cli arguments array
- * @returns - string value of the option
- */
-function getURLValue(index: number, args: string[]): string {
-  const value = getStringValue(index, args);
-  console.log(`Input tileset value: ${value}`);
-  console.log(`Modified tileset value: ${value.replace(/\\/g, '/')}`);
-  return value.replace(/\\/g, '/');
-}
-
-/**
- * Get integer option value from cli arguments
- * @param index - option's name index in the argument's array
- *                The value of the option should be next to name of the option.
- * @param args - cli arguments array
- * @returns - number value of the option
- */
-function getIntegerValue(index: number, args: string[]): number {
-  const stringValue: string = getStringValue(index, args);
-  const result: number = Number.parseInt(stringValue);
-  if (isFinite(result)) {
-    return result;
-  }
-  return NaN;
-}
-
-function getBooleanValue(index: number, args: string[]): boolean {
-  const stringValue: string = getStringValue(index, args).toLowerCase().trim();
-  if (['--no-draco', '--split-nodes'].includes(args[index]) && !stringValue) {
-    return false;
-  }
-  if (!stringValue || stringValue === 'true') {
-    return true;
-  }
-  return false;
-}

package/src/deps-installer/deps-installer.ts

@@ -3,6 +3,8 @@ import {ZipLoader} from '@loaders.gl/zip';
 import {writeFile} from '../lib/utils/file-utils';
 import {join} from 'path';
 import {ChildProcessProxy} from '@loaders.gl/worker-utils';
+import {DRACO_EXTERNAL_LIBRARIES, DRACO_EXTERNAL_LIBRARY_URLS} from '@loaders.gl/draco';
+import {BASIS_EXTERNAL_LIBRARIES} from '@loaders.gl/textures';
 
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
@@ -23,7 +25,7 @@ export class DepsInstaller {
    *    This path is '' by default and is not used by tile-converter.
    *    It is used in tests to prevent rewriting actual workers during tests running
    */
-  async install(path: string = '', workersPath: string = ''): Promise<void> {
+  async install(path: string = ''): Promise<void> {
     console.log('Installing "EGM2008-5" model...'); // eslint-disable-line no-console
     const fileMap = await load(PGM_LINK, ZipLoader, {});
 
@@ -34,38 +36,82 @@ export class DepsInstaller {
 
     await writeFile(depsPath, new Uint8Array(fileMap['geoids/egm2008-5.pgm']), 'egm2008-5.pgm');
 
-    console.log('Installing "I3S Content Loader worker"'); // eslint-disable-line no-console
-    await this.installWorker('i3s', 'i3s-content-worker-node.js', workersPath);
+    console.log('Installing "I3S Content Loader" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('i3s', 'i3s-content-worker-node.js');
 
-    console.log('Installing "Draco Loader worker"'); // eslint-disable-line no-console
-    await this.installWorker('draco', 'draco-worker-node.js', workersPath);
+    console.log('Installing "Draco Loader" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('draco', 'draco-worker-node.js');
 
-    console.log('Installing "Basis Loader worker"'); // eslint-disable-line no-console
-    await this.installWorker('textures', 'basis-worker-node.js', workersPath);
+    console.log('Installing "Draco Writer" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('draco', 'draco-writer-worker-node.js');
+
+    console.log('Installing "Basis Loader" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', 'basis-worker-node.js');
+
+    console.log('Installing "KTX2 Basis Writer" worker'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', 'ktx2-basis-writer-worker-node.js');
+
+    console.log('Installing "Draco decoder" library'); // eslint-disable-line no-console
+    await this.installFromUrl(
+      DRACO_EXTERNAL_LIBRARY_URLS[DRACO_EXTERNAL_LIBRARIES.DECODER],
+      'draco',
+      DRACO_EXTERNAL_LIBRARIES.DECODER
+    );
+    await this.installFromUrl(
+      DRACO_EXTERNAL_LIBRARY_URLS[DRACO_EXTERNAL_LIBRARIES.DECODER_WASM],
+      'draco',
+      DRACO_EXTERNAL_LIBRARIES.DECODER_WASM
+    );
+
+    console.log('Installing "Draco encoder" library'); // eslint-disable-line no-console
+    await this.installFromUrl(
+      DRACO_EXTERNAL_LIBRARY_URLS[DRACO_EXTERNAL_LIBRARIES.ENCODER],
+      'draco',
+      DRACO_EXTERNAL_LIBRARIES.ENCODER
+    );
+
+    console.log('Installing "Basis transcoder" library'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.TRANSCODER, 'libs');
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.TRANSCODER_WASM, 'libs');
+
+    console.log('Installing "Basis encoder" library'); // eslint-disable-line no-console
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.ENCODER, 'libs');
+    await this.installFromNpm('textures', BASIS_EXTERNAL_LIBRARIES.ENCODER_WASM, 'libs');
 
     console.log('Installing "join-images" npm package');
     const childProcess = new ChildProcessProxy();
     await childProcess.start({
-      command: 'npm',
+      command: process.platform === 'win32' ? 'npm.cmd' : 'npm',
       // `npm install sharp join-images` works unstable. It fails because installed `sharp` version
       // may be different from the version required by `join-images`. Pointing to specific versions
       // resolve this issue
       arguments: ['install', 'sharp@0.30.4', 'join-images@1.1.3'],
-      wait: 0
+      wait: 0,
+      ignoreStderr: true
     });
 
     console.log('All dependencies were installed succesfully.'); // eslint-disable-line no-console
   }
 
-  private async installWorker(module: string, name: string, extraPath: string) {
+  private async installFromNpm(module: string, name: string, extraPath: string = '') {
     const fileResponse = await fetchFile(
-      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${name}`
+      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${extraPath}/${name}`
     );
     const fileData = await fileResponse.arrayBuffer();
     if (!fileData) {
       return;
     }
-    const path = join(process.cwd(), extraPath, 'modules', module, 'dist');
+    const path = join(process.cwd(), 'modules', module, 'dist', extraPath);
+    await writeFile(path, fileData, name);
+  }
+
+  private async installFromUrl(url: string, module: string, name: string) {
+    const fileResponse = await fetchFile(url);
+    const fileData = await fileResponse.arrayBuffer();
+    if (!fileData) {
+      return;
+    }
+    const path = join(process.cwd(), 'modules', module, 'dist', 'libs');
     await writeFile(path, fileData, name);
   }
 }

package/src/i3s-converter/helpers/batch-ids-extensions.ts

@@ -1,30 +1,64 @@
 import {GLTFAccessorPostprocessed, GLTFMeshPrimitivePostprocessed} from '@loaders.gl/gltf';
 import type {NumericArray} from '@loaders.gl/loader-utils';
 import type {
-  GLTF_EXT_feature_metadata_attribute,
-  GLTF_EXT_feature_metadata_primitive
+  GLTF_EXT_feature_metadata_FeatureIdTexture,
+  GLTF_EXT_feature_metadata_GLTF,
+  GLTF_EXT_feature_metadata_Primitive
 } from '@loaders.gl/gltf';
+
+import type {GLTF_EXT_mesh_features} from '@loaders.gl/gltf';
+
 import {TypedArray} from '@math.gl/core';
-import {TextureImageProperties} from '../../i3s-attributes-worker';
+import {TextureImageProperties} from '../types';
+import {emod} from '@loaders.gl/math';
+import {EXT_MESH_FEATURES, EXT_FEATURE_METADATA} from '@loaders.gl/gltf';
+import {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 
-const EXT_MESH_FEATURES = 'EXT_mesh_features';
-const EXT_FEATURE_METADATA = 'EXT_feature_metadata';
+/**
+ * Get featureTexture by a metadata class.
+ * Metadata classes come from a structural metadata extesion (EXT_feature_metadata or EXT_structural_metadata).
+ * The glTF might contain multiple texel-level metadata textures related to different classes. Having only one metadata class
+ * selected to convert to I3S, we have to pick only one texture to convert to per-vertex property.
+ * @param tileContent - 3d tile content
+ * @param metadataClass - user selected feature metadata class name
+ * @returns featureTexture key
+ */
+export function getTextureByMetadataClass(
+  tileContent: Tiles3DTileContent,
+  metadataClass?: string
+): string | null {
+  const extFeatureMetadata = tileContent.gltf?.extensions?.[
+    EXT_FEATURE_METADATA
+  ] as GLTF_EXT_feature_metadata_GLTF;
+  if (!extFeatureMetadata?.featureTextures) {
+    return null;
+  }
+  for (const textureKey in extFeatureMetadata.featureTextures) {
+    const texture = extFeatureMetadata.featureTextures[textureKey];
+    if (texture.class === metadataClass) {
+      return textureKey;
+    }
+  }
+  return null;
+}
 
 /**
  * Getting batchIds from 3DTilesNext extensions.
  * @param attributes - gltf accessors
  * @param primitive - gltf primitive data
  * @param images - gltf texture images
+ * @param featureTexture - feature texture key
+ * @return array of batch IDs
  */
 export function handleBatchIdsExtensions(
   attributes: {
     [key: string]: GLTFAccessorPostprocessed;
   },
   primitive: GLTFMeshPrimitivePostprocessed,
-  images: (TextureImageProperties | null)[]
+  images: (TextureImageProperties | null)[],
+  featureTexture: string | null
 ): NumericArray {
   const extensions = primitive?.extensions;
-
   if (!extensions) {
     return [];
   }
@@ -34,12 +68,12 @@ export function handleBatchIdsExtensions(
       case EXT_FEATURE_METADATA:
         return handleExtFeatureMetadataExtension(
           attributes,
-          extensionData as GLTF_EXT_feature_metadata_primitive,
-          images
+          extensionData as GLTF_EXT_feature_metadata_Primitive,
+          images,
+          featureTexture
         );
       case EXT_MESH_FEATURES:
-        console.warn('EXT_mesh_features extension is not supported yet');
-        return [];
+        return handleExtMeshFeaturesExtension(attributes, extensionData as GLTF_EXT_mesh_features);
       default:
         return [];
     }
@@ -48,19 +82,43 @@ export function handleBatchIdsExtensions(
   return [];
 }
 
+/**
+ * Getting batchIds from EXT_mesh_features extensions.
+ * @param attributes - gltf accessors
+ * @param extMeshFeatures - EXT_mesh_features extension
+ * @returns an array of attribute values
+ */
+function handleExtMeshFeaturesExtension(
+  attributes: {
+    [key: string]: GLTFAccessorPostprocessed;
+  },
+  extMeshFeatures: GLTF_EXT_mesh_features
+): NumericArray {
+  for (let ids of extMeshFeatures.featureIds) {
+    if (typeof ids.propertyTable !== 'undefined') {
+      // propertyTable is an index that can be 0
+      // return the first featureID set that corresponts to property table.
+      return ids.data as NumericArray;
+    }
+  }
+  return [];
+}
+
 /**
  * Get batchIds from EXT_feature_metadata extension.
- * Docs - https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_feature_metadata
- * @param attributes
- * @param extFeatureMetadata
- * @param textures
+ * @see - https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_feature_metadata
+ * @param attributes - glTF attributes
+ * @param extFeatureMetadata - primitive-level EXT_FEATURE_METADATA extension data
+ * @param textures - texture images
+ * @param featureTexture - feature texture key
  */
 function handleExtFeatureMetadataExtension(
   attributes: {
     [key: string]: GLTFAccessorPostprocessed;
   },
-  extFeatureMetadata: GLTF_EXT_feature_metadata_primitive,
-  images: (TextureImageProperties | null)[]
+  extFeatureMetadata: GLTF_EXT_feature_metadata_Primitive,
+  images: (TextureImageProperties | null)[],
+  featureTexture: string | null
 ): NumericArray {
   // Take only first extension object to get batchIds attribute name.
   const featureIdAttribute = extFeatureMetadata?.featureIdAttributes?.[0];
@@ -93,10 +151,6 @@ function handleExtFeatureMetadataExtension(
     return generateBatchIdsFromTexture(featureIdTexture, textureCoordinates, images);
   }
 
-  // Take only first extension texture to get batchIds from the root EXT_feature_metadata object.
-  const featureTexture =
-    extFeatureMetadata?.featureTextures && extFeatureMetadata?.featureTextures[0];
-
   if (featureTexture) {
     const batchIdsAttribute = attributes[featureTexture];
     return batchIdsAttribute.value;
@@ -107,7 +161,7 @@ function handleExtFeatureMetadataExtension(
 
 /**
  * Generates implicit feature ids
- * Spec - https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_feature_metadata#implicit-feature-ids
+ * @see - https://github.com/CesiumGS/glTF/tree/3d-tiles-next/extensions/2.0/Vendor/EXT_feature_metadata#implicit-feature-ids
  * @param featuresCount
  * @param constant
  * @param devisor
@@ -146,7 +200,7 @@ function generateImplicitFeatureIds(
  * @param featureIdTextures
  */
 function generateBatchIdsFromTexture(
-  featureIdTexture: GLTF_EXT_feature_metadata_attribute,
+  featureIdTexture: GLTF_EXT_feature_metadata_FeatureIdTexture,
   textureCoordinates: TypedArray,
   images: (TextureImageProperties | null)[]
 ) {
@@ -191,12 +245,3 @@ function generateBatchIdsFromTexture(
 
   return batchIds;
 }
-
-/**
- * Handle UVs if they are out of range [0,1].
- * @param n
- * @param m
- */
-function emod(n: number): number {
-  return ((n % 1) + 1) % 1;
-}

package/src/i3s-converter/helpers/coordinate-converter.ts

@@ -103,7 +103,6 @@ export function convertPositionsToVectors(positions: Float32Array): Vector3[] {
 
 /**
  * Convert common coordinate to fullExtent https://github.com/Esri/i3s-spec/blob/master/docs/1.8/fullExtent.cmn.md
- * @param
  * @param boundingVolume
  * @returns - fullExtent object
  */
@@ -142,7 +141,7 @@ export function convertBoundingVolumeToI3SFullExtent(
 /**
  * Creates oriented boundinb box from mbs.
  * @param mbs - Minimum Bounding Sphere
- * @returns - Oriented BOunding Box
+ * @returns - Oriented Bounding Box
  */
 export function createObbFromMbs(mbs: Mbs): Obb {
   const radius = mbs[3];

package/src/i3s-converter/helpers/feature-attributes.ts

@@ -9,26 +9,26 @@ import {
 } from '@loaders.gl/i3s';
 
 /**
- * Takes attributes from property table based on featureIds.
+ * Takes attributes from property table based on featureIdsMap.
  * If there is no property value for particular featureId (index) the property will be null.
  * Example:
  * Initial data:
- *   OBJECTID: [0, 1, 5]
+ *   OBJECTID: {0: 0, 3: 33, 4: 333}
  *   component: ['Windows', 'Frames', 'Wall', 'Roof', 'Skylight']
  * Result:
- *   OBJECTID: [0, 1, 5]
- *   component: ['Windows', 'Frames', 'null']
- * @param featureIds
+ *   OBJECTID: [0, 33, 333]
+ *   component: ['Windows', 'Roof', 'Skylight']
+ * @param featureIdsMap
  * @param propertyTable
  */
 export function flattenPropertyTableByFeatureIds(
-  featureIds: number[],
+  featureIdsMap: Record<string, number>,
   propertyTable: FeatureTableJson
 ): FeatureTableJson {
   const resultPropertyTable: FeatureTableJson = {};
   for (const propertyName in propertyTable) {
     const properties = propertyTable[propertyName];
-    resultPropertyTable[propertyName] = getPropertiesByFeatureIds(properties, featureIds);
+    resultPropertyTable[propertyName] = getPropertiesByFeatureIds(properties, featureIdsMap);
   }
 
   return resultPropertyTable;
@@ -37,14 +37,19 @@ export function flattenPropertyTableByFeatureIds(
 /**
  * Getting properties by featureId index
  * @param properties
- * @param featureIds
+ * @param featureIdsMap
  */
-function getPropertiesByFeatureIds(properties: any[], featureIds: number[]): any[] {
-  const resultProperties: any = [];
-
-  for (const featureId of featureIds) {
-    const property = properties[featureId] || null;
-    resultProperties.push(property);
+function getPropertiesByFeatureIds(
+  properties: unknown[],
+  featureIdsMap: Record<string, number>
+): unknown[] {
+  const resultProperties: unknown[] = [];
+
+  if (properties) {
+    for (const featureIdKey in featureIdsMap) {
+      const property = properties[featureIdKey] || null;
+      resultProperties.push(property);
+    }
   }
 
   return resultProperties;
@@ -64,7 +69,7 @@ export function checkPropertiesLength(
   let needFlatten = false;
 
   for (const attribute of Object.values(propertyTable)) {
-    if (featureIds.length !== attribute.length) {
+    if (!featureIds || !attribute || featureIds.length !== attribute.length) {
       needFlatten = true;
     }
   }
@@ -83,13 +88,13 @@ const OBJECT_ID_TYPE = 'OBJECTID';
 /**
  * Get the attribute type for attributeStorageInfo https://github.com/Esri/i3s-spec/blob/master/docs/1.7/attributeStorageInfo.cmn.md
  * @param key - attribute's key
- * @param attribute - attribute's type in propertyTable
+ * @param attribute - attribute taken from propertyTable
  */
-export function getAttributeType(key: string, attribute: string): string {
+export function getAttributeType(key: string, attribute: unknown): string {
   if (key === OBJECT_ID_TYPE) {
     return OBJECT_ID_TYPE;
   }
-  if (typeof attribute === STRING_TYPE) {
+  if (typeof attribute === STRING_TYPE || typeof attribute === 'bigint') {
     return STRING_TYPE;
   } else if (typeof attribute === 'number') {
     return Number.isInteger(attribute) ? SHORT_INT_TYPE : DOUBLE_TYPE;