@loaders.gl/tile-converter 4.0.0-alpha.23 → 4.0.0-alpha.25

package/src/i3s-converter/helpers/geometry-converter.ts

@@ -1,14 +1,13 @@
 import type {FeatureTableJson, Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 import type {
-  GLTF_EXT_mesh_features,
-  GLTF_EXT_structural_metadata,
   GLTFAccessorPostprocessed,
   GLTFMaterialPostprocessed,
   GLTFNodePostprocessed,
   GLTFMeshPrimitivePostprocessed,
   GLTFMeshPostprocessed,
   GLTFTexturePostprocessed,
-  GLTF_EXT_feature_metadata_GLTF
+  GLTF_EXT_feature_metadata_GLTF,
+  GLTF_EXT_structural_metadata_GLTF
 } from '@loaders.gl/gltf';
 
 import {Vector3, Matrix4, Vector4} from '@math.gl/core';
@@ -50,7 +49,12 @@ import type {GLTFAttributesData, TextureImageProperties, TypedArrayConstructor}
 import {generateSyntheticIndices} from '../../lib/utils/geometry-utils';
 import {BoundingSphere, OrientedBoundingBox} from '@math.gl/culling';
 
-import {EXT_MESH_FEATURES, EXT_FEATURE_METADATA, EXT_STRUCTURAL_METADATA} from '@loaders.gl/gltf';
+import {
+  EXT_FEATURE_METADATA,
+  EXT_STRUCTURAL_METADATA,
+  getPropertyTableFromExtFeatureMetadata,
+  getPropertyTableFromExtStructuralMetadata
+} from '@loaders.gl/gltf';
 
 // Spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.7/pbrMetallicRoughness.cmn.md
 const DEFAULT_ROUGHNESS_FACTOR = 1;
@@ -244,12 +248,18 @@ async function _makeNodeResources({
   const {faceRange, featureIds, positions, normals, colors, uvRegions, texCoords, featureCount} =
     generateAttributes(convertedAttributes);
 
-  if (tileContent.batchTableJson) {
-    makeFeatureIdsUnique(
+  let featureIdsMap: Record<string, number> = {};
+  if (propertyTable) {
+    /**
+     * 3DTiles has featureIndices unique only for one tile.
+     * In I3S featureIds are unique layer-wide. We create featureIds from all feature properties.
+     * If 3DTiles features has equal set of properties they are considered as same feature in I3S.
+     */
+    featureIdsMap = makeFeatureIdsUnique(
       featureIds,
       convertedAttributes.featureIndices,
       featuresHashArray,
-      tileContent.batchTableJson
+      propertyTable
     );
   }
 
@@ -291,6 +301,7 @@ async function _makeNodeResources({
   if (attributeStorageInfo && propertyTable) {
     attributes = convertPropertyTableToAttributeBuffers(
       featureIds,
+      featureIdsMap,
       propertyTable,
       attributeStorageInfo
     );
@@ -378,7 +389,7 @@ export async function convertAttributes(
 }
 
 /**
- * Gltf has hierarchical structure of nodes. This function converts nodes starting from those which are in gltf scene object.
+ * glTF has hierarchical structure of nodes. This function converts nodes starting from those which are in gltf scene object.
  *   The goal is applying tranformation matrix for all children. Functions "convertNodes" and "convertNode" work together recursively.
  * @param nodes - gltf nodes array
  * @param images - gltf images array
@@ -1046,7 +1057,7 @@ function convertMaterial(sourceMaterial: GLTFMaterialPostprocessed): I3SMaterial
 
 /**
  * Converts from `alphaMode` material property from GLTF to I3S format
- * @param gltfAlphaMode Gltf material `alphaMode` property
+ * @param gltfAlphaMode glTF material `alphaMode` property
  * @returns I3SMaterialDefinition.alphaMode property
  */
 function convertAlphaMode(
@@ -1252,17 +1263,18 @@ function generateImageId(texture: GLTFTexturePostprocessed, nodeId: number) {
  * @param featureIndices
  * @param featuresHashArray
  * @param batchTable
- * @returns {void}
+ * @returns propertyTable indices to map featureIds
  */
 function makeFeatureIdsUnique(
   featureIds: number[],
   featureIndices: number[],
   featuresHashArray: string[],
   batchTable: {[key: string]: any}
-) {
+): Record<string, number> {
   const replaceMap = getFeaturesReplaceMap(featureIds, batchTable, featuresHashArray);
   replaceIndicesByUnique(featureIndices, replaceMap);
   replaceIndicesByUnique(featureIds, replaceMap);
+  return replaceMap;
 }
 
 /**
@@ -1276,8 +1288,8 @@ function getFeaturesReplaceMap(
   featureIds: any[],
   batchTable: object,
   featuresHashArray: any[]
-): Record<string, any> {
-  const featureMap: Record<string, any> = {};
+): Record<string, number> {
+  const featureMap: Record<string, number> = {};
 
   for (let index = 0; index < featureIds.length; index++) {
     const oldFeatureId = featureIds[index];
@@ -1329,7 +1341,7 @@ function getOrCreateUniqueFeatureId(
  * @param featureMap
  * @returns
  */
-function replaceIndicesByUnique(indicesArray: any[], featureMap: Record<string, []>) {
+function replaceIndicesByUnique(indicesArray: number[], featureMap: Record<string, number>) {
   for (let index = 0; index < indicesArray.length; index++) {
     indicesArray[index] = featureMap[indicesArray[index]];
   }
@@ -1344,6 +1356,7 @@ function replaceIndicesByUnique(indicesArray: any[], featureMap: Record<string,
  */
 function convertPropertyTableToAttributeBuffers(
   featureIds: number[],
+  featureIdsMap: Record<string, number>,
   propertyTable: FeatureTableJson,
   attributeStorageInfo: AttributeStorageInfo[]
 ): any[] {
@@ -1351,7 +1364,7 @@ function convertPropertyTableToAttributeBuffers(
 
   const needFlattenPropertyTable = checkPropertiesLength(featureIds, propertyTable);
   const properties = needFlattenPropertyTable
-    ? flattenPropertyTableByFeatureIds(featureIds, propertyTable)
+    ? flattenPropertyTableByFeatureIds(featureIdsMap, propertyTable)
     : propertyTable;
 
   const propertyTableWithObjectIds = {
@@ -1597,7 +1610,7 @@ function generateFeatureIndexAttribute(
  * For example it can be batchTable for b3dm files or property table in gLTF extension.
  * @param tileContent - 3DTiles tile content
  * @param metadataClass - user selected feature metadata class name
- * @return batch table from b3dm / feature properties from EXT_FEATURE_METADATA, EXT_MESH_FEATURES or EXT_STRUCTURAL_METADATA
+ * @return batch table from b3dm / feature properties from EXT_FEATURE_METADATA or EXT_STRUCTURAL_METADATA.
  */
 export function getPropertyTable(
   tileContent: Tiles3DTileContent | null,
@@ -1607,7 +1620,7 @@ export function getPropertyTable(
     return null;
   }
   let propertyTable: FeatureTableJson | null;
-  const batchTableJson = tileContent?.batchTableJson;
+  const batchTableJson = tileContent.batchTableJson;
 
   if (batchTableJson) {
     return batchTableJson;
@@ -1616,16 +1629,9 @@ export function getPropertyTable(
   const {extensionName, extension} = getPropertyTableExtension(tileContent);
 
   switch (extensionName) {
-    case EXT_MESH_FEATURES: {
-      propertyTable = getPropertyTableFromExtMeshFeatures(
-        extension as GLTF_EXT_mesh_features,
-        metadataClass
-      );
-      return propertyTable;
-    }
     case EXT_STRUCTURAL_METADATA: {
       propertyTable = getPropertyTableFromExtStructuralMetadata(
-        extension as GLTF_EXT_structural_metadata,
+        extension as GLTF_EXT_structural_metadata_GLTF,
         metadataClass
       );
       return propertyTable;
@@ -1648,18 +1654,9 @@ export function getPropertyTable(
  */
 function getPropertyTableExtension(tileContent: Tiles3DTileContent): {
   extensionName: null | string;
-  extension:
-    | string
-    | GLTF_EXT_feature_metadata_GLTF
-    | GLTF_EXT_structural_metadata
-    | GLTF_EXT_mesh_features
-    | null;
+  extension: string | GLTF_EXT_feature_metadata_GLTF | GLTF_EXT_structural_metadata_GLTF | null;
 } {
-  const extensionsWithPropertyTables = [
-    EXT_FEATURE_METADATA,
-    EXT_STRUCTURAL_METADATA,
-    EXT_MESH_FEATURES
-  ];
+  const extensionsWithPropertyTables = [EXT_FEATURE_METADATA, EXT_STRUCTURAL_METADATA];
   const extensionsUsed = tileContent?.gltf?.extensionsUsed;
 
   if (!extensionsUsed) {
@@ -1670,6 +1667,12 @@ function getPropertyTableExtension(tileContent: Tiles3DTileContent): {
   for (const extensionItem of tileContent?.gltf?.extensionsUsed || []) {
     if (extensionsWithPropertyTables.includes(extensionItem)) {
       extensionName = extensionItem;
+      /*
+        It returns the first extension containing the property table.
+        We assume that there can be only one extension containing the property table:
+        either EXT_FEATURE_METADATA, which is a depricated extension,
+        or EXT_STRUCTURAL_METADATA.
+      */
       break;
     }
   }
@@ -1680,147 +1683,8 @@ function getPropertyTableExtension(tileContent: Tiles3DTileContent): {
 
   const extension = tileContent?.gltf?.extensions?.[extensionName] as
     | string // EXT_mesh_features doesn't have global metadata
-    | GLTF_EXT_feature_metadata_GLTF;
+    | GLTF_EXT_feature_metadata_GLTF
+    | GLTF_EXT_structural_metadata_GLTF;
 
   return {extensionName, extension};
 }
-
-/**
- * Handle EXT_feature_metadata to get property table
- * @param extension - global level of EXT_FEATURE_METADATA extension
- * @param metadataClass - user selected feature metadata class name
- * @returns {FeatureTableJson | null} Property table or null if the extension can't be handled properly.
- */
-function getPropertyTableFromExtFeatureMetadata(
-  extension: GLTF_EXT_feature_metadata_GLTF,
-  metadataClass?: string
-): FeatureTableJson | null {
-  if (extension?.featureTables) {
-    /**
-     * Take only first feature table to generate attributes storage info object.
-     * TODO: Think about getting data from all feature tables?
-     * It can be tricky just because 3dTiles is able to have multiple featureId attributes and multiple feature tables.
-     * In I3S we should decide which featureIds attribute will be passed to geometry data.
-     */
-    const firstFeatureTableName = Object.keys(extension.featureTables)?.[0];
-
-    if (firstFeatureTableName) {
-      const featureTable = extension?.featureTables[firstFeatureTableName];
-      const propertyTable = {};
-
-      for (const propertyName in featureTable.properties) {
-        propertyTable[propertyName] = featureTable.properties[propertyName].data;
-      }
-
-      return propertyTable;
-    }
-  }
-
-  if (extension?.featureTextures) {
-    let featureTexture: string | undefined;
-    for (const textureKey in extension.featureTextures) {
-      const texture = extension.featureTextures[textureKey];
-      if (texture.class === metadataClass) {
-        featureTexture = textureKey;
-      }
-    }
-
-    if (typeof featureTexture === 'string') {
-      const featureTable = extension?.featureTextures[featureTexture];
-      const propertyTable = {};
-
-      for (const propertyName in featureTable.properties) {
-        propertyTable[propertyName] = featureTable.properties[propertyName].data;
-      }
-
-      return propertyTable;
-    }
-  }
-
-  console.warn(
-    "The I3S converter couldn't handle EXT_feature_metadata extension: There is neither featureTables, no featureTextures in the extension."
-  );
-  return null;
-}
-
-/**
- * Handle EXT_structural_metadata to get property table
- * @param extension - global level of EXT_STRUCTURAL_METADATA extension
- * @param metadataClass - user selected feature metadata class name
- * @returns {FeatureTableJson | null} Property table or null if the extension can't be handled properly.
- */
-function getPropertyTableFromExtStructuralMetadata(
-  extension: GLTF_EXT_structural_metadata,
-  metadataClass?: string
-): FeatureTableJson | null {
-  if (extension?.propertyTables) {
-    /**
-     * Take only first feature table to generate attributes storage info object.
-     * TODO: Think about getting data from all feature tables?
-     * It can be tricky just because 3dTiles is able to have multiple featureId attributes and multiple feature tables.
-     * In I3S we should decide which featureIds attribute will be passed to geometry data.
-     */
-    const firstPropertyTable = extension?.propertyTables[0];
-    const propertyTableWithData = {};
-
-    for (const propertyName in firstPropertyTable.properties) {
-      propertyTableWithData[propertyName] = firstPropertyTable.properties[propertyName].data;
-    }
-
-    return propertyTableWithData;
-  }
-
-  if (extension?.propertyTextures) {
-    /**
-     * Take only first feature table to generate attributes storage info object.
-     * TODO: Think about getting data from all feature tables?
-     * It can be tricky just because 3dTiles is able to have multiple featureId attributes and multiple feature tables.
-     * In I3S we should decide which featureIds attribute will be passed to geometry data.
-     */
-    if (extension?.propertyTextures) {
-      const firstPropertyTexture = extension?.propertyTextures[0];
-      const propertyTableWithData = {};
-
-      for (const propertyName in firstPropertyTexture.properties) {
-        propertyTableWithData[propertyName] = firstPropertyTexture.properties[propertyName].data;
-      }
-
-      return propertyTableWithData;
-    }
-  }
-
-  console.warn(
-    "The I3S converter couldn't handle EXT_structural_metadata extension: There is neither propertyTables, no propertyTextures in the extension."
-  );
-  return null;
-}
-
-/**
- * Handle EXT_mesh_features to get property table
- * @param extension - global level of EXT_MESH_FEATURES extension
- * @param metadataClass - user selected feature metadata class name
- * @returns {FeatureTableJson | null} Property table or null if the extension can't be handled properly.
- */
-function getPropertyTableFromExtMeshFeatures(
-  extension: GLTF_EXT_mesh_features,
-  metadataClass?: string
-): FeatureTableJson | null {
-  if (extension?.featureIds) {
-    const firstFeatureId = extension?.featureIds[0];
-    const propertyTableWithData = {};
-
-    // When firstFeatureId.propertyTable is defined, the property data will be taken from EXT_structural_metadata extension
-    if (!firstFeatureId.propertyTable) {
-      console.warn(
-        'Should be implemented as we have the tileset with Ext_mesh_features not linked with EXT_structural_metadata extension'
-      );
-    }
-
-    return propertyTableWithData;
-  }
-
-  console.warn(
-    "The I3S converter couldn't handle EXT_mesh_features extension: There is no featureIds in the extension."
-  );
-  return null;
-}

package/src/i3s-converter/helpers/preprocess-3d-tiles.ts

@@ -1,6 +1,12 @@
 import {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 import {GLTFPrimitiveModeString, PreprocessData} from '../types';
-import {GLTF, GLTFLoader, GLTF_EXT_feature_metadata_GLTF} from '@loaders.gl/gltf';
+import {
+  EXT_STRUCTURAL_METADATA,
+  GLTF,
+  GLTFLoader,
+  GLTF_EXT_feature_metadata_GLTF,
+  GLTF_EXT_structural_metadata_GLTF
+} from '@loaders.gl/gltf';
 import {parse} from '@loaders.gl/core';
 import {EXT_FEATURE_METADATA} from '@loaders.gl/gltf';
 
@@ -43,8 +49,8 @@ export const analyzeTileContent = async (
   if (!gltf) {
     return defaultResult;
   }
-  const meshTopologyTypes = getMeshTypesFromGltf(gltf);
-  const metadataClasses = getMetadataClassesFromGltf(gltf);
+  const meshTopologyTypes = getMeshTypesFromGLTF(gltf);
+  const metadataClasses = getMetadataClassesFromGLTF(gltf);
   return {
     meshTopologyTypes,
     metadataClasses
@@ -56,7 +62,7 @@ export const analyzeTileContent = async (
  * @param gltfJson - JSON part of GLB content
  * @returns array of mesh types found
  */
-const getMeshTypesFromGltf = (gltfJson: GLTF): Set<GLTFPrimitiveModeString> => {
+const getMeshTypesFromGLTF = (gltfJson: GLTF): Set<GLTFPrimitiveModeString> => {
   const result: Set<GLTFPrimitiveModeString> = new Set();
   for (const mesh of gltfJson.meshes || []) {
     for (const primitive of mesh.primitives) {
@@ -72,17 +78,32 @@ const getMeshTypesFromGltf = (gltfJson: GLTF): Set<GLTFPrimitiveModeString> => {
 
 /**
  * Get feature metadata classes from glTF
+ * The tileset might contain multiple metadata classes provided by EXT_feature_metadata and EXT_structural_metadata extensions.
+ * Every class is a set of properties. But I3S can consume only one set of properties.
+ * On the pre-process we collect all classes from the tileset in order to show the prompt to select one class for conversion to I3S.
  * @param gltfJson - JSON part of GLB content
  * @returns array of classes
  */
-const getMetadataClassesFromGltf = (gltfJson: GLTF): Set<string> => {
+const getMetadataClassesFromGLTF = (gltfJson: GLTF): Set<string> => {
   const result: Set<string> = new Set();
 
-  const classes = (gltfJson.extensions?.[EXT_FEATURE_METADATA] as GLTF_EXT_feature_metadata_GLTF)
-    ?.schema?.classes;
+  // Try to parse from EXT_feature_metadata
+  const extFeatureMetadataClasses = (
+    gltfJson.extensions?.[EXT_FEATURE_METADATA] as GLTF_EXT_feature_metadata_GLTF
+  )?.schema?.classes;
 
-  if (classes) {
-    for (const classKey of Object.keys(classes)) {
+  if (extFeatureMetadataClasses) {
+    for (const classKey of Object.keys(extFeatureMetadataClasses)) {
+      result.add(classKey);
+    }
+  }
+
+  // Try to parse from EXT_structural_metadata
+  const extStructuralMetadataClasses = (
+    gltfJson.extensions?.[EXT_STRUCTURAL_METADATA] as GLTF_EXT_structural_metadata_GLTF
+  )?.schema?.classes;
+  if (extStructuralMetadataClasses) {
+    for (const classKey of Object.keys(extStructuralMetadataClasses)) {
       result.add(classKey);
     }
   }

package/src/i3s-converter/i3s-converter.ts

@@ -656,7 +656,6 @@ export default class I3SConverter {
         We will append new attributes only in case the property table is updated.
         According to ver 1.9 (see https://github.com/Esri/i3s-spec/blob/master/docs/1.9/attributeStorageInfo.cmn.md):
         "The attributeStorageInfo object describes the structure of the binary attribute data resource of a layer, which is the same for every node in the layer."
-        But the specification of ver 2.1 doesn't have such a requirement ("...the same for every node...")
       */
       this._convertPropertyTableToNodeAttributes(propertyTable);
     }
@@ -1167,7 +1166,6 @@ export default class I3SConverter {
         We will append new attributes only in case the property table is updated.
         According to ver 1.9 (see https://github.com/Esri/i3s-spec/blob/master/docs/1.9/attributeStorageInfo.cmn.md):
         "The attributeStorageInfo object describes the structure of the binary attribute data resource of a layer, which is the same for every node in the layer."
-        But the specification of ver 2.1 doesn't have such a requirement ("...the same for every node...")
       */
       const found = this.layers0!.attributeStorageInfo!.find((element) => element.name === key);
       if (!found) {

package/src/i3s-converter/types.ts

@@ -106,10 +106,8 @@ export type GeometryAttributes = {
   featureCount: number;
 };
 
-/** Geometry attributes specific for the particular feature */
-export type GroupedByFeatureIdAttributes = {
-  /** Feature Id */
-  featureId: number;
+/** Geometry attributes applicable for reordering by featureId */
+export type GroupedAttributes = {
   /** POSITION attribute value */
   positions: Float32Array;
   /** NORMAL attribute value */
@@ -122,6 +120,12 @@ export type GroupedByFeatureIdAttributes = {
   texCoords: Float32Array;
 };
 
+/** Geometry attributes specific for the particular feature */
+export type GroupedByFeatureIdAttributes = GroupedAttributes & {
+  /** Feature Id */
+  featureId: number;
+};
+
 /** Shared resources made from GLTF material */
 export type SharedResourcesArrays = {
   /** material definitions list https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitionInfo.cmn.md */
@@ -183,7 +187,12 @@ export enum GLTFPrimitiveModeString {
 export type PreprocessData = {
   /** Mesh topology types used in gltf primitives of the tileset */
   meshTopologyTypes: Set<GLTFPrimitiveModeString>;
-  /** Featrue metadata classes found in glTF extensions */
+  /**
+   * Featrue metadata classes found in glTF extensions
+   * The tileset might contain multiple metadata classes provided by EXT_feature_metadata and EXT_structural_metadata extensions.
+   * Every class is a set of properties. But I3S can consume only one set of properties.
+   * On the pre-process we collect all classes from the tileset in order to show the prompt to select one class for conversion to I3S.
+   */
   metadataClasses: Set<string>;
 };
 

package/src/pgm-loader.ts

@@ -1,24 +1,32 @@
-import type {LoaderWithParser} from '@loaders.gl/loader-utils';
-import {parsePGM} from '@math.gl/geoid';
+import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import {Geoid, parsePGM} from '@math.gl/geoid';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+export {Geoid};
+
+export type PGMLoaderOptions = LoaderOptions & {
+  pgm?: {
+    cubic?: boolean;
+  };
+};
+
 /**
  * Loader for PGM - Netpbm grayscale image format
  */
-export const PGMLoader: LoaderWithParser = {
+export const PGMLoader: LoaderWithParser<Geoid, never, PGMLoaderOptions> = {
   name: 'PGM - Netpbm grayscale image format',
   id: 'pgm',
   module: 'tile-converter',
   version: VERSION,
   mimeTypes: ['image/x-portable-graymap'],
-  // @ts-expect-error LoaderOptions does not have cubic parameter
-  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options),
+  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options?.pgm || {}),
   extensions: ['pgm'],
   options: {
-    // TODO - use pgm namespace
-    cubic: false
+    pgm: {
+      cubic: false
+    }
   }
 };