@loaders.gl/tile-converter 4.0.0-alpha.22 → 4.0.0-alpha.24

package/src/i3s-converter/helpers/geometry-converter.ts

@@ -1,5 +1,7 @@
 import type {FeatureTableJson, Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 import type {
+  GLTF_EXT_mesh_features,
+  GLTF_EXT_structural_metadata,
   GLTFAccessorPostprocessed,
   GLTFMaterialPostprocessed,
   GLTFNodePostprocessed,
@@ -47,7 +49,8 @@ import {GL} from '@loaders.gl/math';
 import type {GLTFAttributesData, TextureImageProperties, TypedArrayConstructor} from '../types';
 import {generateSyntheticIndices} from '../../lib/utils/geometry-utils';
 import {BoundingSphere, OrientedBoundingBox} from '@math.gl/culling';
-import {EXT_FEATURE_METADATA, EXT_MESH_FEATURES} from '../../constants';
+
+import {EXT_MESH_FEATURES, EXT_FEATURE_METADATA, EXT_STRUCTURAL_METADATA} from '@loaders.gl/gltf';
 
 // Spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.7/pbrMetallicRoughness.cmn.md
 const DEFAULT_ROUGHNESS_FACTOR = 1;
@@ -241,12 +244,18 @@ async function _makeNodeResources({
   const {faceRange, featureIds, positions, normals, colors, uvRegions, texCoords, featureCount} =
     generateAttributes(convertedAttributes);
 
-  if (tileContent.batchTableJson) {
-    makeFeatureIdsUnique(
+  let featureIdsMap: Record<string, number> = {};
+  if (propertyTable) {
+    /**
+     * 3DTiles has featureIndices unique only for one tile.
+     * In I3S featureIds are unique layer-wide. We create featureIds from all feature properties.
+     * If 3DTiles features has equal set of properties they are considered as same feature in I3S.
+     */
+    featureIdsMap = makeFeatureIdsUnique(
       featureIds,
       convertedAttributes.featureIndices,
       featuresHashArray,
-      tileContent.batchTableJson
+      propertyTable
     );
   }
 
@@ -288,6 +297,7 @@ async function _makeNodeResources({
   if (attributeStorageInfo && propertyTable) {
     attributes = convertPropertyTableToAttributeBuffers(
       featureIds,
+      featureIdsMap,
       propertyTable,
       attributeStorageInfo
     );
@@ -1249,17 +1259,18 @@ function generateImageId(texture: GLTFTexturePostprocessed, nodeId: number) {
  * @param featureIndices
  * @param featuresHashArray
  * @param batchTable
- * @returns {void}
+ * @returns propertyTable indices to map featureIds
  */
 function makeFeatureIdsUnique(
   featureIds: number[],
   featureIndices: number[],
   featuresHashArray: string[],
   batchTable: {[key: string]: any}
-) {
+): Record<string, number> {
   const replaceMap = getFeaturesReplaceMap(featureIds, batchTable, featuresHashArray);
   replaceIndicesByUnique(featureIndices, replaceMap);
   replaceIndicesByUnique(featureIds, replaceMap);
+  return replaceMap;
 }
 
 /**
@@ -1273,8 +1284,8 @@ function getFeaturesReplaceMap(
   featureIds: any[],
   batchTable: object,
   featuresHashArray: any[]
-): Record<string, any> {
-  const featureMap: Record<string, any> = {};
+): Record<string, number> {
+  const featureMap: Record<string, number> = {};
 
   for (let index = 0; index < featureIds.length; index++) {
     const oldFeatureId = featureIds[index];
@@ -1326,7 +1337,7 @@ function getOrCreateUniqueFeatureId(
  * @param featureMap
  * @returns
  */
-function replaceIndicesByUnique(indicesArray: any[], featureMap: Record<string, []>) {
+function replaceIndicesByUnique(indicesArray: number[], featureMap: Record<string, number>) {
   for (let index = 0; index < indicesArray.length; index++) {
     indicesArray[index] = featureMap[indicesArray[index]];
   }
@@ -1341,6 +1352,7 @@ function replaceIndicesByUnique(indicesArray: any[], featureMap: Record<string,
  */
 function convertPropertyTableToAttributeBuffers(
   featureIds: number[],
+  featureIdsMap: Record<string, number>,
   propertyTable: FeatureTableJson,
   attributeStorageInfo: AttributeStorageInfo[]
 ): any[] {
@@ -1348,7 +1360,7 @@ function convertPropertyTableToAttributeBuffers(
 
   const needFlattenPropertyTable = checkPropertiesLength(featureIds, propertyTable);
   const properties = needFlattenPropertyTable
-    ? flattenPropertyTableByFeatureIds(featureIds, propertyTable)
+    ? flattenPropertyTableByFeatureIds(featureIdsMap, propertyTable)
     : propertyTable;
 
   const propertyTableWithObjectIds = {
@@ -1358,10 +1370,12 @@ function convertPropertyTableToAttributeBuffers(
 
   for (const propertyName in propertyTableWithObjectIds) {
     const type = getAttributeType(propertyName, attributeStorageInfo);
-    const value = propertyTableWithObjectIds[propertyName];
-    const attributeBuffer = generateAttributeBuffer(type, value);
+    if (type) {
+      const value = propertyTableWithObjectIds[propertyName];
+      const attributeBuffer = generateAttributeBuffer(type, value);
 
-    attributeBuffers.push(attributeBuffer);
+      attributeBuffers.push(attributeBuffer);
+    }
   }
 
   return attributeBuffers;
@@ -1401,6 +1415,20 @@ function generateAttributeBuffer(type: string, value: any): ArrayBuffer {
  */
 function getAttributeType(key: string, attributeStorageInfo: any[]): string {
   const attribute = attributeStorageInfo.find((attr) => attr.name === key);
+  if (!attribute) {
+    console.error(
+      `attribute is null, key=${key}, attributeStorageInfo=${JSON.stringify(
+        attributeStorageInfo,
+        null,
+        2
+      )}`
+    );
+    return '';
+  }
+  if (!attribute.attributeValues) {
+    console.error(`attributeValues is null, attribute=${attribute}`);
+    return '';
+  }
   return attribute.attributeValues.valueType;
 }
 
@@ -1577,8 +1605,8 @@ function generateFeatureIndexAttribute(
  * Find property table in tile
  * For example it can be batchTable for b3dm files or property table in gLTF extension.
  * @param tileContent - 3DTiles tile content
- * @param metadataClass - - user selected feature metadata class name
- * @return batch table from b3dm / feature properties from EXT_FEATURE_METADATA
+ * @param metadataClass - user selected feature metadata class name
+ * @return batch table from b3dm / feature properties from EXT_FEATURE_METADATA, EXT_MESH_FEATURES or EXT_STRUCTURAL_METADATA
  */
 export function getPropertyTable(
   tileContent: Tiles3DTileContent | null,
@@ -1587,7 +1615,7 @@ export function getPropertyTable(
   if (!tileContent) {
     return null;
   }
-
+  let propertyTable: FeatureTableJson | null;
   const batchTableJson = tileContent?.batchTableJson;
 
   if (batchTableJson) {
@@ -1598,14 +1626,25 @@ export function getPropertyTable(
 
   switch (extensionName) {
     case EXT_MESH_FEATURES: {
-      console.warn('The I3S converter does not yet support the EXT_mesh_features extension');
-      return null;
+      propertyTable = getPropertyTableFromExtMeshFeatures(
+        extension as GLTF_EXT_mesh_features,
+        metadataClass
+      );
+      return propertyTable;
+    }
+    case EXT_STRUCTURAL_METADATA: {
+      propertyTable = getPropertyTableFromExtStructuralMetadata(
+        extension as GLTF_EXT_structural_metadata,
+        metadataClass
+      );
+      return propertyTable;
     }
     case EXT_FEATURE_METADATA: {
-      return getPropertyTableFromExtFeatureMetadata(
+      propertyTable = getPropertyTableFromExtFeatureMetadata(
         extension as GLTF_EXT_feature_metadata_GLTF,
         metadataClass
       );
+      return propertyTable;
     }
     default:
       return null;
@@ -1618,9 +1657,18 @@ export function getPropertyTable(
  */
 function getPropertyTableExtension(tileContent: Tiles3DTileContent): {
   extensionName: null | string;
-  extension: string | GLTF_EXT_feature_metadata_GLTF | null;
+  extension:
+    | string
+    | GLTF_EXT_feature_metadata_GLTF
+    | GLTF_EXT_structural_metadata
+    | GLTF_EXT_mesh_features
+    | null;
 } {
-  const extensionsWithPropertyTables = [EXT_FEATURE_METADATA, EXT_MESH_FEATURES];
+  const extensionsWithPropertyTables = [
+    EXT_FEATURE_METADATA,
+    EXT_STRUCTURAL_METADATA,
+    EXT_MESH_FEATURES
+  ];
   const extensionsUsed = tileContent?.gltf?.extensionsUsed;
 
   if (!extensionsUsed) {
@@ -1649,6 +1697,8 @@ function getPropertyTableExtension(tileContent: Tiles3DTileContent): {
 /**
  * Handle EXT_feature_metadata to get property table
  * @param extension - global level of EXT_FEATURE_METADATA extension
+ * @param metadataClass - user selected feature metadata class name
+ * @returns {FeatureTableJson | null} Property table or null if the extension can't be handled properly.
  */
 function getPropertyTableFromExtFeatureMetadata(
   extension: GLTF_EXT_feature_metadata_GLTF,
@@ -1701,3 +1751,85 @@ function getPropertyTableFromExtFeatureMetadata(
   );
   return null;
 }
+
+/**
+ * Handle EXT_structural_metadata to get property table
+ * @param extension - global level of EXT_STRUCTURAL_METADATA extension
+ * @param metadataClass - user selected feature metadata class name
+ * @returns {FeatureTableJson | null} Property table or null if the extension can't be handled properly.
+ */
+function getPropertyTableFromExtStructuralMetadata(
+  extension: GLTF_EXT_structural_metadata,
+  metadataClass?: string
+): FeatureTableJson | null {
+  if (extension?.propertyTables) {
+    /**
+     * Take only first feature table to generate attributes storage info object.
+     * TODO: Think about getting data from all feature tables?
+     * It can be tricky just because 3dTiles is able to have multiple featureId attributes and multiple feature tables.
+     * In I3S we should decide which featureIds attribute will be passed to geometry data.
+     */
+    const firstPropertyTable = extension?.propertyTables[0];
+    const propertyTableWithData = {};
+
+    for (const propertyName in firstPropertyTable.properties) {
+      propertyTableWithData[propertyName] = firstPropertyTable.properties[propertyName].data;
+    }
+
+    return propertyTableWithData;
+  }
+
+  if (extension?.propertyTextures) {
+    /**
+     * Take only first feature table to generate attributes storage info object.
+     * TODO: Think about getting data from all feature tables?
+     * It can be tricky just because 3dTiles is able to have multiple featureId attributes and multiple feature tables.
+     * In I3S we should decide which featureIds attribute will be passed to geometry data.
+     */
+    if (extension?.propertyTextures) {
+      const firstPropertyTexture = extension?.propertyTextures[0];
+      const propertyTableWithData = {};
+
+      for (const propertyName in firstPropertyTexture.properties) {
+        propertyTableWithData[propertyName] = firstPropertyTexture.properties[propertyName].data;
+      }
+
+      return propertyTableWithData;
+    }
+  }
+
+  console.warn(
+    "The I3S converter couldn't handle EXT_structural_metadata extension: There is neither propertyTables, no propertyTextures in the extension."
+  );
+  return null;
+}
+
+/**
+ * Handle EXT_mesh_features to get property table
+ * @param extension - global level of EXT_MESH_FEATURES extension
+ * @param metadataClass - user selected feature metadata class name
+ * @returns {FeatureTableJson | null} Property table or null if the extension can't be handled properly.
+ */
+function getPropertyTableFromExtMeshFeatures(
+  extension: GLTF_EXT_mesh_features,
+  metadataClass?: string
+): FeatureTableJson | null {
+  if (extension?.featureIds) {
+    const firstFeatureId = extension?.featureIds[0];
+    const propertyTableWithData = {};
+
+    // When firstFeatureId.propertyTable is defined, the property data will be taken from EXT_structural_metadata extension
+    if (!firstFeatureId.propertyTable) {
+      console.warn(
+        'Should be implemented as we have the tileset with Ext_mesh_features not linked with EXT_structural_metadata extension'
+      );
+    }
+
+    return propertyTableWithData;
+  }
+
+  console.warn(
+    "The I3S converter couldn't handle EXT_mesh_features extension: There is no featureIds in the extension."
+  );
+  return null;
+}

package/src/i3s-converter/helpers/node-index-document.ts

@@ -323,7 +323,11 @@ export class NodeIndexDocument {
         parentNode.converter.layers0?.attributeStorageInfo?.length
       ) {
         node.attributeData = [];
-        for (let index = 0; index < attributes.length; index++) {
+        const minimumLength =
+          attributes.length < parentNode.converter.layers0.attributeStorageInfo.length
+            ? attributes.length
+            : parentNode.converter.layers0.attributeStorageInfo.length;
+        for (let index = 0; index < minimumLength; index++) {
           const folderName = parentNode.converter.layers0.attributeStorageInfo[index].key;
           node.attributeData.push({href: `./attributes/${folderName}/0`});
         }

package/src/i3s-converter/helpers/preprocess-3d-tiles.ts

@@ -2,7 +2,7 @@ import {Tiles3DTileContent} from '@loaders.gl/3d-tiles';
 import {GLTFPrimitiveModeString, PreprocessData} from '../types';
 import {GLTF, GLTFLoader, GLTF_EXT_feature_metadata_GLTF} from '@loaders.gl/gltf';
 import {parse} from '@loaders.gl/core';
-import {EXT_FEATURE_METADATA} from '../../constants';
+import {EXT_FEATURE_METADATA} from '@loaders.gl/gltf';
 
 /**
  * glTF primitive modes

package/src/i3s-converter/i3s-converter.ts

@@ -12,7 +12,8 @@ import type {
   SceneLayer3D,
   BoundingVolumes,
   MaxScreenThresholdSQ,
-  NodeInPage
+  NodeInPage,
+  AttributeStorageInfo
 } from '@loaders.gl/i3s';
 import {load, encode, isBrowser} from '@loaders.gl/core';
 import {CesiumIonLoader, Tiles3DLoader} from '@loaders.gl/3d-tiles';
@@ -649,7 +650,14 @@ export default class I3SConverter {
 
     const propertyTable = getPropertyTable(tileContent, this.options.metadataClass);
 
-    if (propertyTable && !this.layers0?.attributeStorageInfo?.length) {
+    if (propertyTable) {
+      /*
+        Call the convertion procedure even if the node attributes have been already created.
+        We will append new attributes only in case the property table is updated.
+        According to ver 1.9 (see https://github.com/Esri/i3s-spec/blob/master/docs/1.9/attributeStorageInfo.cmn.md):
+        "The attributeStorageInfo object describes the structure of the binary attribute data resource of a layer, which is the same for every node in the layer."
+        But the specification of ver 2.1 doesn't have such a requirement ("...the same for every node...")
+      */
       this._convertPropertyTableToNodeAttributes(propertyTable);
     }
 
@@ -735,7 +743,7 @@ export default class I3SConverter {
    * @param boundingVolume - initialized bounding volume of the source tile
    * @param tileContent - content of the source tile
    * @param parentId - id of parent node in node pages
-   * @param propertyTable - batch table from b3dm / feature properties from EXT_FEATURE_METADATA
+   * @param propertyTable - batch table from b3dm / feature properties from EXT_FEATURE_METADATA, EXT_MESH_FEATURES or EXT_STRUCTURAL_METADATA
    * @returns - converted node resources
    */
   private async _convertResources(
@@ -1068,7 +1076,12 @@ export default class I3SConverter {
     slpkChildPath: string
   ): Promise<void> {
     if (attributes?.length && this.layers0?.attributeStorageInfo?.length) {
-      for (let index = 0; index < attributes.length; index++) {
+      const minimumLength =
+        attributes.length < this.layers0.attributeStorageInfo.length
+          ? attributes.length
+          : this.layers0.attributeStorageInfo.length;
+
+      for (let index = 0; index < minimumLength; index++) {
         const folderName = this.layers0.attributeStorageInfo[index].key;
         const fileBuffer = new Uint8Array(attributes[index]);
 
@@ -1150,19 +1163,31 @@ export default class I3SConverter {
     };
 
     for (const key in propertyTableWithObjectId) {
-      const firstAttribute = propertyTableWithObjectId[key][0];
-      const attributeType = getAttributeType(key, firstAttribute);
-
-      const storageAttribute = createdStorageAttribute(attributeIndex, key, attributeType);
-      const fieldAttributeType = getFieldAttributeType(attributeType);
-      const fieldAttribute = createFieldAttribute(key, fieldAttributeType);
-      const popupInfo = createPopupInfo(propertyTableWithObjectId);
-
-      this.layers0!.attributeStorageInfo!.push(storageAttribute);
-      this.layers0!.fields!.push(fieldAttribute);
-      this.layers0!.popupInfo = popupInfo;
-      this.layers0!.layerType = _3D_OBJECT_LAYER_TYPE;
-
+      /*
+        We will append new attributes only in case the property table is updated.
+        According to ver 1.9 (see https://github.com/Esri/i3s-spec/blob/master/docs/1.9/attributeStorageInfo.cmn.md):
+        "The attributeStorageInfo object describes the structure of the binary attribute data resource of a layer, which is the same for every node in the layer."
+        But the specification of ver 2.1 doesn't have such a requirement ("...the same for every node...")
+      */
+      const found = this.layers0!.attributeStorageInfo!.find((element) => element.name === key);
+      if (!found) {
+        const firstAttribute = propertyTableWithObjectId[key][0];
+        const attributeType = getAttributeType(key, firstAttribute);
+
+        const storageAttribute: AttributeStorageInfo = createdStorageAttribute(
+          attributeIndex,
+          key,
+          attributeType
+        );
+        const fieldAttributeType = getFieldAttributeType(attributeType);
+        const fieldAttribute = createFieldAttribute(key, fieldAttributeType);
+        const popupInfo = createPopupInfo(propertyTableWithObjectId);
+
+        this.layers0!.attributeStorageInfo!.push(storageAttribute);
+        this.layers0!.fields!.push(fieldAttribute);
+        this.layers0!.popupInfo = popupInfo;
+        this.layers0!.layerType = _3D_OBJECT_LAYER_TYPE;
+      }
       attributeIndex += 1;
     }
   }

package/src/i3s-converter/types.ts

@@ -106,10 +106,8 @@ export type GeometryAttributes = {
   featureCount: number;
 };
 
-/** Geometry attributes specific for the particular feature */
-export type GroupedByFeatureIdAttributes = {
-  /** Feature Id */
-  featureId: number;
+/** Geometry attributes applicable for reordering by featureId */
+export type GroupedAttributes = {
   /** POSITION attribute value */
   positions: Float32Array;
   /** NORMAL attribute value */
@@ -122,6 +120,12 @@ export type GroupedByFeatureIdAttributes = {
   texCoords: Float32Array;
 };
 
+/** Geometry attributes specific for the particular feature */
+export type GroupedByFeatureIdAttributes = GroupedAttributes & {
+  /** Feature Id */
+  featureId: number;
+};
+
 /** Shared resources made from GLTF material */
 export type SharedResourcesArrays = {
   /** material definitions list https://github.com/Esri/i3s-spec/blob/master/docs/1.8/materialDefinitionInfo.cmn.md */

package/src/i3s-server/controllers/slpk-controller.ts

@@ -1,6 +1,6 @@
 import '@loaders.gl/polyfills';
 import {parseSLPK} from '@loaders.gl/i3s';
-import {FileHandleFile} from '@loaders.gl/zip';
+import {FileHandleFile} from '@loaders.gl/loader-utils';
 
 let slpkArchive;
 

package/src/pgm-loader.ts

@@ -1,24 +1,32 @@
-import type {LoaderWithParser} from '@loaders.gl/loader-utils';
-import {parsePGM} from '@math.gl/geoid';
+import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import {Geoid, parsePGM} from '@math.gl/geoid';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+export {Geoid};
+
+export type PGMLoaderOptions = LoaderOptions & {
+  pgm?: {
+    cubic?: boolean;
+  };
+};
+
 /**
  * Loader for PGM - Netpbm grayscale image format
  */
-export const PGMLoader: LoaderWithParser = {
+export const PGMLoader: LoaderWithParser<Geoid, never, PGMLoaderOptions> = {
   name: 'PGM - Netpbm grayscale image format',
   id: 'pgm',
   module: 'tile-converter',
   version: VERSION,
   mimeTypes: ['image/x-portable-graymap'],
-  // @ts-expect-error LoaderOptions does not have cubic parameter
-  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options),
+  parse: async (arrayBuffer, options) => parsePGM(new Uint8Array(arrayBuffer), options?.pgm || {}),
   extensions: ['pgm'],
   options: {
-    // TODO - use pgm namespace
-    cubic: false
+    pgm: {
+      cubic: false
+    }
   }
 };

package/src/slpk-extractor/slpk-extractor.ts

@@ -2,7 +2,8 @@ import {isBrowser} from '@loaders.gl/core';
 
 import {BROWSER_ERROR_MESSAGE} from '../constants';
 import {path} from '@loaders.gl/loader-utils';
-import {FileHandleFile, parseZipLocalFileHeader} from '@loaders.gl/zip';
+import {FileHandleFile} from '@loaders.gl/loader-utils';
+import {parseZipLocalFileHeader} from '@loaders.gl/zip';
 import {GZipCompression} from '@loaders.gl/compression';
 import {writeFile} from '../lib/utils/file-utils';