@loaders.gl/tile-converter 3.3.0-alpha.3 → 3.3.0-alpha.4

package/src/i3s-converter/i3s-converter.ts

@@ -2,19 +2,13 @@ import type {Tile3D, Tileset3DProps} from '@loaders.gl/tiles';
 import type {FeatureTableJson} from '@loaders.gl/3d-tiles';
 import type {WriteQueueItem} from '../lib/utils/write-queue';
 import type {
-  AttributeStorageInfo,
   SceneLayer3D,
   BoundingVolumes,
   Node3DIndexDocument,
   NodeReference,
   MaxScreenThresholdSQ,
   NodeInPage,
-  LodSelection,
-  Attribute,
-  ESRIField,
-  Field,
-  PopupInfo,
-  FieldInfo
+  LodSelection
 } from '@loaders.gl/i3s';
 import {load, encode, fetchFile, getLoaderOptions, isBrowser} from '@loaders.gl/core';
 import {Tileset3D} from '@loaders.gl/tiles';
@@ -59,6 +53,13 @@ import {DracoWriterWorker} from '@loaders.gl/draco';
 import WriteQueue from '../lib/utils/write-queue';
 import {I3SAttributesWorker} from '../i3s-attributes-worker';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
+import {
+  createdStorageAttribute,
+  createFieldAttribute,
+  createPopupInfo,
+  getAttributeType,
+  getFieldAttributeType
+} from './helpers/feature-attributes';
 
 const ION_DEFAULT_TOKEN =
   process.env?.IonToken || // eslint-disable-line
@@ -66,10 +67,6 @@ const ION_DEFAULT_TOKEN =
 const HARDCODED_NODES_PER_PAGE = 64;
 const _3D_TILES = '3DTILES';
 const _3D_OBJECT_LAYER_TYPE = '3DObject';
-const STRING_TYPE = 'string';
-const SHORT_INT_TYPE = 'Int32';
-const DOUBLE_TYPE = 'double';
-const OBJECT_ID_TYPE = 'OBJECTID';
 const REFRESH_TOKEN_TIMEOUT = 1800; // 30 minutes in seconds
 const CESIUM_DATASET_PREFIX = 'https://';
 // const FS_FILE_TOO_LARGE = 'ERR_FS_FILE_TOO_LARGE';
@@ -190,7 +187,18 @@ export default class I3SConverter {
 
     try {
       const preloadOptions = await this._fetchPreloadOptions();
-      const tilesetOptions: Tileset3DProps = {loadOptions: {basis: {format: 'rgba32'}}};
+      const tilesetOptions: Tileset3DProps = {
+        loadOptions: {
+          _nodeWorkers: true,
+          reuseWorkers: true,
+          basis: {format: 'rgba32'},
+          'basis-nodejs': {
+            format: 'rgba32',
+            workerUrl: './modules/textures/dist/basis-nodejs-worker.js'
+          },
+          'draco-nodejs': {workerUrl: './modules/draco/dist/draco-nodejs-worker.js'}
+        }
+      };
       if (preloadOptions.headers) {
         tilesetOptions.loadOptions!.fetch = {headers: preloadOptions.headers};
       }
@@ -462,6 +470,74 @@ export default class I3SConverter {
     await this._addNeighborsAndWriteFile(data.parentNode, childNodes);
   }
 
+  /**
+   * Convert nested subtree of 3DTiles dataset
+   * @param param0
+   * @param param0.sourceTile - source 3DTile data
+   * @param param0.parentNode - parent I3S node
+   * @param param0.childNodes - child I3S nodes
+   * @param param0.parentId - parent node ID
+   * @param param0.level - tree level
+   */
+  private async convertNestedTileset({
+    sourceTile,
+    parentNode,
+    childNodes,
+    parentId,
+    level
+  }: {
+    childNodes: NodeReference[];
+    sourceTile: TileHeader;
+    parentNode: Node3DIndexDocument;
+    parentId: number;
+    level: number;
+  }) {
+    await this.sourceTileset!._loadTile(sourceTile);
+    await this._addChildren({
+      parentNode,
+      sourceTiles: sourceTile.children,
+      childNodes,
+      parentId,
+      level: level + 1
+    });
+    await sourceTile.unloadContent();
+  }
+
+  /**
+   * Convert 3DTiles tile to I3S node
+   * @param param0
+   * @param param0.sourceTile - source 3DTile data
+   * @param param0.parentNode - parent I3S node
+   * @param param0.childNodes - child I3S nodes
+   * @param param0.parentId - parent node ID
+   * @param param0.level - tree level
+   */
+  private async convertNode({
+    sourceTile,
+    parentNode,
+    childNodes,
+    parentId,
+    level
+  }: {
+    childNodes: NodeReference[];
+    sourceTile: TileHeader;
+    parentNode: Node3DIndexDocument;
+    parentId: number;
+    level: number;
+  }) {
+    const children = await this._createNode(parentNode, sourceTile, parentId, level);
+    parentNode.children = parentNode.children || [];
+    for (const child of children) {
+      parentNode.children.push({
+        id: child.id,
+        href: `../${child.path}`,
+        obb: child.obb,
+        mbs: child.mbs
+      });
+      childNodes.push(child);
+    }
+  }
+
   /**
    * Add child nodes recursively and write them to files
    * @param data - arguments
@@ -482,30 +558,18 @@ export default class I3SConverter {
     if (this.options.maxDepth && level > this.options.maxDepth) {
       return;
     }
+
+    const promises: Promise<void>[] = [];
+
     for (const sourceTile of sourceTiles) {
       if (sourceTile.type === 'json') {
-        await this.sourceTileset!._loadTile(sourceTile);
-        await this._addChildren({
-          parentNode,
-          sourceTiles: sourceTile.children,
-          childNodes,
-          parentId,
-          level: level + 1
-        });
-        await sourceTile.unloadContent();
+        promises.push(
+          this.convertNestedTileset({sourceTile, parentNode, childNodes, parentId, level})
+        );
       } else {
-        const children = await this._createNode(parentNode, sourceTile, parentId, level);
-        parentNode.children = parentNode.children || [];
-        for (const child of children) {
-          parentNode.children.push({
-            id: child.id,
-            href: `../${child.path}`,
-            obb: child.obb,
-            mbs: child.mbs
-          });
-          childNodes.push(child);
-        }
+        promises.push(this.convertNode({sourceTile, parentNode, childNodes, parentId, level}));
       }
+      await Promise.all(promises);
       if (sourceTile.id) {
         console.log(sourceTile.id); // eslint-disable-line
       }
@@ -572,13 +636,13 @@ export default class I3SConverter {
 
     let boundingVolumes = createBoundingVolumes(sourceTile, this.geoidHeightModel!);
 
-    const propertyTable = getPropertyTable(sourceTile);
+    const propertyTable = getPropertyTable(sourceTile.content);
 
     if (propertyTable && !this.layers0?.attributeStorageInfo?.length) {
       this._convertPropertyTableToNodeAttributes(propertyTable);
     }
 
-    const resourcesData = await this._convertResources(sourceTile, propertyTable);
+    const resourcesData = await this._convertResources(sourceTile, parentId, propertyTable);
 
     const nodes: Node3DIndexDocument[] = [];
     const nodesInPage: NodeInPage[] = [];
@@ -606,7 +670,7 @@ export default class I3SConverter {
         (val) => val.metricType === 'maxScreenThresholdSQ'
       ) || {maxError: 0};
 
-      const nodeInPage = this._createNodeInNodePages(
+      const nodeInPage = this._updateNodeInNodePages(
         maxScreenThresholdSQ,
         boundingVolumes,
         sourceTile,
@@ -651,6 +715,8 @@ export default class I3SConverter {
   /**
    * Convert tile to one or more I3S nodes
    * @param sourceTile - source tile (3DTile)
+   * @param parentId - id of parent node in node pages
+   * @param propertyTable - batch table from b3dm / feature properties from EXT_FEATURE_METADATA
    * result.geometry - ArrayBuffer with geometry attributes
    * result.compressedGeometry - ArrayBuffer with compressed (draco) geometry
    * result.texture - texture image
@@ -662,14 +728,20 @@ export default class I3SConverter {
    */
   private async _convertResources(
     sourceTile: TileHeader,
+    parentId: number,
     propertyTable: FeatureTableJson | null
   ): Promise<I3SConvertedResources[] | null> {
     if (!this.isContentSupported(sourceTile)) {
       return null;
     }
+    const draftObb = {
+      center: [],
+      halfSize: [],
+      quaternion: []
+    };
     const resourcesData = await convertB3dmToI3sGeometry(
       sourceTile.content,
-      Number(this.nodePages.nodesCounter),
+      () => this.nodePages.push({index: 0, obb: draftObb}, parentId),
       propertyTable,
       this.featuresHashArray,
       this.layers0?.attributeStorageInfo,
@@ -682,7 +754,7 @@ export default class I3SConverter {
   }
 
   /**
-   * Create a new node object (https://github.com/Esri/i3s-spec/blob/master/docs/1.7/node.cmn.md)
+   * Update node object (https://github.com/Esri/i3s-spec/blob/master/docs/1.7/node.cmn.md)
    * in node pages (https://github.com/Esri/i3s-spec/blob/master/docs/1.7/nodePage.cmn.md)
    * @param maxScreenThresholdSQ - Level of Details (LOD) metric
    * @param boundingVolumes - Bounding volumes
@@ -695,7 +767,7 @@ export default class I3SConverter {
    * @param resources.featureCount - number of features
    * @return the node object in node pages
    */
-  private _createNodeInNodePages(
+  private _updateNodeInNodePages(
     maxScreenThresholdSQ: MaxScreenThresholdSQ,
     boundingVolumes: BoundingVolumes,
     sourceTile: TileHeader,
@@ -723,7 +795,15 @@ export default class I3SConverter {
         }
       };
     }
-    const nodeId = this.nodePages.push(nodeInPage, parentId);
+
+    let nodeId = resources.nodeId;
+    if (nodeId) {
+      this.nodePages.updateAll(nodeId, nodeInPage);
+      const node = this.nodePages.getNodeById(nodeId);
+      this.nodePages.updateResourceInMesh(node);
+    } else {
+      nodeId = this.nodePages.push(nodeInPage, parentId);
+    }
 
     if (meshMaterial) {
       this.nodePages.updateMaterialByNodeId(nodeId, this._findOrCreateMaterial(meshMaterial));
@@ -743,7 +823,7 @@ export default class I3SConverter {
       this.nodePages.updateFeatureCountByNodeId(nodeId, featureCount);
     }
 
-    return nodeInPage;
+    return this.nodePages.getNodeById(nodeId);
   }
 
   /**
@@ -1067,115 +1147,6 @@ export default class I3SConverter {
     return newMaterialId;
   }
 
-  /**
-   * Generate storage attribute for map segmentation.
-   * @param attributeIndex - order index of attribute (f_0, f_1 ...).
-   * @param key - attribute key from propertyTable.
-   * @param attributeType - attribute type.
-   * @return Updated storageAttribute.
-   */
-  private _createdStorageAttribute(
-    attributeIndex: number,
-    key: string,
-    attributeType: Attribute
-  ): AttributeStorageInfo {
-    const storageAttribute = {
-      key: `f_${attributeIndex}`,
-      name: key,
-      ordering: ['attributeValues'],
-      header: [{property: 'count', valueType: 'UInt32'}],
-      attributeValues: {valueType: 'Int32', valuesPerElement: 1}
-    };
-
-    switch (attributeType) {
-      case OBJECT_ID_TYPE:
-        this._setupIdAttribute(storageAttribute);
-        break;
-      case STRING_TYPE:
-        this._setupStringAttribute(storageAttribute);
-        break;
-      case DOUBLE_TYPE:
-        this._setupDoubleAttribute(storageAttribute);
-        break;
-      case SHORT_INT_TYPE:
-        break;
-      default:
-        this._setupStringAttribute(storageAttribute);
-    }
-
-    return storageAttribute;
-  }
-
-  /**
-   * Get the attribute type for attributeStorageInfo https://github.com/Esri/i3s-spec/blob/master/docs/1.7/attributeStorageInfo.cmn.md
-   * @param key - attribute's key
-   * @param attribute - attribute's type in propertyTable
-   */
-  private getAttributeType(key: string, attribute: string): string {
-    if (key === OBJECT_ID_TYPE) {
-      return OBJECT_ID_TYPE;
-    }
-    if (typeof attribute === STRING_TYPE) {
-      return STRING_TYPE;
-    } else if (typeof attribute === 'number') {
-      return Number.isInteger(attribute) ? SHORT_INT_TYPE : DOUBLE_TYPE;
-    }
-    return STRING_TYPE;
-  }
-
-  /**
-   * Setup storage attribute as string.
-   * @param storageAttribute - attribute for map segmentation.
-   */
-  private _setupStringAttribute(storageAttribute: AttributeStorageInfo): void {
-    storageAttribute.ordering!.unshift('attributeByteCounts');
-    storageAttribute.header.push({property: 'attributeValuesByteCount', valueType: 'UInt32'});
-    storageAttribute.attributeValues = {
-      valueType: 'String',
-      encoding: 'UTF-8',
-      valuesPerElement: 1
-    };
-    storageAttribute.attributeByteCounts = {
-      valueType: 'UInt32',
-      valuesPerElement: 1
-    };
-  }
-
-  /**
-   * Setup Id attribute for map segmentation.
-   * @param storageAttribute - attribute for map segmentation .
-   */
-  private _setupIdAttribute(storageAttribute: AttributeStorageInfo): void {
-    storageAttribute.attributeValues = {
-      valueType: 'Oid32',
-      valuesPerElement: 1
-    };
-  }
-
-  /**
-   * Setup double attribute for map segmentation.
-   * @param storageAttribute - attribute for map segmentation .
-   */
-  private _setupDoubleAttribute(storageAttribute: AttributeStorageInfo): void {
-    storageAttribute.attributeValues = {
-      valueType: 'Float64',
-      valuesPerElement: 1
-    };
-  }
-
-  /**
-   * Setup field attribute for map segmentation.
-   * @param key - attribute for map segmentation.
-   * @param fieldAttributeType - esri attribute type ('esriFieldTypeString' or 'esriFieldTypeOID').
-   */
-  private _createFieldAttribute(key: string, fieldAttributeType: ESRIField): Field {
-    return {
-      name: key,
-      type: fieldAttributeType,
-      alias: key
-    };
-  }
-
   /**
    * Do conversion of 3DTiles property table to I3s node attributes.
    * @param propertyTable - Table with layer meta data.
@@ -1189,12 +1160,12 @@ export default class I3SConverter {
 
     for (const key in propertyTableWithObjectId) {
       const firstAttribute = propertyTableWithObjectId[key][0];
-      const attributeType = this.getAttributeType(key, firstAttribute);
+      const attributeType = getAttributeType(key, firstAttribute);
 
-      const storageAttribute = this._createdStorageAttribute(attributeIndex, key, attributeType);
-      const fieldAttributeType = this._getFieldAttributeType(attributeType);
-      const fieldAttribute = this._createFieldAttribute(key, fieldAttributeType);
-      const popupInfo = this._createPopupInfo(propertyTableWithObjectId);
+      const storageAttribute = createdStorageAttribute(attributeIndex, key, attributeType);
+      const fieldAttributeType = getFieldAttributeType(attributeType);
+      const fieldAttribute = createFieldAttribute(key, fieldAttributeType);
+      const popupInfo = createPopupInfo(propertyTableWithObjectId);
 
       this.layers0!.attributeStorageInfo!.push(storageAttribute);
       this.layers0!.fields!.push(fieldAttribute);
@@ -1205,62 +1176,6 @@ export default class I3SConverter {
     }
   }
 
-  /**
-   * Find and return attribute type based on key form propertyTable.
-   * @param attributeType
-   */
-  private _getFieldAttributeType(attributeType: Attribute): ESRIField {
-    switch (attributeType) {
-      case OBJECT_ID_TYPE:
-        return 'esriFieldTypeOID';
-      case STRING_TYPE:
-        return 'esriFieldTypeString';
-      case SHORT_INT_TYPE:
-        return 'esriFieldTypeInteger';
-      case DOUBLE_TYPE:
-        return 'esriFieldTypeDouble';
-      default:
-        return 'esriFieldTypeString';
-    }
-  }
-
-  /**
-   * Generate popup info to show metadata on the map.
-   * @param propertyTable - table data with OBJECTID.
-   * @return data for correct rendering of popup.
-   */
-  private _createPopupInfo(propertyTable: FeatureTableJson): PopupInfo {
-    const title = '{OBJECTID}';
-    const mediaInfos = [];
-    const fieldInfos: FieldInfo[] = [];
-    const popupElements: {
-      fieldInfos: FieldInfo[];
-      type: string;
-    }[] = [];
-    const expressionInfos = [];
-
-    for (const key in propertyTable) {
-      fieldInfos.push({
-        fieldName: key,
-        visible: true,
-        isEditable: false,
-        label: key
-      });
-    }
-    popupElements.push({
-      fieldInfos,
-      type: 'fields'
-    });
-
-    return {
-      title,
-      mediaInfos,
-      popupElements,
-      fieldInfos,
-      expressionInfos
-    };
-  }
-
   /**
    * Print statistics in the end of conversion
    * @param params - output files data

package/src/i3s-converter/types.ts

@@ -8,6 +8,10 @@ import {ImageDataType} from '@loaders.gl/images';
 
 /** Converted resources for specific node */
 export type I3SConvertedResources = {
+  /**
+   * Node id
+   */
+  nodeId?: number;
   /** Non-compressed geometry buffer that have structure met
    * https://github.com/Esri/i3s-spec/blob/master/docs/1.8/defaultGeometrySchema.cmn.md
    * (Geometry buffer)