@loaders.gl/tile-converter 3.3.0-alpha.3 → 3.3.0-alpha.4

package/src/3d-tiles-converter/3d-tiles-converter.ts

@@ -15,12 +15,11 @@ import {convertScreenThresholdToGeometricError} from '../lib/utils/lod-conversio
 import {writeFile, removeDir} from '../lib/utils/file-utils';
 import {calculateFilesSize, timeConverter} from '../lib/utils/statistic-utills';
 import {TILESET as tilesetTemplate} from './json-templates/tileset';
-import B3dmConverter from './helpers/b3dm-converter';
 import {createObbFromMbs} from '../i3s-converter/helpers/coordinate-converter';
 import {
   I3SAttributesData,
-  Tile3dAttributesWorker
-  /*transform3DTilesAttributesOnWorker*/
+  Tile3dAttributesWorker,
+  transform3DTilesAttributesOnWorker
 } from '../3d-tiles-attributes-worker';
 import {getWorkerURL, WorkerFarm} from '@loaders.gl/worker-utils';
 import {BROWSER_ERROR_MESSAGE} from '../constants';
@@ -85,6 +84,11 @@ export default class Tiles3DConverter {
 
     this.sourceTileset = new Tileset3D(sourceTilesetJson, {
       loadOptions: {
+        _nodeWorkers: true,
+        reuseWorkers: true,
+        'i3s-content-nodejs': {
+          workerUrl: './modules/i3s/dist/i3s-content-nodejs-worker.js'
+        },
         i3s: {coordinateSystem: COORDINATE_SYSTEM.LNGLAT_OFFSETS, decodeTextures: false}
       }
     });
@@ -124,6 +128,67 @@ export default class Tiles3DConverter {
     workerFarm.destroy();
   }
 
+  /**
+   * Convert particular I3S Node
+   * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
+   * @param parentNode object in resulting tileset
+   * @param level a current level of a tree depth
+   * @param childNodeInfo child node to convert
+   */
+  private async convertChildNode(
+    parentSourceNode: Tile3D,
+    parentNode: Node3D,
+    level: number,
+    childNodeInfo: NodeReference
+  ): Promise<void> {
+    const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
+    parentSourceNode.children.push(sourceChild);
+    if (sourceChild.contentUrl) {
+      await this.sourceTileset!._loadTile(sourceChild);
+      this.vertexCounter += sourceChild.content.vertexCount;
+
+      let featureAttributes: FeatureAttribute | null = null;
+      if (this.attributeStorageInfo) {
+        featureAttributes = await this._loadChildAttributes(sourceChild, this.attributeStorageInfo);
+      }
+
+      if (!sourceChild.header.obb) {
+        sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
+      }
+
+      const boundingVolume = {
+        box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
+      };
+      const child: Node3D = {
+        boundingVolume,
+        geometricError: convertScreenThresholdToGeometricError(sourceChild),
+        children: []
+      };
+
+      const i3sAttributesData: I3SAttributesData = {
+        tileContent: sourceChild.content,
+        textureFormat: sourceChild?.header?.textureFormat
+      };
+
+      const b3dm = await transform3DTilesAttributesOnWorker(i3sAttributesData, {
+        source: this.workerSource.tile3dWorkerSource,
+        featureAttributes
+      });
+
+      child.content = {
+        uri: `${sourceChild.id}.b3dm`,
+        boundingVolume
+      };
+      await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
+      parentNode.children.push(child);
+
+      sourceChild.unloadContent();
+      await this._addChildren(sourceChild, child, level + 1);
+    } else {
+      await this._addChildren(sourceChild, parentNode, level + 1);
+    }
+  }
+
   /**
    * The recursive function of traversal of a nodes tree
    * @param parentSourceNode the parent node tile object (@loaders.gl/tiles/Tile3D)
@@ -138,60 +203,11 @@ export default class Tiles3DConverter {
     if (this.options.maxDepth && level > this.options.maxDepth) {
       return;
     }
+    const promises: Promise<void>[] = [];
     for (const childNodeInfo of parentSourceNode.header.children || []) {
-      const sourceChild = await this._loadChildNode(parentSourceNode, childNodeInfo);
-      parentSourceNode.children.push(sourceChild);
-      if (sourceChild.contentUrl) {
-        await this.sourceTileset!._loadTile(sourceChild);
-        this.vertexCounter += sourceChild.content.vertexCount;
-
-        let featureAttributes: FeatureAttribute | null = null;
-        if (this.attributeStorageInfo) {
-          featureAttributes = await this._loadChildAttributes(
-            sourceChild,
-            this.attributeStorageInfo
-          );
-        }
-
-        if (!sourceChild.header.obb) {
-          sourceChild.header.obb = createObbFromMbs(sourceChild.header.mbs);
-        }
-
-        const boundingVolume = {
-          box: i3sObbTo3dTilesObb(sourceChild.header.obb, this.geoidHeightModel)
-        };
-        const child: Node3D = {
-          boundingVolume,
-          geometricError: convertScreenThresholdToGeometricError(sourceChild),
-          children: []
-        };
-
-        const i3sAttributesData: I3SAttributesData = {
-          tileContent: sourceChild.content,
-          textureFormat: sourceChild?.header?.textureFormat
-        };
-
-        // TODO Uncomment when 3d-tiles-attributes-worker will be published on CDN.
-        // const b3dm = await transform3DTilesAttributesOnWorker(i3sAttributesData, {
-        //   source: this.workerSource.tile3dWorkerSource,
-        //   featureAttributes
-        // });
-
-        const b3dm = await new B3dmConverter().convert(i3sAttributesData, featureAttributes);
-
-        child.content = {
-          uri: `${sourceChild.id}.b3dm`,
-          boundingVolume
-        };
-        await writeFile(this.tilesetPath, new Uint8Array(b3dm), `${sourceChild.id}.b3dm`);
-        parentNode.children.push(child);
-
-        sourceChild.unloadContent();
-        await this._addChildren(sourceChild, child, level + 1);
-      } else {
-        await this._addChildren(sourceChild, parentNode, level + 1);
-      }
+      promises.push(this.convertChildNode(parentSourceNode, parentNode, level, childNodeInfo));
     }
+    await Promise.all(promises);
   }
 
   /**

package/src/deps-installer/deps-installer.ts

@@ -1,8 +1,11 @@
-import {load} from '@loaders.gl/core';
+import {load, fetchFile} from '@loaders.gl/core';
 import {ZipLoader} from '@loaders.gl/zip';
 import {writeFile} from '../lib/utils/file-utils';
 import {join} from 'path';
 
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'beta';
+
 const PGM_LINK = 'https://raw.githubusercontent.com/visgl/deck.gl-data/master/egm/egm2008-5.zip';
 
 /**
@@ -15,8 +18,11 @@ export class DepsInstaller {
   /**
    * Run instalation
    * @param path destination folder
+   * @param workersPath destination folder for workers.
+   *    This path is '' by default and is not used by tile-converter.
+   *    It is used in tests to prevent rewriting actual workers during tests running
    */
-  async install(path: string = ''): Promise<void> {
+  async install(path: string = '', workersPath: string = ''): Promise<void> {
     console.log('Installing "EGM2008-5" model...'); // eslint-disable-line no-console
     const fileMap = await load(PGM_LINK, ZipLoader, {});
 
@@ -27,6 +33,27 @@ export class DepsInstaller {
 
     await writeFile(depsPath, new Uint8Array(fileMap['geoids/egm2008-5.pgm']), 'egm2008-5.pgm');
 
+    console.log('Installing "I3S Content Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('i3s', 'i3s-content-nodejs-worker.js', workersPath);
+
+    console.log('Installing "Draco Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('draco', 'draco-nodejs-worker.js', workersPath);
+
+    console.log('Installing "Basis Loader worker"'); // eslint-disable-line no-console
+    await this.installWorker('textures', 'basis-nodejs-worker.js', workersPath);
+
     console.log('All dependencies were installed succesfully.'); // eslint-disable-line no-console
   }
+
+  private async installWorker(module: string, name: string, extraPath: string) {
+    const fileResponse = await fetchFile(
+      `https://unpkg.com/@loaders.gl/${module}@${VERSION}/dist/${name}`
+    );
+    const fileData = await fileResponse.arrayBuffer();
+    if (!fileData) {
+      return;
+    }
+    const path = join(process.cwd(), extraPath, 'modules', module, 'dist');
+    await writeFile(path, fileData, name);
+  }
 }

package/src/i3s-attributes-worker.ts

@@ -1,6 +1,7 @@
 import type {WorkerObject} from '@loaders.gl/worker-utils';
 import type {ConvertedAttributes} from './i3s-converter/types';
 import type {Matrix4, Vector3} from '@math.gl/core';
+import type {GLTFImagePostprocessed, GLTFNodePostprocessed} from '@loaders.gl/gltf';
 
 import {processOnWorker} from '@loaders.gl/worker-utils';
 
@@ -9,13 +10,16 @@ import {processOnWorker} from '@loaders.gl/worker-utils';
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 export type I3SAttributesWorkerOptions = {
+  _nodeWorkers: boolean;
+  reuseWorkers: boolean;
   useCartesianPositions: boolean;
   source: string;
 };
 
 export type B3DMAttributesData = {
   gltfMaterials?: {id: string}[];
-  nodes: any;
+  nodes: GLTFNodePostprocessed[];
+  images: GLTFImagePostprocessed[];
   cartographicOrigin: Vector3;
   cartesianModelMatrix: Matrix4;
 };

package/src/i3s-converter/helpers/batch-ids-extensions.ts

@@ -1,5 +1,8 @@
-import type {GLTFAccessorPostprocessed} from 'modules/gltf/src/lib/types/gltf-types';
-import type {Image, MeshPrimitive} from 'modules/gltf/src/lib/types/gltf-postprocessed-schema';
+import {
+  GLTFAccessorPostprocessed,
+  GLTFImagePostprocessed,
+  GLTFMeshPrimitivePostprocessed
+} from '@loaders.gl/gltf';
 import type {
   GLTF_EXT_feature_metadata_attribute,
   GLTF_EXT_feature_metadata_primitive
@@ -18,8 +21,8 @@ export function handleBatchIdsExtensions(
   attributes: {
     [key: string]: GLTFAccessorPostprocessed;
   },
-  primitive: MeshPrimitive,
-  images: Image[]
+  primitive: GLTFMeshPrimitivePostprocessed,
+  images: GLTFImagePostprocessed[]
 ): number[] {
   const extensions = primitive?.extensions;
 
@@ -58,7 +61,7 @@ function handleExtFeatureMetadataExtension(
     [key: string]: GLTFAccessorPostprocessed;
   },
   extFeatureMetadata: GLTF_EXT_feature_metadata_primitive,
-  images: Image[]
+  images: GLTFImagePostprocessed[]
 ): number[] {
   // Take only first extension object to get batchIds attribute name.
   const featureIdAttribute = extFeatureMetadata?.featureIdAttributes?.[0];
@@ -149,8 +152,12 @@ function generateImplicitFeatureIds(
 function generateBatchIdsFromTexture(
   featureIdTexture: GLTF_EXT_feature_metadata_attribute,
   textureCoordinates: Float32Array,
-  images: Image[]
+  images: GLTFImagePostprocessed[]
 ) {
+  if (!images?.length) {
+    return [];
+  }
+
   const CHANNELS_MAP = {
     r: 0,
     g: 1,

package/src/i3s-converter/helpers/coordinate-converter.ts

@@ -8,7 +8,7 @@ import {
   makeBoundingSphereFromPoints,
   BoundingSphere
 } from '@math.gl/culling';
-import TileHeader from '@loaders.gl/tiles/src/tileset/tile-3d';
+import {Tile3D} from '@loaders.gl/tiles';
 import {Geoid} from '@math.gl/geoid';
 
 /**
@@ -17,7 +17,7 @@ import {Geoid} from '@math.gl/geoid';
  * @param geoidHeightModel
  * @returns - Bounding volumes object
  */
-export function createBoundingVolumes(tile: TileHeader, geoidHeightModel: Geoid): BoundingVolumes {
+export function createBoundingVolumes(tile: Tile3D, geoidHeightModel: Geoid): BoundingVolumes {
   let radius;
   let halfSize;
   let quaternion;

package/src/i3s-converter/helpers/feature-attributes.ts

@@ -1,4 +1,12 @@
 import type {FeatureTableJson} from '@loaders.gl/3d-tiles';
+import {
+  Attribute,
+  AttributeStorageInfo,
+  ESRIField,
+  Field,
+  FieldInfo,
+  PopupInfo
+} from '@loaders.gl/i3s';
 
 /**
  * Takes attributes from property table based on featureIds.
@@ -63,3 +71,177 @@ export function checkPropertiesLength(
 
   return needFlatten;
 }
+
+/** String data type name for feature attributes */
+const STRING_TYPE = 'string';
+/** Integer data type name for feature attributes */
+const SHORT_INT_TYPE = 'Int32';
+/** Double data type name for feature attributes */
+const DOUBLE_TYPE = 'double';
+/** Type of attribute that is linked with feature ids */
+const OBJECT_ID_TYPE = 'OBJECTID';
+/**
+ * Get the attribute type for attributeStorageInfo https://github.com/Esri/i3s-spec/blob/master/docs/1.7/attributeStorageInfo.cmn.md
+ * @param key - attribute's key
+ * @param attribute - attribute's type in propertyTable
+ */
+export function getAttributeType(key: string, attribute: string): string {
+  if (key === OBJECT_ID_TYPE) {
+    return OBJECT_ID_TYPE;
+  }
+  if (typeof attribute === STRING_TYPE) {
+    return STRING_TYPE;
+  } else if (typeof attribute === 'number') {
+    return Number.isInteger(attribute) ? SHORT_INT_TYPE : DOUBLE_TYPE;
+  }
+  return STRING_TYPE;
+}
+
+/**
+ * Generate storage attribute for map segmentation.
+ * @param attributeIndex - order index of attribute (f_0, f_1 ...).
+ * @param key - attribute key from propertyTable.
+ * @param attributeType - attribute type.
+ * @return Updated storageAttribute.
+ */
+export function createdStorageAttribute(
+  attributeIndex: number,
+  key: string,
+  attributeType: Attribute
+): AttributeStorageInfo {
+  const storageAttribute = {
+    key: `f_${attributeIndex}`,
+    name: key,
+    ordering: ['attributeValues'],
+    header: [{property: 'count', valueType: 'UInt32'}],
+    attributeValues: {valueType: 'Int32', valuesPerElement: 1}
+  };
+
+  switch (attributeType) {
+    case OBJECT_ID_TYPE:
+      setupIdAttribute(storageAttribute);
+      break;
+    case STRING_TYPE:
+      setupStringAttribute(storageAttribute);
+      break;
+    case DOUBLE_TYPE:
+      setupDoubleAttribute(storageAttribute);
+      break;
+    case SHORT_INT_TYPE:
+      break;
+    default:
+      setupStringAttribute(storageAttribute);
+  }
+
+  return storageAttribute;
+}
+
+/**
+ * Find and return attribute type based on key form propertyTable.
+ * @param attributeType
+ */
+export function getFieldAttributeType(attributeType: Attribute): ESRIField {
+  switch (attributeType) {
+    case OBJECT_ID_TYPE:
+      return 'esriFieldTypeOID';
+    case STRING_TYPE:
+      return 'esriFieldTypeString';
+    case SHORT_INT_TYPE:
+      return 'esriFieldTypeInteger';
+    case DOUBLE_TYPE:
+      return 'esriFieldTypeDouble';
+    default:
+      return 'esriFieldTypeString';
+  }
+}
+
+/**
+ * Setup field attribute for map segmentation.
+ * @param key - attribute for map segmentation.
+ * @param fieldAttributeType - esri attribute type ('esriFieldTypeString' or 'esriFieldTypeOID').
+ */
+export function createFieldAttribute(key: string, fieldAttributeType: ESRIField): Field {
+  return {
+    name: key,
+    type: fieldAttributeType,
+    alias: key
+  };
+}
+
+/**
+ * Generate popup info to show metadata on the map.
+ * @param propertyTable - table data with OBJECTID.
+ * @return data for correct rendering of popup.
+ */
+export function createPopupInfo(propertyTable: FeatureTableJson): PopupInfo {
+  const title = '{OBJECTID}';
+  const mediaInfos = [];
+  const fieldInfos: FieldInfo[] = [];
+  const popupElements: {
+    fieldInfos: FieldInfo[];
+    type: string;
+  }[] = [];
+  const expressionInfos = [];
+
+  for (const key in propertyTable) {
+    fieldInfos.push({
+      fieldName: key,
+      visible: true,
+      isEditable: false,
+      label: key
+    });
+  }
+  popupElements.push({
+    fieldInfos,
+    type: 'fields'
+  });
+
+  return {
+    title,
+    mediaInfos,
+    popupElements,
+    fieldInfos,
+    expressionInfos
+  };
+}
+
+/**
+ * Setup storage attribute as string.
+ * @param storageAttribute - attribute for map segmentation.
+ */
+function setupStringAttribute(storageAttribute: AttributeStorageInfo): void {
+  // @ts-expect-error
+  storageAttribute.ordering.unshift('attributeByteCounts');
+  storageAttribute.header.push({property: 'attributeValuesByteCount', valueType: 'UInt32'});
+  storageAttribute.attributeValues = {
+    valueType: 'String',
+    encoding: 'UTF-8',
+    valuesPerElement: 1
+  };
+  storageAttribute.attributeByteCounts = {
+    valueType: 'UInt32',
+    valuesPerElement: 1
+  };
+}
+
+/**
+ * Setup Id attribute for map segmentation.
+ * @param storageAttribute - attribute for map segmentation .
+ */
+function setupIdAttribute(storageAttribute: AttributeStorageInfo): void {
+  storageAttribute.attributeValues = {
+    valueType: 'Oid32',
+    valuesPerElement: 1
+  };
+}
+
+/**
+ * Setup double attribute for map segmentation.
+ * @param storageAttribute - attribute for map segmentation .
+ */
+function setupDoubleAttribute(storageAttribute: AttributeStorageInfo): void {
+  storageAttribute.attributeValues = {
+    valueType: 'Float64',
+    valuesPerElement: 1
+  };
+}