@loaders.gl/i3s 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/lib/helpers/i3s-nodepages-tiles.ts

@@ -1,21 +1,34 @@
 import {load} from '@loaders.gl/core';
 import {getSupportedGPUTextureFormats, selectSupportedBasisFormat} from '@loaders.gl/textures';
-import {Tileset, NodePage} from '../../types';
 import {I3SNodePageLoader} from '../../i3s-node-page-loader';
 import {normalizeTileNonUrlData} from '../parsers/parse-i3s';
 import {getUrlWithToken, generateTilesetAttributeUrls} from '../utils/url-utils';
+import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import {
+  I3STilesetHeader,
+  LodSelection,
+  NodePage,
+  NodeInPage,
+  Obb,
+  MeshMaterial,
+  I3SMaterialDefinition,
+  I3STextureFormat,
+  MeshGeometry,
+  I3STileHeader
+} from '../../types';
 
 /**
  * class I3SNodePagesTiles - loads nodePages and form i3s tiles from them
  */
 export default class I3SNodePagesTiles {
-  tileset: Tileset;
+  tileset: I3STilesetHeader;
   nodePages: NodePage[] = [];
   pendingNodePages: {promise: Promise<NodePage>; status: 'Pending' | 'Done'}[] = [];
   nodesPerPage: number;
-  options: {[key: string]: any};
-  lodSelectionMetricType: any;
-  textureDefinitionsSelectedFormats: any[] = [];
+  options: LoaderOptions;
+  lodSelectionMetricType?: string;
+  textureDefinitionsSelectedFormats: ({format: I3STextureFormat; name: string} | null)[] = [];
+  nodesInNodePages: number;
   private textureLoaderOptions: {[key: string]: any} = {};
 
   /**
@@ -24,11 +37,12 @@ export default class I3SNodePagesTiles {
    * @param tileset - i3s tileset header ('layers/0')
    * @param options - i3s loader options
    */
-  constructor(tileset: Tileset, options: object) {
+  constructor(tileset: I3STilesetHeader, options: LoaderOptions) {
     this.tileset = {...tileset}; // spread the tileset to avoid circular reference
-    this.nodesPerPage = tileset.nodePages.nodesPerPage;
-    this.lodSelectionMetricType = tileset.nodePages.lodSelectionMetricType;
+    this.nodesPerPage = tileset.nodePages?.nodesPerPage || 64;
+    this.lodSelectionMetricType = tileset.nodePages?.lodSelectionMetricType;
     this.options = options;
+    this.nodesInNodePages = 0;
 
     this.initSelectedFormatsForTextureDefinitions(tileset);
   }
@@ -37,11 +51,12 @@ export default class I3SNodePagesTiles {
    * Loads some nodePage and return a particular node from it
    * @param id - id of node through all node pages
    */
-  async getNodeById(id: number) {
+  async getNodeById(id: number): Promise<NodeInPage> {
     const pageIndex = Math.floor(id / this.nodesPerPage);
     if (!this.nodePages[pageIndex] && !this.pendingNodePages[pageIndex]) {
       const nodePageUrl = getUrlWithToken(
         `${this.tileset.url}/nodepages/${pageIndex}`,
+        // @ts-expect-error this.options is not properly typed
         this.options.i3s?.token
       );
       this.pendingNodePages[pageIndex] = {
@@ -49,6 +64,7 @@ export default class I3SNodePagesTiles {
         promise: load(nodePageUrl, I3SNodePageLoader, this.options)
       };
       this.nodePages[pageIndex] = await this.pendingNodePages[pageIndex].promise;
+      this.nodesInNodePages += this.nodePages[pageIndex].nodes.length;
       this.pendingNodePages[pageIndex].status = 'Done';
     }
     if (this.pendingNodePages[pageIndex].status === 'Pending') {
@@ -62,35 +78,39 @@ export default class I3SNodePagesTiles {
    * Forms tile header using node and tileset data
    * @param id - id of node through all node pages
    */
-  // eslint-disable-next-line complexity
-  async formTileFromNodePages(id: number) {
-    const node = await this.getNodeById(id);
-    const children: any[] = [];
+  // eslint-disable-next-line complexity, max-statements
+  async formTileFromNodePages(id: number): Promise<I3STileHeader> {
+    const node: NodeInPage = await this.getNodeById(id);
+    const children: {id: string; obb: Obb}[] = [];
+    const childNodesPromises: Promise<NodeInPage>[] = [];
     for (const child of node.children || []) {
-      const childNode = await this.getNodeById(child);
+      childNodesPromises.push(this.getNodeById(child));
+    }
+
+    const childNodes = await Promise.all(childNodesPromises);
+    for (const childNode of childNodes) {
       children.push({
-        id: child,
+        id: childNode.index.toString(),
         obb: childNode.obb
       });
     }
 
-    let contentUrl = null;
-    let textureUrl: string | null = null;
-    let materialDefinition = null;
-    let textureFormat = 'jpeg';
+    let contentUrl: string | undefined;
+    let textureUrl: string | undefined;
+    let materialDefinition: I3SMaterialDefinition | undefined;
+    let textureFormat: I3STextureFormat = 'jpg';
     let attributeUrls: string[] = [];
-    let isDracoGeometry = false;
+    let isDracoGeometry: boolean = false;
 
     if (node && node.mesh) {
       // Get geometry resource URL and type (compressed / non-compressed)
       const {url, isDracoGeometry: isDracoGeometryResult} = (node.mesh.geometry &&
-        this.getContentUrl(node.mesh.geometry)) || {url: null, isDracoGeometry: null};
+        this.getContentUrl(node.mesh.geometry)) || {isDracoGeometry: false};
       contentUrl = url;
       isDracoGeometry = isDracoGeometryResult;
 
-      const [textureData, nodeMaterialDefinition] = this.getInformationFromMaterial(
-        node.mesh.material
-      );
+      const {textureData, materialDefinition: nodeMaterialDefinition} =
+        this.getInformationFromMaterial(node.mesh.material);
       materialDefinition = nodeMaterialDefinition;
       textureFormat = textureData.format || textureFormat;
       if (textureData.name) {
@@ -105,7 +125,7 @@ export default class I3SNodePagesTiles {
     const lodSelection = this.getLodSelection(node);
 
     return normalizeTileNonUrlData({
-      id,
+      id: id.toString(),
       lodSelection,
       obb: node.obb,
       contentUrl,
@@ -121,16 +141,18 @@ export default class I3SNodePagesTiles {
 
   /**
    * Forms url and type of geometry resource by nodepage's data and `geometryDefinitions` in the tileset
-   * @param {Object} meshGeometryData - data about the node's mesh from the nodepage
-   * @returns {Object} -
+   * @param - data about the node's mesh from the nodepage
+   * @returns -
    *   {string} url - url to the geometry resource
    *   {boolean} isDracoGeometry - whether the geometry resource contain DRACO compressed geometry
    */
-  private getContentUrl(meshGeometryData) {
-    let result = {};
+  private getContentUrl(meshGeometryData: MeshGeometry) {
+    let result: {url: string; isDracoGeometry: boolean} | null = null;
+    // @ts-ignore
     const geometryDefinition = this.tileset.geometryDefinitions[meshGeometryData.definition];
     let geometryIndex = -1;
     // Try to find DRACO geometryDefinition of `useDracoGeometry` option is set
+    // @ts-expect-error this.options is not properly typed
     if (this.options.i3s && this.options.i3s.useDracoGeometry) {
       geometryIndex = geometryDefinition.geometryBuffers.findIndex(
         (buffer) => buffer.compressedAttributes && buffer.compressedAttributes.encoding === 'draco'
@@ -156,21 +178,21 @@ export default class I3SNodePagesTiles {
 
   /**
    * Forms 1.6 compatible LOD selection object from a nodepage's node data
-   * @param {Object} node - a node from nodepage
-   * @returns {Object[]} - Array of object of following properties:
-   *   {string} metricType - the label of the LOD metric
-   *   {number} maxError - the value of the metric
+   * @param node - a node from nodepage
+   * @returns- Array of LodSelection
    */
-  private getLodSelection(node): object[] {
-    const lodSelection: object[] = [];
+  private getLodSelection(node: NodeInPage): LodSelection[] {
+    const lodSelection: LodSelection[] = [];
     if (this.lodSelectionMetricType === 'maxScreenThresholdSQ') {
       lodSelection.push({
         metricType: 'maxScreenThreshold',
+        // @ts-ignore
         maxError: Math.sqrt(node.lodThreshold / (Math.PI * 0.25))
       });
     }
     lodSelection.push({
       metricType: this.lodSelectionMetricType,
+      // @ts-ignore
       maxError: node.lodThreshold
     });
     return lodSelection;
@@ -178,43 +200,47 @@ export default class I3SNodePagesTiles {
 
   /**
    * Returns information about texture and material from `materialDefinitions`
-   * @param {Object} material - material data from nodepage
-   * @returns {Object[]} - Couple [textureData, materialDefinition]
+   * @param material - material data from nodepage
+   * @returns - Couple {textureData, materialDefinition}
    * {string} textureData.name - path name of the texture
    * {string} textureData.format - format of the texture
    * materialDefinition - PBR-like material definition from `materialDefinitions`
    */
-  private getInformationFromMaterial(material) {
-    const textureDataDefault = {name: null, format: null};
+  private getInformationFromMaterial(material: MeshMaterial) {
+    const informationFromMaterial: {
+      textureData: {name: string | null; format?: I3STextureFormat};
+      materialDefinition?: I3SMaterialDefinition;
+    } = {textureData: {name: null}};
+
     if (material) {
-      const materialDefinition = this.tileset.materialDefinitions[material.definition];
-      const textureSetDefinitionIndex =
-        materialDefinition &&
-        materialDefinition.pbrMetallicRoughness &&
-        materialDefinition.pbrMetallicRoughness.baseColorTexture &&
-        materialDefinition.pbrMetallicRoughness.baseColorTexture.textureSetDefinitionId;
-      if (textureSetDefinitionIndex || textureSetDefinitionIndex === 0) {
-        const textureData =
-          this.textureDefinitionsSelectedFormats[textureSetDefinitionIndex] || textureDataDefault;
-        return [textureData, materialDefinition];
+      const materialDefinition = this.tileset.materialDefinitions?.[material.definition];
+      if (materialDefinition) {
+        informationFromMaterial.materialDefinition = materialDefinition;
+        const textureSetDefinitionIndex =
+          materialDefinition?.pbrMetallicRoughness?.baseColorTexture?.textureSetDefinitionId;
+
+        if (typeof textureSetDefinitionIndex === 'number') {
+          informationFromMaterial.textureData =
+            this.textureDefinitionsSelectedFormats[textureSetDefinitionIndex] ||
+            informationFromMaterial.textureData;
+        }
       }
-      return [textureDataDefault, materialDefinition];
     }
-    return [textureDataDefault, null];
+    return informationFromMaterial;
   }
 
   /**
    * Sets preferable and supported format for each textureDefinition of the tileset
-   * @param {Object} tileset - I3S layer data
-   * @returns {void}
+   * @param tileset - I3S layer data
+   * @returns
    */
-  private initSelectedFormatsForTextureDefinitions(tileset) {
+  private initSelectedFormatsForTextureDefinitions(tileset: I3STilesetHeader): void {
     this.textureDefinitionsSelectedFormats = [];
     const possibleI3sFormats = this.getSupportedTextureFormats();
     const textureSetDefinitions = tileset.textureSetDefinitions || [];
     for (const textureSetDefinition of textureSetDefinitions) {
       const formats = (textureSetDefinition && textureSetDefinition.formats) || [];
-      let selectedFormat: {format: string; name: string} | null = null;
+      let selectedFormat: {name: string; format: I3STextureFormat} | null = null;
       for (const i3sFormat of possibleI3sFormats) {
         const format = formats.find((value) => value.format === i3sFormat);
         if (format) {
@@ -239,8 +265,9 @@ export default class I3SNodePagesTiles {
    * Returns the array of supported texture format
    * @returns list of format strings
    */
-  private getSupportedTextureFormats(): string[] {
-    const formats: string[] = [];
+  private getSupportedTextureFormats(): I3STextureFormat[] {
+    const formats: I3STextureFormat[] = [];
+    // @ts-expect-error this.options is not properly typed
     if (!this.options.i3s || this.options.i3s.useCompressedTextures) {
       // I3S 1.7 selection
       const supportedCompressedFormats = getSupportedGPUTextureFormats();

package/src/lib/parsers/constants.ts

@@ -1,50 +1,84 @@
 import GL from '@luma.gl/constants';
+import {DATA_TYPE} from '../../types';
 
-export const TYPE_ARRAY_MAP = {
-  UInt8: Uint8Array,
-  UInt16: Uint16Array,
-  UInt32: Uint32Array,
-  Float32: Float32Array,
-  UInt64: Float64Array
-};
+export function getConstructorForDataFormat(dataType: string) {
+  switch (dataType) {
+    case DATA_TYPE.UInt8:
+      return Uint8Array;
+    case DATA_TYPE.UInt16:
+      return Uint16Array;
+    case DATA_TYPE.UInt32:
+      return Uint32Array;
+    case DATA_TYPE.Float32:
+      return Float32Array;
+    case DATA_TYPE.UInt64:
+      return Float64Array;
+    default:
+      throw new Error(`parse i3s tile content: unknown type of data: ${dataType}`);
+  }
+}
 
-export const GL_TYPE_MAP = {
+export const GL_TYPE_MAP: {[key: string]: number} = {
   UInt8: GL.UNSIGNED_BYTE,
-  UInt16: GL.UNSIGNED_INT,
+  UInt16: GL.UNSIGNED_SHORT,
   Float32: GL.FLOAT,
   UInt32: GL.UNSIGNED_INT,
   UInt64: GL.DOUBLE
 };
-
-export const I3S_NAMED_VERTEX_ATTRIBUTES = {
-  position: 'position',
-  normal: 'normal',
-  uv0: 'uv0',
-  color: 'color',
-  region: 'region'
-};
-
-export const I3S_NAMED_GEOMETRY_ATTRIBUTES = {
-  vertexAttributes: 'vertexAttributes',
-  featureAttributeOrder: 'featureAttributeOrder',
-  featureAttributes: 'featureAttributes'
-};
-
-export const I3S_NAMED_HEADER_ATTRIBUTES = {
-  header: 'header',
-  vertexCount: 'vertexCount',
-  featureCount: 'featureCount'
-};
-
-export const SIZEOF = {
-  UInt8: 1,
-  UInt16: 2,
-  UInt32: 4,
-  Float32: 4,
-  UInt64: 8
-};
+/**
+ * Returns how many bytes a type occupies
+ * @param dataType
+ * @returns
+ */
+export function sizeOf(dataType: string): number {
+  switch (dataType) {
+    case DATA_TYPE.UInt8:
+      return 1;
+    case DATA_TYPE.UInt16:
+    case DATA_TYPE.Int16:
+      return 2;
+    case DATA_TYPE.UInt32:
+    case DATA_TYPE.Int32:
+    case DATA_TYPE.Float32:
+      return 4;
+    case DATA_TYPE.UInt64:
+    case DATA_TYPE.Int64:
+    case DATA_TYPE.Float64:
+      return 8;
+    default:
+      throw new Error(`parse i3s tile content: unknown size of data: ${dataType}`);
+  }
+}
 
 export const STRING_ATTRIBUTE_TYPE = 'String';
 export const OBJECT_ID_ATTRIBUTE_TYPE = 'Oid32';
 export const FLOAT_64_TYPE = 'Float64';
 export const INT_16_ATTRIBUTE_TYPE = 'Int16';
+
+// https://github.com/visgl/deck.gl/blob/9548f43cba2234a1f4877b6b17f6c88eb35b2e08/modules/core/src/lib/constants.js#L27
+// Describes the format of positions
+export enum COORDINATE_SYSTEM {
+  /**
+   * `LNGLAT` if rendering into a geospatial viewport, `CARTESIAN` otherwise
+   */
+  DEFAULT = -1,
+  /**
+   * Positions are interpreted as [lng, lat, elevation]
+   * lng lat are degrees, elevation is meters. distances as meters.
+   */
+  LNGLAT = 1,
+  /**
+   * Positions are interpreted as meter offsets, distances as meters
+   */
+  METER_OFFSETS = 2,
+  /**
+   * Positions are interpreted as lng lat offsets: [deltaLng, deltaLat, elevation]
+   * deltaLng, deltaLat are delta degrees, elevation is meters.
+   * distances as meters.
+   */
+  LNGLAT_OFFSETS = 3,
+  /**
+   * Non-geospatial
+   */
+  CARTESIAN = 0
+}

package/src/lib/parsers/parse-arcgis-webscene.ts

@@ -0,0 +1,102 @@
+import {JSONLoader, load} from '@loaders.gl/core';
+import type {ArcGisWebSceneData, OperationalLayer} from '../../types';
+
+/**
+ * WKID, or Well-Known ID, of the CRS. Specify either WKID or WKT of the CRS.
+ * Spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.8/spatialReference.cmn.md
+ */
+const SUPPORTED_WKID = 4326;
+
+const ARCGIS_SCENE_SERVER_LAYER_TYPE = 'ArcGISSceneServiceLayer';
+const BUILDING_SCENE_LAYER = 'BuildingSceneLayer';
+const INTEGRATED_MESH_LAYER = 'IntegratedMeshLayer';
+const GROUP_LAYER = 'GroupLayer';
+
+/**
+ * Supported layers list
+ * Possible operational layers in WebScene: https://developers.arcgis.com/web-scene-specification/objects/operationalLayers/
+ */
+const SUPPORTED_LAYERS_TYPES = [
+  ARCGIS_SCENE_SERVER_LAYER_TYPE,
+  INTEGRATED_MESH_LAYER,
+  BUILDING_SCENE_LAYER,
+  GROUP_LAYER
+];
+
+const NO_AVAILABLE_SUPPORTED_LAYERS_ERROR = 'NO_AVAILABLE_SUPPORTED_LAYERS_ERROR';
+const NOT_SUPPORTED_CRS_ERROR = 'NOT_SUPPORTED_CRS_ERROR';
+
+/**
+ * Parses ArcGIS WebScene
+ * @param data
+ */
+export async function parseWebscene(data: ArrayBuffer): Promise<ArcGisWebSceneData> {
+  const layer0 = JSON.parse(new TextDecoder().decode(data));
+  const {operationalLayers} = layer0;
+  const {layers, unsupportedLayers} = await parseOperationalLayers(operationalLayers, true);
+
+  if (!layers.length) {
+    throw new Error(NO_AVAILABLE_SUPPORTED_LAYERS_ERROR);
+  }
+
+  return {
+    header: layer0,
+    layers,
+    unsupportedLayers
+  };
+}
+
+/**
+ * Recursively parses WebScene operational layers.
+ * @param layersList
+ */
+async function parseOperationalLayers(
+  layersList: OperationalLayer[],
+  needToCheckCRS: boolean
+): Promise<{layers: OperationalLayer[]; unsupportedLayers: OperationalLayer[]}> {
+  const layers: OperationalLayer[] = [];
+  let unsupportedLayers: OperationalLayer[] = [];
+
+  for (let index = 0; index < layersList.length; index++) {
+    const layer = layersList[index];
+    const isLayerSupported = SUPPORTED_LAYERS_TYPES.includes(layer.layerType);
+
+    if (isLayerSupported) {
+      if (needToCheckCRS && layer.layerType !== GROUP_LAYER) {
+        await checkSupportedIndexCRS(layer);
+        needToCheckCRS = false;
+      }
+
+      layers.push(layer);
+    } else {
+      unsupportedLayers.push(layer);
+    }
+
+    if (layer.layers?.length) {
+      const {layers: childLayers, unsupportedLayers: childUnsupportedLayers} =
+        await parseOperationalLayers(layer.layers, needToCheckCRS);
+      layer.layers = childLayers;
+      unsupportedLayers = [...unsupportedLayers, ...childUnsupportedLayers];
+    }
+  }
+
+  return {layers, unsupportedLayers};
+}
+
+/**
+ * Check if layer has supported CRS
+ * @param layer
+ */
+async function checkSupportedIndexCRS(layer: OperationalLayer) {
+  try {
+    const layerJson = await load(layer.url, JSONLoader);
+    // @ts-expect-error
+    const wkid = layerJson?.spatialReference?.wkid;
+
+    if (wkid !== SUPPORTED_WKID) {
+      throw new Error(NOT_SUPPORTED_CRS_ERROR);
+    }
+  } catch (error) {
+    throw error;
+  }
+}

package/src/lib/parsers/parse-i3s-attribute.ts

@@ -1,3 +1,7 @@
+// loaders.gl, MIT license
+
+import {TypedArray} from '@loaders.gl/schema';
+
 import {
   STRING_ATTRIBUTE_TYPE,
   OBJECT_ID_ATTRIBUTE_TYPE,
@@ -5,13 +9,16 @@ import {
   INT_16_ATTRIBUTE_TYPE
 } from './constants';
 
+type Attribute = string[] | TypedArray | null;
+export type I3STileAttributes = Record<string, Attribute>;
+
 /**
  * Get particular tile and creates attribute object inside.
- * @param {ArrayBuffer} arrayBuffer
+ * @param  arrayBuffer
  * @param {Object} options
  * @returns {Promise<object>}
  */
-export async function parseI3STileAttribute(arrayBuffer, options) {
+export function parseI3STileAttribute(arrayBuffer: ArrayBuffer, options): I3STileAttributes {
   const {attributeName, attributeType} = options;
 
   if (!attributeName) {
@@ -25,10 +32,10 @@ export async function parseI3STileAttribute(arrayBuffer, options) {
 /**
  * Parse attributes based on attribute type.
  * @param {String} attributeType
- * @param {ArrayBuffer} arrayBuffer
- * @returns {any}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseAttribute(attributeType, arrayBuffer) {
+function parseAttribute(attributeType, arrayBuffer: ArrayBuffer): Attribute {
   switch (attributeType) {
     case STRING_ATTRIBUTE_TYPE:
       return parseStringsAttribute(arrayBuffer);
@@ -46,10 +53,10 @@ function parseAttribute(attributeType, arrayBuffer) {
 /**
  * Parse short number attribute.
  * Short Integer spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.7/attributeStorageInfo.cmn.md
- * @param {ArrayBuffer} arrayBuffer
- * @returns {Uint32Array}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseShortNumberAttribute(arrayBuffer) {
+function parseShortNumberAttribute(arrayBuffer: ArrayBuffer): Uint32Array {
   const countOffset = 4;
   return new Uint32Array(arrayBuffer, countOffset);
 }
@@ -57,10 +64,10 @@ function parseShortNumberAttribute(arrayBuffer) {
 /**
  * Parse Int16 short number attribute.
  * Parsing of such data is not documented. Added to handle Building Scene Layer Tileset attributes data.
- * @param {ArrayBuffer} arrayBuffer
- * @returns {Int16Array}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseInt16ShortNumberAttribute(arrayBuffer) {
+function parseInt16ShortNumberAttribute(arrayBuffer: ArrayBuffer): Int16Array {
   const countOffset = 4;
   return new Int16Array(arrayBuffer, countOffset);
 }
@@ -68,10 +75,10 @@ function parseInt16ShortNumberAttribute(arrayBuffer) {
 /**
  * Parse float attribute.
  * Double Spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.7/attributeStorageInfo.cmn.md
- * @param {ArrayBuffer} arrayBuffer
- * @returns {Float64Array}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseFloatAttribute(arrayBuffer) {
+function parseFloatAttribute(arrayBuffer: ArrayBuffer): Float64Array {
   const countOffset = 8;
   return new Float64Array(arrayBuffer, countOffset);
 }

package/src/lib/parsers/parse-i3s-building-scene-layer.ts

@@ -34,7 +34,7 @@ function parseSublayersTree(
 
   for (let index = 0; index < sublayers.length; index++) {
     const subLayer = sublayers[index];
-    const {id, layerType, ...rest} = subLayer;
+    const {id, layerType, visibility = true, ...rest} = subLayer;
 
     // Add support only for 3DObject layer type for I3S purposes.
     if (layerType === OBJECT_3D_LAYER_TYPE) {
@@ -44,6 +44,7 @@ function parseSublayersTree(
         url: sublayerUrl,
         id,
         layerType,
+        visibility,
         ...rest
       });
     }