@loaders.gl/i3s 4.0.0-alpha.5 → 4.0.0-alpha.7

package/src/lib/parsers/parse-arcgis-webscene.ts

@@ -0,0 +1,102 @@
+import {JSONLoader, load} from '@loaders.gl/core';
+import type {ArcGisWebSceneData, OperationalLayer} from '../../types';
+
+/**
+ * WKID, or Well-Known ID, of the CRS. Specify either WKID or WKT of the CRS.
+ * Spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.8/spatialReference.cmn.md
+ */
+const SUPPORTED_WKID = 4326;
+
+const ARCGIS_SCENE_SERVER_LAYER_TYPE = 'ArcGISSceneServiceLayer';
+const BUILDING_SCENE_LAYER = 'BuildingSceneLayer';
+const INTEGRATED_MESH_LAYER = 'IntegratedMeshLayer';
+const GROUP_LAYER = 'GroupLayer';
+
+/**
+ * Supported layers list
+ * Possible operational layers in WebScene: https://developers.arcgis.com/web-scene-specification/objects/operationalLayers/
+ */
+const SUPPORTED_LAYERS_TYPES = [
+  ARCGIS_SCENE_SERVER_LAYER_TYPE,
+  INTEGRATED_MESH_LAYER,
+  BUILDING_SCENE_LAYER,
+  GROUP_LAYER
+];
+
+const NO_AVAILABLE_SUPPORTED_LAYERS_ERROR = 'NO_AVAILABLE_SUPPORTED_LAYERS_ERROR';
+const NOT_SUPPORTED_CRS_ERROR = 'NOT_SUPPORTED_CRS_ERROR';
+
+/**
+ * Parses ArcGIS WebScene
+ * @param data
+ */
+export async function parseWebscene(data: ArrayBuffer): Promise<ArcGisWebSceneData> {
+  const layer0 = JSON.parse(new TextDecoder().decode(data));
+  const {operationalLayers} = layer0;
+  const {layers, unsupportedLayers} = await parseOperationalLayers(operationalLayers, true);
+
+  if (!layers.length) {
+    throw new Error(NO_AVAILABLE_SUPPORTED_LAYERS_ERROR);
+  }
+
+  return {
+    header: layer0,
+    layers,
+    unsupportedLayers
+  };
+}
+
+/**
+ * Recursively parses WebScene operational layers.
+ * @param layersList
+ */
+async function parseOperationalLayers(
+  layersList: OperationalLayer[],
+  needToCheckCRS: boolean
+): Promise<{layers: OperationalLayer[]; unsupportedLayers: OperationalLayer[]}> {
+  const layers: OperationalLayer[] = [];
+  let unsupportedLayers: OperationalLayer[] = [];
+
+  for (let index = 0; index < layersList.length; index++) {
+    const layer = layersList[index];
+    const isLayerSupported = SUPPORTED_LAYERS_TYPES.includes(layer.layerType);
+
+    if (isLayerSupported) {
+      if (needToCheckCRS && layer.layerType !== GROUP_LAYER) {
+        await checkSupportedIndexCRS(layer);
+        needToCheckCRS = false;
+      }
+
+      layers.push(layer);
+    } else {
+      unsupportedLayers.push(layer);
+    }
+
+    if (layer.layers?.length) {
+      const {layers: childLayers, unsupportedLayers: childUnsupportedLayers} =
+        await parseOperationalLayers(layer.layers, needToCheckCRS);
+      layer.layers = childLayers;
+      unsupportedLayers = [...unsupportedLayers, ...childUnsupportedLayers];
+    }
+  }
+
+  return {layers, unsupportedLayers};
+}
+
+/**
+ * Check if layer has supported CRS
+ * @param layer
+ */
+async function checkSupportedIndexCRS(layer: OperationalLayer) {
+  try {
+    const layerJson = await load(layer.url, JSONLoader);
+    // @ts-expect-error
+    const wkid = layerJson?.spatialReference?.wkid;
+
+    if (wkid !== SUPPORTED_WKID) {
+      throw new Error(NOT_SUPPORTED_CRS_ERROR);
+    }
+  } catch (error) {
+    throw error;
+  }
+}

package/src/lib/parsers/parse-i3s-attribute.ts

@@ -1,3 +1,7 @@
+// loaders.gl, MIT license
+
+import {TypedArray} from '@loaders.gl/schema';
+
 import {
   STRING_ATTRIBUTE_TYPE,
   OBJECT_ID_ATTRIBUTE_TYPE,
@@ -5,13 +9,16 @@ import {
   INT_16_ATTRIBUTE_TYPE
 } from './constants';
 
+type Attribute = string[] | TypedArray | null;
+export type I3STileAttributes = Record<string, Attribute>;
+
 /**
  * Get particular tile and creates attribute object inside.
- * @param {ArrayBuffer} arrayBuffer
+ * @param  arrayBuffer
  * @param {Object} options
  * @returns {Promise<object>}
  */
-export async function parseI3STileAttribute(arrayBuffer, options) {
+export function parseI3STileAttribute(arrayBuffer: ArrayBuffer, options): I3STileAttributes {
   const {attributeName, attributeType} = options;
 
   if (!attributeName) {
@@ -25,10 +32,10 @@ export async function parseI3STileAttribute(arrayBuffer, options) {
 /**
  * Parse attributes based on attribute type.
  * @param {String} attributeType
- * @param {ArrayBuffer} arrayBuffer
- * @returns {any}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseAttribute(attributeType, arrayBuffer) {
+function parseAttribute(attributeType, arrayBuffer: ArrayBuffer): Attribute {
   switch (attributeType) {
     case STRING_ATTRIBUTE_TYPE:
       return parseStringsAttribute(arrayBuffer);
@@ -46,10 +53,10 @@ function parseAttribute(attributeType, arrayBuffer) {
 /**
  * Parse short number attribute.
  * Short Integer spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.7/attributeStorageInfo.cmn.md
- * @param {ArrayBuffer} arrayBuffer
- * @returns {Uint32Array}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseShortNumberAttribute(arrayBuffer) {
+function parseShortNumberAttribute(arrayBuffer: ArrayBuffer): Uint32Array {
   const countOffset = 4;
   return new Uint32Array(arrayBuffer, countOffset);
 }
@@ -57,10 +64,10 @@ function parseShortNumberAttribute(arrayBuffer) {
 /**
  * Parse Int16 short number attribute.
  * Parsing of such data is not documented. Added to handle Building Scene Layer Tileset attributes data.
- * @param {ArrayBuffer} arrayBuffer
- * @returns {Int16Array}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseInt16ShortNumberAttribute(arrayBuffer) {
+function parseInt16ShortNumberAttribute(arrayBuffer: ArrayBuffer): Int16Array {
   const countOffset = 4;
   return new Int16Array(arrayBuffer, countOffset);
 }
@@ -68,10 +75,10 @@ function parseInt16ShortNumberAttribute(arrayBuffer) {
 /**
  * Parse float attribute.
  * Double Spec - https://github.com/Esri/i3s-spec/blob/master/docs/1.7/attributeStorageInfo.cmn.md
- * @param {ArrayBuffer} arrayBuffer
- * @returns {Float64Array}
+ * @param  arrayBuffer
+ * @returns
  */
-function parseFloatAttribute(arrayBuffer) {
+function parseFloatAttribute(arrayBuffer: ArrayBuffer): Float64Array {
   const countOffset = 8;
   return new Float64Array(arrayBuffer, countOffset);
 }

package/src/lib/parsers/parse-i3s-tile-content.ts

@@ -8,19 +8,22 @@ import {DracoLoader, DracoMesh} from '@loaders.gl/draco';
 import {BasisLoader, CompressedTextureLoader} from '@loaders.gl/textures';
 
 import {
-  I3STilesetHeader,
-  I3STileHeader,
   FeatureAttribute,
   VertexAttribute,
   I3SMeshAttributes,
   I3SMeshAttribute,
   TileContentTexture,
   HeaderAttributeProperty,
-  I3SMaterialDefinition
+  I3SMaterialDefinition,
+  I3STileContent,
+  I3STileOptions,
+  I3STilesetOptions
 } from '../../types';
 import {getUrlWithToken} from '../utils/url-utils';
 
 import {GL_TYPE_MAP, getConstructorForDataFormat, sizeOf, COORDINATE_SYSTEM} from './constants';
+import {I3SLoaderOptions} from '../../i3s-loader';
+import {customizeColors} from '../utils/customizeColors';
 
 const scratchVector = new Vector3([0, 0, 0]);
 
@@ -42,80 +45,90 @@ const I3S_ATTRIBUTE_TYPE = 'i3s-attribute-type';
 
 export async function parseI3STileContent(
   arrayBuffer: ArrayBuffer,
-  tile: I3STileHeader,
-  tileset: I3STilesetHeader,
+  tileOptions: I3STileOptions,
+  tilesetOptions: I3STilesetOptions,
   options?: LoaderOptions,
   context?: LoaderContext
-) {
-  tile.content = tile.content || {};
-  tile.content.featureIds = tile.content.featureIds || null;
-
-  tile.content.attributes = {};
+): Promise<I3STileContent> {
+  const content: I3STileContent = {
+    attributes: {},
+    indices: null,
+    featureIds: [],
+    vertexCount: 0,
+    modelMatrix: new Matrix4(),
+    coordinateSystem: 0,
+    byteLength: 0,
+    texture: null
+  };
 
-  if (tile.textureUrl) {
-    const url = getUrlWithToken(tile.textureUrl, options?.i3s?.token);
-    const loader = getLoaderForTextureFormat(tile.textureFormat);
-    const response = await fetch(url);
+  if (tileOptions.textureUrl) {
+    // @ts-expect-error options is not properly typed
+    const url = getUrlWithToken(tileOptions.textureUrl, options?.i3s?.token);
+    const loader = getLoaderForTextureFormat(tileOptions.textureFormat);
+    const response = await fetch(url, options?.fetch as RequestInit);
     const arrayBuffer = await response.arrayBuffer();
 
+    // @ts-expect-error options is not properly typed
     if (options?.i3s.decodeTextures) {
       if (loader === ImageLoader) {
-        const options = {...tile.textureLoaderOptions, image: {type: 'data'}};
+        const options = {...tileOptions.textureLoaderOptions, image: {type: 'data'}};
         try {
           // @ts-ignore context must be defined
           // Image constructor is not supported in worker thread.
           // Do parsing image data on the main thread by using context to avoid worker issues.
-          tile.content.texture = await context.parse(arrayBuffer, options);
+          content.texture = await context.parse(arrayBuffer, options);
         } catch (e) {
           // context object is different between worker and node.js conversion script.
           // To prevent error we parse data in ordinary way if it is not parsed by using context.
-          tile.content.texture = await parse(arrayBuffer, loader, options);
+          // @ts-expect-error
+          content.texture = await parse(arrayBuffer, loader, options);
         }
       } else if (loader === CompressedTextureLoader || loader === BasisLoader) {
-        let texture = await load(arrayBuffer, loader, tile.textureLoaderOptions);
+        let texture = await load(arrayBuffer, loader, tileOptions.textureLoaderOptions);
         if (loader === BasisLoader) {
+          // @ts-expect-error
           texture = texture[0];
         }
-        tile.content.texture = {
+        content.texture = {
           compressed: true,
           mipmaps: false,
+          // @ts-expect-error
           width: texture[0].width,
+          // @ts-expect-error
           height: texture[0].height,
+          // @ts-expect-error
           data: texture
         };
       }
     } else {
-      tile.content.texture = arrayBuffer;
+      content.texture = arrayBuffer;
     }
   }
 
-  tile.content.material = makePbrMaterial(tile.materialDefinition, tile.content.texture);
-  if (tile.content.material) {
-    tile.content.texture = null;
+  content.material = makePbrMaterial(tileOptions.materialDefinition, content.texture);
+  if (content.material) {
+    content.texture = null;
   }
 
-  return await parseI3SNodeGeometry(arrayBuffer, tile, tileset, options);
+  return await parseI3SNodeGeometry(arrayBuffer, content, tileOptions, tilesetOptions, options);
 }
 
 /* eslint-disable max-statements */
 async function parseI3SNodeGeometry(
   arrayBuffer: ArrayBuffer,
-  tile: I3STileHeader,
-  tileset: I3STilesetHeader,
-  options?: LoaderOptions
-) {
-  if (!tile.content) {
-    return tile;
-  }
-
-  const content = tile.content;
+  content: I3STileContent,
+  tileOptions: I3STileOptions,
+  tilesetOptions: I3STilesetOptions,
+  options?: I3SLoaderOptions
+): Promise<I3STileContent> {
+  const contentByteLength = arrayBuffer.byteLength;
   let attributes: I3SMeshAttributes;
   let vertexCount: number;
   let byteOffset: number = 0;
   let featureCount: number = 0;
   let indices: TypedArray | undefined;
 
-  if (tile.isDracoGeometry) {
+  if (tileOptions.isDracoGeometry) {
     const decompressedGeometry: DracoMesh = await parse(arrayBuffer, DracoLoader, {
       draco: {
         attributeNameEntry: I3S_ATTRIBUTE_TYPE
@@ -155,9 +168,9 @@ async function parseI3SNodeGeometry(
       ordering: attributesOrder,
       featureAttributes,
       featureAttributeOrder
-    } = tileset.store.defaultGeometrySchema;
+    } = tilesetOptions.store.defaultGeometrySchema;
     // First 8 bytes reserved for header (vertexCount and featureCount)
-    const headers = parseHeaders(tileset, arrayBuffer);
+    const headers = parseHeaders(arrayBuffer, tilesetOptions);
     byteOffset = headers.byteOffset;
     vertexCount = headers.vertexCount;
     featureCount = headers.featureCount;
@@ -187,7 +200,7 @@ async function parseI3SNodeGeometry(
     !options?.i3s?.coordinateSystem ||
     options.i3s.coordinateSystem === COORDINATE_SYSTEM.METER_OFFSETS
   ) {
-    const enuMatrix = parsePositions(attributes.position, tile);
+    const enuMatrix = parsePositions(attributes.position, tileOptions);
     content.modelMatrix = enuMatrix.invert();
     content.coordinateSystem = COORDINATE_SYSTEM.METER_OFFSETS;
   } else {
@@ -195,17 +208,25 @@ async function parseI3SNodeGeometry(
     content.coordinateSystem = COORDINATE_SYSTEM.LNGLAT_OFFSETS;
   }
 
+  attributes.color = await customizeColors(
+    attributes.color,
+    attributes.id,
+    tileOptions,
+    tilesetOptions,
+    options
+  );
+
   content.attributes = {
     positions: attributes.position,
     normals: attributes.normal,
     colors: normalizeAttribute(attributes.color), // Normalize from UInt8
     texCoords: attributes.uv0,
-    uvRegions: normalizeAttribute(attributes.uvRegion) // Normalize from UInt16
+    uvRegions: normalizeAttribute(attributes.uvRegion || attributes.region) // Normalize from UInt16
   };
   content.indices = indices || null;
 
   if (attributes.id && attributes.id.value) {
-    tile.content.featureIds = attributes.id.value;
+    content.featureIds = attributes.id.value;
   }
 
   // Remove undefined attributes
@@ -216,9 +237,9 @@ async function parseI3SNodeGeometry(
   }
 
   content.vertexCount = vertexCount;
-  content.byteLength = arrayBuffer.byteLength;
+  content.byteLength = contentByteLength;
 
-  return tile;
+  return content;
 }
 
 /**
@@ -273,12 +294,12 @@ function normalizeAttribute(attribute: I3SMeshAttribute): I3SMeshAttribute {
   return attribute;
 }
 
-function parseHeaders(tileset: I3STilesetHeader, arrayBuffer: ArrayBuffer) {
+function parseHeaders(arrayBuffer: ArrayBuffer, options: I3STilesetOptions) {
   let byteOffset = 0;
   // First 8 bytes reserved for header (vertexCount and featurecount)
   let vertexCount = 0;
   let featureCount = 0;
-  for (const {property, type} of tileset.store.defaultGeometrySchema.header) {
+  for (const {property, type} of options.store.defaultGeometrySchema.header) {
     const TypedArrayTypeHeader = getConstructorForDataFormat(type);
     switch (property) {
       case HeaderAttributeProperty.vertexCount:
@@ -307,7 +328,7 @@ function normalizeAttributes(
   arrayBuffer: ArrayBuffer,
   byteOffset: number,
   vertexAttributes: VertexAttribute | FeatureAttribute,
-  vertexCount: number,
+  attributeCount: number,
   attributesOrder: string[]
 ) {
   const attributes: I3SMeshAttributes = {};
@@ -317,44 +338,46 @@ function normalizeAttributes(
     if (vertexAttributes[attribute]) {
       const {valueType, valuesPerElement}: {valueType: string; valuesPerElement: number} =
         vertexAttributes[attribute];
-      // update count and byteOffset count by calculating from defaultGeometrySchema + binnary content
-      const count = vertexCount;
       // protect from arrayBuffer read overunns by NOT assuming node has regions always even though its declared in defaultGeometrySchema.
       // In i3s 1.6: client is required to decide that based on ./shared resource of the node (materialDefinitions.[Mat_id].params.vertexRegions == true)
       // In i3s 1.7 the property has been rolled into the 3d scene layer json/node pages.
       // Code below does not account when the bytelength is actually bigger than
       // the calculated value (b\c the tile potentially could have mesh segmentation information).
       // In those cases tiles without regions could fail or have garbage values.
-      if (byteOffset + count * valuesPerElement > arrayBuffer.byteLength) {
-        break;
-      }
-      const buffer = arrayBuffer.slice(byteOffset);
-      let value: TypedArray;
-
-      if (valueType === 'UInt64') {
-        value = parseUint64Values(buffer, count * valuesPerElement, sizeOf(valueType));
-      } else {
-        const TypedArrayType = getConstructorForDataFormat(valueType);
-        value = new TypedArrayType(buffer, 0, count * valuesPerElement);
-      }
+      if (
+        byteOffset + attributeCount * valuesPerElement * sizeOf(valueType) <=
+        arrayBuffer.byteLength
+      ) {
+        const buffer = arrayBuffer.slice(byteOffset);
+        let value: TypedArray;
+
+        if (valueType === 'UInt64') {
+          value = parseUint64Values(buffer, attributeCount * valuesPerElement, sizeOf(valueType));
+        } else {
+          const TypedArrayType = getConstructorForDataFormat(valueType);
+          value = new TypedArrayType(buffer, 0, attributeCount * valuesPerElement);
+        }
 
-      attributes[attribute] = {
-        value,
-        type: GL_TYPE_MAP[valueType],
-        size: valuesPerElement
-      };
-
-      switch (attribute) {
-        case 'color':
-          attributes.color.normalized = true;
-          break;
-        case 'position':
-        case 'region':
-        case 'normal':
-        default:
-      }
+        attributes[attribute] = {
+          value,
+          type: GL_TYPE_MAP[valueType],
+          size: valuesPerElement
+        };
 
-      byteOffset = byteOffset + count * valuesPerElement * sizeOf(valueType);
+        switch (attribute) {
+          case 'color':
+            attributes.color.normalized = true;
+            break;
+          case 'position':
+          case 'region':
+          case 'normal':
+          default:
+        }
+
+        byteOffset = byteOffset + attributeCount * valuesPerElement * sizeOf(valueType);
+      } else if (attribute !== 'uv0') {
+        break;
+      }
     }
   }
 
@@ -391,8 +414,8 @@ function parseUint64Values(
   return new Uint32Array(values);
 }
 
-function parsePositions(attribute: I3SMeshAttribute, tile: I3STileHeader): Matrix4 {
-  const mbs = tile.mbs;
+function parsePositions(attribute: I3SMeshAttribute, options: I3STileOptions): Matrix4 {
+  const mbs = options.mbs;
   const value = attribute.value;
   const metadata = attribute.metadata;
   const enuMatrix = new Matrix4();

package/src/lib/parsers/parse-i3s.ts

@@ -13,7 +13,7 @@ import {
 } from '../../types';
 import type {LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
 
-export function normalizeTileData(tile : Node3DIndexDocument, options : LoaderOptions, context: LoaderContext): I3STileHeader {
+export function normalizeTileData(tile : Node3DIndexDocument, context: LoaderContext): I3STileHeader {
   const url: string = context.url || '';
   let contentUrl: string | undefined;
   if (tile.geometryData) {
@@ -82,12 +82,17 @@ export async function normalizeTilesetData(tileset : I3STilesetHeader, options :
 
   if (tileset.nodePages) {
     tileset.nodePagesTile = new I3SNodePagesTiles(tileset, options);
-    tileset.root = await tileset.nodePagesTile.formTileFromNodePages(0);
+    tileset.root = tileset.nodePagesTile.formTileFromNodePages(0);
   } else {
+    // @ts-expect-error options is not properly typed
     const rootNodeUrl = getUrlWithToken(`${tileset.url}/nodes/root`, options.i3s?.token);
     // eslint-disable-next-line no-use-before-define
     tileset.root = await load(rootNodeUrl, tileset.loader, {
-      i3s: {loadContent: false, isTileHeader: true, isTileset: false}
+      ...options,
+      i3s: {
+        // @ts-expect-error options is not properly typed
+        ...options.i3s,
+        loadContent: false, isTileHeader: true, isTileset: false}
     });
   }
 

package/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -0,0 +1,61 @@
+import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
+import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
+import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {SLPKArchive} from './slpk-archieve';
+
+/**
+ * Returns one byte from the provided buffer at the provided position
+ * @param offset - position where to read
+ * @param buffer - buffer to read
+ * @returns one byte from the provided buffer at the provided position
+ */
+const getByteAt = (offset: number, buffer: DataView): number => {
+  return buffer.getUint8(buffer.byteOffset + offset);
+};
+
+export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
+  const archive = new DataView(data);
+  const cdFileHeaderSignature = [80, 75, 1, 2];
+
+  const searchWindow = [
+    getByteAt(archive.byteLength - 1, archive),
+    getByteAt(archive.byteLength - 2, archive),
+    getByteAt(archive.byteLength - 3, archive),
+    undefined
+  ];
+
+  let hashCDOffset = 0;
+
+  // looking for the last record in the central directory
+  for (let i = archive.byteLength - 4; i > -1; i--) {
+    searchWindow[3] = searchWindow[2];
+    searchWindow[2] = searchWindow[1];
+    searchWindow[1] = searchWindow[0];
+    searchWindow[0] = getByteAt(i, archive);
+    if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
+      hashCDOffset = i;
+      break;
+    }
+  }
+
+  const cdFileHeader = parseZipCDFileHeader(hashCDOffset, archive);
+
+  const textDecoder = new TextDecoder();
+  if (textDecoder.decode(cdFileHeader.fileName) !== '@specialIndexFileHASH128@') {
+    throw new Error('No hash file in slpk');
+  }
+
+  const localFileHeader = parseZipLocalFileHeader(cdFileHeader.localHeaderOffset, archive);
+
+  const fileDataOffset = localFileHeader.fileDataOffset;
+  const hashFile = archive.buffer.slice(
+    fileDataOffset,
+    fileDataOffset + localFileHeader.compressedSize
+  );
+
+  if (!hashFile) {
+    throw new Error('No hash file in slpk');
+  }
+
+  return await new SLPKArchive(data, hashFile).getFile(options.path ?? '');
+}