@loaders.gl/i3s 4.0.0-alpha.9 → 4.0.0-beta.1

package/src/lib/parsers/constants.ts

@@ -1,17 +1,16 @@
 import GL from '@luma.gl/constants';
-import {DATA_TYPE} from '../../types';
 
 export function getConstructorForDataFormat(dataType: string) {
   switch (dataType) {
-    case DATA_TYPE.UInt8:
+    case 'UInt8':
       return Uint8Array;
-    case DATA_TYPE.UInt16:
+    case 'UInt16':
       return Uint16Array;
-    case DATA_TYPE.UInt32:
+    case 'UInt32':
       return Uint32Array;
-    case DATA_TYPE.Float32:
+    case 'Float32':
       return Float32Array;
-    case DATA_TYPE.UInt64:
+    case 'UInt64':
       return Float64Array;
     default:
       throw new Error(`parse i3s tile content: unknown type of data: ${dataType}`);
@@ -32,18 +31,18 @@ export const GL_TYPE_MAP: {[key: string]: number} = {
  */
 export function sizeOf(dataType: string): number {
   switch (dataType) {
-    case DATA_TYPE.UInt8:
+    case 'UInt8':
       return 1;
-    case DATA_TYPE.UInt16:
-    case DATA_TYPE.Int16:
+    case 'UInt16':
+    case 'Int16':
       return 2;
-    case DATA_TYPE.UInt32:
-    case DATA_TYPE.Int32:
-    case DATA_TYPE.Float32:
+    case 'UInt32':
+    case 'Int32':
+    case 'Float32':
       return 4;
-    case DATA_TYPE.UInt64:
-    case DATA_TYPE.Int64:
-    case DATA_TYPE.Float64:
+    case 'UInt64':
+    case 'Int64':
+    case 'Float64':
       return 8;
     default:
       throw new Error(`parse i3s tile content: unknown size of data: ${dataType}`);

package/src/lib/parsers/parse-arcgis-webscene.ts

@@ -1,5 +1,5 @@
 import {JSONLoader, load} from '@loaders.gl/core';
-import type {ArcGisWebSceneData, OperationalLayer} from '../../types';
+import type {ArcGISWebSceneData, OperationalLayer} from '../../types';
 
 /**
  * WKID, or Well-Known ID, of the CRS. Specify either WKID or WKT of the CRS.
@@ -30,7 +30,7 @@ const NOT_SUPPORTED_CRS_ERROR = 'NOT_SUPPORTED_CRS_ERROR';
  * Parses ArcGIS WebScene
  * @param data
  */
-export async function parseWebscene(data: ArrayBuffer): Promise<ArcGisWebSceneData> {
+export async function parseWebscene(data: ArrayBuffer): Promise<ArcGISWebSceneData> {
   const layer0 = JSON.parse(new TextDecoder().decode(data));
   const {operationalLayers} = layer0;
   const {layers, unsupportedLayers} = await parseOperationalLayers(operationalLayers, true);

package/src/lib/parsers/parse-i3s-tile-content.ts

@@ -2,7 +2,7 @@ import type {TypedArray} from '@loaders.gl/schema';
 import {load, parse} from '@loaders.gl/core';
 import {Vector3, Matrix4} from '@math.gl/core';
 import {Ellipsoid} from '@math.gl/geospatial';
-import type {LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
+import {LoaderOptions, LoaderContext, parseFromContext} from '@loaders.gl/loader-utils';
 import {ImageLoader} from '@loaders.gl/images';
 import {DracoLoader, DracoMesh} from '@loaders.gl/draco';
 import {BasisLoader, CompressedTextureLoader} from '@loaders.gl/textures';
@@ -70,33 +70,31 @@ export async function parseI3STileContent(
 
     // @ts-expect-error options is not properly typed
     if (options?.i3s.decodeTextures) {
+      // TODO - replace with switch
       if (loader === ImageLoader) {
         const options = {...tileOptions.textureLoaderOptions, image: {type: 'data'}};
         try {
-          // @ts-ignore context must be defined
           // Image constructor is not supported in worker thread.
           // Do parsing image data on the main thread by using context to avoid worker issues.
-          content.texture = await context.parse(arrayBuffer, options);
+          const texture = await parseFromContext(arrayBuffer, [], options, context!);
+          // @ts-expect-error
+          content.texture = texture;
         } catch (e) {
           // context object is different between worker and node.js conversion script.
           // To prevent error we parse data in ordinary way if it is not parsed by using context.
-          // @ts-expect-error
-          content.texture = await parse(arrayBuffer, loader, options);
+          const texture = await parse(arrayBuffer, loader, options, context);
+          content.texture = texture;
         }
       } else if (loader === CompressedTextureLoader || loader === BasisLoader) {
         let texture = await load(arrayBuffer, loader, tileOptions.textureLoaderOptions);
         if (loader === BasisLoader) {
-          // @ts-expect-error
           texture = texture[0];
         }
         content.texture = {
           compressed: true,
           mipmaps: false,
-          // @ts-expect-error
           width: texture[0].width,
-          // @ts-expect-error
           height: texture[0].height,
-          // @ts-expect-error
           data: texture
         };
       }

package/src/lib/parsers/parse-i3s.ts

@@ -9,9 +9,12 @@ import {
   I3STileHeader,
   Mbs,
   I3SMinimalNodeData,
-  Node3DIndexDocument
+  Node3DIndexDocument,
+  SceneLayer3D,
+  I3SParseOptions
 } from '../../types';
 import type {LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
+import { I3SLoader } from '../../i3s-loader';
 
 export function normalizeTileData(tile : Node3DIndexDocument, context: LoaderContext): I3STileHeader {
   const url: string = context.url || '';
@@ -30,11 +33,15 @@ export function normalizeTileData(tile : Node3DIndexDocument, context: LoaderCon
     attributeUrls = generateTileAttributeUrls(url, tile);
   }
 
+  const children = tile.children || [];
+
   return normalizeTileNonUrlData({
     ...tile,
+    children,
     url,
     contentUrl,
     textureUrl,
+    textureFormat: 'jpg', // `jpg` format will cause `ImageLoader` usage that will be able to handle `png` as well
     attributeUrls,
     isDracoGeometry: false
   });
@@ -67,27 +74,27 @@ export function normalizeTileNonUrlData(tile : I3SMinimalNodeData): I3STileHeade
 
   const lodMetricType = tile.lodSelection?.[0].metricType;
   const lodMetricValue = tile.lodSelection?.[0].maxError;
-  const transformMatrix = tile.transform;
   const type = TILE_TYPE.MESH;
   /**
    * I3S specification supports only REPLACE
    */
   const refine = TILE_REFINEMENT.REPLACE;
 
-  return {...tile, mbs, boundingVolume, lodMetricType, lodMetricValue, transformMatrix, type, refine};
+  return {...tile, mbs, boundingVolume, lodMetricType, lodMetricValue, type, refine};
 }
 
-export async function normalizeTilesetData(tileset : I3STilesetHeader, options : LoaderOptions, context: LoaderContext) {
-  tileset.url = context.url;
-
+export async function normalizeTilesetData(tileset : SceneLayer3D, options : LoaderOptions, context: LoaderContext): Promise<I3STileHeader | I3STilesetHeader> {
+  const url = context.url;
+  let nodePagesTile: I3SNodePagesTiles | undefined;
+  let root: I3STileHeader | I3STilesetHeader;
   if (tileset.nodePages) {
-    tileset.nodePagesTile = new I3SNodePagesTiles(tileset, options);
-    tileset.root = tileset.nodePagesTile.formTileFromNodePages(0);
+    nodePagesTile = new I3SNodePagesTiles(tileset, url, options);
+    root = await nodePagesTile.formTileFromNodePages(0);
   } else {
-    // @ts-expect-error options is not properly typed
-    const rootNodeUrl = getUrlWithToken(`${tileset.url}/nodes/root`, options.i3s?.token);
+    const parseOptions = options.i3s as I3SParseOptions;
+    const rootNodeUrl = getUrlWithToken(`${url}/nodes/root`, parseOptions.token);
     // eslint-disable-next-line no-use-before-define
-    tileset.root = await load(rootNodeUrl, tileset.loader, {
+    root = await load(rootNodeUrl, I3SLoader, {
       ...options,
       i3s: {
         // @ts-expect-error options is not properly typed
@@ -96,11 +103,16 @@ export async function normalizeTilesetData(tileset : I3STilesetHeader, options :
     });
   }
 
-  // base path that non-absolute paths in tileset are relative to.
-  tileset.basePath = tileset.url;
-  tileset.type = TILESET_TYPE.I3S;
-
-  // populate from root node
-  tileset.lodMetricType = tileset.root.lodMetricType;
-  tileset.lodMetricValue = tileset.root.lodMetricValue;
+  return {
+    ...tileset,
+    loader: I3SLoader,
+    url,
+    basePath: url,
+    type: TILESET_TYPE.I3S,
+    nodePagesTile,
+    // @ts-expect-error
+    root,
+    lodMetricType: root.lodMetricType,
+    lodMetricValue: root.lodMetricValue
+  }
 }

package/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -1,64 +1,52 @@
-import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
-import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
-import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {FileProvider} from '@loaders.gl/loader-utils';
+import {
+  parseZipCDFileHeader,
+  cdSignature as cdHeaderSignature,
+  parseZipLocalFileHeader,
+  searchFromTheEnd,
+  HashElement,
+  parseHashFile,
+  generateHashInfo
+} from '@loaders.gl/zip';
 import {SLPKArchive} from './slpk-archieve';
 
 /**
- * Returns one byte from the provided buffer at the provided position
- * @param offset - position where to read
- * @param buffer - buffer to read
- * @returns one byte from the provided buffer at the provided position
+ * Creates slpk file handler from raw file
+ * @param fileProvider raw file data
+ * @param cb is called with information message during parsing
+ * @returns slpk file handler
  */
-const getByteAt = (offset: number, buffer: DataView): number => {
-  return buffer.getUint8(buffer.byteOffset + offset);
-};
-
-export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
-  const archive = new DataView(data);
-  const cdFileHeaderSignature = [80, 75, 1, 2];
-
-  const searchWindow = [
-    getByteAt(archive.byteLength - 1, archive),
-    getByteAt(archive.byteLength - 2, archive),
-    getByteAt(archive.byteLength - 3, archive),
-    undefined
-  ];
-
-  let hashCDOffset = 0;
-
-  // looking for the last record in the central directory
-  for (let i = archive.byteLength - 4; i > -1; i--) {
-    searchWindow[3] = searchWindow[2];
-    searchWindow[2] = searchWindow[1];
-    searchWindow[1] = searchWindow[0];
-    searchWindow[0] = getByteAt(i, archive);
-    if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
-      hashCDOffset = i;
-      break;
+export const parseSLPK = async (
+  fileProvider: FileProvider,
+  cb?: (msg: string) => void
+): Promise<SLPKArchive> => {
+  const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature);
+
+  const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
+
+  let hashData: HashElement[];
+  if (cdFileHeader?.fileName !== '@specialIndexFileHASH128@') {
+    cb?.('SLPK doesnt contain hash file');
+    hashData = await generateHashInfo(fileProvider);
+    cb?.('hash info has been composed according to central directory records');
+  } else {
+    cb?.('SLPK contains hash file');
+    const localFileHeader = await parseZipLocalFileHeader(
+      cdFileHeader.localHeaderOffset,
+      fileProvider
+    );
+    if (!localFileHeader) {
+      throw new Error('corrupted SLPK');
     }
-  }
 
-  const cdFileHeader = parseZipCDFileHeader(hashCDOffset, archive);
+    const fileDataOffset = localFileHeader.fileDataOffset;
+    const hashFile = await fileProvider.slice(
+      fileDataOffset,
+      fileDataOffset + localFileHeader.compressedSize
+    );
 
-  const textDecoder = new TextDecoder();
-  if (textDecoder.decode(cdFileHeader.fileName) !== '@specialIndexFileHASH128@') {
-    throw new Error('No hash file in slpk');
+    hashData = parseHashFile(hashFile);
   }
 
-  const localFileHeader = parseZipLocalFileHeader(cdFileHeader.localHeaderOffset, archive);
-
-  const fileDataOffset = localFileHeader.fileDataOffset;
-  const hashFile = archive.buffer.slice(
-    fileDataOffset,
-    fileDataOffset + localFileHeader.compressedSize
-  );
-
-  if (!hashFile) {
-    throw new Error('No hash file in slpk');
-  }
-
-  return await new SLPKArchive(data, hashFile).getFile(
-    options.slpk?.path ?? '',
-    options.slpk?.pathMode
-  );
-}
+  return new SLPKArchive(fileProvider, hashData);
+};

package/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -1,20 +1,9 @@
-import {processOnWorker} from '@loaders.gl/worker-utils';
 import md5 from 'md5';
-import {CompressionWorker} from '@loaders.gl/compression';
-import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
-
-/** Element of hash array */
-type HashElement = {
-  /**
-   * File name hash
-   */
-  hash: Buffer;
-  /**
-   * File offset in the archive
-   */
-  offset: number;
-};
+import {FileProvider} from '@loaders.gl/loader-utils';
+import {parseZipLocalFileHeader, HashElement, findBin} from '@loaders.gl/zip';
+import {GZipCompression} from '@loaders.gl/compression';
 
+/** Description of real paths for different file types */
 const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
   {
     test: /^$/,
@@ -25,7 +14,7 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
     extensions: ['.json.gz']
   },
   {
-    test: /^nodes\/\d+$/,
+    test: /^nodes\/(\d+|root)$/,
     extensions: ['/3dNodeIndexDocument.json.gz']
   },
   {
@@ -54,37 +43,11 @@ const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
  * Class for handling information about slpk file
  */
 export class SLPKArchive {
-  slpkArchive: DataView;
-  hashArray: {hash: Buffer; offset: number}[];
-  constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) {
-    this.slpkArchive = new DataView(slpkArchiveBuffer);
-    this.hashArray = this.parseHashFile(hashFile);
-  }
-
-  /**
-   * Reads hash file from buffer and returns it in ready-to-use form
-   * @param hashFile - bufer containing hash file
-   * @returns Array containing file info
-   */
-  private parseHashFile(hashFile: ArrayBuffer): HashElement[] {
-    const hashFileBuffer = Buffer.from(hashFile);
-    const hashArray: HashElement[] = [];
-    for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
-      const offsetBuffer = new DataView(
-        hashFileBuffer.buffer.slice(
-          hashFileBuffer.byteOffset + i + 16,
-          hashFileBuffer.byteOffset + i + 24
-        )
-      );
-      const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true);
-      hashArray.push({
-        hash: Buffer.from(
-          hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
-        ),
-        offset
-      });
-    }
-    return hashArray;
+  private slpkArchive: FileProvider;
+  private hashArray: HashElement[];
+  constructor(slpkArchive: FileProvider, hashFile: HashElement[]) {
+    this.slpkArchive = slpkArchive;
+    this.hashArray = hashFile;
   }
 
   /**
@@ -114,7 +77,7 @@ export class SLPKArchive {
       if (decompressedFile) {
         return Buffer.from(decompressedFile);
       }
-      const fileWithoutCompression = this.getFileBytes(path);
+      const fileWithoutCompression = await this.getFileBytes(path);
       if (fileWithoutCompression) {
         return Buffer.from(fileWithoutCompression);
       }
@@ -129,17 +92,19 @@ export class SLPKArchive {
    * @returns buffer with the file data
    */
   private async getDataByPath(path: string): Promise<ArrayBuffer | undefined> {
-    const data = this.getFileBytes(path);
+    // sometimes paths are not in lower case when hash file is created,
+    // so first we're looking for lower case file name and then for original one
+    let data = await this.getFileBytes(path.toLocaleLowerCase());
+    if (!data) {
+      data = await this.getFileBytes(path);
+    }
     if (!data) {
       return undefined;
     }
     if (/\.gz$/.test(path)) {
-      const decompressedData = await processOnWorker(CompressionWorker, data, {
-        compression: 'gzip',
-        operation: 'decompress',
-        _workerType: 'test',
-        gzip: {}
-      });
+      const compression = new GZipCompression();
+
+      const decompressedData = await compression.decompress(data);
       return decompressedData;
     }
     return Buffer.from(data);
@@ -150,23 +115,26 @@ export class SLPKArchive {
    * @param path - path inside the archive
    * @returns buffer with the raw file data
    */
-  private getFileBytes(path: string): ArrayBuffer | undefined {
+  private async getFileBytes(path: string): Promise<ArrayBuffer | undefined> {
     const nameHash = Buffer.from(md5(path), 'hex');
-    const fileInfo = this.hashArray.find((val) => Buffer.compare(val.hash, nameHash) === 0);
+    const fileInfo = findBin(nameHash, this.hashArray); // implement binary search
     if (!fileInfo) {
       return undefined;
     }
 
-    const localFileHeader = parseZipLocalFileHeader(
-      this.slpkArchive.byteOffset + fileInfo?.offset,
-      this.slpkArchive
-    );
+    const localFileHeader = await parseZipLocalFileHeader(fileInfo.offset, this.slpkArchive);
+    if (!localFileHeader) {
+      return undefined;
+    }
 
-    const compressedFile = this.slpkArchive.buffer.slice(
+    const compressedFile = this.slpkArchive.slice(
       localFileHeader.fileDataOffset,
       localFileHeader.fileDataOffset + localFileHeader.compressedSize
     );
 
     return compressedFile;
   }
+  findBin(nameHash: Buffer) {
+    throw new Error('Method not implemented.');
+  }
 }

package/src/lib/utils/customize-/321/201olors.ts

@@ -5,6 +5,7 @@ import {load} from '@loaders.gl/core';
 import {getAttributeValueType, I3SAttributeLoader} from '../../i3s-attribute-loader';
 import {I3SLoaderOptions} from '../../i3s-loader';
 import {getUrlWithToken} from './url-utils';
+import {I3STileAttributes} from '../parsers/parse-i3s-attribute';
 
 /**
  * Modify vertex colors array to visualize 3D objects in a attribute driven way
@@ -127,7 +128,7 @@ async function loadFeatureAttributeData(
   {attributeUrls}: I3STileOptions,
   {attributeStorageInfo}: I3STilesetOptions,
   options?: I3SLoaderOptions
-): Promise<{[key: string]: string[] | Uint32Array | Uint16Array | Float64Array | null} | null> {
+): Promise<I3STileAttributes | null> {
   const attributeIndex = attributeStorageInfo.findIndex(({name}) => attributeName === name);
   if (attributeIndex === -1) {
     return null;
@@ -139,6 +140,5 @@ async function loadFeatureAttributeData(
     attributeType
   });
 
-  // @ts-expect-error TODO action engine
   return objectIdAttributeData;
 }

package/src/lib/utils/url-utils.ts

@@ -1,4 +1,4 @@
-import {Node3DIndexDocument} from '../../types';
+import {Node3DIndexDocument, SceneLayer3D} from '../../types';
 
 /**
  * Generates url with token if it is exists.
@@ -29,13 +29,14 @@ export function generateTileAttributeUrls(url: string, tile: Node3DIndexDocument
 
 /**
  * Generates attribute urls for tileset based on tileset and resource
- * @param {Object} tileset
- * @param {number} resource
+ * @param tileset - tileset metadata
+ * @param url - tileset base url
+ * @param resource - resource id per I3S spec
  * @returns {Array}
  */
-export function generateTilesetAttributeUrls(tileset, resource) {
+export function generateTilesetAttributeUrls(tileset: SceneLayer3D, url: string, resource: number) {
   const attributeUrls: string[] = [];
-  const {attributeStorageInfo, url} = tileset;
+  const {attributeStorageInfo = []} = tileset;
 
   for (let index = 0; index < attributeStorageInfo.length; index++) {
     const fileName = attributeStorageInfo[index].key;