@loaders.gl/i3s 4.0.0-alpha.1 → 4.0.0-alpha.11

package/src/lib/parsers/parse-i3s.ts

@@ -4,79 +4,95 @@ import {load} from '@loaders.gl/core';
 import {TILE_TYPE, TILE_REFINEMENT, TILESET_TYPE} from '@loaders.gl/tiles';
 import I3SNodePagesTiles from '../helpers/i3s-nodepages-tiles';
 import {generateTileAttributeUrls, getUrlWithToken} from '../utils/url-utils';
+import {
+  I3STilesetHeader,
+  I3STileHeader,
+  Mbs,
+  I3SMinimalNodeData,
+  Node3DIndexDocument
+} from '../../types';
+import type {LoaderOptions, LoaderContext} from '@loaders.gl/loader-utils';
 
-export function normalizeTileData(tile, options, context) {
-  tile.url = context.url;
-
-  if (tile.featureData) {
-    tile.featureUrl = `${tile.url}/${tile.featureData[0].href}`;
-  }
-
+export function normalizeTileData(tile : Node3DIndexDocument, context: LoaderContext): I3STileHeader {
+  const url: string = context.url || '';
+  let contentUrl: string | undefined;
   if (tile.geometryData) {
-    tile.contentUrl = `${tile.url}/${tile.geometryData[0].href}`;
+    contentUrl = `${url}/${tile.geometryData[0].href}`;
   }
 
+  let textureUrl: string | undefined;
   if (tile.textureData) {
-    tile.textureUrl = `${tile.url}/${tile.textureData[0].href}`;
+    textureUrl = `${url}/${tile.textureData[0].href}`;
   }
 
+  let attributeUrls: string[] | undefined;
   if (tile.attributeData) {
-    tile.attributeUrls = generateTileAttributeUrls(tile);
+    attributeUrls = generateTileAttributeUrls(url, tile);
   }
 
-  return normalizeTileNonUrlData(tile);
+  return normalizeTileNonUrlData({
+    ...tile,
+    url,
+    contentUrl,
+    textureUrl,
+    attributeUrls,
+    isDracoGeometry: false
+  });
 }
 
-export function normalizeTileNonUrlData(tile) {
-  const box = tile.obb
-    ? [
-      ...Ellipsoid.WGS84.cartographicToCartesian(tile.obb.center), // cartesian center of box
-      ...tile.obb.halfSize, // halfSize
-      ...tile.obb.quaternion // quaternion
-    ]
-    : undefined;
-  let sphere;
+export function normalizeTileNonUrlData(tile : I3SMinimalNodeData): I3STileHeader {
+  const boundingVolume: {box?: number[]; sphere?: number[]} = {};
+  let mbs: Mbs = [0, 0, 0, 1];
   if (tile.mbs) {
-    sphere = [
+    mbs = tile.mbs;
+    boundingVolume.sphere = [
       ...Ellipsoid.WGS84.cartographicToCartesian(tile.mbs.slice(0, 3)), // cartesian center of sphere
       tile.mbs[3] // radius of sphere
+    ] as Mbs;
+  } else if (tile.obb) {
+    boundingVolume.box = [
+      ...Ellipsoid.WGS84.cartographicToCartesian(tile.obb.center), // cartesian center of box
+      ...tile.obb.halfSize, // halfSize
+      ...tile.obb.quaternion // quaternion
     ];
-  } else if (box) {
     const obb = new OrientedBoundingBox().fromCenterHalfSizeQuaternion(
-      box.slice(0, 3),
+      boundingVolume.box.slice(0, 3),
       tile.obb.halfSize,
       tile.obb.quaternion
     );
     const boundingSphere = obb.getBoundingSphere();
-    sphere = [...boundingSphere.center, boundingSphere.radius];
-    tile.mbs = [...tile.obb.center, boundingSphere.radius];
+    boundingVolume.sphere = [...boundingSphere.center , boundingSphere.radius] as Mbs;
+    mbs = [...tile.obb.center, boundingSphere.radius] as Mbs;
   }
 
-  tile.boundingVolume = {
-    sphere,
-    box
-  };
-  tile.lodMetricType = tile.lodSelection[0].metricType;
-  tile.lodMetricValue = tile.lodSelection[0].maxError;
-  tile.transformMatrix = tile.transform;
-  tile.type = TILE_TYPE.MESH;
-  // TODO only support replacement for now
-  tile.refine = TILE_REFINEMENT.REPLACE;
+  const lodMetricType = tile.lodSelection?.[0].metricType;
+  const lodMetricValue = tile.lodSelection?.[0].maxError;
+  const transformMatrix = tile.transform;
+  const type = TILE_TYPE.MESH;
+  /**
+   * I3S specification supports only REPLACE
+   */
+  const refine = TILE_REFINEMENT.REPLACE;
 
-  return tile;
+  return {...tile, mbs, boundingVolume, lodMetricType, lodMetricValue, transformMatrix, type, refine};
 }
 
-export async function normalizeTilesetData(tileset, options, context) {
+export async function normalizeTilesetData(tileset : I3STilesetHeader, options : LoaderOptions, context: LoaderContext) {
   tileset.url = context.url;
 
   if (tileset.nodePages) {
     tileset.nodePagesTile = new I3SNodePagesTiles(tileset, options);
-    tileset.root = await tileset.nodePagesTile.formTileFromNodePages(0);
+    tileset.root = tileset.nodePagesTile.formTileFromNodePages(0);
   } else {
+    // @ts-expect-error options is not properly typed
     const rootNodeUrl = getUrlWithToken(`${tileset.url}/nodes/root`, options.i3s?.token);
     // eslint-disable-next-line no-use-before-define
     tileset.root = await load(rootNodeUrl, tileset.loader, {
-      i3s: {loadContent: false, isTileHeader: true, isTileset: false}
+      ...options,
+      i3s: {
+        // @ts-expect-error options is not properly typed
+        ...options.i3s,
+        loadContent: false, isTileHeader: true, isTileset: false}
     });
   }
 

package/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -0,0 +1,72 @@
+import type {SLPKLoaderOptions} from '../../../i3s-slpk-loader';
+import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
+import {parseZipCDFileHeader} from '../parse-zip/cd-file-header';
+import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {SLPKArchive} from './slpk-archieve';
+
+/**
+ * Returns one byte from the provided buffer at the provided position
+ * @param offset - position where to read
+ * @param buffer - buffer to read
+ * @returns one byte from the provided buffer at the provided position
+ */
+const getByteAt = (offset: number, buffer: DataView): number => {
+  return buffer.getUint8(buffer.byteOffset + offset);
+};
+
+export async function parseSLPK(data: ArrayBuffer, options: SLPKLoaderOptions = {}) {
+  const archive = new DataView(data);
+  const cdFileHeaderSignature = [80, 75, 1, 2];
+
+  const searchWindow = [
+    getByteAt(archive.byteLength - 1, archive),
+    getByteAt(archive.byteLength - 2, archive),
+    getByteAt(archive.byteLength - 3, archive),
+    undefined
+  ];
+
+  let hashCDOffset = 0;
+
+  // looking for the last record in the central directory
+  for (let i = archive.byteLength - 4; i > -1; i--) {
+    searchWindow[3] = searchWindow[2];
+    searchWindow[2] = searchWindow[1];
+    searchWindow[1] = searchWindow[0];
+    searchWindow[0] = getByteAt(i, archive);
+    if (searchWindow.every((val, index) => val === cdFileHeaderSignature[index])) {
+      hashCDOffset = i;
+      break;
+    }
+  }
+
+  const fileProvider = new DataViewFileProvider(archive);
+
+  const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
+
+  if (cdFileHeader.fileName !== '@specialIndexFileHASH128@') {
+    throw new Error('No hash file in slpk');
+  }
+
+  const localFileHeader = await parseZipLocalFileHeader(
+    cdFileHeader.localHeaderOffset,
+    fileProvider
+  );
+  if (!localFileHeader) {
+    throw new Error('No hash file in slpk');
+  }
+
+  const fileDataOffset = localFileHeader.fileDataOffset;
+  const hashFile = archive.buffer.slice(
+    fileDataOffset,
+    fileDataOffset + localFileHeader.compressedSize
+  );
+
+  if (!hashFile) {
+    throw new Error('No hash file in slpk');
+  }
+
+  return await new SLPKArchive(data, hashFile).getFile(
+    options.slpk?.path ?? '',
+    options.slpk?.pathMode
+  );
+}

package/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -0,0 +1,173 @@
+import md5 from 'md5';
+import {parseZipLocalFileHeader} from '../parse-zip/local-file-header';
+import {DataViewFileProvider} from '../parse-zip/buffer-file-provider';
+import {GZipCompression} from '@loaders.gl/compression';
+
+/** Element of hash array */
+type HashElement = {
+  /**
+   * File name hash
+   */
+  hash: Buffer;
+  /**
+   * File offset in the archive
+   */
+  offset: number;
+};
+
+/** Description of real paths for different file types */
+const PATH_DESCRIPTIONS: {test: RegExp; extensions: string[]}[] = [
+  {
+    test: /^$/,
+    extensions: ['3dSceneLayer.json.gz']
+  },
+  {
+    test: /^nodepages\/\d+$/,
+    extensions: ['.json.gz']
+  },
+  {
+    test: /^nodes\/\d+$/,
+    extensions: ['/3dNodeIndexDocument.json.gz']
+  },
+  {
+    test: /^nodes\/\d+\/textures\/.+$/,
+    extensions: ['.jpg', '.png', '.bin.dds.gz', '.ktx']
+  },
+  {
+    test: /^nodes\/\d+\/geometries\/\d+$/,
+    extensions: ['.bin.gz', '.draco.gz']
+  },
+  {
+    test: /^nodes\/\d+\/attributes\/f_\d+\/\d+$/,
+    extensions: ['.bin.gz']
+  },
+  {
+    test: /^statistics\/f_\d+\/\d+$/,
+    extensions: ['.json.gz']
+  },
+  {
+    test: /^nodes\/\d+\/shared$/,
+    extensions: ['/sharedResource.json.gz']
+  }
+];
+
+/**
+ * Class for handling information about slpk file
+ */
+export class SLPKArchive {
+  slpkArchive: DataView;
+  hashArray: {hash: Buffer; offset: number}[];
+  constructor(slpkArchiveBuffer: ArrayBuffer, hashFile: ArrayBuffer) {
+    this.slpkArchive = new DataView(slpkArchiveBuffer);
+    this.hashArray = this.parseHashFile(hashFile);
+  }
+
+  /**
+   * Reads hash file from buffer and returns it in ready-to-use form
+   * @param hashFile - bufer containing hash file
+   * @returns Array containing file info
+   */
+  private parseHashFile(hashFile: ArrayBuffer): HashElement[] {
+    const hashFileBuffer = Buffer.from(hashFile);
+    const hashArray: HashElement[] = [];
+    for (let i = 0; i < hashFileBuffer.buffer.byteLength; i = i + 24) {
+      const offsetBuffer = new DataView(
+        hashFileBuffer.buffer.slice(
+          hashFileBuffer.byteOffset + i + 16,
+          hashFileBuffer.byteOffset + i + 24
+        )
+      );
+      const offset = offsetBuffer.getUint32(offsetBuffer.byteOffset, true);
+      hashArray.push({
+        hash: Buffer.from(
+          hashFileBuffer.subarray(hashFileBuffer.byteOffset + i, hashFileBuffer.byteOffset + i + 16)
+        ),
+        offset
+      });
+    }
+    return hashArray;
+  }
+
+  /**
+   * Returns file with the given path from slpk archive
+   * @param path - path inside the slpk
+   * @param mode - currently only raw mode supported
+   * @returns buffer with ready to use file
+   */
+  async getFile(path: string, mode: 'http' | 'raw' = 'raw'): Promise<Buffer> {
+    if (mode === 'http') {
+      const extensions = PATH_DESCRIPTIONS.find((val) => val.test.test(path))?.extensions;
+      if (extensions) {
+        let data: ArrayBuffer | undefined;
+        for (const ext of extensions) {
+          data = await this.getDataByPath(`${path}${ext}`);
+          if (data) {
+            break;
+          }
+        }
+        if (data) {
+          return Buffer.from(data);
+        }
+      }
+    }
+    if (mode === 'raw') {
+      const decompressedFile = await this.getDataByPath(`${path}.gz`);
+      if (decompressedFile) {
+        return Buffer.from(decompressedFile);
+      }
+      const fileWithoutCompression = await this.getFileBytes(path);
+      if (fileWithoutCompression) {
+        return Buffer.from(fileWithoutCompression);
+      }
+    }
+
+    throw new Error('No such file in the archieve');
+  }
+
+  /**
+   * returning uncompressed data for paths that ends with .gz and raw data for all other paths
+   * @param path - path inside the archive
+   * @returns buffer with the file data
+   */
+  private async getDataByPath(path: string): Promise<ArrayBuffer | undefined> {
+    const data = await this.getFileBytes(path);
+    if (!data) {
+      return undefined;
+    }
+    if (/\.gz$/.test(path)) {
+      const compression = new GZipCompression();
+
+      const decompressedData = await compression.decompress(data);
+      return decompressedData;
+    }
+    return Buffer.from(data);
+  }
+
+  /**
+   * Trying to get raw file data by adress
+   * @param path - path inside the archive
+   * @returns buffer with the raw file data
+   */
+  private async getFileBytes(path: string): Promise<ArrayBuffer | undefined> {
+    const nameHash = Buffer.from(md5(path), 'hex');
+    const fileInfo = this.hashArray.find((val) => Buffer.compare(val.hash, nameHash) === 0);
+    if (!fileInfo) {
+      return undefined;
+    }
+
+    const localFileHeader = await parseZipLocalFileHeader(
+      this.slpkArchive.byteOffset + fileInfo?.offset,
+      new DataViewFileProvider(this.slpkArchive)
+    );
+    if (!localFileHeader) {
+      return undefined;
+    }
+
+    const compressedFile = this.slpkArchive.buffer.slice(
+      localFileHeader.fileDataOffset,
+      localFileHeader.fileDataOffset + localFileHeader.compressedSize
+    );
+
+    return compressedFile;
+  }
+}

package/src/lib/parsers/parse-zip/buffer-file-provider.ts

@@ -0,0 +1,55 @@
+import {FileProvider} from './file-provider';
+
+/**
+ * Provides file data using DataView
+ */
+export class DataViewFileProvider implements FileProvider {
+  /**
+   * The DataView from which data is provided
+   */
+  private file: DataView;
+
+  constructor(file: DataView) {
+    this.file = file;
+  }
+
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: number): Promise<number> {
+    return Promise.resolve(this.file.getUint8(offset));
+  }
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: number): Promise<number> {
+    return Promise.resolve(this.file.getUint16(offset, true));
+  }
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint32(offset: number): Promise<number> {
+    return Promise.resolve(this.file.getUint32(offset, true));
+  }
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: number, endOffset: number): Promise<ArrayBuffer> {
+    return Promise.resolve(this.file.buffer.slice(startOffset, endOffset));
+  }
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  get length() {
+    return this.file.byteLength;
+  }
+}

package/src/lib/parsers/parse-zip/cd-file-header.ts

@@ -0,0 +1,86 @@
+import {FileProvider} from './file-provider';
+
+/**
+ * zip central directory file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipCDFileHeader = {
+  /** Compressed size */
+  compressedSize: number;
+  /** Uncompressed size */
+  uncompressedSize: number;
+  /** File name length */
+  fileNameLength: number;
+  /** File name */
+  fileName: string;
+  /** Extra field offset */
+  extraOffset: number;
+  /** Relative offset of local file header */
+  localHeaderOffset: number;
+};
+
+/**
+ * Parses central directory file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipCDFileHeader = async (
+  headerOffset: number,
+  buffer: FileProvider
+): Promise<ZipCDFileHeader> => {
+  const offsets = {
+    CD_COMPRESSED_SIZE_OFFSET: 20,
+    CD_UNCOMPRESSED_SIZE_OFFSET: 24,
+    CD_FILE_NAME_LENGTH_OFFSET: 28,
+    CD_EXTRA_FIELD_LENGTH_OFFSET: 30,
+    CD_LOCAL_HEADER_OFFSET_OFFSET: 42,
+    CD_FILE_NAME_OFFSET: 46
+  };
+
+  const compressedSize = await buffer.getUint32(headerOffset + offsets.CD_COMPRESSED_SIZE_OFFSET);
+
+  const uncompressedSize = await buffer.getUint32(
+    headerOffset + offsets.CD_UNCOMPRESSED_SIZE_OFFSET
+  );
+
+  const fileNameLength = await buffer.getUint16(headerOffset + offsets.CD_FILE_NAME_LENGTH_OFFSET);
+
+  const fileName = new TextDecoder().decode(
+    await buffer.slice(
+      headerOffset + offsets.CD_FILE_NAME_OFFSET,
+      headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength
+    )
+  );
+
+  const extraOffset = headerOffset + offsets.CD_FILE_NAME_OFFSET + fileNameLength;
+
+  const oldFormatOffset = await buffer.getUint32(
+    headerOffset + offsets.CD_LOCAL_HEADER_OFFSET_OFFSET
+  );
+
+  let fileDataOffset = oldFormatOffset;
+  if (fileDataOffset === 0xffffffff) {
+    let offsetInZip64Data = 4;
+    // looking for info that might be also be in zip64 extra field
+    if (compressedSize === 0xffffffff) {
+      offsetInZip64Data += 8;
+    }
+    if (uncompressedSize === 0xffffffff) {
+      offsetInZip64Data += 8;
+    }
+
+    // getUint32 needs to be replaced with getBigUint64 for archieves bigger than 2gb
+    fileDataOffset = await buffer.getUint32(extraOffset + offsetInZip64Data); // setting it to the one from zip64
+  }
+  const localHeaderOffset = fileDataOffset;
+
+  return {
+    compressedSize,
+    uncompressedSize,
+    fileNameLength,
+    fileName,
+    extraOffset,
+    localHeaderOffset
+  };
+};

package/src/lib/parsers/parse-zip/file-provider.ts

@@ -0,0 +1,34 @@
+/**
+ * Interface for providing file data
+ */
+export interface FileProvider {
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: number): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: number): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: number): Promise<number>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: number, endOffset: number): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: number;
+}

package/src/lib/parsers/parse-zip/local-file-header.ts

@@ -0,0 +1,65 @@
+import {FileProvider} from './file-provider';
+
+/**
+ * zip local file header info
+ * according to https://en.wikipedia.org/wiki/ZIP_(file_format)
+ */
+export type ZipLocalFileHeader = {
+  /** File name length */
+  fileNameLength: number;
+  /** File name */
+  fileName: string;
+  /** Extra field length */
+  extraFieldLength: number;
+  /** Offset of the file data */
+  fileDataOffset: number;
+  /** Compressed size */
+  compressedSize: number;
+};
+
+const offsets = {
+  COMPRESSED_SIZE_OFFSET: 18,
+  FILE_NAME_LENGTH_OFFSET: 26,
+  EXTRA_FIELD_LENGTH_OFFSET: 28,
+  FILE_NAME_OFFSET: 30
+};
+
+const signature = Buffer.from([0x50, 0x4b, 0x03, 0x04]);
+
+/**
+ * Parses local file header of zip file
+ * @param headerOffset - offset in the archive where header starts
+ * @param buffer - buffer containing whole array
+ * @returns Info from the header
+ */
+export const parseZipLocalFileHeader = async (
+  headerOffset: number,
+  buffer: FileProvider
+): Promise<ZipLocalFileHeader | undefined> => {
+  if (Buffer.from(await buffer.slice(headerOffset, headerOffset + 4)).compare(signature) !== 0) {
+    return Promise.resolve(undefined);
+  }
+
+  const fileNameLength = await buffer.getUint16(headerOffset + offsets.FILE_NAME_LENGTH_OFFSET);
+
+  const fileName = new TextDecoder().decode(
+    await buffer.slice(
+      headerOffset + offsets.FILE_NAME_OFFSET,
+      headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength
+    )
+  );
+  const extraFieldLength = await buffer.getUint16(headerOffset + offsets.EXTRA_FIELD_LENGTH_OFFSET);
+
+  const fileDataOffset =
+    headerOffset + offsets.FILE_NAME_OFFSET + fileNameLength + extraFieldLength;
+
+  const compressedSize = await buffer.getUint32(headerOffset + offsets.COMPRESSED_SIZE_OFFSET);
+
+  return {
+    fileNameLength,
+    fileName,
+    extraFieldLength,
+    fileDataOffset,
+    compressedSize
+  };
+};