@loaders.gl/3d-tiles 4.0.0-alpha.8 → 4.0.0-beta.1

package/src/cesium-ion-loader.ts

@@ -1,4 +1,4 @@
-import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {LoaderOptions, LoaderWithParser} from '@loaders.gl/loader-utils';
 import {Tiles3DLoader} from './tiles-3d-loader';
 import {getIonTilesetMetadata} from './lib/ion/ion';
 
@@ -18,7 +18,7 @@ async function preload(url, options = {}) {
 /**
  * Loader for 3D tiles from Cesium ION
  */
-export const CesiumIonLoader: LoaderWithParser = {
+export const CesiumIonLoader: LoaderWithParser<unknown, never, LoaderOptions> = {
   ...Tiles3DLoader,
   id: 'cesium-ion',
   name: 'Cesium Ion',
@@ -33,7 +33,6 @@ export const CesiumIonLoader: LoaderWithParser = {
   },
   options: {
     'cesium-ion': {
-      // @ts-expect-error
       ...Tiles3DLoader.options['3d-tiles'],
       accessToken: null
     }

package/src/index.ts

@@ -2,6 +2,9 @@
 export {Tiles3DLoader} from './tiles-3d-loader';
 export {CesiumIonLoader} from './cesium-ion-loader';
 export {Tile3DSubtreeLoader} from './tile-3d-subtree-loader';
+export type {Tiles3DArchiveFileLoaderOptions} from './3d-tiles-archive-loader';
+export {Tiles3DArchiveFileLoader} from './3d-tiles-archive-loader';
+export {Tiles3DArchiveFileSystem} from './lib/filesystems/tiles-3d-archive-file-system';
 
 // WRITERS
 export {Tile3DWriter} from './tile-3d-writer';

package/src/lib/filesystems/tiles-3d-archive-file-system.ts

@@ -0,0 +1,97 @@
+import {FileProvider} from '@loaders.gl/loader-utils';
+import {
+  ZipFileSystem,
+  cdSignature as cdHeaderSignature,
+  searchFromTheEnd,
+  parseZipCDFileHeader,
+  HashElement,
+  parseHashFile,
+  parseZipLocalFileHeader
+} from '@loaders.gl/zip';
+import {Tiles3DArchive} from '../../3d-tiles-archive/3d-tiles-archive-archive';
+
+/**
+ * FileSystem adapter for a 3tz (3D tiles archive format) file
+ * Holds FileProvider object that provides random access to archived files.
+ * The difference from ZipFileSystem is usage of `@3dtilesIndex1@` index file that increases
+ * access speed to archived files
+ * @see https://github.com/erikdahlstrom/3tz-specification/blob/master/Specification.md
+ */
+export class Tiles3DArchiveFileSystem extends ZipFileSystem {
+  hashData?: HashElement[] | null;
+
+  /**
+   * Constructor
+   * @param file - instance of FileProvider or file path string
+   */
+  constructor(file: FileProvider | string) {
+    super(file);
+  }
+
+  /**
+   * Implementation of fetch against this file system.
+   * It tries to take `@3dtilesIndex1@` file from the archive and use it
+   * for faster access to archived files
+   * @param filename - name of a file
+   * @returns - Response with file data
+   */
+  async fetch(filename: string): Promise<Response> {
+    const fileProvider = await this.fileProvider;
+    if (!fileProvider) {
+      throw new Error('No data detected in the zip archive');
+    }
+    await this.parseHashFile();
+    if (this.hashData) {
+      const archive = new Tiles3DArchive(fileProvider, this.hashData);
+
+      const fileData = await archive.getFile(filename);
+      const response = new Response(fileData);
+      Object.defineProperty(response, 'url', {value: `${this.fileName || ''}/${filename}`});
+      return response;
+    }
+    return super.fetch(filename);
+  }
+
+  /**
+   * Try to get and parse '@3dtilesIndex1@' file, that allows to get direct access
+   * to files inside the archive
+   * @returns void
+   */
+  private async parseHashFile(): Promise<void> {
+    if (this.hashData !== undefined) {
+      return;
+    }
+
+    const fileProvider = await this.fileProvider;
+    if (!fileProvider) {
+      throw new Error('No data detected in the zip archive');
+    }
+
+    const hashCDOffset = await searchFromTheEnd(fileProvider, cdHeaderSignature);
+
+    const cdFileHeader = await parseZipCDFileHeader(hashCDOffset, fileProvider);
+
+    // '@3dtilesIndex1@' is index file that must be the last in the archive. It allows
+    // to improve load and read performance when the archive contains a very large number
+    // of files.
+    if (cdFileHeader?.fileName === '@3dtilesIndex1@') {
+      const localFileHeader = await parseZipLocalFileHeader(
+        cdFileHeader.localHeaderOffset,
+        fileProvider
+      );
+      if (!localFileHeader) {
+        throw new Error('corrupted 3tz');
+      }
+
+      const fileDataOffset = localFileHeader.fileDataOffset;
+      const hashFile = await fileProvider.slice(
+        fileDataOffset,
+        fileDataOffset + localFileHeader.compressedSize
+      );
+
+      this.hashData = parseHashFile(hashFile);
+    } else {
+      this.hashData = null;
+    }
+  }
+}

package/src/lib/ion/ion.ts

@@ -34,7 +34,7 @@ export async function getIonAssets(accessToken) {
   assert(accessToken);
   const url = CESIUM_ION_URL;
   const headers = {Authorization: `Bearer ${accessToken}`};
-  const response = await fetchFile(url, {fetch: {headers}});
+  const response = await fetchFile(url, {headers});
   if (!response.ok) {
     throw new Error(response.statusText);
   }
@@ -49,7 +49,7 @@ export async function getIonAssetMetadata(accessToken, assetId) {
   const url = `${CESIUM_ION_URL}/${assetId}`;
   // https://cesium.com/docs/rest-api/#operation/getAsset
   // Retrieves metadata information about a specific asset.
-  let response = await fetchFile(`${url}`, {fetch: {headers}});
+  let response = await fetchFile(`${url}`, {headers});
   if (!response.ok) {
     throw new Error(response.statusText);
   }
@@ -57,7 +57,7 @@ export async function getIonAssetMetadata(accessToken, assetId) {
 
   // https://cesium.com/docs/rest-api/#operation/getAssetEndpoint
   // Retrieves information and credentials that allow you to access the tiled asset data for visualization and analysis.
-  response = await fetchFile(`${url}/endpoint`, {fetch: {headers}});
+  response = await fetchFile(`${url}/endpoint`, {headers});
   if (!response.ok) {
     throw new Error(response.statusText);
   }

package/src/lib/parsers/helpers/parse-3d-implicit-tiles.ts

@@ -1,11 +1,14 @@
 import type {Availability, Tile3DBoundingVolume, Subtree} from '../../../types';
 import {Tile3DSubtreeLoader} from '../../../tile-3d-subtree-loader';
 import {load} from '@loaders.gl/core';
+import {default as log} from '@probe.gl/log';
 
 import {getS2CellIdFromToken, getS2ChildCellId, getS2TokenFromCellId} from '../../utils/s2/index';
 import type {S2VolumeInfo} from '../../utils/obb/s2-corners-to-obb';
 import {convertS2BoundingVolumetoOBB} from '../../utils/obb/s2-corners-to-obb';
 import Long from 'long';
+import {Tiles3DLoaderOptions} from '../../../tiles-3d-loader';
+import {ImplicitOptions} from '../parse-3d-tile-header';
 
 const QUADTREE_DEVISION_COUNT = 4;
 const OCTREE_DEVISION_COUNT = 8;
@@ -83,15 +86,16 @@ function getChildS2VolumeBox(
 // eslint-disable-next-line max-statements
 export async function parseImplicitTiles(params: {
   subtree: Subtree;
-  options: any;
+  implicitOptions: ImplicitOptions;
   parentData?: {mortonIndex: number; x: number; y: number; z: number};
   childIndex?: number;
   level?: number;
   globalData?: {level: number; mortonIndex: number; x: number; y: number; z: number};
   s2VolumeBox?: S2VolumeBox;
+  loaderOptions: Tiles3DLoaderOptions;
 }) {
   const {
-    options,
+    implicitOptions,
     parentData = {
       mortonIndex: 0,
       x: 0,
@@ -106,7 +110,8 @@ export async function parseImplicitTiles(params: {
       y: 0,
       z: 0
     },
-    s2VolumeBox
+    s2VolumeBox,
+    loaderOptions
   } = params;
   let {subtree, level = 0} = params;
   const {
@@ -116,43 +121,56 @@ export async function parseImplicitTiles(params: {
     contentUrlTemplate,
     subtreesUriTemplate,
     basePath
-  } = options;
-
+  } = implicitOptions;
   const tile = {children: [], lodMetricValue: 0, contentUrl: ''};
 
+  if (!maximumLevel) {
+    // eslint-disable-next-line no-console
+    log.once(
+      `Missing 'maximumLevel' or 'availableLevels' property. The subtree ${contentUrlTemplate} won't be loaded...`
+    );
+    return tile;
+  }
+
+  const lev = level + globalData.level;
+  if (lev > maximumLevel) {
+    return tile;
+  }
+
   const childrenPerTile = SUBDIVISION_COUNT_MAP[subdivisionScheme];
+  const bitsPerTile = Math.log2(childrenPerTile);
 
-  const childX = childIndex & 0b01;
-  const childY = (childIndex >> 1) & 0b01;
-  const childZ = (childIndex >> 2) & 0b01;
+  // childIndex is in range [0,4] for quadtrees and [0, 7] for octrees
+  const childX = childIndex & 0b01; // Get first bit for X
+  const childY = (childIndex >> 1) & 0b01; // Get second bit for Y
+  const childZ = (childIndex >> 2) & 0b01; // Get third bit for Z
 
   const levelOffset = (childrenPerTile ** level - 1) / (childrenPerTile - 1);
-  let childTileMortonIndex = concatBits(parentData.mortonIndex, childIndex);
+  let childTileMortonIndex = concatBits(parentData.mortonIndex, childIndex, bitsPerTile);
   let tileAvailabilityIndex = levelOffset + childTileMortonIndex;
 
   // Local tile coordinates
-  let childTileX = concatBits(parentData.x, childX);
-  let childTileY = concatBits(parentData.y, childY);
-  let childTileZ = concatBits(parentData.z, childZ);
+  let childTileX = concatBits(parentData.x, childX, 1);
+  let childTileY = concatBits(parentData.y, childY, 1);
+  let childTileZ = concatBits(parentData.z, childZ, 1);
 
   let isChildSubtreeAvailable = false;
 
-  if (level + 1 > subtreeLevels) {
+  if (level >= subtreeLevels) {
     isChildSubtreeAvailable = getAvailabilityResult(
       subtree.childSubtreeAvailability,
       childTileMortonIndex
     );
   }
 
-  const x = concatBits(globalData.x, childTileX);
-  const y = concatBits(globalData.y, childTileY);
-  const z = concatBits(globalData.z, childTileZ);
-  const lev = level + globalData.level;
+  const x = concatBits(globalData.x, childTileX, level * bitsPerTile);
+  const y = concatBits(globalData.y, childTileY, level * bitsPerTile);
+  const z = concatBits(globalData.z, childTileZ, level * bitsPerTile);
 
   if (isChildSubtreeAvailable) {
     const subtreePath = `${basePath}/${subtreesUriTemplate}`;
     const childSubtreeUrl = replaceContentUrlTemplate(subtreePath, lev, x, y, z);
-    const childSubtree = await load(childSubtreeUrl, Tile3DSubtreeLoader);
+    const childSubtree = await load(childSubtreeUrl, Tile3DSubtreeLoader, loaderOptions);
 
     subtree = childSubtree;
 
@@ -172,7 +190,7 @@ export async function parseImplicitTiles(params: {
 
   const isTileAvailable = getAvailabilityResult(subtree.tileAvailability, tileAvailabilityIndex);
 
-  if (!isTileAvailable || level > maximumLevel) {
+  if (!isTileAvailable) {
     return tile;
   }
 
@@ -198,11 +216,12 @@ export async function parseImplicitTiles(params: {
     // Recursive calling...
     const childTileParsed = await parseImplicitTiles({
       subtree,
-      options,
+      implicitOptions,
+      loaderOptions,
       parentData: pData,
       childIndex: index,
       level: childTileLevel,
-      globalData,
+      globalData: {...globalData},
       s2VolumeBox: childS2VolumeBox
     });
 
@@ -213,7 +232,7 @@ export async function parseImplicitTiles(params: {
         childTileParsed,
         globalLevel,
         childCoordinates,
-        options,
+        implicitOptions,
         s2VolumeBox
       );
       // @ts-ignore
@@ -224,13 +243,37 @@ export async function parseImplicitTiles(params: {
   return tile;
 }
 
-function getAvailabilityResult(availabilityData: Availability, index: number): boolean {
-  if ('constant' in availabilityData) {
-    return Boolean(availabilityData.constant);
+/**
+ * Check tile availability in the bitstream array
+ * @param availabilityData - tileAvailability / contentAvailability / childSubtreeAvailability object
+ * @param index - index in the bitstream array
+ * @returns
+ */
+function getAvailabilityResult(
+  availabilityData: Availability | Availability[],
+  index: number
+): boolean {
+  let availabilityObject: Availability;
+  if (Array.isArray(availabilityData)) {
+    /** TODO: we don't support `3DTILES_multiple_contents` extension at the moment.
+     * https://github.com/CesiumGS/3d-tiles/blob/main/extensions/3DTILES_implicit_tiling/README.md#multiple-contents
+     * Take first item in the array
+     */
+    availabilityObject = availabilityData[0];
+    if (availabilityData.length > 1) {
+      // eslint-disable-next-line no-console
+      log.once('Not supported extension "3DTILES_multiple_contents" has been detected');
+    }
+  } else {
+    availabilityObject = availabilityData;
+  }
+
+  if ('constant' in availabilityObject) {
+    return Boolean(availabilityObject.constant);
   }
 
-  if (availabilityData.explicitBitstream) {
-    return getBooleanValueFromBitstream(index, availabilityData.explicitBitstream);
+  if (availabilityObject.explicitBitstream) {
+    return getBooleanValueFromBitstream(index, availabilityObject.explicitBitstream);
   }
 
   return false;
@@ -248,7 +291,7 @@ function formatTileData(
   tile,
   level: number,
   childCoordinates: {childTileX: number; childTileY: number; childTileZ: number},
-  options: any,
+  options: ImplicitOptions,
   s2VolumeBox?: S2VolumeBox
 ) {
   const {
@@ -337,11 +380,12 @@ function calculateBoundingVolumeForChildTile(
 
 /**
  * Do binary concatenation
- * @param first
- * @param second
+ * @param higher - number to put to higher part of result
+ * @param lower - number to put to lower part of result
+ * @param shift - number of bits to shift lower number
  */
-function concatBits(first: number, second: number): number {
-  return parseInt(first.toString(2) + second.toString(2), 2);
+function concatBits(higher: number, lower: number, shift: number): number {
+  return (higher << shift) + lower;
 }
 
 /**

package/src/lib/parsers/helpers/parse-3d-tile-gltf-view.ts

@@ -8,7 +8,7 @@
 // - Also, should we have hard dependency on gltf module or use injection or auto-discovery for gltf parser?
 
 import {GLTFLoader, postProcessGLTF, _getMemoryUsageGLTF} from '@loaders.gl/gltf';
-import {LoaderContext, sliceArrayBuffer} from '@loaders.gl/loader-utils';
+import {LoaderContext, sliceArrayBuffer, parseFromContext} from '@loaders.gl/loader-utils';
 import {Tiles3DTileContent} from '../../../types';
 import {Tiles3DLoaderOptions} from '../../../tiles-3d-loader';
 
@@ -74,15 +74,20 @@ export async function extractGLTF(
     if (!context) {
       return;
     }
-    const {parse, fetch} = context;
     if (tile.gltfUrl) {
+      const {fetch} = context;
       const response = await fetch(tile.gltfUrl, options);
       tile.gltfArrayBuffer = await response.arrayBuffer();
       tile.gltfByteOffset = 0;
     }
     if (tile.gltfArrayBuffer) {
       // TODO - Should handle byteOffset... However, not used now...
-      const gltfWithBuffers = await parse(tile.gltfArrayBuffer, GLTFLoader, options, context);
+      const gltfWithBuffers = await parseFromContext(
+        tile.gltfArrayBuffer,
+        GLTFLoader,
+        options,
+        context
+      );
       tile.gltf = postProcessGLTF(gltfWithBuffers);
       tile.gpuMemoryUsageInBytes = _getMemoryUsageGLTF(tile.gltf);
       delete tile.gltfArrayBuffer;

package/src/lib/parsers/helpers/parse-3d-tile-subtree.ts

@@ -1,4 +1,4 @@
-import type {Subtree, ExplicitBitstream} from '../../../types';
+import type {Subtree, Availability} from '../../../types';
 import type {LoaderContext, LoaderOptions} from '@loaders.gl/loader-utils';
 
 const SUBTREE_FILE_MAGIC = 0x74627573;
@@ -42,79 +42,54 @@ export default async function parse3DTilesSubtree(
     internalBinaryBuffer = data.slice(24 + jsonByteLength);
   }
 
-  if ('bufferView' in subtree.tileAvailability) {
-    subtree.tileAvailability.explicitBitstream = await getExplicitBitstream(
+  await loadExplicitBitstream(subtree, subtree.tileAvailability, internalBinaryBuffer, context);
+  if (Array.isArray(subtree.contentAvailability)) {
+    for (const contentAvailability of subtree.contentAvailability) {
+      await loadExplicitBitstream(subtree, contentAvailability, internalBinaryBuffer, context);
+    }
+  } else {
+    await loadExplicitBitstream(
       subtree,
-      'tileAvailability',
-      internalBinaryBuffer,
-      context
-    );
-  }
-
-  if ('bufferView' in subtree.contentAvailability) {
-    subtree.contentAvailability.explicitBitstream = await getExplicitBitstream(
-      subtree,
-      'contentAvailability',
-      internalBinaryBuffer,
-      context
-    );
-  }
-
-  if ('bufferView' in subtree.childSubtreeAvailability) {
-    subtree.childSubtreeAvailability.explicitBitstream = await getExplicitBitstream(
-      subtree,
-      'childSubtreeAvailability',
+      subtree.contentAvailability,
       internalBinaryBuffer,
       context
     );
   }
+  await loadExplicitBitstream(
+    subtree,
+    subtree.childSubtreeAvailability,
+    internalBinaryBuffer,
+    context
+  );
 
   return subtree;
 }
 
 /**
- * Get url for bitstream downloading
- * @param bitstreamRelativeUri
- * @param basePath
- * @returns
- */
-function resolveBufferUri(bitstreamRelativeUri: string, basePath: string): string {
-  const hasProtocol = basePath.startsWith('http');
-
-  if (hasProtocol) {
-    const resolvedUri = new URL(bitstreamRelativeUri, basePath);
-    return decodeURI(resolvedUri.toString());
-  }
-
-  /**
-   * Adding http protocol only for new URL constructor usage.
-   * It allows to resolve relative paths like ../../example with basePath.
-   */
-  const basePathWithProtocol = `http://${basePath}`;
-  const resolvedUri = new URL(bitstreamRelativeUri, basePathWithProtocol);
-  /**
-   * Drop protocol and use just relative path.
-   */
-  return `/${resolvedUri.host}${resolvedUri.pathname}`;
-}
-
-/**
- * Get explicit bitstream for subtree availability data.
- * @param subtree
- * @param name
- * @param internalBinaryBuffer
+ * Load explicit bitstream for subtree availability data.
+ * @param subtree - subtree data
+ * @param availabilityObject - tileAvailability / contentAvailability / childSubtreeAvailability object
+ * @param internalBinaryBuffer - subtree binary buffer
+ * @param context - loaders.gl context
  */
-async function getExplicitBitstream(
+async function loadExplicitBitstream(
   subtree: Subtree,
-  name: string,
+  availabilityObject: Availability,
   internalBinaryBuffer: ArrayBuffer,
   context: LoaderContext | undefined
-): Promise<ExplicitBitstream> {
-  const bufferViewIndex = subtree[name].bufferView;
+): Promise<void> {
+  const bufferViewIndex = Number.isFinite(availabilityObject.bitstream)
+    ? availabilityObject.bitstream
+    : availabilityObject.bufferView;
+
+  if (typeof bufferViewIndex !== 'number') {
+    return;
+  }
+
   const bufferView = subtree.bufferViews[bufferViewIndex];
   const buffer = subtree.buffers[bufferView.buffer];
 
-  if (!context?.url || !context.fetch) {
+  if (!context?.baseUrl) {
     throw new Error('Url is not provided');
   }
 
@@ -124,14 +99,21 @@ async function getExplicitBitstream(
 
   // External bitstream loading
   if (buffer.uri) {
-    const bufferUri = resolveBufferUri(buffer.uri, context?.url);
+    const bufferUri = `${context?.baseUrl || ''}/${buffer.uri}`;
     const response = await context.fetch(bufferUri);
     const data = await response.arrayBuffer();
-    // Return view of bitstream.
-    return new Uint8Array(data, bufferView.byteOffset, bufferView.byteLength);
+    availabilityObject.explicitBitstream = new Uint8Array(
+      data,
+      bufferView.byteOffset,
+      bufferView.byteLength
+    );
+    return;
   }
-  // Return view of bitstream.
-  return new Uint8Array(internalBinaryBuffer, bufferView.byteOffset, bufferView.byteLength);
+  availabilityObject.explicitBitstream = new Uint8Array(
+    internalBinaryBuffer,
+    bufferView.byteOffset,
+    bufferView.byteLength
+  );
 }
 
 /**

package/src/lib/parsers/parse-3d-tile-composite.ts

@@ -38,7 +38,7 @@ export async function parseComposite3DTile(
   // extract each tile from the byte stream
   tile.tiles = [];
   while (tile.tiles.length < tile.tilesLength && (tile.byteLength || 0) - byteOffset > 12) {
-    const subtile = {};
+    const subtile: Tiles3DTileContent = {shape: 'tile3d'};
     tile.tiles.push(subtile);
     byteOffset = await parse3DTile(arrayBuffer, byteOffset, options, context, subtile);
     // TODO - do we need to add any padding in between tiles?

package/src/lib/parsers/parse-3d-tile-gltf.ts

@@ -1,6 +1,8 @@
-import type {LoaderContext} from '@loaders.gl/loader-utils';
-import type {Tiles3DLoaderOptions} from '../../tiles-3d-loader';
+// loaders.gl, MIT license
+
+import {parseFromContext, LoaderContext} from '@loaders.gl/loader-utils';
 import {_getMemoryUsageGLTF, GLTFLoader, postProcessGLTF} from '@loaders.gl/gltf';
+import type {Tiles3DLoaderOptions} from '../../tiles-3d-loader';
 import {Tiles3DTileContent} from '../../types';
 
 export async function parseGltf3DTile(
@@ -8,26 +10,25 @@ export async function parseGltf3DTile(
   arrayBuffer: ArrayBuffer,
   options?: Tiles3DLoaderOptions,
   context?: LoaderContext
-): Promise<void> {
+): Promise<number> {
   // Set flags
   // glTF models need to be rotated from Y to Z up
   // https://github.com/AnalyticalGraphicsInc/3d-tiles/tree/master/specification#y-up-to-z-up
   tile.rotateYtoZ = true;
   // Save gltf up axis
-  tile.gltfUpAxis =
-    options?.['3d-tiles'] && options['3d-tiles'].assetGltfUpAxis
-      ? options['3d-tiles'].assetGltfUpAxis
-      : 'Y';
+  tile.gltfUpAxis = options?.['3d-tiles']?.assetGltfUpAxis
+    ? options['3d-tiles'].assetGltfUpAxis
+    : 'Y';
 
   if (options?.['3d-tiles']?.loadGLTF) {
     if (!context) {
-      return;
+      return arrayBuffer.byteLength;
     }
-    const {parse} = context;
-    const gltfWithBuffers = await parse(arrayBuffer, GLTFLoader, options, context);
+    const gltfWithBuffers = await parseFromContext(arrayBuffer, GLTFLoader, options, context);
     tile.gltf = postProcessGLTF(gltfWithBuffers);
     tile.gpuMemoryUsageInBytes = _getMemoryUsageGLTF(tile.gltf);
   } else {
     tile.gltfArrayBuffer = arrayBuffer;
   }
+  return arrayBuffer.byteLength;
 }