@loaders.gl/tile-converter 5.0.0-alpha.1 → 5.0.0-alpha.11

package/src/v5/browser-sink.ts

@@ -0,0 +1,130 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TileConversionError} from './conversion-api.js';
+import type {TileConversionReport, TileConversionSink} from './conversion-api.js';
+
+const BLOB_SIZE_GETTER = Object.getOwnPropertyDescriptor(Blob.prototype, 'size')?.get;
+
+/** One named browser output resource. */
+export interface BrowserTileConversionResource {
+  /** Stable relative output path supplied by the target profile. */
+  readonly resourceId: string;
+  /** Parts combined into a Blob after the total output budget is checked. */
+  readonly parts: readonly BlobPart[];
+  /** MIME type recorded on the resulting Blob. */
+  readonly contentType?: string;
+}
+
+/** A named output resource retained by the bounded browser sink. */
+export interface BrowserTileConversionFile {
+  /** Stable relative output path supplied by the target profile. */
+  readonly resourceId: string;
+  /** Browser-native data for download or application-provided packaging. */
+  readonly blob: Blob;
+}
+
+/** Memory limit for a browser conversion sink. */
+export interface BoundedMemoryTileConversionSinkOptions {
+  /** Maximum combined Blob size retained by this sink, captured at creation, in bytes. */
+  readonly maxTotalBytes: number;
+}
+
+/** Browser sink with read access to its bounded output resources. */
+export interface BoundedMemoryTileConversionSink
+  extends TileConversionSink<BrowserTileConversionResource> {
+  /** Returns the retained files in stable resource-ID order. */
+  getFiles(): readonly BrowserTileConversionFile[];
+}
+
+/**
+ * Creates a browser-only sink that retains output as Blobs under a total byte limit.
+ *
+ * The sink does not download or archive files. Applications can use `getFiles()` after
+ * `convertTileset` completes and package or download the returned Blobs as needed.
+ *
+ * @param options - Maximum in-memory output size.
+ * @returns A sink that stores browser-native Blobs and clears them on abort.
+ */
+export function createBoundedMemoryTileConversionSink(
+  options: BoundedMemoryTileConversionSinkOptions
+): BoundedMemoryTileConversionSink {
+  const {maxTotalBytes} = options;
+  if (!Number.isSafeInteger(maxTotalBytes) || maxTotalBytes < 0) {
+    throw new TileConversionError(
+      'INVALID_OUTPUT_MEMORY_LIMIT',
+      'The browser output memory limit must be a non-negative safe integer'
+    );
+  }
+
+  const files = new Map<string, BrowserTileConversionFile>();
+  let totalBytes = 0;
+  let closed = false;
+
+  return {
+    getFiles() {
+      return [...files.values()].sort((left, right) =>
+        left.resourceId < right.resourceId ? -1 : left.resourceId > right.resourceId ? 1 : 0
+      );
+    },
+    async write(resource, signal) {
+      if (signal?.aborted) {
+        throw signal.reason ?? new DOMException('The operation was aborted', 'AbortError');
+      }
+      if (closed) {
+        throw new TileConversionError(
+          'RESOURCE_SINK_CLOSED',
+          'Cannot write after sink finalization'
+        );
+      }
+      if (!resource.resourceId || files.has(resource.resourceId)) {
+        throw new TileConversionError(
+          'INVALID_BROWSER_RESOURCE_ID',
+          'Browser output resource IDs must be non-empty and unique'
+        );
+      }
+
+      const byteLength = resource.parts.reduce((total, part) => total + measureBlobPart(part), 0);
+      if (!Number.isSafeInteger(byteLength) || totalBytes + byteLength > maxTotalBytes) {
+        throw new TileConversionError(
+          'OUTPUT_MEMORY_LIMIT_EXCEEDED',
+          `Browser output exceeds the configured ${maxTotalBytes} byte memory limit`
+        );
+      }
+
+      const blob = new Blob([...resource.parts], {type: resource.contentType});
+      files.set(resource.resourceId, {resourceId: resource.resourceId, blob});
+      totalBytes += blob.size;
+    },
+    async finalize(_report: TileConversionReport) {
+      if (closed) {
+        throw new TileConversionError('RESOURCE_SINK_CLOSED', 'Sink has already been finalized');
+      }
+      closed = true;
+    },
+    async abort() {
+      files.clear();
+      totalBytes = 0;
+      closed = true;
+    }
+  };
+}
+
+/** Measures supported Blob parts without copying them into a Blob first. */
+function measureBlobPart(part: BlobPart): number {
+  if (typeof part === 'string') {
+    return new TextEncoder().encode(part).byteLength;
+  }
+  if (BLOB_SIZE_GETTER) {
+    try {
+      return BLOB_SIZE_GETTER.call(part) as number;
+    } catch {
+      // Keep checking the remaining supported Blob part types.
+    }
+  }
+  if (part instanceof ArrayBuffer || ArrayBuffer.isView(part)) {
+    return part.byteLength;
+  }
+  throw new TileConversionError('UNSUPPORTED_BROWSER_RESOURCE', 'Unsupported browser Blob part');
+}

package/src/v5/browser-source.ts

@@ -0,0 +1,155 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TileConversionError} from './conversion-api.js';
+import type {TileConversionSource} from './conversion-api.js';
+
+/** Metadata available before a browser resource is read. */
+export interface BrowserTileConversionInspection {
+  /** Stable resource path assigned to the input tile. */
+  readonly resourceId: string;
+  /** Optional MIME type supplied by the caller or source Blob. */
+  readonly contentType?: string;
+}
+
+/** Raw bytes read from one browser tile resource. */
+export interface BrowserTileConversionInputResource {
+  /** Stable resource path assigned to the input tile. */
+  readonly resourceId: string;
+  /** Complete resource bytes for a profile-specific codec. */
+  readonly data: Uint8Array;
+  /** Optional MIME type supplied by the caller, Blob, or URL response. */
+  readonly contentType?: string;
+}
+
+/** Browser input and size limit for a single-resource conversion. */
+export interface BrowserTileConversionSourceOptions {
+  /** Input URL or already available browser Blob. */
+  readonly input: string | Blob;
+  /** Stable path to associate with the input resource. */
+  readonly resourceId: string;
+  /** Maximum number of input bytes held in memory. */
+  readonly maxInputBytes: number;
+  /** Optional MIME type override. */
+  readonly contentType?: string;
+}
+
+/**
+ * Creates a browser source for one mesh resource provided as a URL or Blob.
+ *
+ * URL responses are read incrementally and canceled if they exceed `maxInputBytes`.
+ * Tileset JSON traversal and format decoding remain profile-specific source and codec work.
+ *
+ * @param options - Input, stable resource ID, and maximum input size.
+ * @returns A source that yields one complete resource to a v5 codec.
+ */
+export function createBrowserTileConversionSource(
+  options: BrowserTileConversionSourceOptions
+): TileConversionSource<BrowserTileConversionInspection, BrowserTileConversionInputResource> {
+  if (
+    !options.resourceId ||
+    !Number.isSafeInteger(options.maxInputBytes) ||
+    options.maxInputBytes < 0
+  ) {
+    throw new TileConversionError(
+      'INVALID_BROWSER_SOURCE_OPTIONS',
+      'Browser sources require a resource ID and a non-negative safe input byte limit'
+    );
+  }
+
+  return {
+    async inspect(signal) {
+      throwIfAborted(signal);
+      const contentType = options.contentType ?? getBlobContentType(options.input);
+      return {
+        resourceId: options.resourceId,
+        ...(contentType ? {contentType} : {})
+      };
+    },
+    async *read(inspection, signal) {
+      throwIfAborted(signal);
+      const input = options.input;
+      let data: Uint8Array;
+      let contentType = inspection.contentType;
+
+      if (typeof input === 'string') {
+        const response = await fetch(input, {signal});
+        if (!response.ok) {
+          throw new TileConversionError(
+            'BROWSER_SOURCE_FETCH_FAILED',
+            `Could not fetch browser input (${response.status} ${response.statusText})`
+          );
+        }
+        contentType = contentType ?? response.headers.get('content-type') ?? undefined;
+        data = await readResponseBytes(response, options.maxInputBytes, signal);
+      } else {
+        if (input.size > options.maxInputBytes) {
+          throw createInputLimitError(options.maxInputBytes);
+        }
+        data = new Uint8Array(await input.arrayBuffer());
+      }
+
+      throwIfAborted(signal);
+      yield {
+        resourceId: inspection.resourceId,
+        data,
+        ...(contentType ? {contentType} : {})
+      };
+    }
+  };
+}
+
+/** Reads a response stream while enforcing the input byte limit. */
+async function readResponseBytes(
+  response: Response,
+  maxInputBytes: number,
+  signal?: AbortSignal
+): Promise<Uint8Array> {
+  const reader = response.body?.getReader();
+  if (!reader) {
+    throw new TileConversionError(
+      'BROWSER_SOURCE_STREAM_UNAVAILABLE',
+      'Browser URL inputs must provide a readable response stream'
+    );
+  }
+
+  const data = new Uint8Array(maxInputBytes);
+  let totalBytes = 0;
+  try {
+    while (true) {
+      throwIfAborted(signal);
+      const {done, value} = await reader.read();
+      if (done) break;
+      if (value.byteLength > maxInputBytes - totalBytes) {
+        await reader.cancel();
+        throw createInputLimitError(maxInputBytes);
+      }
+      data.set(value, totalBytes);
+      totalBytes += value.byteLength;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  return data.subarray(0, totalBytes);
+}
+
+/** Returns a Blob's MIME type without requiring a same-realm Blob constructor. */
+function getBlobContentType(input: string | Blob): string | undefined {
+  return typeof input === 'string' ? undefined : input.type || undefined;
+}
+
+/** Creates the typed error used when browser input exceeds its configured limit. */
+function createInputLimitError(maxInputBytes: number): TileConversionError {
+  return new TileConversionError(
+    'INPUT_RESOURCE_TOO_LARGE',
+    `Browser input exceeds the configured ${maxInputBytes} byte memory limit`
+  );
+}
+
+/** Throws the signal reason when a browser source operation has been canceled. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw signal.reason ?? new DOMException('The operation was aborted', 'AbortError');
+  }
+}

package/src/v5/browser-tileset-source.ts

@@ -0,0 +1,328 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TileConversionError} from './conversion-api.js';
+import type {TileConversionSource} from './conversion-api.js';
+
+type TilesetContent = {uri?: string; url?: string};
+type TilesetTile = {
+  content?: TilesetContent;
+  contents?: TilesetContent[];
+  children?: TilesetTile[];
+  transform?: number[];
+  implicitTiling?: unknown;
+  extensions?: Record<string, unknown>;
+};
+type TilesetDocument = {asset?: {version?: string}; root?: TilesetTile};
+
+/** Input options for bounded traversal of one explicit 3D Tiles tileset. */
+export interface BrowserTilesetConversionSourceOptions {
+  /** URL of the root tileset JSON document. */
+  readonly input: string;
+  /** Maximum bytes retained across the tileset JSON and all content resources. */
+  readonly maxInputBytes: number;
+  /** Maximum number of content resources to visit. */
+  readonly maxInputResources: number;
+  /** Fetch implementation, defaulting to the browser global. */
+  readonly fetcher?: typeof fetch;
+}
+
+/** Inspection result for an explicit 3D Tiles source. */
+export interface BrowserTilesetConversionInspection {
+  /** Absolute URL of the root tileset document. */
+  readonly rootUrl: string;
+  /** Parsed root document used for deterministic content traversal. */
+  readonly tileset: TilesetDocument;
+  /** Ordered content descriptors. */
+  readonly resources: readonly BrowserTilesetResourceDescriptor[];
+  /** Root document bytes charged against the input limit. */
+  readonly tilesetBytes: number;
+}
+
+/** Placement metadata for one ordered content resource. */
+export interface BrowserTilesetResourceDescriptor {
+  /** Unique, stable path for this content placement. */
+  readonly resourceId: string;
+  /** Original URI declared by the tileset. */
+  readonly uri: string;
+  /** Tile child indexes from the tileset root. */
+  readonly tilePath: readonly number[];
+  /** Combined root-to-tile transform in column-major order. */
+  readonly transform: readonly number[];
+}
+
+/** One raw content resource from a browser-readable explicit tileset. */
+export interface BrowserTilesetConversionInputResource extends BrowserTilesetResourceDescriptor {
+  /** Complete content bytes for an injected profile-specific codec. */
+  readonly data: Uint8Array;
+  /** Optional content type reported by the response. */
+  readonly contentType?: string;
+}
+
+/**
+ * Creates a bounded browser source for explicit 3D Tiles 1.0 and 1.1 tilesets.
+ *
+ * Traversal visits each declared content placement once in depth-first order. Implicit tiling and
+ * nested external tilesets are rejected; content decoding and conversion remain codec work.
+ *
+ * @param options - Tileset URL and aggregate byte and resource limits.
+ * @returns A source yielding ordered raw tile resources with placement transforms.
+ */
+export function createBrowserTilesetConversionSource(
+  options: BrowserTilesetConversionSourceOptions
+): TileConversionSource<BrowserTilesetConversionInspection, BrowserTilesetConversionInputResource> {
+  if (
+    !Number.isSafeInteger(options.maxInputBytes) ||
+    options.maxInputBytes < 1 ||
+    !Number.isSafeInteger(options.maxInputResources) ||
+    options.maxInputResources < 1
+  ) {
+    throw new TileConversionError(
+      'INVALID_BROWSER_SOURCE_OPTIONS',
+      'Browser tileset sources require positive byte and resource limits'
+    );
+  }
+
+  const fetcher = options.fetcher ?? fetch;
+  return {
+    async inspect(signal) {
+      throwIfAborted(signal);
+      const rootUrl = new URL(options.input).href;
+      const response = await fetcher(rootUrl, {signal});
+      if (!response.ok) {
+        throw new TileConversionError(
+          'BROWSER_SOURCE_FETCH_FAILED',
+          `Could not fetch browser tileset (${response.status} ${response.statusText})`
+        );
+      }
+      const data = await readResponseBytes(response, options.maxInputBytes, signal);
+      let tileset: TilesetDocument;
+      try {
+        tileset = JSON.parse(new TextDecoder().decode(data)) as TilesetDocument;
+      } catch {
+        throw new TileConversionError('INVALID_BROWSER_TILESET', 'Tileset input is not valid JSON');
+      }
+      if (
+        !isRecord(tileset) ||
+        !['1.0', '1.1'].includes(tileset.asset?.version || '') ||
+        !isRecord(tileset.root)
+      ) {
+        throw new TileConversionError(
+          'UNSUPPORTED_BROWSER_TILESET',
+          'Browser tileset traversal supports explicit 3D Tiles 1.0 and 1.1 documents'
+        );
+      }
+      const resources = collectResources(tileset.root, options.maxInputResources);
+      return {rootUrl, tileset, resources, tilesetBytes: data.byteLength};
+    },
+    async *read(inspection, signal) {
+      let inputBytes = inspection.tilesetBytes;
+      for (const descriptor of inspection.resources) {
+        throwIfAborted(signal);
+        const resourceUrl = resolveResourceUrl(descriptor.uri, inspection.rootUrl);
+        const response = await fetcher(resourceUrl, {signal});
+        if (!response.ok) {
+          throw new TileConversionError(
+            'BROWSER_SOURCE_FETCH_FAILED',
+            `Could not fetch browser tile content (${response.status} ${response.statusText})`,
+            [
+              {
+                code: 'BROWSER_SOURCE_FETCH_FAILED',
+                message: resourceUrl,
+                severity: 'error',
+                resourceId: descriptor.resourceId
+              }
+            ]
+          );
+        }
+        const remainingBytes = options.maxInputBytes - inputBytes;
+        const data = await readResponseBytes(response, remainingBytes, signal);
+        inputBytes += data.byteLength;
+        const contentType = response.headers.get('content-type') ?? undefined;
+        if (isJsonContent(data, contentType)) {
+          throw new TileConversionError(
+            'UNSUPPORTED_BROWSER_TILESET_CONTENT',
+            `Nested tilesets and JSON tile content are not supported: ${descriptor.uri}`
+          );
+        }
+        yield {...descriptor, data, ...(contentType ? {contentType} : {})};
+      }
+    }
+  };
+}
+
+/** Collects explicit tile contents in declaration order and fails on unbounded formats. */
+function collectResources(
+  root: TilesetTile,
+  maxInputResources: number
+): BrowserTilesetResourceDescriptor[] {
+  const resources: BrowserTilesetResourceDescriptor[] = [];
+  const stack: Array<{tile: TilesetTile; path: number[]; transform: number[]}> = [
+    {tile: root, path: [], transform: identityMatrix()}
+  ];
+  while (stack.length > 0) {
+    const {tile, path, transform: parentTransform} = stack.pop()!;
+    if (!isRecord(tile)) {
+      throw new TileConversionError('INVALID_BROWSER_TILESET', 'Tile entries must be objects');
+    }
+    if (
+      tile.implicitTiling !== undefined ||
+      tile.extensions?.['3DTILES_implicit_tiling'] !== undefined
+    ) {
+      throw new TileConversionError(
+        'UNSUPPORTED_BROWSER_TILESET',
+        'Browser tileset traversal does not yet support implicit tiling'
+      );
+    }
+    if (tile.contents !== undefined && !Array.isArray(tile.contents)) {
+      throw new TileConversionError('INVALID_BROWSER_TILESET', 'Tile contents must be an array');
+    }
+    if (tile.children !== undefined && !Array.isArray(tile.children)) {
+      throw new TileConversionError('INVALID_BROWSER_TILESET', 'Tile children must be an array');
+    }
+    const transform = multiplyMatrices(parentTransform, normalizeMatrix(tile.transform));
+    const contents = tile.contents ?? (tile.content ? [tile.content] : []);
+    for (let contentIndex = 0; contentIndex < contents.length; contentIndex++) {
+      const content = contents[contentIndex];
+      if (!isRecord(content)) {
+        throw new TileConversionError(
+          'INVALID_BROWSER_TILESET',
+          'Tile content entries must be objects'
+        );
+      }
+      const uri = content.uri ?? content.url;
+      if (typeof uri !== 'string' || !uri) {
+        throw new TileConversionError(
+          'INVALID_BROWSER_TILESET',
+          `Tile content at ${path.join('/')} has no URI`
+        );
+      }
+      if (resources.length >= maxInputResources) {
+        throw new TileConversionError(
+          'INPUT_RESOURCE_COUNT_EXCEEDED',
+          `Browser tileset exceeds the configured ${maxInputResources} resource limit`
+        );
+      }
+      resources.push({
+        resourceId: `tile-${path.length ? path.join('-') : 'root'}-content-${contentIndex}`,
+        uri,
+        tilePath: path,
+        transform
+      });
+    }
+    const children = tile.children ?? [];
+    for (let childIndex = children.length - 1; childIndex >= 0; childIndex--) {
+      stack.push({
+        tile: children[childIndex],
+        path: [...path, childIndex],
+        transform
+      });
+    }
+  }
+  return resources;
+}
+
+/** Reads one response into a single buffer without exceeding its remaining byte budget. */
+async function readResponseBytes(
+  response: Response,
+  maximumBytes: number,
+  signal?: AbortSignal
+): Promise<Uint8Array> {
+  const reader = response.body?.getReader();
+  if (!reader) {
+    throw new TileConversionError(
+      'BROWSER_SOURCE_STREAM_UNAVAILABLE',
+      'Browser inputs must provide a readable response stream'
+    );
+  }
+  const data = new Uint8Array(maximumBytes);
+  let byteLength = 0;
+  try {
+    while (true) {
+      throwIfAborted(signal);
+      const {done, value} = await reader.read();
+      if (done) break;
+      if (value.byteLength > maximumBytes - byteLength) {
+        await reader.cancel();
+        throw new TileConversionError(
+          'INPUT_RESOURCE_TOO_LARGE',
+          `Browser tileset input exceeds the configured ${maximumBytes} byte remaining limit`
+        );
+      }
+      data.set(value, byteLength);
+      byteLength += value.byteLength;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+  return data.subarray(0, byteLength);
+}
+
+/** Detects JSON payloads that may declare an unsupported external tileset. */
+function isJsonContent(data: Uint8Array, contentType?: string): boolean {
+  if (contentType?.includes('json')) return true;
+  let byteOffset = data[0] === 0xef && data[1] === 0xbb && data[2] === 0xbf ? 3 : 0;
+  while ([0x09, 0x0a, 0x0d, 0x20].includes(data[byteOffset])) byteOffset++;
+  return data[byteOffset] === 0x7b;
+}
+
+/** Resolves a tile URI and carries root query parameters to same-origin relative resources. */
+function resolveResourceUrl(uri: string, rootUrl: string): string {
+  const root = new URL(rootUrl);
+  const resource = new URL(uri, root);
+  if (resource.origin === root.origin) {
+    for (const [name, value] of root.searchParams) {
+      if (!resource.searchParams.has(name)) resource.searchParams.append(name, value);
+    }
+  }
+  return resource.href;
+}
+
+/** Returns whether a JSON value is a non-array object. */
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return Boolean(value && typeof value === 'object' && !Array.isArray(value));
+}
+
+/** Returns an identity matrix in column-major order. */
+function identityMatrix(): number[] {
+  return [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1];
+}
+
+/** Validates and copies a tile's optional column-major transform. */
+function normalizeMatrix(matrix?: number[]): number[] {
+  if (matrix === undefined) return identityMatrix();
+  if (
+    !Array.isArray(matrix) ||
+    matrix.length !== 16 ||
+    matrix.some(value => !Number.isFinite(value))
+  ) {
+    throw new TileConversionError(
+      'INVALID_BROWSER_TILESET',
+      'Tile transform must contain 16 finite numbers'
+    );
+  }
+  return matrix;
+}
+
+/** Multiplies column-major 4x4 matrices. */
+function multiplyMatrices(left: number[], right: number[]): number[] {
+  const result = new Array<number>(16);
+  for (let column = 0; column < 4; column++) {
+    for (let row = 0; row < 4; row++) {
+      result[column * 4 + row] =
+        left[row] * right[column * 4] +
+        left[4 + row] * right[column * 4 + 1] +
+        left[8 + row] * right[column * 4 + 2] +
+        left[12 + row] * right[column * 4 + 3];
+    }
+  }
+  return result;
+}
+
+/** Throws the signal reason when a browser source operation has been canceled. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw signal.reason ?? new DOMException('The operation was aborted', 'AbortError');
+  }
+}