@loaders.gl/tile-converter 4.5.3 → 5.0.0-alpha.10

package/src/v5/conversion-api.ts

@@ -0,0 +1,287 @@
+/** A diagnostic produced while inspecting, converting, or validating tiles. */
+export interface TileConversionDiagnostic {
+  /** Stable machine-readable diagnostic code. */
+  readonly code: string;
+  /** Human-readable explanation of the diagnostic. */
+  readonly message: string;
+  /** Severity used by callers to decide whether the result is acceptable. */
+  readonly severity: 'info' | 'warning' | 'error';
+  /** Optional resource that produced the diagnostic. */
+  readonly resourceId?: string;
+}
+
+/** Typed failure raised when a conversion policy or validation gate rejects a resource. */
+export class TileConversionError extends Error {
+  /** Stable machine-readable failure code. */
+  readonly code: string;
+  /** Diagnostics associated with this failure. */
+  readonly diagnostics: readonly TileConversionDiagnostic[];
+
+  /** Creates a typed conversion failure. */
+  constructor(
+    code: string,
+    message: string,
+    diagnostics: readonly TileConversionDiagnostic[] = []
+  ) {
+    super(message);
+    this.name = 'TileConversionError';
+    this.code = code;
+    this.diagnostics = diagnostics;
+  }
+}
+
+/** Progress reported by the portable conversion core. */
+export interface TileConversionProgress {
+  /** Current conversion phase. */
+  readonly phase: 'inspect' | 'convert' | 'validate' | 'finalize';
+  /** Number of input resources consumed so far. */
+  readonly inputResources: number;
+  /** Number of output resources written so far. */
+  readonly outputResources: number;
+  /** Number of input bytes consumed so far. */
+  readonly inputBytes: number;
+  /** Number of output bytes written so far. */
+  readonly outputBytes: number;
+}
+
+/** Summary returned after a tileset conversion completes. */
+export interface TileConversionReport {
+  /** Completed conversion state. Failed conversions reject with their cause. */
+  readonly state: 'completed';
+  /** Number of input resources consumed. */
+  readonly inputResources: number;
+  /** Number of output resources written. */
+  readonly outputResources: number;
+  /** Number of input bytes consumed. */
+  readonly inputBytes: number;
+  /** Number of output bytes written. */
+  readonly outputBytes: number;
+  /** Largest output resource written. */
+  readonly largestOutputResourceBytes: number;
+  /** Diagnostics reported during conversion and validation. */
+  readonly diagnostics: readonly TileConversionDiagnostic[];
+}
+
+/** Portable source adapter that resolves and reads a tileset. */
+export interface TileConversionSource<TInspection, TInput> {
+  /** Inspect the source and return its profile-specific metadata. */
+  inspect(signal?: AbortSignal): Promise<TInspection>;
+  /** Stream input resources in deterministic traversal order. */
+  read(inspection: TInspection, signal?: AbortSignal): AsyncIterable<TInput>;
+}
+
+/** Output adapter whose awaited writes apply backpressure to conversion. */
+export interface TileConversionSink<TOutput> {
+  /** Write one completed output resource and resolve when it can be released. */
+  write(resource: TOutput, signal?: AbortSignal): Promise<void>;
+  /** Finalize the destination after every resource has been written. */
+  finalize(report: TileConversionReport): Promise<void>;
+  /** Abort and clean up a partial destination after a failure. */
+  abort(reason: unknown): Promise<void>;
+}
+
+/** Codec adapter used to convert one input resource into streamed outputs. */
+export interface TileConversionCodec<TInspection, TInput, TOutput> {
+  /** Convert one input resource into zero or more output resources. */
+  convert(
+    resource: TInput,
+    inspection: TInspection,
+    signal?: AbortSignal
+  ): AsyncIterable<TOutput> | Promise<AsyncIterable<TOutput>>;
+  /** Optionally validate each output resource before it is written. */
+  validateOutput?(
+    resource: TOutput,
+    signal?: AbortSignal
+  ): Promise<readonly TileConversionDiagnostic[]>;
+}
+
+/** Options for the portable `convertTileset` operation. */
+export interface ConvertTilesetOptions<TInspection, TInput, TOutput> {
+  /** Injected input resolver and resource reader. */
+  readonly source: TileConversionSource<TInspection, TInput>;
+  /** Injected output codec. */
+  readonly codec: TileConversionCodec<TInspection, TInput, TOutput>;
+  /** Injected destination writer. */
+  readonly sink: TileConversionSink<TOutput>;
+  /** Return the decoded input byte size as a nonnegative safe integer. */
+  readonly measureInputBytes: (resource: TInput) => number;
+  /** Return the byte size of an output resource. */
+  readonly measureOutputBytes: (resource: TOutput) => number;
+  /** Reject a decoded input resource above this byte limit before encoding. Defaults to no limit. */
+  readonly maxInputResourceBytes?: number;
+  /** Reject any output resource larger than this configured byte limit. */
+  readonly maxOutputResourceBytes?: number;
+  /** Cancel inspection, conversion, validation, or writes. */
+  readonly signal?: AbortSignal;
+  /** Receive progress updates after each resource transition. */
+  readonly onProgress?: (progress: TileConversionProgress) => void;
+}
+
+/** Options for a standalone tileset validation pass. */
+export interface ValidateTilesetOptions<TTileset> {
+  /** Tileset or conversion result to validate. */
+  readonly tileset: TTileset;
+  /** Injected profile-specific validator. */
+  readonly validate: (
+    tileset: TTileset,
+    signal?: AbortSignal
+  ) => Promise<readonly TileConversionDiagnostic[]>;
+  /** Cancel validation. */
+  readonly signal?: AbortSignal;
+}
+
+/** Result of validating a tileset. */
+export interface TileValidationReport {
+  /** False when at least one error diagnostic was reported. */
+  readonly valid: boolean;
+  /** Diagnostics returned by the injected validator. */
+  readonly diagnostics: readonly TileConversionDiagnostic[];
+}
+
+/** Inspects a tileset with an injected, platform-specific source adapter. */
+export async function inspectTileset<TInspection, TInput>(
+  source: TileConversionSource<TInspection, TInput>,
+  signal?: AbortSignal
+): Promise<TInspection> {
+  throwIfAborted(signal);
+  const inspection = await source.inspect(signal);
+  throwIfAborted(signal);
+  return inspection;
+}
+
+/**
+ * Converts a stream of tileset resources through injected source, codec, and sink adapters.
+ * Decoded inputs are measured and checked against `maxInputResourceBytes` before encoding.
+ * This does not limit source decoding allocations or content retained by a source.
+ * Writes are awaited one resource at a time, keeping queued output bytes bounded by the
+ * largest emitted resource and the configured `maxOutputResourceBytes` limit.
+ */
+export async function convertTileset<TInspection, TInput, TOutput>(
+  options: ConvertTilesetOptions<TInspection, TInput, TOutput>
+): Promise<TileConversionReport> {
+  const {
+    source,
+    codec,
+    sink,
+    measureInputBytes,
+    measureOutputBytes,
+    maxInputResourceBytes = Number.POSITIVE_INFINITY,
+    maxOutputResourceBytes = Number.POSITIVE_INFINITY,
+    signal,
+    onProgress
+  } = options;
+  let inputResources = 0;
+  let outputResources = 0;
+  let inputBytes = 0;
+  let outputBytes = 0;
+  let largestOutputResourceBytes = 0;
+  const diagnostics: TileConversionDiagnostic[] = [];
+  const reportProgress = (phase: TileConversionProgress['phase']): void => {
+    onProgress?.({phase, inputResources, outputResources, inputBytes, outputBytes});
+  };
+
+  try {
+    if (
+      maxInputResourceBytes !== Number.POSITIVE_INFINITY &&
+      (!Number.isSafeInteger(maxInputResourceBytes) || maxInputResourceBytes < 0)
+    ) {
+      throw new TileConversionError(
+        'INVALID_INPUT_RESOURCE_LIMIT',
+        'maxInputResourceBytes must be a nonnegative safe integer or Infinity'
+      );
+    }
+    reportProgress('inspect');
+    const inspection = await inspectTileset(source, signal);
+    for await (const inputResource of source.read(inspection, signal)) {
+      throwIfAborted(signal);
+      const inputResourceBytes = measureInputBytes(inputResource);
+      if (!Number.isSafeInteger(inputResourceBytes) || inputResourceBytes < 0) {
+        throw new TileConversionError(
+          'INVALID_INPUT_RESOURCE_SIZE',
+          'measureInputBytes must return a nonnegative safe integer'
+        );
+      }
+      if (inputResourceBytes > maxInputResourceBytes) {
+        throw new TileConversionError(
+          'INPUT_RESOURCE_TOO_LARGE',
+          `Input resource is ${inputResourceBytes} bytes, exceeding the configured limit of ${maxInputResourceBytes} bytes`
+        );
+      }
+      inputResources++;
+      inputBytes += inputResourceBytes;
+      reportProgress('convert');
+      const convertedResources = await codec.convert(inputResource, inspection, signal);
+      for await (const outputResource of convertedResources) {
+        throwIfAborted(signal);
+        reportProgress('validate');
+        const resourceDiagnostics = (await codec.validateOutput?.(outputResource, signal)) || [];
+        diagnostics.push(...resourceDiagnostics);
+        const outputResourceBytes = measureOutputBytes(outputResource);
+        largestOutputResourceBytes = Math.max(largestOutputResourceBytes, outputResourceBytes);
+        if (outputResourceBytes > maxOutputResourceBytes) {
+          throw new TileConversionError(
+            'OUTPUT_RESOURCE_TOO_LARGE',
+            `Output resource is ${outputResourceBytes} bytes, exceeding the configured limit of ${maxOutputResourceBytes} bytes`
+          );
+        }
+        if (resourceDiagnostics.some(diagnostic => diagnostic.severity === 'error')) {
+          throw new TileConversionError(
+            'OUTPUT_VALIDATION_FAILED',
+            'Output resource validation failed',
+            resourceDiagnostics
+          );
+        }
+        throwIfAborted(signal);
+        await sink.write(outputResource, signal);
+        throwIfAborted(signal);
+        outputResources++;
+        outputBytes += outputResourceBytes;
+        reportProgress('convert');
+      }
+    }
+
+    const report: TileConversionReport = {
+      state: 'completed',
+      inputResources,
+      outputResources,
+      inputBytes,
+      outputBytes,
+      largestOutputResourceBytes,
+      diagnostics
+    };
+    throwIfAborted(signal);
+    reportProgress('finalize');
+    await sink.finalize(report);
+    return report;
+  } catch (error) {
+    try {
+      await sink.abort(error);
+    } catch (abortError) {
+      throw new AggregateError(
+        [error, abortError],
+        'Conversion failed and the destination could not abort'
+      );
+    }
+    throw error;
+  }
+}
+
+/** Validates a tileset through an injected profile-specific validator. */
+export async function validateTileset<TTileset>(
+  options: ValidateTilesetOptions<TTileset>
+): Promise<TileValidationReport> {
+  throwIfAborted(options.signal);
+  const diagnostics = await options.validate(options.tileset, options.signal);
+  throwIfAborted(options.signal);
+  return {
+    valid: !diagnostics.some(diagnostic => diagnostic.severity === 'error'),
+    diagnostics
+  };
+}
+
+/** Throws the signal reason when a conversion operation has been cancelled. */
+function throwIfAborted(signal?: AbortSignal): void {
+  if (signal?.aborted) {
+    throw signal.reason ?? new DOMException('The operation was aborted', 'AbortError');
+  }
+}

package/src/v5/conversion-archive.ts

@@ -0,0 +1,85 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {
+  Tiles3DArchiveWriter,
+  encodeTiles3DArchiveInBatches
+} from '@loaders.gl/3d-tiles/3d-tiles-archive-writer';
+import {SLPKWriter, encodeSLPKArchiveInBatches} from '@loaders.gl/i3s/i3s-slpk-writer';
+import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
+import type {BrowserTileConversionFile} from '@loaders.gl/tile-converter/v5/core';
+
+/** Output profile and required budget for packaging finalized conversion resources. */
+export interface TileConversionArchiveOptions {
+  /** Archive format; resources must already use the selected format's layout and encoding. */
+  readonly format: '3tz' | 'slpk';
+  /** Maximum final archive bytes, including headers/index; not peak serialization memory. */
+  readonly maxArchiveBytes: number;
+  /** Cooperative packaging cancellation; active block reads and header encodes finish first. */
+  readonly signal?: AbortSignal;
+}
+
+/**
+ * Packages finalized, already-authored resources as a bounded 3TZ or SLPK Blob.
+ * 3TZ requires tileset.json; SLPK requires 3dSceneLayer.json.gz and precompressed I3S resources.
+ * Selecting an archive format does not convert geometry, metadata, or coordinate systems.
+ * Applications own input qualification, object URLs, and downloads. Inputs must remain unchanged.
+ * @param files - Unmodified files from a successfully finalized browser sink.
+ * @param options - Archive format and complete output-size budget.
+ * @returns Archive Blob to save using the selected .3tz or .slpk extension.
+ */
+export async function createTileConversionArchive(
+  files: readonly BrowserTileConversionFile[],
+  options: TileConversionArchiveOptions
+): Promise<Blob> {
+  const {resources, format, maxArchiveBytes, signal} = prepareArchiveInput(files, options);
+  const writer = format === '3tz' ? Tiles3DArchiveWriter : SLPKWriter;
+  const archive = await writer.encode(resources, {[format]: {maxArchiveBytes, signal}});
+  return new Blob([archive], {type: writer.mimeTypes[0]});
+}
+
+/**
+ * Streams finalized resources as a bounded 3TZ or SLPK archive without a complete output buffer.
+ * Checks declarations and the full budget on first pull, before output. Await each chunk write
+ * for backpressure; callers finalize storage only after successful completion and discard partial
+ * output on failure, cancellation, or early exit. Resource inputs remain caller-owned and immutable.
+ * Uses payload blocks of at most 64 KiB and retains index/directory metadata and caller inputs.
+ * Each resource is read once for its checksum, then again for output; this is not a total heap guarantee.
+ * @param files - Files from a successfully finalized browser conversion sink.
+ * @param options - Format, complete archive-size budget, and optional cancellation signal.
+ * @returns Uint8Array views to write to application-owned storage, respecting their byte ranges.
+ */
+export async function* encodeTileConversionArchiveInBatches(
+  files: readonly BrowserTileConversionFile[],
+  options: TileConversionArchiveOptions
+): AsyncIterable<Uint8Array<ArrayBuffer>> {
+  const {resources, format, maxArchiveBytes, signal} = prepareArchiveInput(files, options);
+  const encode = format === '3tz' ? encodeTiles3DArchiveInBatches : encodeSLPKArchiveInBatches;
+  yield* encode(resources, {[format]: {maxArchiveBytes, signal}});
+}
+
+/** Captures archive resources/options and preserves the converter's typed input diagnostics. */
+function prepareArchiveInput(
+  files: readonly BrowserTileConversionFile[],
+  options: TileConversionArchiveOptions
+) {
+  const {format, maxArchiveBytes, signal} = options;
+  if (!Number.isSafeInteger(maxArchiveBytes) || maxArchiveBytes < 0) {
+    throw new TileConversionError(
+      'INVALID_ARCHIVE_BYTE_LIMIT',
+      'Archive byte limit must be a non-negative safe integer'
+    );
+  }
+  if (format !== '3tz' && format !== 'slpk') {
+    throw new TileConversionError('INVALID_ARCHIVE_FORMAT', 'Expected 3tz or slpk archive format');
+  }
+  const resources = Object.fromEntries(files.map(file => [file.resourceId, file.blob]));
+  if (Object.keys(resources).length !== files.length) {
+    throw new TileConversionError(
+      'DUPLICATE_ARCHIVE_RESOURCE',
+      'Archive resource identifiers must be unique'
+    );
+  }
+  return {resources, format, maxArchiveBytes, signal};
+}

package/src/v5/core.ts

@@ -0,0 +1,61 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// Portable orchestration and adapters using the existing tiles and math dependencies.
+export {
+  TileConversionError,
+  inspectTileset,
+  convertTileset,
+  validateTileset
+} from './conversion-api.js';
+export type {
+  TileConversionDiagnostic,
+  TileConversionProgress,
+  TileConversionReport,
+  TileConversionSource,
+  TileConversionSink,
+  TileConversionCodec,
+  ConvertTilesetOptions,
+  ValidateTilesetOptions,
+  TileValidationReport
+} from './conversion-api.js';
+export {createBrowserTileConversionSource} from './browser-source.js';
+export type {
+  BrowserTileConversionInspection,
+  BrowserTileConversionInputResource,
+  BrowserTileConversionSourceOptions
+} from './browser-source.js';
+export {traversePointCloudSource} from './point-cloud-source.js';
+export type {PointCloudSourceTile, TraversePointCloudSourceOptions} from './point-cloud-source.js';
+export {
+  createTiles3DConversionSpatialContext,
+  createI3SConversionSpatialContext
+} from './spatial-conversion.js';
+export type {
+  Tiles3DConversionSpatialContext,
+  I3SConversionSpatialContext
+} from './spatial-conversion.js';
+export {createTilesetConversionSource} from './tileset-source.js';
+export type {TilesetConversionSourceOptions} from './tileset-source.js';
+export {createManifestBackedTileConversionSink} from './resource-manifest.js';
+export type {
+  TileResourceManifestEntry,
+  TileResourceManifest,
+  TileResourceManifestStore,
+  ManifestBackedTileConversionSinkOptions
+} from './resource-manifest.js';
+export {createBoundedMemoryTileConversionSink} from './browser-sink.js';
+export type {
+  BrowserTileConversionResource,
+  BrowserTileConversionFile,
+  BoundedMemoryTileConversionSinkOptions,
+  BoundedMemoryTileConversionSink
+} from './browser-sink.js';
+export {createBrowserTilesetConversionSource} from './browser-tileset-source.js';
+export type {
+  BrowserTilesetConversionSourceOptions,
+  BrowserTilesetConversionInspection,
+  BrowserTilesetResourceDescriptor,
+  BrowserTilesetConversionInputResource
+} from './browser-tileset-source.js';