@loaders.gl/tile-converter 5.0.0-alpha.10 → 5.0.0-alpha.12

package/src/v5/point-cloud.ts

@@ -3,7 +3,12 @@
 // Copyright (c) vis.gl contributors
 
 import {Tile3DWriter, TILE3D_TYPE} from '@loaders.gl/3d-tiles';
-import type {Mesh, MeshArrowTable, MeshAttribute} from '@loaders.gl/schema';
+import type {
+  Mesh,
+  MeshArrowTable,
+  MeshAttribute,
+  MeshAttributeQuantizationTransform
+} from '@loaders.gl/schema';
 import {convertTableToMesh} from '@loaders.gl/schema-utils';
 import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
 
@@ -11,12 +16,28 @@ import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
 export interface EncodePointCloudTileOptions {
   /** Optional origin subtracted from absolute positions and stored as the PNTS RTC_CENTER. */
   readonly rtcCenter?: readonly [number, number, number];
+  /** Maximum Euclidean position reconstruction error in source units; omission permits float32 rounding. */
+  readonly maxPositionError?: number;
   /** Optional source-wide RGBA color stored as the PNTS CONSTANT_RGBA property. */
   readonly constantRGBA?: readonly number[];
   /** Optional per-point values indexed by the tile's `BATCH_ID` attribute. */
   readonly batchTableJson?: Readonly<Record<string, readonly (string | number)[]>>;
 }
 
+/** Encoded PNTS and measured placement information from its float32 POSITION stream. */
+export interface EncodedPointCloudTile {
+  /** Self-contained PNTS payload. */
+  readonly pnts: ArrayBuffer;
+  /** Actual number of encoded points. */
+  readonly pointCount: number;
+  /** RTC center added to encoded positions; zero when no RTC center was selected. */
+  readonly rtcCenter: readonly [number, number, number];
+  /** Axis-aligned bounds of encoded positions before adding the RTC center. */
+  readonly localBoundingBox: readonly [readonly number[], readonly number[]];
+  /** Maximum Euclidean error after reconstructing absolute positions, in source units. */
+  readonly maximumPositionError: number;
+}
+
 /**
  * Encodes one mesh or Mesh Arrow point batch as a PNTS resource.
  *
@@ -34,32 +55,34 @@ export function encodePointCloudTile(
   pointBatch: Mesh | MeshArrowTable,
   options: EncodePointCloudTileOptions = {}
 ): ArrayBuffer {
+  return encodePointCloudTileWithMetadata(pointBatch, options).pnts;
+}
+
+/**
+ * Encodes a point batch and measures its decoded float32 bounds and position rounding.
+ * The optional error gate runs before PNTS serialization. Source arrays remain unchanged.
+ * @param pointBatch - One mesh or Arrow point batch.
+ * @param options - RTC center, precision gate, appearance and batch metadata.
+ * @returns PNTS bytes paired with bounds, RTC placement and maximum position error.
+ */
+export function encodePointCloudTileWithMetadata(
+  pointBatch: Mesh | MeshArrowTable,
+  options: EncodePointCloudTileOptions = {}
+): EncodedPointCloudTile {
   const mesh = 'attributes' in pointBatch ? pointBatch : convertTableToMesh(pointBatch);
   validatePointAttributes(mesh.attributes);
   const positionAttribute = mesh.attributes.POSITION;
-  if (!positionAttribute || positionAttribute.size !== 3) {
-    throw new TileConversionError(
-      'POINT_CLOUD_POSITION_REQUIRED',
-      'Point cloud batches must contain a three-component POSITION attribute'
-    );
-  }
-
-  const positions = getPositionValues(positionAttribute, options.rtcCenter);
+  validatePointCloudPosition(positionAttribute);
+  const placement = encodePointPositions(positionAttribute, options);
+  const {positions} = placement;
   const pointCount = positions.length / 3;
-  if (pointCount === 0 || !Number.isInteger(pointCount)) {
-    throw new TileConversionError(
-      'POINT_CLOUD_POSITION_COUNT_INVALID',
-      'Point cloud POSITION must contain one or more xyz triples'
-    );
-  }
-
   const colors = getColorAttribute(mesh.attributes.COLOR_0 || mesh.attributes.COLOR, pointCount);
   const constantRGBA = colors ? null : getConstantRgba(options.constantRGBA);
   const normals = getNormalAttribute(mesh.attributes.NORMAL, pointCount);
   const batchIds = getBatchIds(mesh.attributes.BATCH_ID, pointCount);
   const batchTableJson = validateBatchTable(options.batchTableJson, batchIds);
 
-  return Tile3DWriter.encodeSync(
+  const pnts = Tile3DWriter.encodeSync(
     {
       type: TILE3D_TYPE.POINT_CLOUD,
       featureTableJson:
@@ -74,6 +97,13 @@ export function encodePointCloudTile(
     },
     {}
   );
+  return {
+    pnts,
+    pointCount,
+    rtcCenter: placement.rtcCenter,
+    localBoundingBox: placement.localBoundingBox,
+    maximumPositionError: placement.maximumPositionError
+  };
 }
 
 /** Rejects attributes that this PNTS encoder would otherwise silently discard. */
@@ -114,30 +144,127 @@ function getConstantRgba(
   return [constantRGBA[0], constantRGBA[1], constantRGBA[2], constantRGBA[3]];
 }
 
-/** Decodes positions and offsets them relative to the optional RTC center. */
-function getPositionValues(
-  attribute: MeshAttribute,
-  rtcCenter?: readonly [number, number, number]
-): Float32Array {
-  if (rtcCenter && (rtcCenter.length !== 3 || rtcCenter.some(value => !Number.isFinite(value)))) {
+/** Validates packed point positions and their optional quantization descriptor. */
+function validatePointCloudPosition(
+  attribute: MeshAttribute | undefined
+): asserts attribute is MeshAttribute & {transform?: MeshAttributeQuantizationTransform} {
+  if (
+    !attribute ||
+    attribute.size !== 3 ||
+    attribute.byteOffset ||
+    attribute.byteStride ||
+    attribute.normalized ||
+    attribute.componentType
+  ) {
+    throw new TileConversionError(
+      'POINT_CLOUD_POSITION_REQUIRED',
+      'Point cloud batches must contain packed, non-normalized three-component POSITION values'
+    );
+  }
+
+  if (!attribute.value.length || attribute.value.length % 3) {
+    throw new TileConversionError(
+      'POINT_CLOUD_POSITION_COUNT_INVALID',
+      'Point cloud POSITION must contain one or more xyz triples'
+    );
+  }
+  const transform = attribute.transform;
+  if (
+    transform &&
+    (transform.type !== 'quantization' ||
+      !Number.isInteger(transform.bits) ||
+      transform.bits < 1 ||
+      transform.bits > 32 ||
+      transform.origin.length !== 3 ||
+      transform.origin.some(value => !Number.isFinite(value)) ||
+      !Number.isFinite(transform.range) ||
+      transform.range < 0)
+  ) {
+    throw new TileConversionError(
+      'POINT_CLOUD_POSITION_TRANSFORM_INVALID',
+      'Position quantization requires finite xyz origin, nonnegative range and 1–32 bits'
+    );
+  }
+}
+
+/** Decodes packed point positions to doubles before applying a spatial operation. */
+export function decodePointCloudPositions(attribute: MeshAttribute | undefined): Float64Array {
+  validatePointCloudPosition(attribute);
+  const transform = attribute.transform;
+  return Float64Array.from(attribute.value, (value, index) =>
+    transform
+      ? transform.origin[index % 3] + (value / (2 ** transform.bits - 1)) * transform.range
+      : value
+  );
+}
+
+/** Decodes source positions, rebases them and measures the actual float32 output. */
+function encodePointPositions(
+  attribute: MeshAttribute & {transform?: MeshAttributeQuantizationTransform},
+  options: EncodePointCloudTileOptions
+) {
+  const {rtcCenter = [0, 0, 0], maxPositionError = Infinity} = options;
+  if (rtcCenter.length !== 3 || rtcCenter.some(value => !Number.isFinite(value))) {
     throw new TileConversionError(
       'POINT_CLOUD_RTC_CENTER_INVALID',
       'Point cloud RTC_CENTER must contain three finite values'
     );
   }
+  if (
+    options.maxPositionError !== undefined &&
+    (!Number.isFinite(maxPositionError) || maxPositionError < 0)
+  ) {
+    throw new TileConversionError(
+      'POINT_CLOUD_PRECISION_INVALID',
+      'maxPositionError must be finite and nonnegative'
+    );
+  }
   const values = attribute.value;
-  const result = new Float32Array(values.length);
+  const positions = new Float32Array(values.length);
+  const minimum = [Infinity, Infinity, Infinity];
+  const maximum = [-Infinity, -Infinity, -Infinity];
+  let maximumPositionError = 0;
   const transform = attribute.transform;
-  for (let index = 0; index < values.length; index++) {
-    const component = index % 3;
-    const value = values[index];
-    const absoluteValue =
-      transform?.type === 'quantization'
-        ? transform.origin[component] + (value / (2 ** transform.bits - 1)) * transform.range
+  const errors = [0, 0, 0];
+  for (let point = 0; point < values.length; point += 3) {
+    for (let axis = 0; axis < 3; axis++) {
+      const value = values[point + axis];
+      const absoluteValue = transform
+        ? transform.origin[axis] + (value / (2 ** transform.bits - 1)) * transform.range
         : value;
-    result[index] = absoluteValue - (rtcCenter?.[component] ?? 0);
+      const relativeValue = absoluteValue - rtcCenter[axis];
+      positions[point + axis] = relativeValue;
+      const encodedValue = positions[point + axis];
+      const reconstructed = encodedValue + rtcCenter[axis];
+      if (
+        !Number.isFinite(absoluteValue) ||
+        !Number.isFinite(relativeValue) ||
+        !Number.isFinite(encodedValue) ||
+        !Number.isFinite(reconstructed)
+      ) {
+        throw new TileConversionError(
+          'POINT_CLOUD_POSITION_INVALID',
+          'Point positions and their float32 reconstruction must be finite'
+        );
+      }
+      errors[axis] = reconstructed - absoluteValue;
+      minimum[axis] = Math.min(minimum[axis], encodedValue);
+      maximum[axis] = Math.max(maximum[axis], encodedValue);
+    }
+    maximumPositionError = Math.max(maximumPositionError, Math.hypot(...errors));
+  }
+  if (!Number.isFinite(maximumPositionError) || maximumPositionError > maxPositionError) {
+    throw new TileConversionError(
+      'POINT_CLOUD_PRECISION_EXCEEDED',
+      'Float32 position reconstruction exceeds maxPositionError; select a closer RTC center'
+    );
   }
-  return result;
+  return {
+    positions,
+    rtcCenter: [...rtcCenter] as [number, number, number],
+    localBoundingBox: [minimum, maximum] as const,
+    maximumPositionError
+  };
 }
 
 /** Converts an optional mesh color attribute into the byte colors accepted by PNTS. */

package/src/v5/resource-fetcher.ts

@@ -0,0 +1,113 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {TileConversionError} from './conversion-api.js';
+
+/** One aggregate transport budget shared by selected contents and their dependencies. */
+export interface TileConversionResourceFetcherOptions {
+  /** Maximum total response bytes, including previously inspected root bytes. */
+  readonly maxInputBytes: number;
+  /** Previously consumed input bytes, such as the inspected tileset JSON. */
+  readonly initialInputBytes?: number;
+  /** Application transport; also enables archive-backed or hermetic resolution. */
+  readonly fetcher?: typeof fetch;
+  /** Conversion cancellation, forwarded to every dependency request. */
+  readonly signal?: AbortSignal;
+}
+
+/**
+ * Wraps application fetch with a shared byte budget and conversion cancellation.
+ * Each requested resource is charged, including repeated requests; HTTP(S), Blob and embedded data
+ * URLs are accepted. Response streams are canceled on overflow or failure. The complete bounded
+ * response is retained for codecs; this does not bound their decoded allocations.
+ */
+export function createTileConversionResourceFetcher(
+  options: TileConversionResourceFetcherOptions
+): typeof fetch {
+  let consumed = options.initialInputBytes ?? 0;
+  if (
+    !Number.isSafeInteger(options.maxInputBytes) ||
+    options.maxInputBytes < 1 ||
+    !Number.isSafeInteger(consumed) ||
+    consumed < 0 ||
+    consumed > options.maxInputBytes
+  )
+    throw new TileConversionError(
+      'INVALID_INPUT_MEMORY_LIMIT',
+      'Resource fetch requires a positive aggregate byte budget and valid initial bytes'
+    );
+  const fetcher = options.fetcher ?? fetch;
+  return async (input, requestOptions) => {
+    options.signal?.throwIfAborted();
+    const url = new URL(input instanceof Request ? input.url : String(input));
+    if (!['http:', 'https:', 'blob:', 'data:'].includes(url.protocol))
+      throw new TileConversionError(
+        'EXTERNAL_RESOURCE_UNSUPPORTED',
+        'Conversion transport requires HTTP(S), Blob or embedded data resources'
+      );
+    const requestSignal =
+      requestOptions?.signal ?? (input instanceof Request ? input.signal : undefined);
+    const signal =
+      options.signal && requestSignal
+        ? AbortSignal.any([options.signal, requestSignal])
+        : (options.signal ?? requestSignal);
+    const response = await fetcher(input, {...requestOptions, signal});
+    if (!response.ok) {
+      await response.body?.cancel();
+      throw new TileConversionError(
+        'BROWSER_SOURCE_FETCH_FAILED',
+        `Could not fetch conversion resource (${response.status})`
+      );
+    }
+    const reader = response.body?.getReader();
+    if (!reader)
+      throw new TileConversionError(
+        'BROWSER_SOURCE_STREAM_UNAVAILABLE',
+        'Conversion resources require readable response streams'
+      );
+    /** Cancels an injected transport stream if it does not observe the request signal itself. */
+    const cancelRead = () => {
+      void reader.cancel(signal?.reason).catch(() => {});
+    };
+    signal?.addEventListener('abort', cancelRead, {once: true});
+    const parts: Uint8Array[] = [];
+    let byteLength = 0;
+    try {
+      while (true) {
+        signal?.throwIfAborted();
+        const {done, value} = await reader.read();
+        signal?.throwIfAborted();
+        if (done) break;
+        if (value.byteLength > options.maxInputBytes - consumed)
+          throw new TileConversionError(
+            'INPUT_RESOURCE_TOO_LARGE',
+            'Selected contents and external dependencies exceed the aggregate input byte budget'
+          );
+        consumed += value.byteLength;
+        byteLength += value.byteLength;
+        parts.push(value);
+      }
+    } catch (error) {
+      await reader.cancel(error);
+      throw error;
+    } finally {
+      signal?.removeEventListener('abort', cancelRead);
+      reader.releaseLock();
+    }
+    const bytes = new Uint8Array(byteLength);
+    let offset = 0;
+    for (const part of parts) {
+      bytes.set(part, offset);
+      offset += part.byteLength;
+    }
+    const boundedResponse = new Response(bytes, {
+      status: response.status,
+      statusText: response.statusText,
+      headers: response.headers
+    });
+    // Relative dependencies must resolve against the final URL after redirects.
+    Object.defineProperty(boundedResponse, 'url', {value: response.url});
+    return boundedResponse;
+  };
+}

package/src/v5/resource-manifest.ts

@@ -43,6 +43,17 @@ export interface ManifestBackedTileConversionSinkOptions<TResource> {
   readonly sink: TileConversionSink<TResource>;
   /** Persistence adapter for snapshots written after each resource. */
   readonly manifestStore: TileResourceManifestStore;
+  /** Optional cap on unique checkpoint entries loaded or written; defaults to no cap. */
+  readonly maxResources?: number;
+  /**
+   * Checks destination bytes before skipping a matching checkpoint. Return true only when
+   * stored bytes match the entry's size and fingerprint; false rewrites the resource.
+   * Without this callback the caller guarantees checkpointed destination bytes remain intact.
+   */
+  readonly verifyResource?: (
+    entry: TileResourceManifestEntry,
+    signal?: AbortSignal
+  ) => Promise<boolean>;
   /** Stable profile-specific path or resource key. */
   readonly getResourceId: (resource: TResource) => string;
   /** Byte length used to detect changed content under an existing resource ID. */
@@ -62,7 +73,8 @@ export interface ManifestBackedTileConversionSinkOptions<TResource> {
  * including writes skipped because the matching resource was already present. A graceful abort
  * clears the manifest checkpoint before asking the destination to clean partial output, so the
  * next run starts cleanly. A process interruption that does not call `abort` can resume from its
- * last saved checkpoint.
+ * last saved checkpoint. Optional destination verification repairs missing or changed stored
+ * resources before treating them as complete. maxResources caps unique checkpoint entries.
  *
  * @param options - Destination, manifest store, stable IDs, byte measurement, and fingerprinting.
  * @returns A `TileConversionSink` with resumable per-resource progress.
@@ -70,10 +82,17 @@ export interface ManifestBackedTileConversionSinkOptions<TResource> {
 export async function createManifestBackedTileConversionSink<TResource>(
   options: ManifestBackedTileConversionSinkOptions<TResource>
 ): Promise<TileConversionSink<TResource>> {
+  const maxResources = options.maxResources ?? Infinity;
+  if (maxResources !== Infinity && (!Number.isSafeInteger(maxResources) || maxResources < 0)) {
+    throw new TileConversionError(
+      'INVALID_MANIFEST_RESOURCE_LIMIT',
+      'maxResources must be a nonnegative safe integer or Infinity'
+    );
+  }
   const priorManifest = await options.manifestStore.load();
   const resources = new Map<string, TileResourceManifestEntry>();
   if (priorManifest) {
-    validateManifest(priorManifest);
+    validateManifest(priorManifest, maxResources);
     for (const entry of priorManifest.resources) {
       resources.set(entry.resourceId, entry);
     }
@@ -102,6 +121,7 @@ export async function createManifestBackedTileConversionSink<TResource>(
     if (writeFailed) {
       throw writeFailure;
     }
+    signal?.throwIfAborted();
     const entry = createManifestEntry(options, resource);
     const priorEntry = resources.get(entry.resourceId);
     if (priorEntry) {
@@ -114,7 +134,16 @@ export async function createManifestBackedTileConversionSink<TResource>(
           `Resource "${entry.resourceId}" changed since the saved manifest`
         );
       }
-      return;
+      const verified = options.verifyResource
+        ? await options.verifyResource({...priorEntry}, signal)
+        : true;
+      signal?.throwIfAborted();
+      if (verified === true) return;
+    } else if (resources.size >= maxResources) {
+      throw new TileConversionError(
+        'MANIFEST_RESOURCE_LIMIT_EXCEEDED',
+        'Resource checkpoint exceeds maxResources'
+      );
     }
 
     await options.sink.write(resource, signal);
@@ -179,7 +208,7 @@ function createManifestEntry<TResource>(
 }
 
 /** Checks a loaded manifest before trusting it for restart decisions. */
-function validateManifest(manifest: TileResourceManifest): void {
+function validateManifest(manifest: TileResourceManifest, maxResources: number): void {
   if (
     typeof manifest !== 'object' ||
     manifest === null ||
@@ -189,7 +218,13 @@ function validateManifest(manifest: TileResourceManifest): void {
   ) {
     throw new TileConversionError(
       'UNSUPPORTED_RESOURCE_MANIFEST',
-      `Unsupported or malformed resource manifest version: ${String(manifest.version)}`
+      `Unsupported or malformed resource manifest version: ${String(manifest?.version)}`
+    );
+  }
+  if (manifest.resources.length > maxResources) {
+    throw new TileConversionError(
+      'MANIFEST_RESOURCE_LIMIT_EXCEEDED',
+      'Loaded manifest exceeds maxResources'
     );
   }
   const resourceIds = new Set<string>();