@loaders.gl/tile-converter 5.0.0-alpha.11 → 5.0.0-alpha.14

package/src/v5/point-cloud-potree.ts

@@ -0,0 +1,134 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Mesh, MeshArrowTable} from '@loaders.gl/core/schema';
+import {PointCloudTiler, type PointCloudTilerOptions} from '@loaders.gl/core/schema-utils';
+import {encodePotreeDataset} from '@loaders.gl/tiles-3d/potree';
+import type {
+  PotreeDataset,
+  PotreeWriterNode,
+  PotreeWriterOptions
+} from '@loaders.gl/tiles-3d/potree';
+import {encodePotreeDatasetArchive} from './point-cloud-file.js';
+
+/** Bounded authoring of one decoded dataset in its already placed native coordinates. */
+export interface ConvertPointCloudToPotreeOptions {
+  /** Explicit CRS declaration written verbatim; coordinates are not reprojected. */
+  readonly projection: string;
+  /** Maximum bytes retained across all gathered node attribute buffers. */
+  readonly maxGatheredBytes: number;
+  /** Maximum complete encoded dataset bytes, including metadata and hierarchy. */
+  readonly maxOutputBytes: number;
+  /** Input, index, per-node and node-count limits of the shared dynamic tiler. */
+  readonly tiling?: Omit<PointCloudTilerOptions, 'signal'>;
+  /** Existing version, precision, spacing and hierarchy controls. */
+  readonly potree?: Omit<
+    NonNullable<PotreeWriterOptions['potree']>,
+    'projection' | 'signal' | 'maxOutputBytes'
+  >;
+  /** Also package the completed files as a loaders.gl STORE ZIP32 bundle. */
+  readonly archive?: {readonly maxArchiveBytes: number};
+  /** Cooperative cancellation during indexing, gathering, writing and packaging. */
+  readonly signal?: AbortSignal;
+}
+
+/** Complete Potree files; no partial dataset is returned after a failure. */
+export interface ConvertedPointCloudPotree {
+  /** Exact input row count, emitted once across the additive octree. */
+  readonly pointCount: number;
+  /** Number of connected octree nodes, including an empty root for empty input. */
+  readonly nodeCount: number;
+  /** Metadata and encoded resources; applications own storage. */
+  readonly dataset: PotreeDataset;
+  /** Optional downloadable bundle; applications own object URL and saving lifetimes. */
+  readonly archive?: Blob;
+}
+
+/**
+ * Connects the portable point tiler to the existing Potree 1.x/2.0 dataset writer.
+ * Retains additive octree addresses and every source row exactly once, including terminal
+ * overflow. Input must be immutable, unindexed decoded points already placed in the declared
+ * CRS. No origins, transforms, reprojection, attribute mappings or surface sampling are inferred.
+ * The writer rejects attributes or precision that the selected version cannot represent.
+ * Gathered nodes are bounded separately from source buffers, indexes, encoded output and ZIP
+ * temporaries. This is in-memory authoring, not streaming ingestion or a peak-heap guarantee.
+ * The shared tiler's sample coverage bound is not a qualified viewer LOD/error contract.
+ * @param input - One decoded point Mesh or MeshArrowTable, including loader output.
+ * @param options - Explicit native CRS, resource budgets and existing writer controls.
+ * @returns A complete dataset and optional bundle after releasing the owned tiler.
+ */
+export async function convertPointCloudToPotree(
+  input: Mesh | MeshArrowTable,
+  options: ConvertPointCloudToPotreeOptions
+): Promise<ConvertedPointCloudPotree> {
+  if (typeof options.projection !== 'string' || !options.projection.trim()) {
+    throw new Error('Potree authoring requires an explicit native CRS declaration');
+  }
+  for (const limit of [
+    options.maxGatheredBytes,
+    options.maxOutputBytes,
+    options.archive?.maxArchiveBytes
+  ]) {
+    if (limit !== undefined && (!Number.isSafeInteger(limit) || limit < 1)) {
+      throw new RangeError('Potree authoring byte limits must be positive safe integers');
+    }
+  }
+  // Required budgets must remain required for JavaScript callers too.
+  if (options.maxGatheredBytes === undefined || options.maxOutputBytes === undefined) {
+    throw new RangeError('Potree authoring requires gathered and encoded byte limits');
+  }
+  options.signal?.throwIfAborted();
+  const tiler = new PointCloudTiler(input, {
+    ...options.tiling,
+    maxNodes: Math.min(options.tiling?.maxNodes ?? 100000, options.potree?.maxNodes ?? 100000),
+    maxTileBytes: Math.min(
+      options.tiling?.maxTileBytes ?? 64 * 1024 * 1024,
+      options.maxGatheredBytes
+    ),
+    signal: options.signal
+  });
+  const nodes: PotreeWriterNode[] = [];
+  let gatheredBytes = 0;
+  let pointByteLength = 0;
+  let pointCount: number;
+  try {
+    const pending = [await tiler.getRootNode()];
+    pointCount = tiler.pointCount;
+    while (pending.length) {
+      options.signal?.throwIfAborted();
+      const node = pending.pop()!;
+      // Root gathering is bounded by maxTileBytes; its packed layout sizes later allocations.
+      if (node.pointCount * pointByteLength > options.maxGatheredBytes - gatheredBytes) {
+        throw new RangeError('Potree gathered node byte budget exceeded');
+      }
+      const mesh = await tiler.getTileMesh(node.id);
+      const nodeBytes = Array.from(
+        new Set(Object.values(mesh.attributes).map(attribute => attribute.value.buffer))
+      ).reduce((bytes, buffer) => bytes + buffer.byteLength, 0);
+      if (nodeBytes > options.maxGatheredBytes - gatheredBytes) {
+        throw new RangeError('Potree gathered node byte budget exceeded');
+      }
+      if (node.pointCount) pointByteLength = nodeBytes / node.pointCount;
+      gatheredBytes += nodeBytes;
+      nodes.push({id: node.id, mesh});
+      const children = await tiler.getChildNodes(node.id);
+      for (let index = children.length - 1; index >= 0; index--) pending.push(children[index]);
+    }
+  } finally {
+    tiler.close();
+  }
+  const dataset = await encodePotreeDataset(nodes, {
+    potree: {
+      ...options.potree,
+      projection: options.projection,
+      maxOutputBytes: options.maxOutputBytes,
+      signal: options.signal
+    }
+  });
+  const archive = options.archive
+    ? await encodePotreeDatasetArchive(dataset, options.archive.maxArchiveBytes, options.signal)
+    : undefined;
+  options.signal?.throwIfAborted();
+  return {pointCount, nodeCount: nodes.length, dataset, archive};
+}

package/src/v5/point-cloud-source-encoder.ts

@@ -7,7 +7,7 @@ import type {
   PointCloudCoordinateSystem,
   PointCloudTilesetSource,
   TilesetSpatialReference
-} from '@loaders.gl/tiles';
+} from '@loaders.gl/tiles-3d';
 import {transformPointCloudSourceTile} from './point-cloud-spatial.js';
 import type {PointCloudSpatialOptions} from './point-cloud-spatial.js';
 import {encodePointCloudTileWithMetadata} from './point-cloud.js';

package/src/v5/point-cloud-source.ts

@@ -6,7 +6,7 @@ import type {
   PointCloudTileContent,
   PointCloudTileHeader,
   PointCloudTilesetSource
-} from '@loaders.gl/tiles';
+} from '@loaders.gl/tiles-3d';
 
 /** A source tile and its decoded point content, including null content when no payload exists. */
 export type PointCloudSourceTile = {

package/src/v5/point-cloud-spatial.ts

@@ -3,8 +3,8 @@
 // Copyright (c) vis.gl contributors
 
 import {Matrix4} from '@math.gl/core';
-import {getSpatialCoordinateFrame, createTilesetSpatialReference} from '@loaders.gl/tiles';
-import {convertTableToMesh, makeMeshArrowTable} from '@loaders.gl/schema-utils';
+import {getSpatialCoordinateFrame, createTilesetSpatialReference} from '@loaders.gl/tiles-3d';
+import {convertTableToMesh, makeMeshArrowTable} from '@loaders.gl/core/schema-utils';
 import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
 import type {
   I3SConversionSpatialContext,

package/src/v5/point-cloud-tileset.ts

@@ -0,0 +1,137 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Mesh, MeshArrowTable} from '@loaders.gl/core/schema';
+import type {PointCloudTilerOptions} from '@loaders.gl/core/schema-utils';
+import type {TilesetSpatialReference} from '@loaders.gl/tiles-3d';
+import {PointCloudTileSource} from '@loaders.gl/tiles-3d/point-cloud-tile-source';
+import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
+import type {
+  BrowserTileConversionFile,
+  TileConversionProgress,
+  TileConversionReport
+} from '@loaders.gl/tile-converter/v5/core';
+import {convertPointCloudSource} from './point-cloud-conversion.js';
+import {createPointCloudTilesetSink} from './point-cloud-tileset-sink.js';
+import type {PointCloudTilesetSinkOptions} from './point-cloud-tileset-sink.js';
+import {createTileConversionArchive} from './conversion-archive.js';
+import type {TileConversionArchiveOptions} from './conversion-archive.js';
+
+/** Bounded authoring of a complete decoded ECEF point dataset. */
+export interface ConvertPointCloudToTilesetOptions extends PointCloudTilesetSinkOptions {
+  /** Explicit native EPSG:4978 XYZ meters with ellipsoidal heights. No CRS is inferred. */
+  readonly spatialReference: TilesetSpatialReference;
+  /** In-memory input, index, gathered-tile and node limits; defaults belong to PointCloudTiler. */
+  readonly tiling?: Omit<PointCloudTilerOptions, 'signal'>;
+  /** Also create an indexed 3TZ Blob within this final archive-size budget. */
+  readonly archive?: Pick<TileConversionArchiveOptions, 'maxArchiveBytes'>;
+  /** Cooperative cancellation during indexing, traversal, encoding and packaging. */
+  readonly signal?: AbortSignal;
+  /** PNTS conversion progress; excludes tileset JSON and optional archive packaging. */
+  readonly onProgress?: (progress: TileConversionProgress) => void;
+}
+
+/** Completed point authoring result; no partial files are returned on failure. */
+export interface ConvertedPointCloudTileset {
+  /** Exact number of input points, each emitted once across the complete collection. */
+  readonly pointCount: number;
+  /** PNTS resource counts and bytes; input bytes count gathered Arrow column buffers. */
+  readonly report: TileConversionReport;
+  /** PNTS resources and tileset.json, including JSON in the retained output byte budget. */
+  readonly files: readonly BrowserTileConversionFile[];
+  /** Optional indexed 3TZ Blob. Applications own saving and object URL lifetimes. */
+  readonly archive?: Blob;
+}
+
+/**
+ * Tiles decoded point Mesh/Arrow data and authors a flat additive 3D Tiles 1.0 collection.
+ * All input rows are emitted once, including remaining rows at the tiler's maximum depth.
+ * Per-tile RTC centers reduce float32 rounding; geometricError gates measured reconstruction error.
+ * The flat collection does not preserve octree LOD or provide a sampling-error guarantee.
+ * Input must remain immutable until completion. Arrow normalization can copy buffers; retained
+ * input, row indexes, gathered tiles, output Blobs and optional ZIP temporaries are separate memory.
+ * This is bounded in-memory authoring, not streaming ingestion or an out-of-core converter.
+ * Unsupported PNTS attributes fail; callers must explicitly map or remove them beforehand.
+ * @param input - One decoded unindexed point-list with finite floating-point XYZ positions.
+ * @param options - Explicit ECEF frame, precision and resource limits, and optional 3TZ packaging.
+ * @returns Completed files and report after releasing the privately owned dynamic source.
+ */
+export async function convertPointCloudToTileset(
+  input: Mesh | MeshArrowTable,
+  options: ConvertPointCloudToTilesetOptions
+): Promise<ConvertedPointCloudTileset> {
+  const reference = options.spatialReference;
+  if (
+    !reference ||
+    reference.status !== 'native' ||
+    reference.sourceCrs !== 'EPSG:4978' ||
+    (reference.targetCrs && reference.targetCrs !== 'EPSG:4978') ||
+    reference.coordinateFrame !== 'geocentric' ||
+    reference.axisOrder !== 'xyz' ||
+    reference.heightReference !== 'ellipsoidal' ||
+    reference.verticalUnitScale !== 1 ||
+    reference.units?.some(unit => unit !== 'meter')
+  ) {
+    throw new TileConversionError(
+      'POINT_CLOUD_TILESET_FRAME_UNSUPPORTED',
+      'Point authoring requires explicit native EPSG:4978 xyz meters and ellipsoidal heights'
+    );
+  }
+  if (
+    options.archive &&
+    (!Number.isSafeInteger(options.archive.maxArchiveBytes) || options.archive.maxArchiveBytes < 0)
+  ) {
+    throw new TileConversionError(
+      'INVALID_ARCHIVE_BYTE_LIMIT',
+      'Archive byte limit must be a non-negative safe integer'
+    );
+  }
+  options.signal?.throwIfAborted();
+  const sink = createPointCloudTilesetSink(options);
+  let source: PointCloudTileSource | undefined;
+  try {
+    source = new PointCloudTileSource(input, {
+      pointCloudTiler: {...options.tiling, signal: options.signal, spatialReference: reference}
+    });
+    const {pointCount} = await source.getMetadata();
+    const report = await convertPointCloudSource(source, {
+      sink,
+      signal: options.signal,
+      onProgress: options.onProgress,
+      measureInputBytes: tile => {
+        const table = tile.content?.data.data;
+        return table
+          ? table.schema.fields.reduce(
+              (bytes, _field, index) => bytes + (table.getChildAt(index)?.byteLength ?? 0),
+              0
+            )
+          : 0;
+      },
+      getTileEncodingOptions: tile => ({
+        rtcCenter: [
+          tile.header.boundingVolume.center[0],
+          tile.header.boundingVolume.center[1],
+          tile.header.boundingVolume.center[2]
+        ],
+        maxPositionError: options.geometricError
+      })
+    });
+    options.signal?.throwIfAborted();
+    const files = sink.getFiles();
+    const archive = options.archive
+      ? await createTileConversionArchive(files, {
+          format: '3tz',
+          maxArchiveBytes: options.archive.maxArchiveBytes,
+          signal: options.signal
+        })
+      : undefined;
+    options.signal?.throwIfAborted();
+    return {pointCount, report, files, archive};
+  } catch (error) {
+    await sink.abort(error);
+    throw error;
+  } finally {
+    source?.close();
+  }
+}

package/src/v5/point-cloud.ts

@@ -2,14 +2,14 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {Tile3DWriter, TILE3D_TYPE} from '@loaders.gl/3d-tiles';
+import {Tile3DWriter, TILE3D_TYPE} from '@loaders.gl/tiles-3d/3d-tiles';
 import type {
   Mesh,
   MeshArrowTable,
   MeshAttribute,
   MeshAttributeQuantizationTransform
-} from '@loaders.gl/schema';
-import {convertTableToMesh} from '@loaders.gl/schema-utils';
+} from '@loaders.gl/core/schema';
+import {convertTableToMesh} from '@loaders.gl/core/schema-utils';
 import {TileConversionError} from '@loaders.gl/tile-converter/v5/core';
 
 /** Options for encoding one Arrow or mesh point batch as a 3D Tiles point tile. */

package/src/v5/resource-fetcher.ts

@@ -101,10 +101,13 @@ export function createTileConversionResourceFetcher(
       bytes.set(part, offset);
       offset += part.byteLength;
     }
-    return new Response(bytes, {
+    const boundedResponse = new Response(bytes, {
       status: response.status,
       statusText: response.statusText,
       headers: response.headers
     });
+    // Relative dependencies must resolve against the final URL after redirects.
+    Object.defineProperty(boundedResponse, 'url', {value: response.url});
+    return boundedResponse;
   };
 }

package/src/v5/resource-manifest.ts

@@ -43,6 +43,17 @@ export interface ManifestBackedTileConversionSinkOptions<TResource> {
   readonly sink: TileConversionSink<TResource>;
   /** Persistence adapter for snapshots written after each resource. */
   readonly manifestStore: TileResourceManifestStore;
+  /** Optional cap on unique checkpoint entries loaded or written; defaults to no cap. */
+  readonly maxResources?: number;
+  /**
+   * Checks destination bytes before skipping a matching checkpoint. Return true only when
+   * stored bytes match the entry's size and fingerprint; false rewrites the resource.
+   * Without this callback the caller guarantees checkpointed destination bytes remain intact.
+   */
+  readonly verifyResource?: (
+    entry: TileResourceManifestEntry,
+    signal?: AbortSignal
+  ) => Promise<boolean>;
   /** Stable profile-specific path or resource key. */
   readonly getResourceId: (resource: TResource) => string;
   /** Byte length used to detect changed content under an existing resource ID. */
@@ -62,7 +73,8 @@ export interface ManifestBackedTileConversionSinkOptions<TResource> {
  * including writes skipped because the matching resource was already present. A graceful abort
  * clears the manifest checkpoint before asking the destination to clean partial output, so the
  * next run starts cleanly. A process interruption that does not call `abort` can resume from its
- * last saved checkpoint.
+ * last saved checkpoint. Optional destination verification repairs missing or changed stored
+ * resources before treating them as complete. maxResources caps unique checkpoint entries.
  *
  * @param options - Destination, manifest store, stable IDs, byte measurement, and fingerprinting.
  * @returns A `TileConversionSink` with resumable per-resource progress.
@@ -70,10 +82,17 @@ export interface ManifestBackedTileConversionSinkOptions<TResource> {
 export async function createManifestBackedTileConversionSink<TResource>(
   options: ManifestBackedTileConversionSinkOptions<TResource>
 ): Promise<TileConversionSink<TResource>> {
+  const maxResources = options.maxResources ?? Infinity;
+  if (maxResources !== Infinity && (!Number.isSafeInteger(maxResources) || maxResources < 0)) {
+    throw new TileConversionError(
+      'INVALID_MANIFEST_RESOURCE_LIMIT',
+      'maxResources must be a nonnegative safe integer or Infinity'
+    );
+  }
   const priorManifest = await options.manifestStore.load();
   const resources = new Map<string, TileResourceManifestEntry>();
   if (priorManifest) {
-    validateManifest(priorManifest);
+    validateManifest(priorManifest, maxResources);
     for (const entry of priorManifest.resources) {
       resources.set(entry.resourceId, entry);
     }
@@ -102,6 +121,7 @@ export async function createManifestBackedTileConversionSink<TResource>(
     if (writeFailed) {
       throw writeFailure;
     }
+    signal?.throwIfAborted();
     const entry = createManifestEntry(options, resource);
     const priorEntry = resources.get(entry.resourceId);
     if (priorEntry) {
@@ -114,7 +134,16 @@ export async function createManifestBackedTileConversionSink<TResource>(
           `Resource "${entry.resourceId}" changed since the saved manifest`
         );
       }
-      return;
+      const verified = options.verifyResource
+        ? await options.verifyResource({...priorEntry}, signal)
+        : true;
+      signal?.throwIfAborted();
+      if (verified === true) return;
+    } else if (resources.size >= maxResources) {
+      throw new TileConversionError(
+        'MANIFEST_RESOURCE_LIMIT_EXCEEDED',
+        'Resource checkpoint exceeds maxResources'
+      );
     }
 
     await options.sink.write(resource, signal);
@@ -179,7 +208,7 @@ function createManifestEntry<TResource>(
 }
 
 /** Checks a loaded manifest before trusting it for restart decisions. */
-function validateManifest(manifest: TileResourceManifest): void {
+function validateManifest(manifest: TileResourceManifest, maxResources: number): void {
   if (
     typeof manifest !== 'object' ||
     manifest === null ||
@@ -189,7 +218,13 @@ function validateManifest(manifest: TileResourceManifest): void {
   ) {
     throw new TileConversionError(
       'UNSUPPORTED_RESOURCE_MANIFEST',
-      `Unsupported or malformed resource manifest version: ${String(manifest.version)}`
+      `Unsupported or malformed resource manifest version: ${String(manifest?.version)}`
+    );
+  }
+  if (manifest.resources.length > maxResources) {
+    throw new TileConversionError(
+      'MANIFEST_RESOURCE_LIMIT_EXCEEDED',
+      'Loaded manifest exceeds maxResources'
     );
   }
   const resourceIds = new Set<string>();