@loaders.gl/pointclouds 0.0.0-stage → 5.0.0-beta.10

package/src/las/las-format.ts

@@ -0,0 +1,21 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from '@loaders.gl/core/loader-utils';
+
+/**
+ * LAS (LASer) point cloud format
+ */
+export const LASFormat = {
+  name: 'LAS',
+  id: 'las',
+  module: 'pointclouds',
+  encoding: 'binary',
+  format: 'las',
+  extensions: ['las', 'laz'], // LAZ is the "compressed" flavor of LAS,
+  mimeTypes: ['application/octet-stream'], // TODO - text version?
+  text: false,
+  binary: true,
+  tests: ['LASF']
+} as const satisfies Format;

package/src/las/las-loader-shared.ts

@@ -0,0 +1,126 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader, LoaderOptions} from '@loaders.gl/core/loader-utils';
+import {convertColorArrayToFloat16, convertColorArrayToFloat32} from '@loaders.gl/core/schema';
+import type {Table, RecordBatch} from 'apache-arrow';
+import {LASFormat} from './las-format';
+import type {LASMesh} from './lib/las-types';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+/** Arrow column names that can be selected from LAS point records. */
+export type LASColumnName =
+  | 'POSITION'
+  | 'intensity'
+  | 'classification'
+  | 'synthetic'
+  | 'keyPoint'
+  | 'withheld'
+  | 'overlap'
+  | 'COLOR_0'
+  | 'GPS_TIME'
+  | 'NIR'
+  | 'scanAngle'
+  | 'userData'
+  | 'pointSourceId'
+  | 'returnNumber'
+  | 'numberOfReturns'
+  | 'scannerChannel'
+  | 'scanDirectionFlag'
+  | 'edgeOfFlightLine'
+  | 'WAVEFORM'
+  | 'EXTRA_BYTES';
+
+/** Options accepted by LAS loader implementations. */
+export type LASLoaderOptions = LoaderOptions & {
+  las?: {
+    /** Output representation. */
+    shape?: 'mesh' | 'columnar-table' | 'arrow';
+    /** Store positions in 64-bit floating point arrays. */
+    fp64?: boolean;
+    /** Output color depth or automatic source-depth detection. */
+    colorDepth?: number | string;
+    /** Color storage format. Defaults to uint8norm for backwards compatibility. */
+    colorFormat?: 'uint8norm' | 'float16' | 'float32';
+    /** Arrow columns to decode. POSITION is always included. */
+    columns?: readonly LASColumnName[];
+    /** Decode Extra Bytes descriptors into typed attributes instead of raw bytes. */
+    extraBytes?: 'raw' | 'typed';
+    /** Recover fixed-size LAZ chunks when the LASzip pointer marks an interrupted writer. */
+    recoverMissingChunkTable?: boolean;
+    /** Restore modern fields in LASzip compatibility-mode files, or retain their legacy columns. */
+    compatibilityMode?: 'auto' | 'raw';
+    /** Override the URL to the worker bundle. */
+    workerUrl?: string;
+  };
+  /** Called as point data is decoded on the main thread. */
+  onProgress?: Function;
+};
+
+/** Default output shape for LAS and LAZ loaders. */
+export const DEFAULT_LAS_SHAPE = 'arrow' as const;
+
+/** Resolve the LAS output shape, including the Arrow-primary default. */
+export function getLASShape(options?: LASLoaderOptions): 'mesh' | 'columnar-table' | 'arrow' {
+  return options?.las?.shape || DEFAULT_LAS_SHAPE;
+}
+
+/** Parser-independent LAS loader metadata shared by each loader variant. */
+export const LAS_LOADER_METADATA = {
+  ...LASFormat,
+  defaultShape: 'arrow',
+  supportsNativeArrow: true,
+
+  dataType: null as unknown as LASMesh | Table,
+  batchType: null as unknown as LASMesh | RecordBatch,
+
+  version: VERSION,
+  worker: false,
+  options: {
+    las: {
+      shape: DEFAULT_LAS_SHAPE,
+      fp64: false,
+      colorDepth: 8,
+      colorFormat: 'uint8norm',
+      columns: undefined,
+      extraBytes: 'raw',
+      recoverMissingChunkTable: false,
+      compatibilityMode: 'auto'
+    }
+  }
+} as const satisfies Loader<LASMesh | Table, LASMesh | RecordBatch, LASLoaderOptions>;
+
+/** Apply the requested normalized color representation to a decoded LAS mesh. */
+export function formatLASMeshColors(
+  mesh: LASMesh,
+  colorFormat: 'uint8norm' | 'float16' | 'float32'
+): LASMesh {
+  if (colorFormat === 'uint8norm' || !mesh.attributes.COLOR_0) {
+    return mesh;
+  }
+
+  const colorAttribute = mesh.attributes.COLOR_0;
+  const sourceScale = colorAttribute.value instanceof Uint16Array ? 65535 : 255;
+  const value =
+    colorFormat === 'float16'
+      ? convertColorArrayToFloat16(colorAttribute.value, sourceScale)
+      : convertColorArrayToFloat32(colorAttribute.value, sourceScale);
+  const attributes = {...mesh.attributes};
+
+  for (const [attributeName, attribute] of Object.entries(attributes)) {
+    if (attribute.value === colorAttribute.value) {
+      attributes[attributeName] = {
+        ...attribute,
+        value,
+        normalized: false,
+        ...(colorFormat === 'float16' ? {componentType: 'float16' as const} : {})
+      };
+    }
+  }
+
+  return {...mesh, attributes};
+}

package/src/las/las-loader-types.ts

@@ -0,0 +1,24 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Loader} from '@loaders.gl/core/loader-utils';
+import type {Table, RecordBatch} from 'apache-arrow';
+import type {LASMesh} from './lib/las-types';
+import {LAS_LOADER_METADATA, type LASLoaderOptions} from './las-loader-shared';
+
+export type {LASColumnName, LASLoaderOptions} from './las-loader-shared';
+
+/** Preloads the parser-bearing primary TypeScript LAS loader implementation. */
+async function preload() {
+  const {LASLoaderWithParser} = await import('@loaders.gl/pointclouds/las-loader');
+  return LASLoaderWithParser;
+}
+
+/** Metadata-only loader for the LAS (LASer) point cloud format. */
+export const LASLoader = {
+  ...LAS_LOADER_METADATA,
+  worker: true,
+  workerInBatches: true,
+  preload
+} as const satisfies Loader<LASMesh | Table, LASMesh | RecordBatch, LASLoaderOptions>;

package/src/las/las-loader.ts

@@ -0,0 +1,163 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {paceBatchIterator, type LoaderWithParser} from '@loaders.gl/core/loader-utils';
+import type {Table, RecordBatch} from 'apache-arrow';
+import {makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
+import type {MeshArrowTable} from '@loaders.gl/core/schema';
+import {convertMeshToArrowTableWrapper, convertTableToMesh} from '@loaders.gl/core/schema-utils';
+import {
+  formatLASMeshColors,
+  getLASShape,
+  LAS_LOADER_METADATA,
+  type LASLoaderOptions
+} from './las-loader-shared';
+import {
+  serializeArrowWorkerTable,
+  deserializeArrowWorkerTable
+} from '@loaders.gl/core/arrow-worker-transport';
+import type {LASMesh} from './lib/las-types';
+import {
+  decodeLAZChunkToArrowTable,
+  parseLAS,
+  parseLASInBatches,
+  type LASArrowTable,
+  type LAZChunkArrowTableMetadata
+} from './lib/typescript/parse-las';
+
+type LASWorkerChunkRequest = {
+  metadata: LAZChunkArrowTableMetadata;
+};
+
+type LASWorkerOptions = LASLoaderOptions & {
+  las?: LASLoaderOptions['las'] & {
+    _chunk?: LASWorkerChunkRequest;
+  };
+};
+
+/** Parser-bearing TypeScript-only LAS loader implementation. */
+export const LASColumnLoader = {
+  ...LAS_LOADER_METADATA,
+  dataType: null as unknown as LASMesh | MeshArrowTable,
+  batchType: null as unknown as LASMesh | MeshArrowTable,
+  worker: true,
+  workerInBatches: true,
+  serializeWorkerResult: serializeArrowWorkerTable,
+  serializeWorkerBatch: serializeArrowWorkerTable,
+  deserializeWorkerResult: deserializeLASWorkerTable,
+  deserializeWorkerBatch: deserializeLASWorkerTable,
+  parse: async (arrayBuffer: ArrayBuffer, options?: LASLoaderOptions) =>
+    convertLASMesh(parseLASTable(arrayBuffer, options), options),
+  parseSync: (arrayBuffer: ArrayBuffer, options?: LASLoaderOptions) =>
+    convertLASMesh(parseLASTable(arrayBuffer, options), options),
+  parseInBatches: async function* (arrayBufferIterator, options?: LASLoaderOptions) {
+    yield* convertLASMeshBatches(parseLASInBatches(arrayBufferIterator, options), options);
+  }
+} as const satisfies LoaderWithParser<
+  LASMesh | MeshArrowTable,
+  LASMesh | MeshArrowTable,
+  LASLoaderOptions
+>;
+
+/** Parse a complete LAS file or an internal standalone worker chunk request. */
+function parseLASTable(arrayBuffer: ArrayBuffer, options?: LASLoaderOptions): LASArrowTable {
+  const chunkRequest = (options as LASWorkerOptions | undefined)?.las?._chunk;
+  return chunkRequest
+    ? decodeLAZChunkToArrowTable(arrayBuffer, chunkRequest.metadata, options || {})
+    : parseLAS(arrayBuffer, options);
+}
+
+function convertLASMesh(
+  table: LASArrowTable,
+  options?: LASLoaderOptions
+): LASMesh | MeshArrowTable {
+  const formattedTable =
+    options?.las?.colorFormat === undefined || options.las.colorFormat === 'uint8norm'
+      ? table
+      : {
+          ...convertMeshToArrowTableWrapper(
+            formatLASMeshColors(convertTableToMesh(table) as LASMesh, options.las.colorFormat)
+          ),
+          loader: table.loader,
+          loaderData: table.loaderData,
+          progress: table.progress
+        };
+  if (getLASShape(options) === 'arrow') {
+    return formattedTable;
+  }
+  return {
+    ...(convertTableToMesh(formattedTable) as LASMesh),
+    loader: formattedTable.loader,
+    loaderData: formattedTable.loaderData,
+    progress: formattedTable.progress
+  } as LASMesh & {progress?: number};
+}
+
+async function* convertLASMeshBatches(
+  tableBatches: AsyncIterable<LASArrowTable>,
+  options?: LASLoaderOptions
+): AsyncIterable<LASMesh | MeshArrowTable> {
+  for await (const table of paceBatchIterator(tableBatches, {
+    batchMaxRowsPerSecond: options?.core?.batchMaxRowsPerSecond,
+    getRowCount: batch => batch.data.numRows,
+    signal: options?.signal
+  })) {
+    yield convertLASMesh(table, options);
+  }
+}
+
+/** Hydrates transferred columns and applies the requested mesh and color representation on the caller. */
+function deserializeLASWorkerTable(
+  result: unknown,
+  options?: LASLoaderOptions
+): LASMesh | MeshArrowTable {
+  return convertLASMesh(deserializeArrowWorkerTable(result), options);
+}
+
+/** Exposes native Arrow while retaining the column worker and mesh compatibility paths. */
+export const LASLoaderWithParser = {
+  ...LASColumnLoader,
+  dataType: LAS_LOADER_METADATA.dataType,
+  batchType: LAS_LOADER_METADATA.batchType,
+  parse: async (input: ArrayBuffer, options?: LASLoaderOptions) =>
+    toPublicLASResult(await LASColumnLoader.parse(input, options)),
+  parseSync: (input: ArrayBuffer, options?: LASLoaderOptions) =>
+    toPublicLASResult(LASColumnLoader.parseSync(input, options)),
+  async *parseInBatches(input, options) {
+    for await (const result of LASColumnLoader.parseInBatches(input, options)) {
+      if ('shape' in result) yield* getLASRecordBatches(result);
+      else yield result;
+    }
+  },
+  deserializeWorkerResult: (result: unknown, options?: LASLoaderOptions) =>
+    toPublicLASResult(deserializeLASWorkerTable(result, options)),
+  deserializeWorkerBatch(result: unknown, options?: LASLoaderOptions) {
+    const decoded = deserializeLASWorkerTable(result, options);
+    return 'shape' in decoded ? getLASRecordBatches(decoded)[0] : decoded;
+  }
+} as const satisfies LoaderWithParser<LASMesh | Table, LASMesh | RecordBatch, LASLoaderOptions>;
+
+/** Unwraps the Arrow table with its complete file header. */
+export function toPublicLASResult(result: LASMesh | MeshArrowTable): LASMesh | Table {
+  if (!('shape' in result)) return result;
+  const header = JSON.stringify((result as LASArrowTable).loaderData, (_key, value) =>
+    typeof value === 'bigint'
+      ? value.toString()
+      : ArrayBuffer.isView(value)
+        ? Array.from(value as Uint8Array)
+        : value
+  );
+  if (header) {
+    result.data.schema.metadata.set('loaders.gl.las.header', header);
+    for (const batch of result.data.batches)
+      batch.schema.metadata.set('loaders.gl.las.header', header);
+  }
+  return result.data;
+}
+
+/** Unwraps the record batches, retaining schema on empty LAS results. */
+export function getLASRecordBatches(result: MeshArrowTable): RecordBatch[] {
+  const table = toPublicLASResult(result) as Table;
+  return table.batches.length ? table.batches : [makeArrowMetadataBatch(table.schema)];
+}