@loaders.gl/pointclouds 0.0.0-stage → 5.0.0-beta.10

package/src/pcd/lib/pcd-types.ts

@@ -0,0 +1,34 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Mesh} from '@loaders.gl/core/schema';
+
+type BoundingBox = [[number, number, number], [number, number, number]];
+
+export type PCDHeader = {
+  data: any;
+  headerLen: number;
+  str: string;
+  version: number;
+  fields: string[];
+  size: number[];
+  type: null | string[];
+  count: null | number[];
+  width: number;
+  height: number;
+  viewpoint: null | string;
+  points: number;
+  offset: {[index: string]: number};
+  rowSize: number;
+  vertexCount: number;
+  boundingBox: BoundingBox;
+};
+
+/**  */
+export type PCDMesh = Mesh & {
+  loader: 'pcd';
+  loaderData: PCDHeader;
+  topology: 'point-list';
+  mode: 0;
+};

package/src/pcd/pcd-format.ts

@@ -0,0 +1,21 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Format} from '@loaders.gl/core/loader-utils';
+
+/**
+ * PCD (Point Cloud Data) file format
+ */
+export const PCDFormat = {
+  name: 'PCD (Point Cloud Data)',
+  id: 'pcd',
+  module: 'pointclouds',
+  encoding: 'text',
+  format: 'pcd',
+  extensions: ['pcd'],
+  mimeTypes: ['text/plain'],
+  category: 'pointcloud',
+  text: true,
+  binary: true
+} as const satisfies Format;

package/src/pcd/pcd-loader-types.ts

@@ -0,0 +1,52 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Table, RecordBatch} from 'apache-arrow';
+import type {Loader, LoaderOptions} from '@loaders.gl/core/loader-utils';
+import type {PCDMesh} from './lib/pcd-types';
+import {PCDFormat} from './pcd-format';
+import {serializeArrowWorkerTable} from '@loaders.gl/core/arrow-worker-transport';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type PCDLoaderOptions = LoaderOptions & {
+  pcd?: {
+    /** Output shape. Defaults to a Mesh Arrow table. */
+    shape?: 'mesh' | 'arrow';
+    /** Color storage format. Defaults to uint8norm for backwards compatibility. */
+    colorFormat?: 'uint8norm' | 'float16' | 'float32';
+    /** Override the URL to the worker bundle (by default loads from unpkg.com) */
+    workerUrl?: string;
+  };
+};
+
+/**
+ * Preloads the parser-bearing PCD loader implementation.
+ */
+async function preload() {
+  const {PCDLoaderWithParser} = await import('@loaders.gl/pointclouds/pcd-loader');
+  return PCDLoaderWithParser;
+}
+
+/**
+ * Metadata-only worker loader for PCD - Point Cloud Data
+ */
+export const PCDLoader = {
+  defaultShape: 'arrow',
+  supportsNativeArrow: true,
+  ...PCDFormat,
+  dataType: null as unknown as PCDMesh | Table,
+  batchType: null as unknown as PCDMesh | RecordBatch,
+  version: VERSION,
+  worker: true,
+  /** Enables the stateful input/output batch protocol on the built-in worker. */
+  workerInBatches: true,
+  serializeWorkerResult: serializeArrowWorkerTable,
+  options: {
+    pcd: {shape: 'arrow', colorFormat: 'uint8norm'}
+  },
+  preload
+} as const satisfies Loader<PCDMesh | Table, PCDMesh | RecordBatch, PCDLoaderOptions>;

package/src/pcd/pcd-loader.ts

@@ -0,0 +1,247 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {refineArrowTableBatches, makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
+import type {Table, RecordBatch} from 'apache-arrow';
+import type {PCDLoaderOptions} from './pcd-loader-types';
+
+import {
+  concatenateArrayBuffersAsync,
+  makeTableScanBatch,
+  type LoaderWithParser
+} from '@loaders.gl/core/loader-utils';
+import type {ArrowTableBatch, MeshArrowTable} from '@loaders.gl/core/schema';
+import {
+  deserializeArrowWorkerTable,
+  serializeArrowWorkerTable
+} from '@loaders.gl/core/arrow-worker-transport';
+import {convertMeshToArrowTableWrapper, convertTableToMesh} from '@loaders.gl/core/schema-utils';
+import type {PCDMesh} from './lib/pcd-types';
+import {parsePCD, parsePCDHeader} from './lib/parse-pcd';
+import {PCDLoader as PCDLoaderMetadata} from './pcd-loader-types';
+
+const {preload: _PCDLoaderPreload, ...PCDLoaderMetadataWithoutPreload} = PCDLoaderMetadata;
+
+type PCDParsedBatch = PCDMesh | ArrowTableBatch;
+
+function convertPCDMesh(mesh: PCDMesh, options?: PCDLoaderOptions): PCDMesh | MeshArrowTable {
+  const table = convertMeshToArrowTableWrapper(mesh);
+  return options?.pcd?.shape === 'mesh'
+    ? convertPCDTableToMesh(table, mesh)
+    : {...table, loader: mesh.loader, loaderData: mesh.loaderData};
+}
+
+/**
+ * Loader for PCD - Point Cloud Data
+ */
+export const PCDColumnLoader = {
+  ...PCDLoaderMetadataWithoutPreload,
+  dataType: null as unknown as PCDMesh | MeshArrowTable,
+  batchType: null as unknown as PCDMesh | ArrowTableBatch,
+  parse: async (arrayBuffer, options) => convertPCDMesh(parsePCD(arrayBuffer, options), options),
+  parseSync: (arrayBuffer, options) => convertPCDMesh(parsePCD(arrayBuffer, options), options),
+  parseInBatches: (input, options) =>
+    refineArrowTableBatches(parsePCDWithMetadataInBatches(input, options)),
+  serializeWorkerResult: serializeArrowWorkerTable,
+  deserializeWorkerResult: deserializePCDWorkerResult,
+  serializeWorkerBatch: serializeArrowWorkerTable,
+  deserializeWorkerBatch: deserializePCDWorkerBatch
+} as const satisfies LoaderWithParser<PCDMesh | MeshArrowTable, PCDParsedBatch, PCDLoaderOptions>;
+
+/** Returns a numeric batch size when batching has an explicit row count. */
+function getNumericBatchSize(options?: PCDLoaderOptions): number | undefined {
+  const batchSize = options?.core?.batchSize;
+  return typeof batchSize === 'number' ? batchSize : undefined;
+}
+
+function* parsePCDASCIIInBatches(
+  data: ArrayBuffer,
+  options?: PCDLoaderOptions,
+  batchSize?: number
+): Iterable<PCDParsedBatch> {
+  const textData = new TextDecoder().decode(data);
+  const pcdHeader = parsePCDHeader(textData);
+  const lines = textData
+    .slice(pcdHeader.headerLen)
+    .split(/\r?\n/)
+    .filter(line => line.trim().length > 0);
+  const normalizedBatchSize = batchSize || lines.length || 1;
+
+  for (let rowIndex = 0; rowIndex < lines.length; rowIndex += normalizedBatchSize) {
+    const batchLines = lines.slice(rowIndex, rowIndex + normalizedBatchSize);
+    const batchText = `${makePCDHeaderText(pcdHeader, batchLines.length)}${batchLines.join('\n')}\n`;
+    yield makePCDBatch(parsePCD(new TextEncoder().encode(batchText).buffer, options), options);
+  }
+}
+
+function* parsePCDBinaryInBatches(
+  data: ArrayBuffer,
+  options?: PCDLoaderOptions,
+  batchSize?: number
+): Iterable<PCDParsedBatch> {
+  const textData = new TextDecoder().decode(data);
+  const pcdHeader = parsePCDHeader(textData);
+  const normalizedBatchSize = batchSize || pcdHeader.points || 1;
+  const dataBytes = new Uint8Array(data);
+
+  for (let pointIndex = 0; pointIndex < pcdHeader.points; pointIndex += normalizedBatchSize) {
+    const pointCount = Math.min(normalizedBatchSize, pcdHeader.points - pointIndex);
+    const headerBytes = new TextEncoder().encode(makePCDHeaderText(pcdHeader, pointCount));
+    const rowByteOffset = pcdHeader.headerLen + pointIndex * pcdHeader.rowSize;
+    const rowByteLength = pointCount * pcdHeader.rowSize;
+    const batchBytes = new Uint8Array(headerBytes.length + rowByteLength);
+    batchBytes.set(headerBytes, 0);
+    batchBytes.set(
+      dataBytes.subarray(rowByteOffset, rowByteOffset + rowByteLength),
+      headerBytes.length
+    );
+    yield makePCDBatch(parsePCD(batchBytes.buffer, options), options);
+  }
+}
+
+function makePCDBatch(mesh: PCDMesh, options?: PCDLoaderOptions): PCDParsedBatch {
+  const table = convertMeshToArrowTableWrapper(mesh);
+  if (options?.pcd?.shape === 'mesh') {
+    return convertPCDTableToMesh(table, mesh);
+  }
+
+  return makeTableScanBatch({
+    ...table,
+    loader: mesh.loader,
+    loaderData: mesh.loaderData
+  } as MeshArrowTable);
+}
+
+/** Convert the parser's Arrow table to legacy PCD mesh output. */
+function convertPCDTableToMesh(table: MeshArrowTable, sourceMesh: PCDMesh): PCDMesh {
+  return {
+    ...(convertTableToMesh(table) as PCDMesh),
+    loader: sourceMesh.loader,
+    loaderData: sourceMesh.loaderData
+  };
+}
+
+function makePCDHeaderText(pcdHeader: any, pointCount: number): string {
+  const lines = pcdHeader.str
+    .split(/\r?\n/)
+    .filter(line => line.trim().length > 0)
+    .map(line => {
+      if (/^WIDTH\s+/i.test(line)) {
+        return `WIDTH ${pointCount}`;
+      }
+      if (/^HEIGHT\s+/i.test(line)) {
+        return 'HEIGHT 1';
+      }
+      if (/^POINTS\s+/i.test(line)) {
+        return `POINTS ${pointCount}`;
+      }
+      return line;
+    });
+
+  if (!lines.some(line => /^POINTS\s+/i.test(line))) {
+    const dataLineIndex = lines.findIndex(line => /^DATA\s+/i.test(line));
+    lines.splice(dataLineIndex >= 0 ? dataLineIndex : lines.length, 0, `POINTS ${pointCount}`);
+  }
+
+  return `${lines.join('\n')}\n`;
+}
+
+/** Hydrates worker columns before reconstructing legacy PCD meshes on the caller. */
+function deserializePCDWorkerResult(
+  result: unknown,
+  options?: PCDLoaderOptions
+): PCDMesh | MeshArrowTable {
+  const table = deserializeArrowWorkerTable(result);
+  return options?.pcd?.shape === 'mesh' ? convertPCDTableToMesh(table, table) : table;
+}
+
+/** Restores the existing PCD batch shape after caller-side hydration. */
+function deserializePCDWorkerBatch(result: unknown, options?: PCDLoaderOptions): PCDParsedBatch {
+  return deserializePCDWorkerResult(result, options) as PCDParsedBatch;
+}
+
+/** Parses PCD batches before shared cumulative metadata refinement. */
+async function* parsePCDWithMetadataInBatches(
+  arrayBuffer:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
+  options?: PCDLoaderOptions
+): AsyncIterable<PCDParsedBatch> {
+  const batchSize = getNumericBatchSize(options);
+  const data = await concatenateArrayBuffersAsync(arrayBuffer);
+  const textData = new TextDecoder().decode(data);
+  const pcdHeader = parsePCDHeader(textData);
+
+  if (pcdHeader.data === 'ascii') {
+    yield* parsePCDASCIIInBatches(data, options, batchSize);
+    return;
+  }
+
+  if (pcdHeader.data === 'binary') {
+    yield* parsePCDBinaryInBatches(data, options, batchSize);
+    return;
+  }
+
+  yield makePCDBatch(parsePCD(data, options), options);
+}
+
+/** Exposes native Arrow output while retaining the worker's column protocol. */
+export const PCDLoaderWithParser = {
+  ...PCDColumnLoader,
+  dataType: PCDLoaderMetadata.dataType,
+  batchType: PCDLoaderMetadata.batchType,
+  parse: async (input: ArrayBuffer, options?: PCDLoaderOptions) =>
+    unwrapPCDTable(await PCDColumnLoader.parse(input, options)),
+  parseSync: (input: ArrayBuffer, options?: PCDLoaderOptions) =>
+    unwrapPCDTable(PCDColumnLoader.parseSync(input, options)),
+  async *parseInBatches(input, options) {
+    for await (const batch of PCDColumnLoader.parseInBatches(input, options)) {
+      if (isColumnBatch(batch)) {
+        batch.data.schema.metadata.set(
+          'loaders.gl.pcd.header',
+          JSON.stringify((batch as ArrowTableBatch & {loaderData?: unknown}).loaderData)
+        );
+        for (const recordBatch of batch.data.batches) {
+          recordBatch.schema.metadata.set(
+            'loaders.gl.pcd.header',
+            batch.data.schema.metadata.get('loaders.gl.pcd.header')!
+          );
+        }
+        if (batch.data.batches.length) yield* batch.data.batches;
+        else yield makeArrowMetadataBatch(batch.data.schema);
+      } else yield batch;
+    }
+  },
+  deserializeWorkerResult: (result: unknown, options?: PCDLoaderOptions) =>
+    unwrapPCDTable(PCDColumnLoader.deserializeWorkerResult(result, options)),
+  deserializeWorkerBatch(result: unknown, options?: PCDLoaderOptions) {
+    const batch = PCDColumnLoader.deserializeWorkerBatch(result, options);
+    if (isColumnBatch(batch)) {
+      const recordBatch = batch.data.batches[0] ?? makeArrowMetadataBatch(batch.data.schema);
+      recordBatch.schema.metadata.set(
+        'loaders.gl.pcd.header',
+        JSON.stringify((batch as ArrowTableBatch & {loaderData?: unknown}).loaderData)
+      );
+      return recordBatch;
+    }
+    return batch;
+  }
+} as const satisfies LoaderWithParser<PCDMesh | Table, PCDMesh | RecordBatch, PCDLoaderOptions>;
+
+/** Returns the Arrow table with its file metadata or the requested mesh. */
+function unwrapPCDTable(result: PCDMesh | MeshArrowTable): PCDMesh | Table {
+  if ('shape' in result && result.shape === 'arrow-table') {
+    result.data.schema.metadata.set(
+      'loaders.gl.pcd.header',
+      JSON.stringify((result as MeshArrowTable & {loaderData?: unknown}).loaderData)
+    );
+    return result.data;
+  }
+  return result as PCDMesh;
+}
+
+/** Identifies the private Arrow batch envelope. */
+function isColumnBatch(batch: PCDMesh | ArrowTableBatch): batch is ArrowTableBatch {
+  return 'shape' in batch && batch.shape === 'arrow-table';
+}

package/src/pcd/pcd-writer.ts

@@ -0,0 +1,156 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Table} from 'apache-arrow';
+import type {WriterOptions, WriterWithEncoder} from '@loaders.gl/core/loader-utils';
+import type {Mesh, MeshArrowTable, MeshAttribute} from '@loaders.gl/core/schema';
+import {convertMeshToArrowTableWrapper, convertTableToMesh} from '@loaders.gl/core/schema-utils';
+import {PCDFormat} from './pcd-format';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+/** Options for `PCDWriter`. */
+export type PCDWriterOptions = WriterOptions & {
+  /** Reserved for future PCD writer options. */
+  pcd?: Record<string, never>;
+};
+
+/**
+ * Writer for the PCD point cloud format.
+ */
+export const PCDWriter = {
+  ...PCDFormat,
+  dataType: null as unknown as Mesh | MeshArrowTable | Table,
+  batchType: null as never,
+  version: VERSION,
+  options: {
+    pcd: {}
+  },
+  text: true,
+  encode: async (data, options) => encodePCDSync(data, options),
+  encodeSync: encodePCDSync,
+  encodeTextSync: encodePCD
+} as const satisfies WriterWithEncoder<Mesh | MeshArrowTable | Table, never, PCDWriterOptions>;
+
+/** Encode mesh category data as PCD bytes. */
+function encodePCDSync(
+  data: Mesh | MeshArrowTable | Table,
+  options?: PCDWriterOptions
+): ArrayBuffer {
+  const text = encodePCD(data, options);
+  return new TextEncoder().encode(text).buffer;
+}
+
+/** Encode mesh category data as ASCII PCD text. */
+function encodePCD(data: Mesh | MeshArrowTable | Table, options?: PCDWriterOptions): string {
+  const mesh = convertTableToMesh(normalizeMeshArrowTable(data));
+  const positionAttribute = getRequiredAttribute(mesh, 'POSITION');
+  const normalAttribute = mesh.attributes.NORMAL;
+  const colorAttribute = mesh.attributes.COLOR_0;
+  const vertexCount = positionAttribute.value.length / positionAttribute.size;
+  const fieldNames = ['x', 'y', 'z'];
+  const fieldSizes = ['4', '4', '4'];
+  const fieldTypes = ['F', 'F', 'F'];
+  const fieldCounts = ['1', '1', '1'];
+
+  if (normalAttribute) {
+    fieldNames.push('normal_x', 'normal_y', 'normal_z');
+    fieldSizes.push('4', '4', '4');
+    fieldTypes.push('F', 'F', 'F');
+    fieldCounts.push('1', '1', '1');
+  }
+
+  if (colorAttribute) {
+    fieldNames.push('rgb');
+    fieldSizes.push('4');
+    fieldTypes.push('F');
+    fieldCounts.push('1');
+  }
+
+  const lines = [
+    '# .PCD v0.7 - Point Cloud Data file generated by loaders.gl',
+    'VERSION 0.7',
+    `FIELDS ${fieldNames.join(' ')}`,
+    `SIZE ${fieldSizes.join(' ')}`,
+    `TYPE ${fieldTypes.join(' ')}`,
+    `COUNT ${fieldCounts.join(' ')}`,
+    `WIDTH ${vertexCount}`,
+    'HEIGHT 1',
+    'VIEWPOINT 0 0 0 1 0 0 0',
+    `POINTS ${vertexCount}`,
+    'DATA ascii'
+  ];
+
+  for (let vertexIndex = 0; vertexIndex < vertexCount; vertexIndex++) {
+    const row = [
+      getComponent(positionAttribute, vertexIndex, 0),
+      getComponent(positionAttribute, vertexIndex, 1),
+      getComponent(positionAttribute, vertexIndex, 2)
+    ];
+
+    if (normalAttribute) {
+      row.push(
+        getComponent(normalAttribute, vertexIndex, 0),
+        getComponent(normalAttribute, vertexIndex, 1),
+        getComponent(normalAttribute, vertexIndex, 2)
+      );
+    }
+
+    if (colorAttribute) {
+      row.push(getPackedRGB(colorAttribute, vertexIndex));
+    }
+
+    lines.push(row.join(' '));
+  }
+
+  return `${lines.join('\n')}\n`;
+}
+
+/** Return mesh data as a MeshArrowTable, converting plain Mesh data first. */
+function normalizeMeshArrowTable(data: Mesh | MeshArrowTable | Table): MeshArrowTable | Table {
+  if (!('attributes' in data)) {
+    return data;
+  }
+  return convertMeshToArrowTableWrapper(data as Mesh);
+}
+
+/** Return a required mesh attribute or throw a format-specific error. */
+function getRequiredAttribute(mesh: Mesh, attributeName: string): MeshAttribute {
+  const attribute = mesh.attributes[attributeName];
+  if (!attribute) {
+    throw new Error(`PCDWriter: ${attributeName} attribute is required`);
+  }
+  return attribute;
+}
+
+/** Return a single attribute component with 0 as the missing component fallback. */
+function getComponent(
+  attribute: MeshAttribute,
+  vertexIndex: number,
+  componentIndex: number
+): number {
+  return attribute.value[vertexIndex * attribute.size + componentIndex] || 0;
+}
+
+/** Return an RGB value packed into the PCD float convention. */
+function getPackedRGB(attribute: MeshAttribute, vertexIndex: number): number {
+  const bytes = new Uint8Array(4);
+  bytes[0] = getColorByte(attribute, vertexIndex, 0);
+  bytes[1] = getColorByte(attribute, vertexIndex, 1);
+  bytes[2] = getColorByte(attribute, vertexIndex, 2);
+  return new DataView(bytes.buffer).getFloat32(0, true);
+}
+
+/** Return one color component as an unsigned byte. */
+function getColorByte(
+  attribute: MeshAttribute,
+  vertexIndex: number,
+  componentIndex: number
+): number {
+  const value = getComponent(attribute, vertexIndex, componentIndex);
+  const byteValue = attribute.normalized || value <= 1 ? value * 255 : value;
+  return Math.max(0, Math.min(255, Math.round(byteValue)));
+}

package/src/pcd/unbundled.ts

@@ -0,0 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+export type {PCDLoaderOptions} from './pcd-loader-types';
+export {PCDLoader} from './pcd-loader-types';

package/src/pcd/workers/pcd-worker.ts

@@ -0,0 +1,24 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {createLoaderWorker} from '@loaders.gl/core/loader-utils';
+import {PCDColumnLoader} from '../pcd-loader';
+
+createLoaderWorker({
+  ...PCDColumnLoader,
+  /** Always returns columnar data; legacy mesh conversion runs on the caller. */
+  parse(input, options) {
+    return PCDColumnLoader.parse(input, {
+      ...options,
+      pcd: {...options?.pcd, shape: 'arrow'}
+    });
+  },
+  /** Keeps one columnar parser alive for the entire worker input stream. */
+  parseInBatches(input, options) {
+    return PCDColumnLoader.parseInBatches(input, {
+      ...options,
+      pcd: {...options?.pcd, shape: 'arrow'}
+    });
+  }
+});

package/src/pcd-loader-types.ts

@@ -0,0 +1 @@
+export * from './pcd/pcd-loader-types.js';

package/src/pcd-loader.ts

@@ -0,0 +1 @@
+export * from './pcd/pcd-loader.js';

package/src/pcd-writer.ts

@@ -0,0 +1 @@
+export * from './pcd/pcd-writer.js';

package/src/ply/bundled.ts

@@ -0,0 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+export type {PLYLoaderOptions} from './ply-loader-types';
+export {PLYLoaderWithParser as PLYLoader} from './ply-loader';

package/src/ply/index.ts

@@ -0,0 +1,11 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// PLYLoader
+export {PLYFormat} from './ply-format';
+
+export type {PLYLoaderOptions} from './ply-loader-types';
+export {PLYLoader} from './ply-loader-types';
+export type {PLYWriterOptions} from './ply-writer';
+export {PLYWriter} from './ply-writer';