@loaders.gl/pointclouds 5.0.0-beta.1 → 5.0.0-beta.10

package/src/pcd-loader-types.ts

@@ -0,0 +1 @@
+export * from './pcd/pcd-loader-types.js';

package/src/pcd-loader.ts

@@ -0,0 +1 @@
+export * from './pcd/pcd-loader.js';

package/src/pcd-writer.ts

@@ -0,0 +1 @@
+export * from './pcd/pcd-writer.js';

package/src/ply/bundled.ts

@@ -3,7 +3,4 @@
 // Copyright (c) vis.gl contributors
 
 export type {PLYLoaderOptions} from './ply-loader-types';
-export {
-  PLYLoaderWithParser as PLYWorkerLoader,
-  PLYLoaderWithParser as PLYLoader
-} from './ply-loader';
+export {PLYLoaderWithParser as PLYLoader} from './ply-loader';

package/src/ply/index.ts

@@ -9,7 +9,3 @@ export type {PLYLoaderOptions} from './ply-loader-types';
 export {PLYLoader} from './ply-loader-types';
 export type {PLYWriterOptions} from './ply-writer';
 export {PLYWriter} from './ply-writer';
-
-// DEPRECATED EXPORTS
-/** @deprecated Use PLYLoader. */
-export {PLYWorkerLoader} from './ply-loader-types';

package/src/ply/lib/get-ply-schema.ts

@@ -24,7 +24,7 @@ export function getPLYSchema(plyHeader: PLYHeader, attributes: MeshAttributes):
  * @param plyHeader
  * @returns
  */
-function makeMetadataFromPlyHeader(plyHeader: PLYHeader): Record<string, string> {
+export function makeMetadataFromPlyHeader(plyHeader: PLYHeader): Record<string, string> {
   /* eslint-disable camelcase */
   const metadata: Record<string, string> = {};
   metadata.ply_comments = JSON.stringify(plyHeader.comments);

package/src/ply/lib/parse-ply-arrow.ts

@@ -9,7 +9,7 @@ import type {
   MeshArrowTable,
   TypedArray
 } from '@loaders.gl/core/schema';
-import {getMeshBoundingBox} from '@loaders.gl/core/schema-utils';
+import {getMeshBoundingBox, serializeArrowSchema} from '@loaders.gl/core/schema-utils';
 import * as arrow from 'apache-arrow';
 import type {ParsePLYOptions} from './parse-ply';
 import {parsePLYHeader} from './parse-ply';
@@ -398,7 +398,7 @@ function convertPLYMeshToArrowTable(mesh: Mesh): MeshArrowTable {
 
   return {
     shape: 'arrow-table',
-    schema: mesh.schema,
+    schema: serializeArrowSchema(arrowSchema),
     data: new arrow.Table(arrowSchema, columns),
     topology: mesh.topology,
     indices: mesh.indices?.value.length ? mesh.indices : undefined

package/src/ply/lib/parse-ply-in-batches.ts

@@ -25,6 +25,7 @@
 //   }
 // });
 
+import {log as loaderLog} from '@loaders.gl/core/loader-utils';
 import {
   makeLineIterator,
   makeTextDecoderIterator,
@@ -32,6 +33,8 @@ import {
   toArrayBufferIterator
 } from '@loaders.gl/core/loader-utils';
 import type {MeshArrowTable} from '@loaders.gl/core/schema';
+import {updateArrowTableMetadata} from '@loaders.gl/core/schema-utils';
+import {makeMetadataFromPlyHeader} from './get-ply-schema';
 import normalizePLY from './normalize-ply';
 import {PLYMesh, PLYHeader, PLYElement, PLYProperty, PLYAttributes} from './ply-types';
 import {
@@ -58,7 +61,7 @@ export async function* parsePLYInBatches(
     | Iterable<ArrayBufferLike | ArrayBufferView>,
   options: any
 ): AsyncIterable<PLYMesh | MeshArrowTable> {
-  if (options?.shape === 'arrow-table') {
+  if (options?.shape === 'arrow') {
     yield* parseBinaryPLYToArrowInBatches(iterator, options);
     return;
   }
@@ -108,7 +111,10 @@ async function* parseBinaryPLYToArrowInBatches(
         );
   }
   for await (const table of tables) {
-    yield {...table, loaderData: header} as MeshArrowTable;
+    yield {
+      ...updateArrowTableMetadata(table, makeMetadataFromPlyHeader(header)),
+      loaderData: header
+    } as MeshArrowTable;
   }
 }
 
@@ -656,8 +662,7 @@ async function parseASCIIHeader(
         break;
 
       default:
-        // eslint-disable-next-line
-        console.log('unhandled', lineType, lineValues);
+        loaderLog.log(0, 'unhandled', lineType, lineValues)();
     }
 
     return false;

package/src/ply/lib/parse-ply.ts

@@ -26,6 +26,7 @@
 // });
 import type {MeshAttribute, MeshAttributes, MeshArrowTable} from '@loaders.gl/core/schema';
 import * as arrow from 'apache-arrow';
+import {getMeshBoundingBox} from '@loaders.gl/core/schema-utils';
 import type {
   PLYMesh,
   PLYHeader,
@@ -49,7 +50,7 @@ export type ParsePLYOptions = {
   propertyNameMapping?: Record<string, string>;
   /** Treat PLY data as a point cloud by reading only the leading vertex element. */
   pointCloud?: boolean;
-  shape?: 'mesh' | 'arrow-table';
+  shape?: 'mesh' | 'arrow';
   /** Color storage format. Defaults to uint8norm for backwards compatibility. */
   colorFormat?: 'uint8norm' | 'float16' | 'float32';
 };
@@ -637,6 +638,8 @@ function makePLYArrowTable(header: PLYHeader, attributes: PLYAttributes): MeshAr
   const schema = getPLYSchema(header, meshAttributes);
   schema.metadata.topology = 'point-list';
   schema.metadata.mode = '0';
+  const boundingBox = getMeshBoundingBox(meshAttributes);
+  if (boundingBox) schema.metadata.boundingBox = JSON.stringify(boundingBox);
 
   const columns: Record<string, arrow.Vector> = {};
   const fields: arrow.Field[] = [];

package/src/ply/ply-loader-types.ts

@@ -3,12 +3,12 @@
 // Copyright (c) vis.gl contributors
 
 // PLY Loader
+import type {Table, RecordBatch} from 'apache-arrow';
 import type {Loader, LoaderOptions} from '@loaders.gl/core/loader-utils';
-import type {MeshArrowTable, ArrowTableBatch} from '@loaders.gl/core/schema';
 import type {PLYMesh} from './lib/ply-types';
 import type {ParsePLYOptions} from './lib/parse-ply';
 import {PLYFormat} from './ply-format';
-import {serializeArrowWorkerTable} from '@loaders.gl/core/schema-utils/arrow-transport';
+import {serializeArrowWorkerTable} from '@loaders.gl/core/arrow-worker-transport';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -17,7 +17,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 export type PLYLoaderOptions = LoaderOptions & {
   ply?: ParsePLYOptions & {
     /** Output shape. Defaults to a Mesh Arrow table. */
-    shape?: 'mesh' | 'arrow-table';
+    shape?: 'mesh' | 'arrow';
     /** Color storage format. Defaults to uint8norm for backwards compatibility. */
     colorFormat?: 'uint8norm' | 'float16' | 'float32';
     /** Treat PLY data as a point cloud by reading only the leading vertex element. */
@@ -31,7 +31,7 @@ export type PLYLoaderOptions = LoaderOptions & {
  * Preloads the parser-bearing PLY loader implementation.
  */
 async function preload() {
-  const {PLYLoaderWithParser} = await import('@loaders.gl/pointclouds/ply/ply-loader');
+  const {PLYLoaderWithParser} = await import('@loaders.gl/pointclouds/ply-loader');
   return PLYLoaderWithParser;
 }
 
@@ -41,8 +41,10 @@ async function preload() {
  * 'https://en.wikipedia.org/wiki/PLY_(file_format)']
  */
 export const PLYLoader = {
-  dataType: null as unknown as PLYMesh | MeshArrowTable,
-  batchType: null as unknown as PLYMesh | ArrowTableBatch,
+  defaultShape: 'arrow',
+  supportsNativeArrow: true,
+  dataType: null as unknown as PLYMesh | Table,
+  batchType: null as unknown as PLYMesh | RecordBatch,
 
   ...PLYFormat,
   // shapes: ['mesh', 'gltf', 'columnar-table'],
@@ -52,10 +54,7 @@ export const PLYLoader = {
   workerInBatches: true,
   serializeWorkerResult: serializeArrowWorkerTable,
   options: {
-    ply: {shape: 'arrow-table', colorFormat: 'uint8norm'}
+    ply: {shape: 'arrow', colorFormat: 'uint8norm'}
   },
   preload
-} as const satisfies Loader<PLYMesh | MeshArrowTable, PLYMesh | ArrowTableBatch, LoaderOptions>;
-
-/** @deprecated Use PLYLoader. */
-export const PLYWorkerLoader = PLYLoader;
+} as const satisfies Loader<PLYMesh | Table, PLYMesh | RecordBatch, LoaderOptions>;

package/src/ply/ply-loader.ts

@@ -2,6 +2,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import {refineArrowTableBatches, makeArrowMetadataBatch} from '@loaders.gl/core/schema-utils';
+import type {Table, RecordBatch} from 'apache-arrow';
 import type {PLYLoaderOptions} from './ply-loader-types';
 
 // PLY Loader
@@ -11,10 +13,10 @@ import type {MeshArrowTable, ArrowTableBatch} from '@loaders.gl/core/schema';
 import {
   deserializeArrowWorkerTable,
   serializeArrowWorkerTable
-} from '@loaders.gl/core/schema-utils/arrow-transport';
+} from '@loaders.gl/core/arrow-worker-transport';
 import type {PLYHeader, PLYMesh} from './lib/ply-types';
 import {
-  convertMeshToTable,
+  convertMeshToArrowTableWrapper,
   convertTableToMesh,
   deduceMeshSchema
 } from '@loaders.gl/core/schema-utils';
@@ -29,7 +31,7 @@ import {PLYLoader as PLYLoaderMetadata} from './ply-loader-types';
 const {preload: _PLYLoaderPreload, ...PLYLoaderMetadataWithoutPreload} = PLYLoaderMetadata;
 
 function convertPLYMesh(mesh: PLYMesh, options?: PLYLoaderOptions): PLYMesh | MeshArrowTable {
-  const table = convertMeshToTable(mesh, 'arrow-table');
+  const table = convertMeshToArrowTableWrapper(mesh);
   return convertPLYTable(table, options, mesh.loaderData);
 }
 
@@ -58,28 +60,15 @@ function parsePLYData(
 /**
  * Loader for PLY - Polygon File Format
  */
-export const PLYLoaderWithParser = {
+export const PLYColumnLoader = {
   ...PLYLoaderMetadataWithoutPreload,
+  dataType: null as unknown as PLYMesh | MeshArrowTable,
+  batchType: null as unknown as PLYMesh | ArrowTableBatch,
   parse: async (arrayBuffer, options) => parsePLYData(arrayBuffer, options),
   parseTextSync: (arrayBuffer, options) => parsePLYData(arrayBuffer, options),
   parseSync: (arrayBuffer, options) => parsePLYData(arrayBuffer, options),
-  parseInBatches: async function* (
-    arrayBuffer:
-      | AsyncIterable<ArrayBufferLike | ArrayBufferView>
-      | Iterable<ArrayBufferLike | ArrayBufferView>,
-    options
-  ) {
-    const plyOptions = {
-      ...options?.ply,
-      batchSize: options?.batchSize ?? options?.core?.batchSize
-    };
-    for await (const meshOrTable of parsePLYInBatches(arrayBuffer, plyOptions)) {
-      const convertedData = isMeshArrowTable(meshOrTable)
-        ? convertPLYTable(meshOrTable, options)
-        : convertPLYMesh(meshOrTable, options);
-      yield isMeshArrowTable(convertedData) ? makeTableScanBatch(convertedData) : convertedData;
-    }
-  },
+  parseInBatches: (input, options) =>
+    refineArrowTableBatches(parsePLYWithMetadataInBatches(input, options)),
   serializeWorkerResult: serializeArrowWorkerTable,
   deserializeWorkerResult: deserializePLYWorkerResult,
   serializeWorkerBatch: serializeArrowWorkerTable,
@@ -101,7 +90,7 @@ function convertPLYTable(
 ): PLYMesh | MeshArrowTable {
   const loaderData = header || table.loaderData || {comments: [], elements: []};
   const result =
-    options?.ply?.shape === 'arrow-table'
+    options?.ply?.shape === 'arrow'
       ? {...table, loader: 'ply' as const, loaderData}
       : {
           ...(convertTableToMesh(table) as PLYMesh),
@@ -118,9 +107,9 @@ function convertPLYTable(
     (isMeshArrowTable(resultData) ? convertTableToMesh(resultData) : resultData) as PLYMesh,
     colorFormat
   );
-  return options?.ply?.shape === 'arrow-table'
+  return options?.ply?.shape === 'arrow'
     ? {
-        ...convertMeshToTable(formattedMesh, 'arrow-table'),
+        ...convertMeshToArrowTableWrapper(formattedMesh),
         loader: 'ply' as const,
         loaderData
       }
@@ -183,3 +172,84 @@ function deserializePLYWorkerBatch(
 ): PLYMesh | ArrowTableBatch {
   return deserializePLYWorkerResult(result, options) as PLYMesh | ArrowTableBatch;
 }
+
+/** Parses PLY batches before shared cumulative metadata refinement. */
+async function* parsePLYWithMetadataInBatches(
+  arrayBuffer:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
+  options?: PLYLoaderOptions
+) {
+  const plyOptions = {
+    ...options?.ply,
+    batchSize: options?.core?.batchSize
+  };
+  for await (const meshOrTable of parsePLYInBatches(arrayBuffer, plyOptions)) {
+    const convertedData = isMeshArrowTable(meshOrTable)
+      ? convertPLYTable(meshOrTable, options)
+      : convertPLYMesh(meshOrTable, options);
+    yield isMeshArrowTable(convertedData) ? makeTableScanBatch(convertedData) : convertedData;
+  }
+}
+
+/** Exposes native Arrow output while retaining the worker's column protocol. */
+export const PLYLoaderWithParser = {
+  ...PLYColumnLoader,
+  dataType: PLYLoaderMetadata.dataType,
+  batchType: PLYLoaderMetadata.batchType,
+  parse: async (input: ArrayBuffer, options?: PLYLoaderOptions) =>
+    unwrapPLYTable(await PLYColumnLoader.parse(input, options)),
+  parseTextSync: (input: string, options?: PLYLoaderOptions) =>
+    unwrapPLYTable(PLYColumnLoader.parseTextSync(input, options)),
+  parseSync: (input: ArrayBuffer, options?: PLYLoaderOptions) =>
+    unwrapPLYTable(PLYColumnLoader.parseSync(input, options)),
+  async *parseInBatches(input, options) {
+    for await (const batch of PLYColumnLoader.parseInBatches(input, options)) {
+      if (isColumnBatch(batch)) {
+        batch.data.schema.metadata.set(
+          'loaders.gl.ply.header',
+          JSON.stringify((batch as ArrowTableBatch & {loaderData?: unknown}).loaderData)
+        );
+        for (const recordBatch of batch.data.batches) {
+          recordBatch.schema.metadata.set(
+            'loaders.gl.ply.header',
+            batch.data.schema.metadata.get('loaders.gl.ply.header')!
+          );
+        }
+        if (batch.data.batches.length) yield* batch.data.batches;
+        else yield makeArrowMetadataBatch(batch.data.schema);
+      } else yield batch;
+    }
+  },
+  deserializeWorkerResult: (result: unknown, options?: PLYLoaderOptions) =>
+    unwrapPLYTable(PLYColumnLoader.deserializeWorkerResult(result, options)),
+  deserializeWorkerBatch(result: unknown, options?: PLYLoaderOptions) {
+    const batch = PLYColumnLoader.deserializeWorkerBatch(result, options);
+    if (isColumnBatch(batch)) {
+      const recordBatch = batch.data.batches[0] ?? makeArrowMetadataBatch(batch.data.schema);
+      recordBatch.schema.metadata.set(
+        'loaders.gl.ply.header',
+        JSON.stringify((batch as ArrowTableBatch & {loaderData?: unknown}).loaderData)
+      );
+      return recordBatch;
+    }
+    return batch;
+  }
+} as const satisfies LoaderWithParser<PLYMesh | Table, PLYMesh | RecordBatch, PLYLoaderOptions>;
+
+/** Returns the Arrow table with its file metadata or the requested mesh. */
+function unwrapPLYTable(result: PLYMesh | MeshArrowTable): PLYMesh | Table {
+  if ('shape' in result && result.shape === 'arrow-table') {
+    result.data.schema.metadata.set(
+      'loaders.gl.ply.header',
+      JSON.stringify((result as MeshArrowTable & {loaderData?: unknown}).loaderData)
+    );
+    return result.data;
+  }
+  return result as PLYMesh;
+}
+
+/** Identifies the private Arrow batch envelope. */
+function isColumnBatch(batch: PLYMesh | ArrowTableBatch): batch is ArrowTableBatch {
+  return 'shape' in batch && batch.shape === 'arrow-table';
+}

package/src/ply/ply-writer.ts

@@ -2,9 +2,10 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import type {Table} from 'apache-arrow';
 import type {WriterOptions, WriterWithEncoder} from '@loaders.gl/core/loader-utils';
 import type {Mesh, MeshArrowTable, MeshAttribute} from '@loaders.gl/core/schema';
-import {convertMeshToTable, convertTableToMesh} from '@loaders.gl/core/schema-utils';
+import {convertMeshToArrowTableWrapper, convertTableToMesh} from '@loaders.gl/core/schema-utils';
 import {PLYFormat} from './ply-format';
 
 // __VERSION__ is injected by babel-plugin-version-inline
@@ -22,7 +23,7 @@ export type PLYWriterOptions = WriterOptions & {
  */
 export const PLYWriter = {
   ...PLYFormat,
-  dataType: null as unknown as Mesh | MeshArrowTable,
+  dataType: null as unknown as Mesh | MeshArrowTable | Table,
   batchType: null as never,
   version: VERSION,
   options: {
@@ -32,16 +33,19 @@ export const PLYWriter = {
   encode: async (data, options) => encodePLYSync(data, options),
   encodeSync: encodePLYSync,
   encodeTextSync: encodePLY
-} as const satisfies WriterWithEncoder<Mesh | MeshArrowTable, never, PLYWriterOptions>;
+} as const satisfies WriterWithEncoder<Mesh | MeshArrowTable | Table, never, PLYWriterOptions>;
 
 /** Encode mesh category data as PLY bytes. */
-function encodePLYSync(data: Mesh | MeshArrowTable, options?: PLYWriterOptions): ArrayBuffer {
+function encodePLYSync(
+  data: Mesh | MeshArrowTable | Table,
+  options?: PLYWriterOptions
+): ArrayBuffer {
   const text = encodePLY(data, options);
   return new TextEncoder().encode(text).buffer;
 }
 
 /** Encode mesh category data as ASCII PLY text. */
-function encodePLY(data: Mesh | MeshArrowTable, options?: PLYWriterOptions): string {
+function encodePLY(data: Mesh | MeshArrowTable | Table, options?: PLYWriterOptions): string {
   const mesh = convertTableToMesh(normalizeMeshArrowTable(data));
   const positionAttribute = getRequiredAttribute(mesh, 'POSITION');
   const normalAttribute = mesh.attributes.NORMAL;
@@ -109,11 +113,11 @@ function encodePLY(data: Mesh | MeshArrowTable, options?: PLYWriterOptions): str
 }
 
 /** Return mesh data as a MeshArrowTable, converting plain Mesh data first. */
-function normalizeMeshArrowTable(data: Mesh | MeshArrowTable): MeshArrowTable {
-  if ('shape' in data && data.shape === 'arrow-table') {
+function normalizeMeshArrowTable(data: Mesh | MeshArrowTable | Table): MeshArrowTable | Table {
+  if (!('attributes' in data)) {
     return data;
   }
-  return convertMeshToTable(data as Mesh, 'arrow-table');
+  return convertMeshToArrowTableWrapper(data as Mesh);
 }
 
 /** Return a required mesh attribute or throw a format-specific error. */

package/src/ply/unbundled.ts

@@ -3,4 +3,4 @@
 // Copyright (c) vis.gl contributors
 
 export type {PLYLoaderOptions} from './ply-loader-types';
-export {PLYWorkerLoader, PLYLoader} from './ply-loader-types';
+export {PLYLoader} from './ply-loader-types';

package/src/ply/workers/ply-worker.ts

@@ -4,18 +4,18 @@
 
 import {createLoaderWorker, makeTableScanBatch} from '@loaders.gl/core/loader-utils';
 import type {MeshArrowTable} from '@loaders.gl/core/schema';
-import {convertMeshToTable} from '@loaders.gl/core/schema-utils';
+import {convertMeshToArrowTableWrapper} from '@loaders.gl/core/schema-utils';
 import type {PLYMesh} from '../lib/ply-types';
 import type {PLYLoaderOptions} from '../ply-loader-types';
-import {PLYLoaderWithParser} from '../ply-loader';
+import {PLYColumnLoader} from '../ply-loader';
 
 createLoaderWorker({
-  ...PLYLoaderWithParser,
+  ...PLYColumnLoader,
   /** Always returns columnar data; legacy mesh conversion runs on the caller. */
   parse(input, options) {
-    return PLYLoaderWithParser.parse(input, {
+    return PLYColumnLoader.parse(input, {
       ...options,
-      ply: {...options?.ply, shape: 'arrow-table'}
+      ply: {...options?.ply, shape: 'arrow'}
     });
   },
   /** Preserves full-mesh parsing while streaming point-cloud vertices as Arrow batches. */
@@ -23,14 +23,14 @@ createLoaderWorker({
     const parseMesh = options?.ply?.shape === 'mesh' && !options?.ply?.pointCloud;
     const parserOptions = parseMesh
       ? options
-      : {...options, ply: {...options?.ply, shape: 'arrow-table' as const}};
-    for await (const batch of PLYLoaderWithParser.parseInBatches(input, parserOptions)) {
+      : {...options, ply: {...options?.ply, shape: 'arrow' as const}};
+    for await (const batch of PLYColumnLoader.parseInBatches(input, parserOptions)) {
       if ('shape' in batch && batch.shape === 'arrow-table') {
         yield batch;
       } else {
         const mesh = batch as PLYMesh;
         yield makeTableScanBatch({
-          ...convertMeshToTable(mesh, 'arrow-table'),
+          ...convertMeshToArrowTableWrapper(mesh),
           loader: mesh.loader,
           loaderData: mesh.loaderData
         } as MeshArrowTable);

package/src/ply-loader-types.ts

@@ -0,0 +1 @@
+export * from './ply/ply-loader-types.js';

package/src/ply-loader.ts

@@ -0,0 +1 @@
+export * from './ply/ply-loader.js';

package/src/ply-writer.ts

@@ -0,0 +1 @@
+export * from './ply/ply-writer.js';