@loaders.gl/pointclouds 0.0.0-stage → 5.0.0-beta.10

package/src/las/workers/las-worker.ts

@@ -0,0 +1,24 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {createLoaderWorker} from '@loaders.gl/core/loader-utils';
+import {LASColumnLoader} from '../las-loader';
+
+createLoaderWorker({
+  ...LASColumnLoader,
+  /** Decodes worker output as transferable columns; caller formatting follows hydration. */
+  parse(input, options) {
+    return LASColumnLoader.parse(input, {
+      ...options,
+      las: {...options?.las, shape: 'arrow', colorFormat: 'uint8norm'}
+    });
+  },
+  /** Keeps the primary TypeScript decoder alive for the full input stream. */
+  parseInBatches(input, options) {
+    return LASColumnLoader.parseInBatches(input, {
+      ...options,
+      las: {...options?.las, shape: 'arrow', colorFormat: 'uint8norm'}
+    });
+  }
+});

package/src/las-copc-loader-types.ts

@@ -0,0 +1 @@
+export * from './las/las-copc-loader-types.js';

package/src/las-copc-loader.ts

@@ -0,0 +1 @@
+export * from './las/las-copc-loader.js';

package/src/las-loader-types.ts

@@ -0,0 +1 @@
+export * from './las/las-loader-types.js';

package/src/las-loader.ts

@@ -0,0 +1 @@
+export * from './las/las-loader.js';

package/src/las-writer.ts

@@ -0,0 +1 @@
+export * from './las/las-writer.js';

package/src/laz-rs-loader-types.ts

@@ -0,0 +1 @@
+export * from './las/laz-rs-loader-types.js';

package/src/laz-rs-loader.ts

@@ -0,0 +1 @@
+export * from './las/laz-rs-loader.js';

package/src/lazperf-loader-types.ts

@@ -0,0 +1 @@
+export * from './las/lazperf-loader-types.js';

package/src/lazperf-loader.ts

@@ -0,0 +1 @@
+export * from './las/lazperf-loader.js';

package/src/pcd/bundled.ts

@@ -0,0 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+export type {PCDLoaderOptions} from './pcd-loader-types';
+export {PCDLoaderWithParser as PCDLoader} from './pcd-loader';

package/src/pcd/index.ts

@@ -0,0 +1,9 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright vis.gl contributors
+
+export type {PCDLoaderOptions} from './pcd-loader-types';
+export {PCDFormat} from './pcd-format';
+export {PCDLoader} from './pcd-loader-types';
+export type {PCDWriterOptions} from './pcd-writer';
+export {PCDWriter} from './pcd-writer';

package/src/pcd/lib/decompress-lzf.ts

@@ -0,0 +1,68 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+// Forked from https://gitlab.com/taketwo/three-pcd-loader/blob/master/decompress-lzf.js
+
+/* eslint-disable */
+/**
+ * @param inData
+ * @param outLength
+ * @returns
+ */
+export function decompressLZF(inData: Uint8Array, outLength: number): Uint8Array {
+  const inLength = inData.length;
+  const outData = new Uint8Array(outLength);
+  let inPtr = 0;
+  let outPtr = 0;
+  let ctrl;
+  let len;
+  let ref;
+
+  do {
+    ctrl = inData[inPtr++];
+
+    if (ctrl < 1 << 5) {
+      ctrl++;
+      if (outPtr + ctrl > outLength) {
+        throw new Error('Output buffer is not large enough');
+      }
+      if (inPtr + ctrl > inLength) {
+        throw new Error('Invalid compressed data');
+      }
+
+      do {
+        outData[outPtr++] = inData[inPtr++];
+      } while (--ctrl);
+    } else {
+      len = ctrl >> 5;
+      ref = outPtr - ((ctrl & 0x1f) << 8) - 1;
+      if (inPtr >= inLength) {
+        throw new Error('Invalid compressed data');
+      }
+
+      if (len === 7) {
+        len += inData[inPtr++];
+        if (inPtr >= inLength) {
+          throw new Error('Invalid compressed data');
+        }
+      }
+
+      ref -= inData[inPtr++];
+      if (outPtr + len + 2 > outLength) {
+        throw new Error('Output buffer is not large enough');
+      }
+      if (ref < 0) {
+        throw new Error('Invalid compressed data');
+      }
+      if (ref >= outPtr) {
+        throw new Error('Invalid compressed data');
+      }
+
+      do {
+        outData[outPtr++] = outData[ref++];
+      } while (--len + 2);
+    }
+  } while (inPtr < inLength);
+
+  return outData;
+}

package/src/pcd/lib/get-pcd-schema.ts

@@ -0,0 +1,66 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import {Schema, Field} from '@loaders.gl/core/schema';
+import type {PCDHeader} from './pcd-types';
+
+/**
+ * Gets schema from PCD header
+ * @param PCDheader
+ * @param metadata
+ * @returns Schema
+ */
+export function getPCDSchema(
+  PCDheader: PCDHeader,
+  metadata: Record<string, string>,
+  colorFormat: 'uint8norm' | 'float16' | 'float32' = 'uint8norm'
+): Schema {
+  const fields: Field[] = [];
+
+  for (const key of Object.keys(PCDheader.offset)) {
+    switch (key) {
+      case 'x':
+        fields.push({
+          name: 'POSITION',
+          type: {type: 'fixed-size-list', listSize: 3, children: [{name: 'xyz', type: 'float32'}]},
+          metadata: {attribute: 'POSITION'}
+        });
+        break;
+      case 'y':
+      case 'z':
+        // ignore
+        break;
+
+      case 'normal_x':
+        fields.push({
+          name: 'NORMAL',
+          type: {type: 'fixed-size-list', listSize: 3, children: [{name: 'xyz', type: 'float32'}]},
+          metadata: {attribute: 'NORMAL'}
+        });
+        break;
+      case 'normal_y':
+      case 'normal_z':
+        // ignore
+        break;
+
+      case 'rgb':
+        const colorType =
+          colorFormat === 'float16' ? 'float16' : colorFormat === 'float32' ? 'float32' : 'uint8';
+        fields.push({
+          name: 'COLOR_0',
+          type: {type: 'fixed-size-list', listSize: 3, children: [{name: 'rgb', type: colorType}]},
+          metadata: {
+            attribute: 'COLOR',
+            ...(colorFormat === 'float16' ? {componentType: 'float16'} : {})
+          }
+        });
+        break;
+
+      default:
+      // TODO - push fields
+    }
+  }
+
+  return {fields, metadata};
+}

package/src/pcd/lib/parse-pcd.ts

@@ -0,0 +1,474 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// PCD Loader, adapted from THREE.js (MIT license)
+// Description: A loader for PCD ascii and binary files.
+// Limitations: Compressed binary files are not supported.
+//
+// Attributions per original THREE.js source file:
+// @author Filipe Caixeta / http://filipecaixeta.com.br
+// @author Mugen87 / https://github.com/Mugen87
+
+import {convertColorArrayToFloat16} from '@loaders.gl/core/schema';
+import type {MeshAttribute, MeshAttributes} from '@loaders.gl/core/schema';
+import {getMeshBoundingBox} from '@loaders.gl/core/schema-utils';
+import {decompressLZF} from './decompress-lzf';
+import {getPCDSchema} from './get-pcd-schema';
+import type {PCDHeader, PCDMesh} from './pcd-types';
+
+type MeshHeader = {
+  vertexCount: number;
+  boundingBox: [[number, number, number], [number, number, number]];
+};
+
+type NormalizedAttributes = {
+  POSITION: {
+    value: Float32Array;
+    size: number;
+  };
+  NORMAL?: {
+    value: Float32Array;
+    size: number;
+  };
+  COLOR_0?: MeshAttribute;
+};
+
+type ParsePCDOptions = {
+  pcd?: {colorFormat?: 'uint8norm' | 'float16' | 'float32'};
+};
+
+type HeaderAttributes = {
+  [attributeName: string]: number[];
+};
+
+const LITTLE_ENDIAN: boolean = true;
+
+/**
+ *
+ * @param data
+ * @returns
+ */
+export function parsePCD(data: ArrayBufferLike, options?: ParsePCDOptions): PCDMesh {
+  // parse header (always ascii format)
+  const textData = new TextDecoder().decode(data);
+  const pcdHeader = parsePCDHeader(textData);
+
+  let attributes: any = {};
+
+  // parse data
+  switch (pcdHeader.data) {
+    case 'ascii':
+      attributes = parsePCDASCII(pcdHeader, textData);
+      break;
+
+    case 'binary':
+      attributes = parsePCDBinary(pcdHeader, data);
+      break;
+
+    case 'binary_compressed':
+      attributes = parsePCDBinaryCompressed(pcdHeader, data);
+      break;
+
+    default:
+      throw new Error(`PCD: ${pcdHeader.data} files are not supported`);
+  }
+
+  attributes = getMeshAttributes(attributes, options?.pcd?.colorFormat);
+
+  const header = getMeshHeader(pcdHeader, attributes);
+
+  const schemaMetadata = Object.fromEntries([
+    ['topology', 'point-list'],
+    ['mode', '0'],
+    ['boundingBox', JSON.stringify(header.boundingBox)]
+  ]);
+
+  const schema = getPCDSchema(pcdHeader, schemaMetadata, options?.pcd?.colorFormat);
+
+  return {
+    loader: 'pcd',
+    loaderData: pcdHeader,
+    header,
+    schema,
+    topology: 'point-list',
+    mode: 0, // POINTS (deprecated)
+    attributes
+  };
+}
+
+// Create a header that contains common data for PointCloud category loaders
+function getMeshHeader(pcdHeader: PCDHeader, attributes: NormalizedAttributes): MeshHeader {
+  if (typeof pcdHeader.width === 'number' && typeof pcdHeader.height === 'number') {
+    const pointCount = pcdHeader.width * pcdHeader.height; // Supports "organized" point sets
+    const boundingBox = getMeshBoundingBox(attributes);
+    if (!boundingBox) throw new Error('PCD point cloud is missing positions');
+    return {
+      vertexCount: pointCount,
+      boundingBox
+    };
+  }
+  return {
+    vertexCount: pcdHeader.vertexCount,
+    boundingBox: pcdHeader.boundingBox
+  };
+}
+
+/**
+ * @param attributes
+ * @returns Normalized attributes
+ */
+function getMeshAttributes(
+  attributes: HeaderAttributes,
+  colorFormat: 'uint8norm' | 'float16' | 'float32' = 'uint8norm'
+): {[attributeName: string]: MeshAttribute} {
+  const normalizedAttributes: MeshAttributes = {
+    POSITION: {
+      // Binary PCD is only 32 bit
+      value: new Float32Array(attributes.position),
+      size: 3
+    }
+  };
+
+  if (attributes.normal && attributes.normal.length > 0) {
+    normalizedAttributes.NORMAL = {
+      value: new Float32Array(attributes.normal),
+      size: 3
+    };
+  }
+
+  if (attributes.color && attributes.color.length > 0) {
+    // TODO - RGBA
+    normalizedAttributes.COLOR_0 = makeColorAttribute(attributes.color, colorFormat);
+  }
+
+  if (attributes.intensity && attributes.intensity.length > 0) {
+    // TODO - RGBA
+    normalizedAttributes.COLOR_0 = makeColorAttribute(attributes.color, colorFormat);
+  }
+
+  if (!normalizedAttributes.COLOR_0 && attributes.label && attributes.label.length > 0) {
+    // TODO - RGBA
+    normalizedAttributes.COLOR_0 = makeColorAttribute(attributes.label, colorFormat);
+  }
+
+  return normalizedAttributes;
+}
+
+function makeColorAttribute(
+  values: number[],
+  colorFormat: 'uint8norm' | 'float16' | 'float32'
+): MeshAttribute {
+  if (colorFormat === 'float16') {
+    return {
+      value: convertColorArrayToFloat16(values, 255),
+      size: 3,
+      componentType: 'float16'
+    };
+  }
+  if (colorFormat === 'float32') {
+    return {value: Float32Array.from(values, value => value / 255), size: 3};
+  }
+  return {value: new Uint8Array(values), size: 3};
+}
+
+/**
+ * Incoming data parsing
+ * @param data
+ * @returns Header
+ */
+/* eslint-disable complexity, max-statements */
+export function parsePCDHeader(data: string): PCDHeader {
+  const result1 = data.search(/[\r\n]DATA\s(\S*)\s/i);
+  const result2 = /[\r\n]DATA\s(\S*)\s/i.exec(data.substr(result1 - 1));
+
+  const pcdHeader: any = {};
+  pcdHeader.data = result2 && result2[1];
+  if (result2 !== null) {
+    pcdHeader.headerLen = (result2 && result2[0].length) + result1;
+  }
+  pcdHeader.str = data.substr(0, pcdHeader.headerLen);
+
+  // remove comments
+
+  pcdHeader.str = pcdHeader.str.replace(/\#.*/gi, '');
+
+  // parse
+
+  pcdHeader.version = /VERSION (.*)/i.exec(pcdHeader.str);
+  pcdHeader.fields = /FIELDS (.*)/i.exec(pcdHeader.str);
+  pcdHeader.size = /SIZE (.*)/i.exec(pcdHeader.str);
+  pcdHeader.type = /TYPE (.*)/i.exec(pcdHeader.str);
+  pcdHeader.count = /COUNT (.*)/i.exec(pcdHeader.str);
+  pcdHeader.width = /WIDTH (.*)/i.exec(pcdHeader.str);
+  pcdHeader.height = /HEIGHT (.*)/i.exec(pcdHeader.str);
+  pcdHeader.viewpoint = /VIEWPOINT (.*)/i.exec(pcdHeader.str);
+  pcdHeader.points = /POINTS (.*)/i.exec(pcdHeader.str);
+
+  // evaluate
+
+  if (pcdHeader.version !== null) {
+    pcdHeader.version = parseFloat(pcdHeader.version[1]);
+  }
+
+  if (pcdHeader.fields !== null) {
+    pcdHeader.fields = pcdHeader.fields[1].split(' ');
+  }
+
+  if (pcdHeader.type !== null) {
+    pcdHeader.type = pcdHeader.type[1].split(' ');
+  }
+
+  if (pcdHeader.width !== null) {
+    pcdHeader.width = parseInt(pcdHeader.width[1], 10);
+  }
+
+  if (pcdHeader.height !== null) {
+    pcdHeader.height = parseInt(pcdHeader.height[1], 10);
+  }
+
+  if (pcdHeader.viewpoint !== null) {
+    pcdHeader.viewpoint = pcdHeader.viewpoint[1];
+  }
+
+  if (pcdHeader.points !== null) {
+    pcdHeader.points = parseInt(pcdHeader.points[1], 10);
+  }
+
+  if (
+    pcdHeader.points === null &&
+    typeof pcdHeader.width === 'number' &&
+    typeof pcdHeader.height === 'number'
+  ) {
+    pcdHeader.points = pcdHeader.width * pcdHeader.height;
+  }
+
+  if (pcdHeader.size !== null) {
+    pcdHeader.size = pcdHeader.size[1].split(' ').map(x => parseInt(x, 10));
+  }
+
+  if (pcdHeader.count !== null) {
+    pcdHeader.count = pcdHeader.count[1].split(' ').map(x => parseInt(x, 10));
+  } else {
+    pcdHeader.count = [];
+    if (pcdHeader.fields !== null) {
+      for (let i = 0; i < pcdHeader.fields.length; i++) {
+        pcdHeader.count.push(1);
+      }
+    }
+  }
+
+  pcdHeader.offset = {};
+
+  let sizeSum = 0;
+  for (let i = 0, l = pcdHeader.fields.length; i < l; i++) {
+    if (pcdHeader.data === 'ascii') {
+      pcdHeader.offset[pcdHeader.fields[i]] = i;
+    } else {
+      pcdHeader.offset[pcdHeader.fields[i]] = sizeSum;
+      sizeSum += pcdHeader.size[i] * pcdHeader.count[i];
+    }
+  }
+
+  // for binary only
+  pcdHeader.rowSize = sizeSum;
+  return pcdHeader;
+}
+
+/**
+ * @param pcdHeader
+ * @param textData
+ * @returns [attributes]
+ */
+// eslint-enable-next-line complexity, max-statements
+function parsePCDASCII(pcdHeader: PCDHeader, textData: string): HeaderAttributes {
+  const position: number[] = [];
+  const normal: number[] = [];
+  const color: number[] = [];
+  const intensity: number[] = [];
+  const label: number[] = [];
+
+  const offset = pcdHeader.offset;
+  const pcdData = textData.substr(pcdHeader.headerLen);
+  const lines = pcdData.split('\n');
+
+  for (let i = 0; i < lines.length; i++) {
+    if (lines[i] !== '') {
+      const line = lines[i].split(' ');
+
+      if (offset.x !== undefined) {
+        position.push(parseFloat(line[offset.x]));
+        position.push(parseFloat(line[offset.y]));
+        position.push(parseFloat(line[offset.z]));
+      }
+
+      if (offset.rgb !== undefined) {
+        const floatValue = parseFloat(line[offset.rgb]);
+        const binaryColor = new Float32Array([floatValue]);
+        const dataview = new DataView(binaryColor.buffer, 0);
+        color.push(dataview.getUint8(0));
+        color.push(dataview.getUint8(1));
+        color.push(dataview.getUint8(2));
+        // TODO - handle alpha channel / RGBA?
+      }
+
+      if (offset.normal_x !== undefined) {
+        normal.push(parseFloat(line[offset.normal_x]));
+        normal.push(parseFloat(line[offset.normal_y]));
+        normal.push(parseFloat(line[offset.normal_z]));
+      }
+
+      if (offset.intensity !== undefined) {
+        intensity.push(parseFloat(line[offset.intensity]));
+      }
+
+      if (offset.label !== undefined) {
+        label.push(parseInt(line[offset.label]));
+      }
+    }
+  }
+
+  return {position, normal, color};
+}
+
+/**
+ * @param pcdHeader
+ * @param data
+ * @returns [attributes]
+ */
+function parsePCDBinary(pcdHeader: PCDHeader, data: ArrayBufferLike): HeaderAttributes {
+  const position: number[] = [];
+  const normal: number[] = [];
+  const color: number[] = [];
+  const intensity: number[] = [];
+  const label: number[] = [];
+
+  const dataview = new DataView(data, pcdHeader.headerLen);
+  const offset = pcdHeader.offset;
+
+  for (let i = 0, row = 0; i < pcdHeader.points; i++, row += pcdHeader.rowSize) {
+    if (offset.x !== undefined) {
+      position.push(dataview.getFloat32(row + offset.x, LITTLE_ENDIAN));
+      position.push(dataview.getFloat32(row + offset.y, LITTLE_ENDIAN));
+      position.push(dataview.getFloat32(row + offset.z, LITTLE_ENDIAN));
+    }
+
+    if (offset.rgb !== undefined) {
+      color.push(dataview.getUint8(row + offset.rgb + 0));
+      color.push(dataview.getUint8(row + offset.rgb + 1));
+      color.push(dataview.getUint8(row + offset.rgb + 2));
+    }
+
+    if (offset.normal_x !== undefined) {
+      normal.push(dataview.getFloat32(row + offset.normal_x, LITTLE_ENDIAN));
+      normal.push(dataview.getFloat32(row + offset.normal_y, LITTLE_ENDIAN));
+      normal.push(dataview.getFloat32(row + offset.normal_z, LITTLE_ENDIAN));
+    }
+
+    if (offset.intensity !== undefined) {
+      intensity.push(dataview.getFloat32(row + offset.intensity, LITTLE_ENDIAN));
+    }
+
+    if (offset.label !== undefined) {
+      label.push(dataview.getInt32(row + offset.label, LITTLE_ENDIAN));
+    }
+  }
+
+  return {position, normal, color, intensity, label};
+}
+
+/** Parse compressed PCD data in in binary_compressed form ( https://pointclouds.org/documentation/tutorials/pcd_file_format.html)
+ * from https://github.com/mrdoob/three.js/blob/master/examples/jsm/loaders/PCDLoader.js
+ * @license MIT (http://opensource.org/licenses/MIT)
+ * @param pcdHeader
+ * @param data
+ * @returns [attributes]
+ */
+// eslint-enable-next-line complexity, max-statements
+function parsePCDBinaryCompressed(pcdHeader: PCDHeader, data: ArrayBufferLike): HeaderAttributes {
+  const position: number[] = [];
+  const normal: number[] = [];
+  const color: number[] = [];
+  const intensity: number[] = [];
+  const label: number[] = [];
+
+  const sizes = new Uint32Array(data.slice(pcdHeader.headerLen, pcdHeader.headerLen + 8));
+  const compressedSize = sizes[0];
+  const decompressedSize = sizes[1];
+  const decompressed = decompressLZF(
+    new Uint8Array(data, pcdHeader.headerLen + 8, compressedSize),
+    decompressedSize
+  );
+  const dataview = new DataView(decompressed.buffer);
+
+  const offset = pcdHeader.offset;
+
+  for (let i = 0; i < pcdHeader.points; i++) {
+    if (offset.x !== undefined) {
+      position.push(
+        dataview.getFloat32(pcdHeader.points * offset.x + pcdHeader.size[0] * i, LITTLE_ENDIAN)
+      );
+      position.push(
+        dataview.getFloat32(pcdHeader.points * offset.y + pcdHeader.size[1] * i, LITTLE_ENDIAN)
+      );
+      position.push(
+        dataview.getFloat32(pcdHeader.points * offset.z + pcdHeader.size[2] * i, LITTLE_ENDIAN)
+      );
+    }
+
+    if (offset.rgb !== undefined) {
+      color.push(dataview.getUint8(pcdHeader.points * offset.rgb + pcdHeader.size[3] * i + 0));
+      color.push(dataview.getUint8(pcdHeader.points * offset.rgb + pcdHeader.size[3] * i + 1));
+      color.push(dataview.getUint8(pcdHeader.points * offset.rgb + pcdHeader.size[3] * i + 2));
+    }
+
+    if (offset.normal_x !== undefined) {
+      normal.push(
+        dataview.getFloat32(
+          pcdHeader.points * offset.normal_x + pcdHeader.size[4] * i,
+          LITTLE_ENDIAN
+        )
+      );
+      normal.push(
+        dataview.getFloat32(
+          pcdHeader.points * offset.normal_y + pcdHeader.size[5] * i,
+          LITTLE_ENDIAN
+        )
+      );
+      normal.push(
+        dataview.getFloat32(
+          pcdHeader.points * offset.normal_z + pcdHeader.size[6] * i,
+          LITTLE_ENDIAN
+        )
+      );
+    }
+
+    if (offset.intensity !== undefined) {
+      const intensityIndex = pcdHeader.fields.indexOf('intensity');
+      intensity.push(
+        dataview.getFloat32(
+          pcdHeader.points * offset.intensity + pcdHeader.size[intensityIndex] * i,
+          LITTLE_ENDIAN
+        )
+      );
+    }
+
+    if (offset.label !== undefined) {
+      const labelIndex = pcdHeader.fields.indexOf('label');
+      label.push(
+        dataview.getInt32(
+          pcdHeader.points * offset.label + pcdHeader.size[labelIndex] * i,
+          LITTLE_ENDIAN
+        )
+      );
+    }
+  }
+
+  return {
+    position,
+    normal,
+    color,
+    intensity,
+    label
+  };
+}