@loaders.gl/pointclouds 0.0.0-stage → 5.0.0-beta.2

package/dist/ply/lib/parse-ply-in-batches.cjs

@@ -0,0 +1,607 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/ply/lib/parse-ply-in-batches.ts
+var parse_ply_in_batches_exports = {};
+__export(parse_ply_in_batches_exports, {
+  parsePLYInBatches: () => parsePLYInBatches
+});
+module.exports = __toCommonJS(parse_ply_in_batches_exports);
+var import_loader_utils = require("@loaders.gl/core/loader-utils");
+var import_normalize_ply = __toESM(require("./normalize-ply.cjs"), 1);
+var import_parse_ply = require("./parse-ply.cjs");
+var import_parse_ply_arrow = require("./parse-ply-arrow.cjs");
+var DEFAULT_BINARY_BATCH_SIZE = 65536;
+var PLY_END_HEADER = "end_header";
+async function* parsePLYInBatches(iterator, options) {
+  if (options?.shape === "arrow-table") {
+    yield* parseBinaryPLYToArrowInBatches(iterator, options);
+    return;
+  }
+  const lineIterator = (0, import_loader_utils.makeLineIterator)((0, import_loader_utils.makeTextDecoderIterator)((0, import_loader_utils.toArrayBufferIterator)(iterator)));
+  const header = await parseASCIIHeader(lineIterator, options);
+  let attributes;
+  switch (header.format) {
+    case "ascii":
+      attributes = await parseASCII(lineIterator, header);
+      break;
+    default:
+      throw new Error("Binary PLY can not yet be parsed in streaming mode");
+  }
+  yield (0, import_normalize_ply.default)(header, attributes, options);
+}
+async function* parseBinaryPLYToArrowInBatches(iterator, options) {
+  const byteIterator = (0, import_loader_utils.toArrayBufferIterator)(iterator)[Symbol.asyncIterator]();
+  const { header, initialBodyBytes } = await readPLYHeader(byteIterator, options);
+  let tables;
+  if (header.format === "ascii") {
+    tables = parseASCIIPLYToArrowInBatches(byteIterator, header, initialBodyBytes, options);
+  } else {
+    const vertexElement = getVertexElement(header, options);
+    tables = vertexElement.properties.some((property) => property.type === "list") ? parseBinaryVariableWidthVertexPLYToArrowInBatches(
+      byteIterator,
+      header,
+      initialBodyBytes,
+      options
+    ) : parseBinaryFixedWidthVertexPLYToArrowInBatches(
+      byteIterator,
+      header,
+      initialBodyBytes,
+      options
+    );
+  }
+  for await (const table of tables) {
+    yield { ...table, loaderData: header };
+  }
+}
+async function* parseBinaryFixedWidthVertexPLYToArrowInBatches(byteIterator, header, initialBodyBytes, options) {
+  const vertexElement = getFixedWidthVertexElement(header, options);
+  const batchSize = getBatchSize(options);
+  const vertexStride = getPLYElementSize(vertexElement);
+  const parsePlan = (0, import_parse_ply.getPLYBinaryPointCloudParsePlan)(header, options);
+  let remainingVertices = vertexElement.count;
+  let pendingBytes = initialBodyBytes;
+  while (true) {
+    while (remainingVertices > 0 && Math.floor(pendingBytes.length / vertexStride) >= batchSize) {
+      const batchVertexCount = Math.min(batchSize, remainingVertices);
+      const batch = makeBinaryPLYArrowBatch(
+        header,
+        pendingBytes,
+        batchVertexCount,
+        vertexStride,
+        options,
+        parsePlan
+      );
+      pendingBytes = pendingBytes.subarray(batchVertexCount * vertexStride);
+      remainingVertices -= batchVertexCount;
+      yield batch;
+    }
+    if (options?.pointCloud && remainingVertices <= 0) {
+      return;
+    }
+    const { value: chunk, done } = await byteIterator.next();
+    if (done) {
+      break;
+    }
+    pendingBytes = concatenateBytes([pendingBytes, getUint8Array(chunk)]);
+  }
+  if (remainingVertices > 0) {
+    const batchVertexCount = Math.min(
+      remainingVertices,
+      Math.floor(pendingBytes.length / vertexStride)
+    );
+    if (batchVertexCount > 0) {
+      yield makeBinaryPLYArrowBatch(
+        header,
+        pendingBytes,
+        batchVertexCount,
+        vertexStride,
+        options,
+        parsePlan
+      );
+    }
+  }
+}
+async function* parseASCIIPLYToArrowInBatches(byteIterator, header, initialBodyBytes, options) {
+  const vertexElement = getVertexElement(header, options);
+  const batchSize = getBatchSize(options);
+  const lineIterator = (0, import_loader_utils.makeLineIterator)(
+    (0, import_loader_utils.makeTextDecoderIterator)(getByteIteratorWithInitialBytes(byteIterator, initialBodyBytes))
+  );
+  let batchLines = [];
+  let remainingVertices = vertexElement.count;
+  for await (const sourceLine of lineIterator) {
+    const line = sourceLine.trim();
+    if (line === "") {
+      continue;
+    }
+    batchLines.push(line);
+    remainingVertices--;
+    if (batchLines.length >= batchSize) {
+      yield makeASCIIPLYArrowBatch(header, batchLines, options);
+      batchLines = [];
+    }
+    if (remainingVertices <= 0) {
+      break;
+    }
+  }
+  if (batchLines.length > 0) {
+    yield makeASCIIPLYArrowBatch(header, batchLines, options);
+  }
+}
+async function* parseBinaryVariableWidthVertexPLYToArrowInBatches(byteIterator, header, initialBodyBytes, options) {
+  const vertexElement = getVertexElement(header, options);
+  const batchSize = getBatchSize(options);
+  const littleEndian = header.format !== "binary_big_endian";
+  let remainingVertices = vertexElement.count;
+  let pendingBytes = initialBodyBytes;
+  let batchBytes = [];
+  let batchByteLength = 0;
+  let batchRowCount = 0;
+  while (remainingVertices > 0) {
+    const rowByteLength = getBinaryElementRowByteLength(pendingBytes, vertexElement, littleEndian);
+    if (rowByteLength === null) {
+      const { value: chunk, done } = await byteIterator.next();
+      if (done) {
+        break;
+      }
+      pendingBytes = concatenateBytes([pendingBytes, getUint8Array(chunk)]);
+      continue;
+    }
+    const rowBytes = pendingBytes.subarray(0, rowByteLength);
+    batchBytes.push(rowBytes);
+    batchByteLength += rowByteLength;
+    batchRowCount++;
+    remainingVertices--;
+    pendingBytes = pendingBytes.subarray(rowByteLength);
+    if (batchRowCount >= batchSize) {
+      yield makeBinaryPLYArrowBatchFromRows(
+        header,
+        batchBytes,
+        batchByteLength,
+        batchRowCount,
+        options
+      );
+      batchBytes = [];
+      batchByteLength = 0;
+      batchRowCount = 0;
+    }
+  }
+  if (batchRowCount > 0) {
+    yield makeBinaryPLYArrowBatchFromRows(
+      header,
+      batchBytes,
+      batchByteLength,
+      batchRowCount,
+      options
+    );
+  }
+}
+async function readPLYHeader(iterator, options) {
+  let headerBytes = new Uint8Array(0);
+  while (true) {
+    const { value, done } = await iterator.next();
+    if (done) {
+      throw new Error("Incomplete PLY header");
+    }
+    headerBytes = concatenateBytes([headerBytes, getUint8Array(value)]);
+    const headerEnd = findHeaderEnd(headerBytes);
+    if (headerEnd === -1) {
+      continue;
+    }
+    const header = (0, import_parse_ply.parsePLYHeader)(getArrayBuffer(headerBytes.subarray(0, headerEnd)), options);
+    return { header, initialBodyBytes: headerBytes.subarray(headerEnd) };
+  }
+}
+function getVertexElement(header, options) {
+  const vertexElement = header.elements[0];
+  if (!options?.pointCloud && header.elements.length !== 1 || vertexElement?.name !== "vertex") {
+    throw new Error("PLY arrow-table batch parsing requires one vertex element");
+  }
+  return vertexElement;
+}
+function getFixedWidthVertexElement(header, options) {
+  const vertexElement = header.elements[0];
+  if (!options?.pointCloud && header.elements.length !== 1 || vertexElement?.name !== "vertex" || vertexElement.properties.some((property) => property.type === "list")) {
+    throw new Error("Binary PLY arrow-table batch parsing requires one scalar vertex element");
+  }
+  return vertexElement;
+}
+function makeBinaryPLYArrowBatch(header, pendingBytes, vertexCount, vertexStride, options, parsePlan) {
+  const vertexElement = header.elements[0];
+  const batchHeader = {
+    ...header,
+    headerLength: 0,
+    elements: [{ ...vertexElement, count: vertexCount }]
+  };
+  const byteLength = vertexCount * vertexStride;
+  const table = (0, import_parse_ply.parsePLYBinaryPointCloudRecordsToArrowTable)(
+    pendingBytes.subarray(0, byteLength),
+    batchHeader,
+    options,
+    parsePlan || void 0
+  );
+  if (!table) {
+    throw new Error("Failed to parse binary PLY Arrow batch");
+  }
+  return table;
+}
+function makeASCIIPLYArrowBatch(header, lines, options) {
+  const batchHeaderText = makePLYHeaderText(header, lines.length, options);
+  const elementTables = (0, import_parse_ply_arrow.parsePLYToElementTables)(`${batchHeaderText}${lines.join("\n")}
+`, options);
+  return (0, import_parse_ply_arrow.convertPLYElementTablesToMeshArrowTable)(elementTables);
+}
+function makeBinaryPLYArrowBatchFromRows(header, rows, byteLength, vertexCount, options) {
+  const batchHeaderText = makePLYHeaderText(header, vertexCount, options);
+  const headerBytes = new TextEncoder().encode(batchHeaderText);
+  const batchBytes = new Uint8Array(headerBytes.length + byteLength);
+  batchBytes.set(headerBytes, 0);
+  let byteOffset = headerBytes.length;
+  for (const row of rows) {
+    batchBytes.set(row, byteOffset);
+    byteOffset += row.length;
+  }
+  const elementTables = (0, import_parse_ply_arrow.parsePLYToElementTables)(getArrayBuffer(batchBytes), options);
+  return (0, import_parse_ply_arrow.convertPLYElementTablesToMeshArrowTable)(elementTables);
+}
+function getBinaryElementRowByteLength(bytes, element, littleEndian) {
+  const dataView = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+  let byteOffset = 0;
+  for (const property of element.properties) {
+    if (property.type === "list") {
+      const countSize = getPLYTypeSize(property.countType);
+      if (byteOffset + countSize > bytes.length) {
+        return null;
+      }
+      const count = readBinaryScalar(dataView, byteOffset, property.countType, littleEndian);
+      byteOffset += countSize;
+      const valuesByteLength = count * getPLYTypeSize(property.itemType);
+      if (byteOffset + valuesByteLength > bytes.length) {
+        return null;
+      }
+      byteOffset += valuesByteLength;
+    } else {
+      const propertySize = getPLYTypeSize(property.type);
+      if (byteOffset + propertySize > bytes.length) {
+        return null;
+      }
+      byteOffset += propertySize;
+    }
+  }
+  return byteOffset;
+}
+function readBinaryScalar(dataView, byteOffset, type, littleEndian) {
+  switch (type) {
+    case "int8":
+    case "char":
+      return dataView.getInt8(byteOffset);
+    case "uint8":
+    case "uchar":
+      return dataView.getUint8(byteOffset);
+    case "int16":
+    case "short":
+      return dataView.getInt16(byteOffset, littleEndian);
+    case "uint16":
+    case "ushort":
+      return dataView.getUint16(byteOffset, littleEndian);
+    case "int32":
+    case "int":
+      return dataView.getInt32(byteOffset, littleEndian);
+    case "uint32":
+    case "uint":
+      return dataView.getUint32(byteOffset, littleEndian);
+    case "float32":
+    case "float":
+      return dataView.getFloat32(byteOffset, littleEndian);
+    case "float64":
+    case "double":
+      return dataView.getFloat64(byteOffset, littleEndian);
+    default:
+      throw new Error(type);
+  }
+}
+function makePLYHeaderText(header, vertexCount, options) {
+  const lines = ["ply", `format ${header.format} ${header.version || "1.0"}`];
+  const elements = options?.pointCloud ? header.elements.slice(0, 1) : header.elements;
+  for (const comment of header.comments) {
+    lines.push(`comment ${comment}`);
+  }
+  for (const element of elements) {
+    lines.push(
+      `element ${element.name} ${element.name === "vertex" ? vertexCount : element.count}`
+    );
+    for (const property of element.properties) {
+      lines.push(
+        property.type === "list" ? `property list ${property.countType} ${property.itemType} ${property.name}` : `property ${property.type} ${property.name}`
+      );
+    }
+  }
+  lines.push("end_header");
+  return `${lines.join("\n")}
+`;
+}
+async function* getByteIteratorWithInitialBytes(iterator, initialBodyBytes) {
+  if (initialBodyBytes.length > 0) {
+    yield getArrayBuffer(initialBodyBytes);
+  }
+  while (true) {
+    const { value, done } = await iterator.next();
+    if (done) {
+      break;
+    }
+    yield getArrayBuffer(getUint8Array(value));
+  }
+}
+function getBatchSize(options) {
+  const batchSize = options?.batchSize;
+  return typeof batchSize === "number" ? batchSize : DEFAULT_BINARY_BATCH_SIZE;
+}
+function getPLYElementSize(element) {
+  return element.properties.reduce((size, property) => size + getPLYTypeSize(property.type), 0);
+}
+function getPLYTypeSize(type) {
+  switch (type) {
+    case "int8":
+    case "uint8":
+    case "char":
+    case "uchar":
+      return 1;
+    case "int16":
+    case "uint16":
+    case "short":
+    case "ushort":
+      return 2;
+    case "int32":
+    case "uint32":
+    case "int":
+    case "uint":
+    case "float32":
+    case "float":
+      return 4;
+    case "float64":
+    case "double":
+      return 8;
+    default:
+      throw new Error(type);
+  }
+}
+function findHeaderEnd(bytes) {
+  const pattern = new TextEncoder().encode(PLY_END_HEADER);
+  const lastPatternStart = bytes.length - pattern.length;
+  for (let byteIndex = 0; byteIndex <= lastPatternStart; byteIndex++) {
+    let matches = true;
+    for (let patternIndex = 0; patternIndex < pattern.length; patternIndex++) {
+      if (bytes[byteIndex + patternIndex] !== pattern[patternIndex]) {
+        matches = false;
+        break;
+      }
+    }
+    if (matches) {
+      let headerEnd = byteIndex + pattern.length;
+      if (headerEnd === bytes.length || bytes[headerEnd] === 13 && headerEnd + 1 === bytes.length) {
+        return -1;
+      }
+      if (bytes[headerEnd] === 13 && bytes[headerEnd + 1] === 10) {
+        headerEnd += 2;
+      } else if (headerEnd < bytes.length && isWhitespaceByte(bytes[headerEnd])) {
+        headerEnd += 1;
+      }
+      return headerEnd;
+    }
+  }
+  return -1;
+}
+function isWhitespaceByte(byte) {
+  return byte === 10 || byte === 13 || byte === 9 || byte === 32;
+}
+function getUint8Array(data) {
+  return ArrayBuffer.isView(data) ? new Uint8Array(data.buffer, data.byteOffset, data.byteLength) : new Uint8Array(data);
+}
+function getArrayBuffer(bytes) {
+  const copy = new Uint8Array(bytes.byteLength);
+  copy.set(bytes);
+  return copy.buffer;
+}
+function concatenateBytes(chunks) {
+  const byteLength = chunks.reduce((length, chunk) => length + chunk.length, 0);
+  const bytes = new Uint8Array(byteLength);
+  let byteOffset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, byteOffset);
+    byteOffset += chunk.length;
+  }
+  return bytes;
+}
+async function parseASCIIHeader(lineIterator, options) {
+  let currentElement;
+  const header = {
+    comments: [],
+    elements: []
+    // headerLength
+  };
+  await (0, import_loader_utils.forEach)(lineIterator, (line) => {
+    line = line.trim();
+    if (line === "end_header") {
+      return true;
+    }
+    if (line === "") {
+      return false;
+    }
+    const lineValues = line.split(/\s+/);
+    const lineType = lineValues.shift();
+    line = lineValues.join(" ");
+    switch (lineType) {
+      case "ply":
+        break;
+      case "format":
+        header.format = lineValues[0];
+        header.version = lineValues[1];
+        break;
+      case "comment":
+        header.comments.push(line);
+        break;
+      case "element":
+        if (currentElement) {
+          header.elements.push(currentElement);
+        }
+        currentElement = {
+          name: lineValues[0],
+          count: parseInt(lineValues[1], 10),
+          properties: []
+        };
+        break;
+      case "property":
+        const property = makePLYElementProperty(lineValues, options.propertyNameMapping);
+        currentElement?.properties.push(property);
+        break;
+      default:
+        console.log("unhandled", lineType, lineValues);
+    }
+    return false;
+  });
+  if (currentElement) {
+    header.elements.push(currentElement);
+  }
+  return header;
+}
+function makePLYElementProperty(propertyValues, propertyNameMapping) {
+  const type = propertyValues[0];
+  switch (type) {
+    case "list":
+      return {
+        type,
+        name: propertyValues[3],
+        countType: propertyValues[1],
+        itemType: propertyValues[2]
+      };
+    default:
+      return {
+        type,
+        name: propertyValues[1]
+      };
+  }
+}
+async function parseASCII(lineIterator, header) {
+  const attributes = {
+    indices: [],
+    vertices: [],
+    normals: [],
+    uvs: [],
+    colors: []
+  };
+  let currentElement = 0;
+  let currentElementCount = 0;
+  for await (let line of lineIterator) {
+    line = line.trim();
+    if (line !== "") {
+      if (currentElementCount >= header.elements[currentElement].count) {
+        currentElement++;
+        currentElementCount = 0;
+      }
+      const element = parsePLYElement(header.elements[currentElement].properties, line);
+      handleElement(attributes, header.elements[currentElement].name, element);
+      currentElementCount++;
+    }
+  }
+  return attributes;
+}
+function parseASCIINumber(n, type) {
+  switch (type) {
+    case "char":
+    case "uchar":
+    case "short":
+    case "ushort":
+    case "int":
+    case "uint":
+    case "int8":
+    case "uint8":
+    case "int16":
+    case "uint16":
+    case "int32":
+    case "uint32":
+      return parseInt(n, 10);
+    case "float":
+    case "double":
+    case "float32":
+    case "float64":
+      return parseFloat(n);
+    default:
+      throw new Error(type);
+  }
+}
+function parsePLYElement(properties, line) {
+  const values = line.split(/\s+/);
+  const element = {};
+  for (let i = 0; i < properties.length; i++) {
+    if (properties[i].type === "list") {
+      const list = [];
+      const n = parseASCIINumber(values.shift(), properties[i].countType);
+      for (let j = 0; j < n; j++) {
+        list.push(parseASCIINumber(values.shift(), properties[i].itemType));
+      }
+      element[properties[i].name] = list;
+    } else {
+      element[properties[i].name] = parseASCIINumber(values.shift(), properties[i].type);
+    }
+  }
+  return element;
+}
+function handleElement(buffer, elementName, element = {}) {
+  switch (elementName) {
+    case "vertex":
+      buffer.vertices.push(element.x, element.y, element.z);
+      if ("nx" in element && "ny" in element && "nz" in element) {
+        buffer.normals.push(element.nx, element.ny, element.nz);
+      }
+      if ("s" in element && "t" in element) {
+        buffer.uvs.push(element.s, element.t);
+      }
+      if ("red" in element && "green" in element && "blue" in element) {
+        buffer.colors.push(element.red, element.green, element.blue);
+      }
+      break;
+    case "face":
+      const vertexIndices = element.vertex_indices || element.vertex_index;
+      if (vertexIndices.length === 3) {
+        buffer.indices.push(vertexIndices[0], vertexIndices[1], vertexIndices[2]);
+      } else if (vertexIndices.length === 4) {
+        buffer.indices.push(vertexIndices[0], vertexIndices[1], vertexIndices[3]);
+        buffer.indices.push(vertexIndices[1], vertexIndices[2], vertexIndices[3]);
+      }
+      break;
+    default:
+      break;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  parsePLYInBatches
+});
+//# sourceMappingURL=parse-ply-in-batches.cjs.map