@loaders.gl/pointclouds 5.0.0-beta.1 → 5.0.0-beta.10

package/dist/las/lib/typescript/parse-las.cjs

@@ -1,7 +1,9 @@
 "use strict";
+var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/las/lib/typescript/parse-las.ts
@@ -30,8 +40,11 @@ __export(parse_las_exports, {
 });
 module.exports = __toCommonJS(parse_las_exports);
 var import_schema_utils = require("@loaders.gl/core/schema-utils");
-var import_loader_utils = require("@loaders.gl/core/loader-utils");
+var arrow = __toESM(require("apache-arrow"), 1);
 var import_get_las_schema = require("../get-las-schema.cjs");
+var import_schema_utils2 = require("@loaders.gl/core/schema-utils");
+var import_loader_utils = require("@loaders.gl/core/loader-utils");
+var import_get_las_schema2 = require("../get-las-schema.cjs");
 var import_las_compatibility = require("../las-compatibility.cjs");
 var import_las_extra_bytes = require("../las-extra-bytes.cjs");
 var DEFAULT_BATCH_SIZE = 1e3 * 100;
@@ -204,6 +217,7 @@ async function* parseLASInBatches(arrayBufferIterator, options = {}) {
   let header = null;
   let sourcePointIndex = 0;
   let totalRead = 0;
+  let lastTable;
   while (true) {
     const next = await inputIterator.next();
     if (next.done) {
@@ -213,6 +227,7 @@ async function* parseLASInBatches(arrayBufferIterator, options = {}) {
     pending = concatenateUint8Arrays(pending, toUint8Array(chunk));
     if (!header) {
       header = tryParseHeader(pending, absoluteOffset);
+      if (header && pending.byteLength < header.pointsOffset) header = null;
       if (header) {
         header.totalToRead = header.pointsCount;
       }
@@ -221,7 +236,61 @@ async function* parseLASInBatches(arrayBufferIterator, options = {}) {
       continue;
     }
     if (header.isCompressed) {
-      yield* parseLAZInBatches(pending, inputIterator, header, options);
+      const extendedOffset = header.metadata?.evlrOffset;
+      let inputOffset = pending.byteLength;
+      const extendedChunks = [];
+      if (extendedOffset && extendedOffset < inputOffset) {
+        extendedChunks.push(pending.subarray(extendedOffset));
+      }
+      const trackedIterator = {
+        async next() {
+          const next2 = await inputIterator.next();
+          if (!next2.done && extendedOffset) {
+            const bytes = toUint8Array(next2.value);
+            const start = Math.max(extendedOffset - inputOffset, 0);
+            if (start < bytes.byteLength) extendedChunks.push(bytes.slice(start));
+            inputOffset += bytes.byteLength;
+          }
+          return next2;
+        }
+      };
+      if (header.pointsCount === 0) {
+        const { laszip } = await readLASZipVLRFromInput(pending, trackedIterator, header);
+        validateTypeScriptLAZSupport(header, laszip);
+        lastTable = makeEmptyLASBatch(header, options);
+      } else {
+        for await (const table of parseLAZInBatches(pending, trackedIterator, header, options)) {
+          lastTable = (0, import_schema_utils.updateArrowTableMetadata)(
+            table,
+            (0, import_get_las_schema.makeMetadataFromLasHeader)({
+              ...header,
+              totalRead: table.loaderData.totalRead,
+              metadata: table.loaderData.metadata || header.metadata
+            })
+          );
+          yield lastTable;
+        }
+      }
+      if (extendedOffset && lastTable && header.metadata?.evlrCount && (lastTable.loaderData.metadata?.evlrs.length ?? 0) < header.metadata.evlrCount) {
+        while (!(await trackedIterator.next()).done) {
+        }
+        const suffix = extendedChunks.reduce(
+          concatenateUint8Arrays,
+          new Uint8Array(0)
+        );
+        const update2 = makeLASMetadataUpdate(
+          lastTable,
+          header,
+          suffix,
+          extendedOffset,
+          lastTable.loaderData.totalRead
+        );
+        if (update2) {
+          yield update2;
+          return;
+        }
+      }
+      if (header.pointsCount === 0 && lastTable) yield lastTable;
       return;
     }
     for (const batch of readAvailablePointBatches(
@@ -236,7 +305,11 @@ async function* parseLASInBatches(arrayBufferIterator, options = {}) {
       totalRead = batch.totalRead;
       pending = batch.pending;
       absoluteOffset = batch.absoluteOffset;
-      yield parseLASArrowTableBatch(batch.arrayBuffer, batch.header, options);
+      lastTable = (0, import_schema_utils.updateArrowTableMetadata)(
+        parseLASArrowTableBatch(batch.arrayBuffer, batch.header, options),
+        (0, import_get_las_schema.makeMetadataFromLasHeader)({ ...header, totalRead })
+      );
+      yield lastTable;
     }
   }
   if (!header) {
@@ -245,6 +318,50 @@ async function* parseLASInBatches(arrayBufferIterator, options = {}) {
   if (totalRead < header.totalToRead) {
     throw new Error("LASLoader: truncated LAS point data");
   }
+  if (!lastTable) {
+    lastTable = makeEmptyLASBatch(header, options);
+    yield lastTable;
+  }
+  const update = makeLASMetadataUpdate(lastTable, header, pending, absoluteOffset, totalRead);
+  if (update) yield update;
+}
+function makeEmptyLASBatch(header, options) {
+  const table = parseLASArrowTableBatch(new ArrayBuffer(0), { ...header, pointsOffset: 0 }, options);
+  return (0, import_schema_utils.updateArrowTableMetadata)(
+    { ...table, loaderData: header },
+    (0, import_get_las_schema.makeMetadataFromLasHeader)(header)
+  );
+}
+function makeLASMetadataUpdate(lastTable, header, pending, absoluteOffset, totalRead) {
+  const metadata = header.metadata;
+  if (metadata?.evlrOffset && metadata.evlrCount) {
+    const relativeOffset = metadata.evlrOffset - absoluteOffset;
+    const records = parseLASExtendedVariableLengthRecords(
+      new DataView(pending.buffer, pending.byteOffset, pending.byteLength),
+      relativeOffset,
+      metadata.evlrCount
+    ).map((record) => ({ ...record, offset: record.offset + absoluteOffset }));
+    const finalMetadata = {
+      ...metadata,
+      evlrs: records,
+      extraBytes: [...metadata.extraBytes],
+      waveformPacketDescriptors: [...metadata.waveformPacketDescriptors]
+    };
+    for (const record of records)
+      parseTypedLASMetadataRecord(
+        { ...record, data: record.data || new Uint8Array() },
+        finalMetadata
+      );
+    resolveLASGeoTIFFKeyDirectory(finalMetadata);
+    const finalHeader = { ...header, totalRead, metadata: finalMetadata };
+    if (records.length > 0) {
+      const data = new arrow.Table([(0, import_schema_utils.makeArrowMetadataBatch)(lastTable.data.schema)]);
+      return (0, import_schema_utils.updateArrowTableMetadata)(
+        { ...lastTable, data, loaderData: finalHeader },
+        (0, import_get_las_schema.makeMetadataFromLasHeader)(finalHeader)
+      );
+    }
+  }
 }
 function* parseLASChunkedIterator(arrayBuffer, batchSize = DEFAULT_BATCH_SIZE, options = {}) {
   const header = parseLASHeader(arrayBuffer);
@@ -696,7 +813,7 @@ function* parseLAZChunkedIterator(arrayBuffer, header, batchSize, parsedLASZipVL
   if (header.pointsFormatId <= 5 || laszip.variableChunks) {
     yield* decodeLAZFileFromCompleteBytes(new Uint8Array(arrayBuffer), header, laszip, {
       ...options,
-      batchSize
+      core: { ...options.core, batchSize }
     });
     return;
   }
@@ -720,7 +837,11 @@ function* parseLAZChunkedIterator(arrayBuffer, header, batchSize, parsedLASZipVL
 }
 function* decodeLAZFileFromCompleteBytes(bytes, header, laszip, options = {}) {
   const batchSize = getBatchSize(options);
-  const outputHeader = { ...header, totalToRead: header.pointsCount };
+  const outputHeader = {
+    ...header,
+    metadata: parseLASMetadata(bytes, header),
+    totalToRead: header.pointsCount
+  };
   const state = createRawPointBatchState(
     batchSize,
     header.pointsStructSize,
@@ -852,7 +973,8 @@ function hasCompleteLASMetadata(arrayBuffer, header) {
   return arrayBuffer.byteLength >= Math.max(vlrEnd, header.pointsOffset);
 }
 function parseLASMetadata(arrayBuffer, header) {
-  const dataView = new DataView(arrayBuffer);
+  const dataView = makeDataView(arrayBuffer);
+  const bytes = new Uint8Array(dataView.buffer, dataView.byteOffset, dataView.byteLength);
   const version = header.versionAsString || "1.0";
   const versionParts = version.split(".").map(Number);
   const isAtLeast14 = versionParts[0] > 1 || versionParts[0] === 1 && versionParts[1] >= 4;
@@ -864,9 +986,9 @@ function parseLASMetadata(arrayBuffer, header) {
     fileSourceId: dataView.getUint16(4, true),
     globalEncoding: dataView.getUint16(6, true),
     waveformDataOffset: versionParts[0] > 1 || versionParts[0] === 1 && versionParts[1] >= 3 ? dataView.getBigUint64(227, true) : void 0,
-    projectId: formatLASProjectId(new Uint8Array(arrayBuffer, 8, 16)),
-    systemIdentifier: readLASString(new Uint8Array(arrayBuffer), 26, 32),
-    generatingSoftware: readLASString(new Uint8Array(arrayBuffer), 58, 32),
+    projectId: formatLASProjectId(bytes.subarray(8, 24)),
+    systemIdentifier: readLASString(bytes, 26, 32),
+    generatingSoftware: readLASString(bytes, 58, 32),
     creationDayOfYear: dataView.getUint16(90, true),
     creationYear: dataView.getUint16(92, true),
     headerSize: header.headerSize,
@@ -918,7 +1040,7 @@ function parseLASVariableLengthRecords(dataView, header) {
   return records;
 }
 function parseLASExtendedVariableLengthRecords(dataView, evlrOffset, evlrCount) {
-  if (!evlrOffset || !evlrCount || evlrOffset >= dataView.byteLength) {
+  if (evlrOffset < 0 || !evlrCount || evlrOffset >= dataView.byteLength) {
     return [];
   }
   const bytes = new Uint8Array(dataView.buffer, dataView.byteOffset, dataView.byteLength);
@@ -1239,9 +1361,9 @@ function makeLASArrowTableFromAttributes(lasHeader, positions, colors, intensiti
       numericAttributes[name] = { ...attribute, value: attribute.value };
     }
   }
-  const schema = (0, import_get_las_schema.getLASSchema)(lasHeader, numericAttributes);
+  const schema = (0, import_get_las_schema2.getLASSchema)(lasHeader, numericAttributes);
   return {
-    ...(0, import_schema_utils.makeMeshArrowTable)(attributes, {
+    ...(0, import_schema_utils2.makeMeshArrowTable)(attributes, {
       schema,
       topology: "point-list",
       mode: 0,
@@ -1474,7 +1596,7 @@ function detectTwoByteColors(dataView, lasHeader, sourcePointIndex, batchSize, c
   return false;
 }
 function getBatchSize(options) {
-  const batchSize = options.batchSize ?? options.core?.batchSize;
+  const batchSize = options.core?.batchSize;
   return typeof batchSize === "number" ? batchSize : DEFAULT_BATCH_SIZE;
 }
 function parseLASZipVLR(bytes, header) {