@loaders.gl/pointclouds 5.0.0-beta.1 → 5.0.0-beta.10

package/dist/las/lib/typescript/parse-las.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-las.d.ts","sourceRoot":"","sources":["../../../../src/las/lib/typescript/parse-las.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,cAAc,EAAiB,MAAM,yBAAyB,CAAC;AAY5E,OAAO,KAAK,EACV,gBAAgB,EAGjB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAC,gBAAgB,EAAC,kCAA+B;AAc7D,OAAO,KAAK,EAGV,SAAS,EAIV,wBAAqB;AAEtB,2FAA2F;AAC3F,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG;IAC3C,qEAAqE;IACrE,MAAM,EAAE,KAAK,CAAC;IACd,oDAAoD;IACpD,UAAU,EAAE,SAAS,CAAC;IACtB,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,WAAW,EAAE,eAAe,CAAC;IAC7B,MAAM,EAAE,SAAS,CAAC;CACnB,CAAC;AAEF,iFAAiF;AACjF,MAAM,MAAM,0BAA0B,GAAG,gBAAgB,GAAG;IAC1D,6DAA6D;IAC7D,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,8DAA8D;IAC9D,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AAoHF,yDAAyD;AACzD,wBAAgB,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,GAAE,gBAAqB,GAAG,aAAa,CA8BhG;AAED,iFAAiF;AACjF,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,WAAW,GAAG,eAAe,EACzC,QAAQ,EAAE,0BAA0B,EACpC,OAAO,EAAE,gBAAgB,GACxB,aAAa,CAoCf;AA4GD,wEAAwE;AACxE,wBAAuB,iBAAiB,CACtC,mBAAmB,EACf,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,EAC/C,OAAO,GAAE,gBAAqB,GAC7B,aAAa,CAAC,aAAa,CAAC,CAoD9B;AAED,6CAA6C;AAC7C,wBAAiB,uBAAuB,CACtC,WAAW,EAAE,WAAW,EACxB,SAAS,GAAE,MAA2B,EACtC,OAAO,GAAE,gBAAqB,GAC7B,QAAQ,CAAC,eAAe,CAAC,CA6B3B;AAED,4FAA4F;AAC5F,wBAAuB,sBAAsB,CAC3C,mBAAmB,EACf,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,EAC/C,OAAO,GAAE,gBAAqB,GAC7B,aAAa,CAAC,eAAe,CAAC,CAkBhC;AA6mBD,yCAAyC;AACzC,wBAAgB,cAAc,CAAC,WAAW,EAAE,eAAe,GAAG,SAAS,CAiGtE;AA+9ED,eAAO,MAAM,qBAAqB,MAAuB,CAAC"}
+{"version":3,"file":"parse-las.d.ts","sourceRoot":"","sources":["../../../../src/las/lib/typescript/parse-las.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAC,cAAc,EAAiB,MAAM,yBAAyB,CAAC;AAY5E,OAAO,KAAK,EACV,gBAAgB,EAGjB,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAC,gBAAgB,EAAC,kCAA+B;AAc7D,OAAO,KAAK,EAGV,SAAS,EAIV,wBAAqB;AAEtB,2FAA2F;AAC3F,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG;IAC3C,qEAAqE;IACrE,MAAM,EAAE,KAAK,CAAC;IACd,oDAAoD;IACpD,UAAU,EAAE,SAAS,CAAC;IACtB,gDAAgD;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,WAAW,EAAE,eAAe,CAAC;IAC7B,MAAM,EAAE,SAAS,CAAC;CACnB,CAAC;AAEF,iFAAiF;AACjF,MAAM,MAAM,0BAA0B,GAAG,gBAAgB,GAAG;IAC1D,6DAA6D;IAC7D,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,8DAA8D;IAC9D,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC,CAAC;AAoHF,yDAAyD;AACzD,wBAAgB,QAAQ,CAAC,WAAW,EAAE,WAAW,EAAE,OAAO,GAAE,gBAAqB,GAAG,aAAa,CA8BhG;AAED,iFAAiF;AACjF,wBAAgB,0BAA0B,CACxC,UAAU,EAAE,WAAW,GAAG,eAAe,EACzC,QAAQ,EAAE,0BAA0B,EACpC,OAAO,EAAE,gBAAgB,GACxB,aAAa,CAoCf;AA4GD,wEAAwE;AACxE,wBAAuB,iBAAiB,CACtC,mBAAmB,EACf,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,EAC/C,OAAO,GAAE,gBAAqB,GAC7B,aAAa,CAAC,aAAa,CAAC,CA6H9B;AAkDD,6CAA6C;AAC7C,wBAAiB,uBAAuB,CACtC,WAAW,EAAE,WAAW,EACxB,SAAS,GAAE,MAA2B,EACtC,OAAO,GAAE,gBAAqB,GAC7B,QAAQ,CAAC,eAAe,CAAC,CA6B3B;AAED,4FAA4F;AAC5F,wBAAuB,sBAAsB,CAC3C,mBAAmB,EACf,aAAa,CAAC,eAAe,GAAG,eAAe,CAAC,GAChD,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAC,EAC/C,OAAO,GAAE,gBAAqB,GAC7B,aAAa,CAAC,eAAe,CAAC,CAkBhC;AAinBD,yCAAyC;AACzC,wBAAgB,cAAc,CAAC,WAAW,EAAE,eAAe,GAAG,SAAS,CAiGtE;AAm+ED,eAAO,MAAM,qBAAqB,MAAuB,CAAC"}

package/dist/las/lib/typescript/parse-las.js

@@ -1,6 +1,9 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
+import { makeArrowMetadataBatch, updateArrowTableMetadata } from '@loaders.gl/core/schema-utils';
+import * as arrow from 'apache-arrow';
+import { makeMetadataFromLasHeader } from "../get-las-schema.js";
 import { makeMeshArrowTable } from '@loaders.gl/core/schema-utils';
 import { BinaryChunkReader, createLAZChunkDecoder, createLAZChunkDecoderCursor, decodeLAZChunk, decodeLAZChunkTable, getLAZChunkDeclaredByteLength, getLAZChunkHeaderByteLength, NeedsMoreData } from '@loaders.gl/core/loader-utils';
 import { getLASSchema } from "../get-las-schema.js";
@@ -141,6 +144,7 @@ export async function* parseLASInBatches(arrayBufferIterator, options = {}) {
     let header = null;
     let sourcePointIndex = 0;
     let totalRead = 0;
+    let lastTable;
     while (true) {
         const next = await inputIterator.next();
         if (next.done) {
@@ -150,6 +154,8 @@ export async function* parseLASInBatches(arrayBufferIterator, options = {}) {
         pending = concatenateUint8Arrays(pending, toUint8Array(chunk));
         if (!header) {
             header = tryParseHeader(pending, absoluteOffset);
+            if (header && pending.byteLength < header.pointsOffset)
+                header = null;
             if (header) {
                 header.totalToRead = header.pointsCount;
             }
@@ -158,7 +164,57 @@ export async function* parseLASInBatches(arrayBufferIterator, options = {}) {
             continue;
         }
         if (header.isCompressed) {
-            yield* parseLAZInBatches(pending, inputIterator, header, options);
+            const extendedOffset = header.metadata?.evlrOffset;
+            let inputOffset = pending.byteLength;
+            const extendedChunks = [];
+            if (extendedOffset && extendedOffset < inputOffset) {
+                extendedChunks.push(pending.subarray(extendedOffset));
+            }
+            const trackedIterator = {
+                async next() {
+                    const next = await inputIterator.next();
+                    if (!next.done && extendedOffset) {
+                        const bytes = toUint8Array(next.value);
+                        const start = Math.max(extendedOffset - inputOffset, 0);
+                        if (start < bytes.byteLength)
+                            extendedChunks.push(bytes.slice(start));
+                        inputOffset += bytes.byteLength;
+                    }
+                    return next;
+                }
+            };
+            if (header.pointsCount === 0) {
+                const { laszip } = await readLASZipVLRFromInput(pending, trackedIterator, header);
+                validateTypeScriptLAZSupport(header, laszip);
+                lastTable = makeEmptyLASBatch(header, options);
+            }
+            else {
+                for await (const table of parseLAZInBatches(pending, trackedIterator, header, options)) {
+                    lastTable = updateArrowTableMetadata(table, makeMetadataFromLasHeader({
+                        ...header,
+                        totalRead: table.loaderData.totalRead,
+                        metadata: table.loaderData.metadata || header.metadata
+                    }));
+                    yield lastTable;
+                }
+            }
+            if (extendedOffset &&
+                lastTable &&
+                header.metadata?.evlrCount &&
+                (lastTable.loaderData.metadata?.evlrs.length ?? 0) < header.metadata.evlrCount) {
+                while (!(await trackedIterator.next()).done) {
+                    // Finish reading trailing records if the decoder stopped at its chunk table.
+                }
+                // The tracked iterator retains only the extended-record suffix.
+                const suffix = extendedChunks.reduce(concatenateUint8Arrays, new Uint8Array(0));
+                const update = makeLASMetadataUpdate(lastTable, header, suffix, extendedOffset, lastTable.loaderData.totalRead);
+                if (update) {
+                    yield update;
+                    return;
+                }
+            }
+            if (header.pointsCount === 0 && lastTable)
+                yield lastTable;
             return;
         }
         for (const batch of readAvailablePointBatches(pending, absoluteOffset, header, sourcePointIndex, totalRead, batchSize)) {
@@ -166,7 +222,8 @@ export async function* parseLASInBatches(arrayBufferIterator, options = {}) {
             totalRead = batch.totalRead;
             pending = batch.pending;
             absoluteOffset = batch.absoluteOffset;
-            yield parseLASArrowTableBatch(batch.arrayBuffer, batch.header, options);
+            lastTable = updateArrowTableMetadata(parseLASArrowTableBatch(batch.arrayBuffer, batch.header, options), makeMetadataFromLasHeader({ ...header, totalRead }));
+            yield lastTable;
         }
     }
     if (!header) {
@@ -175,6 +232,40 @@ export async function* parseLASInBatches(arrayBufferIterator, options = {}) {
     if (totalRead < header.totalToRead) {
         throw new Error('LASLoader: truncated LAS point data');
     }
+    if (!lastTable) {
+        lastTable = makeEmptyLASBatch(header, options);
+        yield lastTable;
+    }
+    const update = makeLASMetadataUpdate(lastTable, header, pending, absoluteOffset, totalRead);
+    if (update)
+        yield update;
+}
+/** Creates the declared point schema and original header metadata for a zero-point stream. */
+function makeEmptyLASBatch(header, options) {
+    const table = parseLASArrowTableBatch(new ArrayBuffer(0), { ...header, pointsOffset: 0 }, options);
+    return updateArrowTableMetadata({ ...table, loaderData: header }, makeMetadataFromLasHeader(header));
+}
+/** Builds a schema-only batch carrying late extended records and their absolute offsets. */
+function makeLASMetadataUpdate(lastTable, header, pending, absoluteOffset, totalRead) {
+    const metadata = header.metadata;
+    if (metadata?.evlrOffset && metadata.evlrCount) {
+        const relativeOffset = metadata.evlrOffset - absoluteOffset;
+        const records = parseLASExtendedVariableLengthRecords(new DataView(pending.buffer, pending.byteOffset, pending.byteLength), relativeOffset, metadata.evlrCount).map(record => ({ ...record, offset: record.offset + absoluteOffset }));
+        const finalMetadata = {
+            ...metadata,
+            evlrs: records,
+            extraBytes: [...metadata.extraBytes],
+            waveformPacketDescriptors: [...metadata.waveformPacketDescriptors]
+        };
+        for (const record of records)
+            parseTypedLASMetadataRecord({ ...record, data: record.data || new Uint8Array() }, finalMetadata);
+        resolveLASGeoTIFFKeyDirectory(finalMetadata);
+        const finalHeader = { ...header, totalRead, metadata: finalMetadata };
+        if (records.length > 0) {
+            const data = new arrow.Table([makeArrowMetadataBatch(lastTable.data.schema)]);
+            return updateArrowTableMetadata({ ...lastTable, data, loaderData: finalHeader }, makeMetadataFromLasHeader(finalHeader));
+        }
+    }
 }
 /** Decode LAS data into raw point chunks. */
 export function* parseLASChunkedIterator(arrayBuffer, batchSize = DEFAULT_BATCH_SIZE, options = {}) {
@@ -535,7 +626,7 @@ function* parseLAZChunkedIterator(arrayBuffer, header, batchSize, parsedLASZipVL
     if (header.pointsFormatId <= 5 || laszip.variableChunks) {
         yield* decodeLAZFileFromCompleteBytes(new Uint8Array(arrayBuffer), header, laszip, {
             ...options,
-            batchSize
+            core: { ...options.core, batchSize }
         });
         return;
     }
@@ -559,7 +650,11 @@ function* parseLAZChunkedIterator(arrayBuffer, header, batchSize, parsedLASZipVL
 }
 function* decodeLAZFileFromCompleteBytes(bytes, header, laszip, options = {}) {
     const batchSize = getBatchSize(options);
-    const outputHeader = { ...header, totalToRead: header.pointsCount };
+    const outputHeader = {
+        ...header,
+        metadata: parseLASMetadata(bytes, header),
+        totalToRead: header.pointsCount
+    };
     const state = createRawPointBatchState(batchSize, header.pointsStructSize, getLAZStreamingDecodeStats(options));
     const chunkTable = readLAZFileChunks(bytes, header, laszip, options);
     let byteOffset = header.pointsOffset + (laszip.compressor === 1 ? 0 : LAZ_CHUNK_TABLE_POINTER_LENGTH);
@@ -691,7 +786,8 @@ function hasCompleteLASMetadata(arrayBuffer, header) {
     return arrayBuffer.byteLength >= Math.max(vlrEnd, header.pointsOffset);
 }
 function parseLASMetadata(arrayBuffer, header) {
-    const dataView = new DataView(arrayBuffer);
+    const dataView = makeDataView(arrayBuffer);
+    const bytes = new Uint8Array(dataView.buffer, dataView.byteOffset, dataView.byteLength);
     const version = header.versionAsString || '1.0';
     const versionParts = version.split('.').map(Number);
     const isAtLeast14 = versionParts[0] > 1 || (versionParts[0] === 1 && versionParts[1] >= 4);
@@ -705,9 +801,9 @@ function parseLASMetadata(arrayBuffer, header) {
         waveformDataOffset: versionParts[0] > 1 || (versionParts[0] === 1 && versionParts[1] >= 3)
             ? dataView.getBigUint64(227, true)
             : undefined,
-        projectId: formatLASProjectId(new Uint8Array(arrayBuffer, 8, 16)),
-        systemIdentifier: readLASString(new Uint8Array(arrayBuffer), 26, 32),
-        generatingSoftware: readLASString(new Uint8Array(arrayBuffer), 58, 32),
+        projectId: formatLASProjectId(bytes.subarray(8, 24)),
+        systemIdentifier: readLASString(bytes, 26, 32),
+        generatingSoftware: readLASString(bytes, 58, 32),
         creationDayOfYear: dataView.getUint16(90, true),
         creationYear: dataView.getUint16(92, true),
         headerSize: header.headerSize,
@@ -757,7 +853,7 @@ function parseLASVariableLengthRecords(dataView, header) {
     return records;
 }
 function parseLASExtendedVariableLengthRecords(dataView, evlrOffset, evlrCount) {
-    if (!evlrOffset || !evlrCount || evlrOffset >= dataView.byteLength) {
+    if (evlrOffset < 0 || !evlrCount || evlrOffset >= dataView.byteLength) {
         return [];
     }
     const bytes = new Uint8Array(dataView.buffer, dataView.byteOffset, dataView.byteLength);
@@ -1305,7 +1401,7 @@ function detectTwoByteColors(dataView, lasHeader, sourcePointIndex, batchSize, c
     return false;
 }
 function getBatchSize(options) {
-    const batchSize = options.batchSize ?? options.core?.batchSize;
+    const batchSize = options.core?.batchSize;
     return typeof batchSize === 'number' ? batchSize : DEFAULT_BATCH_SIZE;
 }
 function parseLASZipVLR(bytes, header) {