@loaders.gl/scan 5.0.0-alpha.3 → 5.0.0-alpha.4

package/dist/dist.min.js

@@ -17574,29 +17574,29 @@ return true;`);
   }
 
   // src/scan-engine.ts
-  var scanBackendLoaders = /* @__PURE__ */ new Map();
+  var scanBackends = /* @__PURE__ */ new Map();
   var arrowScanBackend = Object.freeze({
     name: "arrow",
     query: queryArrowTable,
-    queryAsync: async (sourceTable, options) => queryArrowTable(sourceTable, options),
+    queryAsync: (sourceTable, options) => Promise.resolve().then(queryArrowTable.bind(null, sourceTable, options)),
     explain: explainArrowTableQuery
   });
-  scanBackendLoaders.set("arrow", () => arrowScanBackend);
+  scanBackends.set("arrow", arrowScanBackend);
   function registerScanBackend(name, loader) {
     if (!name || !loader) {
       throw new Error("A scan backend name and loader are required");
     }
-    scanBackendLoaders.set(name, loader);
+    scanBackends.set(name, loader);
   }
   async function createScanEngine(options = {}) {
     const backendName = options.backend || "arrow";
-    const loader = scanBackendLoaders.get(backendName);
-    if (!loader) {
+    const registeredBackend = scanBackends.get(backendName);
+    if (!registeredBackend) {
       throw new Error(
         `Scan backend "${backendName}" is not registered. The built-in backend is "arrow".`
       );
     }
-    const backend = await loader();
+    const backend = typeof registeredBackend === "function" ? await registeredBackend() : registeredBackend;
     if (backend.name !== backendName) {
       throw new Error(
         `Scan backend loader returned "${backend.name}" while "${backendName}" was requested.`

package/dist/hudi.cjs

@@ -20676,6 +20676,201 @@ var init_decoders = __esm({
   }
 });
 
+// ../parquet/src/parquetjs/parser/encoded-pages.ts
+async function scanParquetEncodedPages(buffer, context) {
+  const pages = [];
+  let offset = 0;
+  let dataPageOrdinal = 0;
+  while (offset < buffer.byteLength) {
+    const { pageHeader, length: headerByteLength } = decodePageHeader(buffer, offset);
+    const bodyOffset = offset + headerByteLength;
+    const bodyEnd = bodyOffset + pageHeader.compressed_page_size;
+    if (pageHeader.compressed_page_size < 0 || pageHeader.uncompressed_page_size < 0 || bodyEnd > buffer.byteLength) {
+      throw new Error("Parquet page extends beyond the available column-chunk buffer");
+    }
+    const compressedBody = buffer.subarray(bodyOffset, bodyEnd);
+    verifyEncodedPageChecksum(compressedBody, pageHeader, context.verifyPageChecksums);
+    const pageType = getThriftEnum(PageType, pageHeader.type);
+    if (pageType === "DICTIONARY_PAGE") {
+      pages.push(await createDictionaryPage(compressedBody, pageHeader, context));
+    } else if (pageType === "DATA_PAGE") {
+      pages.push(await createDataPageV1(compressedBody, pageHeader, context, dataPageOrdinal++));
+    } else if (pageType === "DATA_PAGE_V2") {
+      pages.push(await createDataPageV2(compressedBody, pageHeader, context, dataPageOrdinal++));
+    } else {
+      throw new Error(`Unsupported Parquet page type ${pageType}`);
+    }
+    offset = bodyEnd;
+  }
+  return pages;
+}
+async function createDictionaryPage(compressedBody, pageHeader, context) {
+  const dictionaryHeader = pageHeader.dictionary_page_header;
+  if (!dictionaryHeader) {
+    throw new Error("Parquet dictionary page is missing its dictionary header");
+  }
+  const preserveCompression = shouldPreserveCompression(context);
+  const data = preserveCompression ? compressedBody : await decompressPageBody(compressedBody, pageHeader.uncompressed_page_size, context);
+  return Object.freeze({
+    type: "dictionary",
+    pageOrdinal: -1,
+    encoding: getThriftEnum(Encoding, dictionaryHeader.encoding),
+    compression: context.compression,
+    compressionState: preserveCompression ? "compressed" : "decompressed",
+    valueCount: dictionaryHeader.num_values,
+    nonNullValueCount: dictionaryHeader.num_values,
+    data,
+    values: preserveCompression ? void 0 : createSection(0, data.byteLength),
+    compressedByteLength: pageHeader.compressed_page_size,
+    uncompressedByteLength: pageHeader.uncompressed_page_size
+  });
+}
+async function createDataPageV1(compressedBody, pageHeader, context, pageOrdinal) {
+  const dataHeader = pageHeader.data_page_header;
+  if (!dataHeader) {
+    throw new Error("Parquet V1 data page is missing its data header");
+  }
+  const preserveCompression = shouldPreserveCompression(context);
+  const data = preserveCompression ? compressedBody : await decompressPageBody(compressedBody, pageHeader.uncompressed_page_size, context);
+  let repetitionLevels;
+  let definitionLevels;
+  let values;
+  if (!preserveCompression) {
+    let sectionOffset = 0;
+    const repetitionLevelEncoding = getThriftEnum(Encoding, dataHeader.repetition_level_encoding);
+    repetitionLevels = readV1LevelSection(
+      data,
+      sectionOffset,
+      dataHeader.num_values,
+      context.column.rLevelMax,
+      repetitionLevelEncoding
+    );
+    sectionOffset = repetitionLevels.byteOffset + repetitionLevels.byteLength;
+    const definitionLevelEncoding = getThriftEnum(Encoding, dataHeader.definition_level_encoding);
+    definitionLevels = readV1LevelSection(
+      data,
+      sectionOffset,
+      dataHeader.num_values,
+      context.column.dLevelMax,
+      definitionLevelEncoding
+    );
+    sectionOffset = definitionLevels.byteOffset + definitionLevels.byteLength;
+    values = createSection(sectionOffset, data.byteLength - sectionOffset);
+  }
+  return Object.freeze({
+    type: "data-v1",
+    pageOrdinal,
+    encoding: getThriftEnum(Encoding, dataHeader.encoding),
+    repetitionLevelEncoding: getThriftEnum(Encoding, dataHeader.repetition_level_encoding),
+    definitionLevelEncoding: getThriftEnum(Encoding, dataHeader.definition_level_encoding),
+    compression: context.compression,
+    compressionState: preserveCompression ? "compressed" : "decompressed",
+    valueCount: dataHeader.num_values,
+    data,
+    repetitionLevels,
+    definitionLevels,
+    values,
+    compressedByteLength: pageHeader.compressed_page_size,
+    uncompressedByteLength: pageHeader.uncompressed_page_size
+  });
+}
+async function createDataPageV2(compressedBody, pageHeader, context, pageOrdinal) {
+  const dataHeader = pageHeader.data_page_header_v2;
+  if (!dataHeader) {
+    throw new Error("Parquet V2 data page is missing its data header");
+  }
+  const repetitionByteLength = dataHeader.repetition_levels_byte_length;
+  const definitionByteLength = dataHeader.definition_levels_byte_length;
+  const levelsByteLength = repetitionByteLength + definitionByteLength;
+  if (levelsByteLength > compressedBody.byteLength) {
+    throw new Error("Parquet V2 level sections extend beyond the page body");
+  }
+  const valuesAreCompressed = dataHeader.is_compressed !== false && context.compression !== "UNCOMPRESSED";
+  const preserveCompression = valuesAreCompressed && shouldPreserveCompression(context);
+  let data = compressedBody;
+  if (valuesAreCompressed && !preserveCompression) {
+    const uncompressedValuesByteLength = pageHeader.uncompressed_page_size - levelsByteLength;
+    if (uncompressedValuesByteLength < 0) {
+      throw new Error("Invalid Parquet V2 uncompressed value length");
+    }
+    const values = await context.decompressPage(
+      compressedBody.subarray(levelsByteLength),
+      uncompressedValuesByteLength
+    );
+    data = new Uint8Array(levelsByteLength + values.byteLength);
+    data.set(compressedBody.subarray(0, levelsByteLength));
+    data.set(values, levelsByteLength);
+  }
+  return Object.freeze({
+    type: "data-v2",
+    pageOrdinal,
+    encoding: getThriftEnum(Encoding, dataHeader.encoding),
+    repetitionLevelEncoding: "RLE",
+    definitionLevelEncoding: "RLE",
+    compression: context.compression,
+    compressionState: preserveCompression ? "compressed" : "decompressed",
+    valueCount: dataHeader.num_values,
+    nonNullValueCount: dataHeader.num_values - dataHeader.num_nulls,
+    data,
+    repetitionLevels: createSection(0, repetitionByteLength),
+    definitionLevels: createSection(repetitionByteLength, definitionByteLength),
+    values: createSection(levelsByteLength, data.byteLength - levelsByteLength),
+    compressedByteLength: pageHeader.compressed_page_size,
+    uncompressedByteLength: pageHeader.uncompressed_page_size
+  });
+}
+function shouldPreserveCompression(context) {
+  var _a;
+  return context.compression !== "UNCOMPRESSED" && Boolean((_a = context.preserveCompression) == null ? void 0 : _a.has(context.compression));
+}
+async function decompressPageBody(compressedBody, uncompressedByteLength, context) {
+  if (context.compression === "UNCOMPRESSED") return compressedBody;
+  return await context.decompressPage(compressedBody, uncompressedByteLength);
+}
+function readV1LevelSection(data, offset, valueCount, maxLevel, encoding) {
+  if (maxLevel === 0) return createSection(offset, 0);
+  if (encoding === "RLE") {
+    if (offset + 4 > data.byteLength) {
+      throw new Error("Parquet V1 RLE level stream is missing its length prefix");
+    }
+    const byteLength = readUInt32LE(data, offset);
+    const byteOffset = offset + 4;
+    assertSectionBounds(data, byteOffset, byteLength);
+    return createSection(byteOffset, byteLength);
+  }
+  if (encoding === "BIT_PACKED") {
+    const byteLength = Math.ceil(valueCount / 8) * getBitWidth(maxLevel);
+    assertSectionBounds(data, offset, byteLength);
+    return createSection(offset, byteLength);
+  }
+  throw new Error(`Unsupported Parquet V1 level encoding ${encoding}`);
+}
+function createSection(byteOffset, byteLength) {
+  return Object.freeze({ byteOffset, byteLength });
+}
+function assertSectionBounds(data, byteOffset, byteLength) {
+  if (byteOffset < 0 || byteLength < 0 || byteOffset + byteLength > data.byteLength) {
+    throw new Error("Parquet page section extends beyond the page body");
+  }
+}
+function verifyEncodedPageChecksum(compressedBody, pageHeader, verifyPageChecksums = false) {
+  if (!verifyPageChecksums || pageHeader.crc === void 0) return;
+  const expected = pageHeader.crc >>> 0;
+  const actual = crc32(compressedBody);
+  if (actual !== expected) {
+    throw new Error(`Parquet page checksum mismatch: expected ${expected}, calculated ${actual}`);
+  }
+}
+var init_encoded_pages = __esm({
+  "../parquet/src/parquetjs/parser/encoded-pages.ts"() {
+    "use strict";
+    init_parquet_thrift();
+    init_crc32();
+    init_binary_utils();
+    init_read_utils();
+  }
+});
+
 // ../parquet/src/lib/parquet-encryption.ts
 function readParquetEncryptedModule(buffer, offset = 0) {
   if (offset + 4 > buffer.length) {
@@ -20918,6 +21113,7 @@ var init_parquet_reader = __esm({
     "use strict";
     init_schema();
     init_decoders();
+    init_encoded_pages();
     init_shred();
     init_compression2();
     init_constants();
@@ -21404,6 +21600,50 @@ var init_parquet_reader = __esm({
           columnData: Object.fromEntries(columnEntries)
         };
       }
+      /**
+       * Reads selected column chunks as encoded page descriptors without materializing values.
+       *
+       * Page headers, encryption, checksums, and optional decompression remain reader concerns. Value,
+       * dictionary, and level decoding is deliberately left to the downstream consumer.
+       */
+      async readRowGroupEncodedPages(schema, rowGroup, columnList, preserveCompression, signal, rowGroupOrdinal = 0) {
+        await this.resolveColumnMetadata(rowGroup, rowGroupOrdinal, columnList);
+        const selectedColumnChunks = [];
+        for (let columnOrdinal = 0; columnOrdinal < rowGroup.columns.length; columnOrdinal++) {
+          const columnChunk = rowGroup.columns[columnOrdinal];
+          const path = getColumnChunkPath(columnChunk);
+          if (columnList.length > 0 && (!path || fieldIndexOf(columnList, path) < 0)) continue;
+          const physicalColumnOrdinal = getPhysicalColumnOrdinal(columnChunk, columnOrdinal);
+          const metadata = columnChunk.meta_data || await this.getColumnMetadata(columnChunk, rowGroupOrdinal, physicalColumnOrdinal);
+          if (columnList.length === 0 || fieldIndexOf(columnList, metadata.path_in_schema) >= 0) {
+            selectedColumnChunks.push({
+              columnChunk,
+              metadata,
+              columnOrdinal: physicalColumnOrdinal
+            });
+          }
+        }
+        const columns = [];
+        for (let batchStart = 0; batchStart < selectedColumnChunks.length; batchStart += MAXIMUM_CONCURRENT_COLUMN_READS) {
+          columns.push(
+            ...await Promise.all(
+              selectedColumnChunks.slice(batchStart, batchStart + MAXIMUM_CONCURRENT_COLUMN_READS).map(async ({ columnChunk, metadata, columnOrdinal }) => {
+                const encodedColumn = await this.readColumnChunkEncodedPages(
+                  schema,
+                  columnChunk,
+                  preserveCompression,
+                  signal,
+                  rowGroupOrdinal,
+                  columnOrdinal,
+                  metadata
+                );
+                return encodedColumn;
+              })
+            )
+          );
+        }
+        return Object.freeze(columns);
+      }
       /** Reads independently materializable non-repeated columns for one row range using page indexes. */
       async readRowGroupRange(schema, rowGroup, columnList, rowRange, pageLocations, signal, rowGroupOrdinal = 0) {
         await this.resolveColumnMetadata(rowGroup, rowGroupOrdinal, columnList);
@@ -21596,6 +21836,99 @@ var init_parquet_reader = __esm({
         ) : chunkBuffer.subarray(pagesRelativeOffset, pagesRelativeOffset + pagesSize);
         return await decodeDataPages(pagesBuf, { ...context, dictionary });
       }
+      /** Reads one column chunk as encoded dictionaries, levels, and values for deferred decoding. */
+      async readColumnChunkEncodedPages(schema, columnChunk, preserveCompression, signal, rowGroupOrdinal = 0, columnOrdinal = 0, resolvedMetadata) {
+        if (columnChunk.file_path !== void 0 && columnChunk.file_path !== null) {
+          throw new Error("external references are not supported");
+        }
+        const metadata = resolvedMetadata || columnChunk.meta_data;
+        if (!metadata) throw new Error("Parquet column metadata is missing");
+        const field = schema.findField(metadata.path_in_schema);
+        const physicalType = getThriftEnum(Type, metadata.type);
+        if (physicalType !== field.primitiveType) {
+          throw new Error(`chunk type not matching schema: ${physicalType}`);
+        }
+        const compression = getThriftEnum(CompressionCodec, metadata.codec);
+        const dataPageOffset = Number(metadata.data_page_offset);
+        const dictionaryPageOffset = metadata.dictionary_page_offset;
+        const validDictionaryPageOffset = dictionaryPageOffset !== void 0 && Number(dictionaryPageOffset) > 0 ? Number(dictionaryPageOffset) : void 0;
+        const chunkOffset = Math.min(dataPageOffset, validDictionaryPageOffset ?? dataPageOffset);
+        const chunkEnd = chunkOffset + Number(metadata.total_compressed_size);
+        const chunkByteLength = Math.min(
+          this.file.size - chunkOffset,
+          Math.max(0, chunkEnd - chunkOffset)
+        );
+        const chunkBuffer = await this.readBytes(chunkOffset, chunkByteLength, signal);
+        const dataPageRelativeOffset = dataPageOffset - chunkOffset;
+        const dataPageByteLength = Math.min(
+          chunkBuffer.byteLength - dataPageRelativeOffset,
+          Math.max(0, chunkEnd - dataPageOffset)
+        );
+        const context = {
+          column: field,
+          compression,
+          decompressPage: this.getPageDecompressor(compression),
+          preserveCompression,
+          verifyPageChecksums: this.props.verifyPageChecksums
+        };
+        let dictionary;
+        if (validDictionaryPageOffset !== void 0) {
+          const dictionaryRelativeOffset = validDictionaryPageOffset - chunkOffset;
+          const dictionaryByteLength = Math.min(
+            Math.max(0, dataPageOffset - validDictionaryPageOffset),
+            chunkBuffer.byteLength - dictionaryRelativeOffset,
+            this.props.defaultDictionarySize
+          );
+          const dictionaryBuffer = chunkBuffer.subarray(
+            dictionaryRelativeOffset,
+            dictionaryRelativeOffset + dictionaryByteLength
+          );
+          const plaintextDictionaryBuffer = columnChunk.crypto_metadata ? await this.decryptColumnPages(
+            dictionaryBuffer,
+            {
+              type: physicalType,
+              rLevelMax: field.rLevelMax,
+              dLevelMax: field.dLevelMax,
+              compression,
+              column: field
+            },
+            rowGroupOrdinal,
+            columnOrdinal,
+            this.getColumnKeyMetadata(columnChunk),
+            true
+          ) : dictionaryBuffer;
+          dictionary = (await scanParquetEncodedPages(plaintextDictionaryBuffer, context))[0];
+        }
+        const dataPageBuffer = chunkBuffer.subarray(
+          dataPageRelativeOffset,
+          dataPageRelativeOffset + dataPageByteLength
+        );
+        const plaintextDataPageBuffer = columnChunk.crypto_metadata ? await this.decryptColumnPages(
+          dataPageBuffer,
+          {
+            type: physicalType,
+            rLevelMax: field.rLevelMax,
+            dLevelMax: field.dLevelMax,
+            compression,
+            column: field
+          },
+          rowGroupOrdinal,
+          columnOrdinal,
+          this.getColumnKeyMetadata(columnChunk)
+        ) : dataPageBuffer;
+        const pages = await scanParquetEncodedPages(plaintextDataPageBuffer, context);
+        return Object.freeze({
+          path: Object.freeze([...metadata.path_in_schema]),
+          physicalType,
+          typeLength: field.typeLength,
+          maxRepetitionLevel: field.rLevelMax,
+          maxDefinitionLevel: field.dLevelMax,
+          compression,
+          valueCount: Number(metadata.num_values),
+          dictionary,
+          pages: Object.freeze(pages)
+        });
+      }
       /** Decrypts the length-prefixed page header and data modules in one column chunk. */
       async decryptColumnPages(encryptedPages, context, rowGroupOrdinal, columnOrdinal, keyMetadata, includesDictionary = false, dataPageOrdinalStart = 0) {
         if (!this.encryptionContext || !this.props.keyRetriever) {
@@ -22419,6 +22752,7 @@ var init_parquet_source_loader = __esm({
     init_src5();
     init_src2();
     init_src3();
+    init_src4();
     init_get_parquet_schema();
     init_geoparquet_covering();
     init_parquet_bloom_filter();
@@ -22572,6 +22906,73 @@ var init_parquet_source_loader = __esm({
       scan(options = {}) {
         return this.read(options);
       }
+      /**
+       * Reads candidate encoded pages without decoding values or constructing Arrow arrays.
+       *
+       * Footer statistics and Bloom filters prune complete row groups. Any exact predicate or spatial
+       * filter is returned as residual work, and its hidden columns are included in the page batch.
+       */
+      async *readPages(options = {}) {
+        const readOptions = this.getReadOptions({
+          rowGroups: options.rowGroups,
+          columns: options.columns,
+          rowGroupFilter: options.rowGroupFilter,
+          predicate: options.predicate,
+          bbox: options.bbox,
+          geometryColumn: options.geometryColumn,
+          signal: options.signal
+        });
+        const readContext = createReadAbortContext(readOptions.signal);
+        this.activeReadControllers.add(readContext.abortController);
+        try {
+          const initialization = await this.getInitialization(readContext.abortController.signal);
+          await this.getCompressionInitialization();
+          throwIfAborted2(readContext.abortController.signal);
+          const physicalPlan = await this.planRowGroups(
+            initialization,
+            readOptions,
+            readContext.abortController.signal
+          );
+          const projectedColumns = normalizeColumns(readOptions.columns, initialization.schema);
+          const predicateColumns = physicalPlan.predicate ? getParquetPredicateColumns(physicalPlan.predicate) : [];
+          const geoMetadata = readOptions.bbox ? getGeoMetadata(initialization.schema.metadata) : void 0;
+          const geometryColumn = readOptions.bbox ? readOptions.geometryColumn || (geoMetadata == null ? void 0 : geoMetadata.primary_column) : void 0;
+          const filterColumns = [
+            .../* @__PURE__ */ new Set([...predicateColumns, ...geometryColumn ? [geometryColumn] : []])
+          ];
+          const decodedColumns = projectedColumns.length === 0 ? [] : [.../* @__PURE__ */ new Set([...projectedColumns, ...filterColumns])];
+          const columnList = decodedColumns.map((column) => [column]);
+          const preserveCompression = new Set(options.preserveCompression ?? []);
+          const residualFilter = readOptions.predicate || readOptions.bbox ? Object.freeze({
+            predicate: readOptions.predicate,
+            bbox: readOptions.bbox,
+            geometryColumn
+          }) : void 0;
+          for (const rowGroupIndex of physicalPlan.rowGroupIndices) {
+            throwIfAborted2(readContext.abortController.signal);
+            const columns = await initialization.reader.readRowGroupEncodedPages(
+              initialization.parquetSchema,
+              initialization.fileMetadata.row_groups[rowGroupIndex],
+              columnList,
+              preserveCompression,
+              readContext.abortController.signal,
+              rowGroupIndex
+            );
+            yield Object.freeze({
+              shape: "parquet-encoded-pages",
+              rowGroup: initialization.metadata.rowGroups[rowGroupIndex],
+              projectedColumns: Object.freeze([...projectedColumns]),
+              filterColumns: Object.freeze(filterColumns),
+              columns,
+              residualFilter
+            });
+          }
+        } finally {
+          readContext.abortController.abort();
+          readContext.removeSignalListener();
+          this.activeReadControllers.delete(readContext.abortController);
+        }
+      }
       /** Returns a copy of cumulative transport, decode, conversion, and pruning telemetry. */
       getTelemetry() {
         return Object.freeze({ ...this.telemetry });
@@ -24021,7 +24422,7 @@ var HudiTableSource = class extends DataSource {
   }
   async loadDescriptor(signal) {
     var _a;
-    const text = typeof this.data === "string" ? await (await this.fetch(this.data, { headers: (_a = this.options.hudi) == null ? void 0 : _a.headers, signal })).text() : await this.data.text();
+    const text = typeof this.data === "string" ? await (await this.fetch(this.url, { headers: (_a = this.options.hudi) == null ? void 0 : _a.headers, signal })).text() : await this.data.text();
     let descriptor;
     try {
       descriptor = JSON.parse(text);
@@ -24032,6 +24433,24 @@ var HudiTableSource = class extends DataSource {
     if (!Array.isArray(files)) {
       throw new Error("Hudi snapshot descriptor must contain a files array");
     }
+    for (const [index, file] of files.entries()) {
+      if (!file || typeof file !== "object" || Array.isArray(file)) {
+        throw new Error(`Hudi snapshot file at index ${index} must be an object`);
+      }
+      const value2 = file;
+      if (typeof value2.path !== "string" || !value2.path) {
+        throw new Error(`Hudi snapshot file at index ${index} must contain a non-empty string path`);
+      }
+      if (value2.size !== void 0 && (typeof value2.size !== "number" || !Number.isFinite(value2.size))) {
+        throw new Error(`Hudi snapshot file at index ${index} size must be a finite number`);
+      }
+      if (value2.numRecords !== void 0 && (typeof value2.numRecords !== "number" || !Number.isFinite(value2.numRecords))) {
+        throw new Error(`Hudi snapshot file at index ${index} numRecords must be a finite number`);
+      }
+      if (value2.partitionValues !== void 0 && (!value2.partitionValues || typeof value2.partitionValues !== "object" || Array.isArray(value2.partitionValues))) {
+        throw new Error(`Hudi snapshot file at index ${index} partitionValues must be an object`);
+      }
+    }
     const value = descriptor;
     if (value.tableType && value.tableType !== "COPY_ON_WRITE" && value.tableType !== "MERGE_ON_READ") {
       throw new Error(`Unsupported Hudi table type: ${String(value.tableType)}`);