@loaders.gl/scan 5.0.0-alpha.3 → 5.0.0-alpha.5

package/dist/hudi.cjs

@@ -3135,6 +3135,12 @@ function serializeArrowType(arrowType) {
         type: "list",
         children: [serializeArrowField(listField)]
       };
+    case arrow.LargeList:
+      const largeListType = arrowType;
+      return {
+        type: "large-list",
+        children: [serializeArrowField(largeListType.valueField)]
+      };
     case arrow.FixedSizeList:
       const fixedSizeList = arrowType;
       return {
@@ -3148,6 +3154,15 @@ function serializeArrowType(arrowType) {
         type: "struct",
         children: structType.children.map((arrowField) => serializeArrowField(arrowField))
       };
+    case arrow.SparseUnion:
+    case arrow.DenseUnion:
+      const unionType = arrowType;
+      return {
+        type: unionType instanceof arrow.DenseUnion ? "dense-union" : "sparse-union",
+        typeIds: unionType.typeIds,
+        children: unionType.children.map((arrowField) => serializeArrowField(arrowField)),
+        typeIdToChildIndex: { ...unionType.typeIdToChildIndex }
+      };
     case arrow.Dictionary:
       const dictionaryType = arrowType;
       return {
@@ -3176,6 +3191,10 @@ function deserializeArrowType(dataType, options) {
         const field = deserializeArrowField(dataType.children[0], options);
         return new arrow.List(field);
       }
+      case "large-list": {
+        const field = deserializeArrowField(dataType.children[0], options);
+        return new arrow.LargeList(field);
+      }
       case "fixed-size-list": {
         const child = deserializeArrowField(dataType.children[0], options);
         return new arrow.FixedSizeList(dataType.listSize, child);
@@ -3188,6 +3207,13 @@ function deserializeArrowType(dataType, options) {
         );
         return new arrow.Struct(children);
       }
+      case "sparse-union":
+      case "dense-union": {
+        const children = dataType.children.map(
+          (arrowField) => deserializeArrowField(arrowField, options)
+        );
+        return dataType.type === "dense-union" ? new arrow.DenseUnion(dataType.typeIds, children) : new arrow.SparseUnion(dataType.typeIds, children);
+      }
       case "dictionary": {
         return new arrow.Dictionary(
           deserializeArrowType(dataType.dictionary, options),
@@ -3825,8 +3851,85 @@ var init_deprecated = __esm({
   }
 });
 
-// ../schema-utils/src/index.ts
+// ../schema/src/metadata.ts
+function getMetadataValue(metadata, key) {
+  return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
+}
+function setMetadataValue(metadata, key, value) {
+  if (metadata instanceof Map) {
+    metadata.set(key, value);
+  } else {
+    metadata[key] = value;
+  }
+}
+function getGeoMetadata(metadata) {
+  if (!metadata) {
+    return null;
+  }
+  const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
+  const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
+  if (!geoMetadata) {
+    return null;
+  }
+  for (const column of Object.values(geoMetadata.columns || {})) {
+    if (column && typeof column === "object" && typeof column.encoding === "string") {
+      column.encoding = column.encoding.toLowerCase();
+    }
+  }
+  return geoMetadata;
+}
+function unpackGeoMetadata(metadata) {
+  const geoMetadata = getGeoMetadata(metadata);
+  if (!geoMetadata) {
+    return;
+  }
+  const { version: version2, primary_column: primaryColumn, columns } = geoMetadata;
+  if (version2) {
+    setMetadataValue(metadata, "geo.version", version2);
+  }
+  if (primaryColumn) {
+    setMetadataValue(metadata, "geo.primary_column", primaryColumn);
+  }
+  setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
+}
+function unpackJSONStringMetadata(metadata, metadataKey) {
+  const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
+  const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
+  for (const [key, value] of Object.entries(json || {})) {
+    setMetadataValue(
+      metadata,
+      `${metadataKey}.${key}`,
+      typeof value === "string" ? value : JSON.stringify(value)
+    );
+  }
+}
+function parseJSONStringMetadata(stringifiedMetadata) {
+  if (!stringifiedMetadata) {
+    return null;
+  }
+  try {
+    const metadata = JSON.parse(stringifiedMetadata);
+    return metadata && typeof metadata === "object" ? metadata : null;
+  } catch {
+    return null;
+  }
+}
+var init_metadata = __esm({
+  "../schema/src/metadata.ts"() {
+    "use strict";
+  }
+});
+
+// ../schema/src/index.ts
 var init_src3 = __esm({
+  "../schema/src/index.ts"() {
+    "use strict";
+    init_metadata();
+  }
+});
+
+// ../schema-utils/src/index.ts
+var init_src4 = __esm({
   "../schema-utils/src/index.ts"() {
     "use strict";
     init_convert_arrow_schema();
@@ -4815,35 +4918,26 @@ var init_arrow_table_transport = __esm({
   "../arrow/src/lib/utils/arrow-table-transport.ts"() {
     "use strict";
     arrow6 = __toESM(require("apache-arrow"), 1);
-    init_src3();
+    init_src4();
     init_split_arrow_buffers();
   }
 });
 
 // ../gis/src/lib/geoarrow/metadata-utils.ts
-function getMetadataValue(metadata, key) {
-  return metadata instanceof Map ? metadata.get(key) || null : metadata[key] || null;
-}
-function setMetadataValue(metadata, key, value) {
-  if (metadata instanceof Map) {
-    metadata.set(key, value);
-  } else {
-    metadata[key] = value;
-  }
-}
 var init_metadata_utils = __esm({
   "../gis/src/lib/geoarrow/metadata-utils.ts"() {
     "use strict";
+    init_src3();
   }
 });
 
 // ../gis/src/lib/geoarrow/geoparquet-metadata.ts
-function getGeoMetadata(metadata) {
+function getGeoMetadata2(metadata) {
   if (!metadata) {
     return null;
   }
   const stringifiedGeoMetadata = getMetadataValue(metadata, "geo");
-  const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata(stringifiedGeoMetadata);
+  const geoMetadata = stringifiedGeoMetadata && parseJSONStringMetadata2(stringifiedGeoMetadata);
   if (!geoMetadata) {
     return null;
   }
@@ -4854,32 +4948,7 @@ function getGeoMetadata(metadata) {
   }
   return geoMetadata;
 }
-function unpackGeoMetadata(metadata) {
-  const geoMetadata = getGeoMetadata(metadata);
-  if (!geoMetadata) {
-    return;
-  }
-  const { version: version2, primary_column, columns } = geoMetadata;
-  if (version2) {
-    setMetadataValue(metadata, "geo.version", version2);
-  }
-  if (primary_column) {
-    setMetadataValue(metadata, "geo.primary_column", primary_column);
-  }
-  setMetadataValue(metadata, "geo.columns", Object.keys(columns || {}).join(","));
-}
-function unpackJSONStringMetadata(metadata, metadataKey) {
-  const stringifiedMetadata = getMetadataValue(metadata, metadataKey);
-  const json = stringifiedMetadata && parseJSONStringMetadata(stringifiedMetadata);
-  for (const [key, value] of Object.entries(json || {})) {
-    setMetadataValue(
-      metadata,
-      `${metadataKey}.${key}`,
-      typeof value === "string" ? value : JSON.stringify(value)
-    );
-  }
-}
-function parseJSONStringMetadata(stringifiedMetadata) {
+function parseJSONStringMetadata2(stringifiedMetadata) {
   if (!stringifiedMetadata) {
     return null;
   }
@@ -4901,16 +4970,15 @@ var init_geoparquet_metadata = __esm({
 });
 
 // ../gis/src/index.ts
-var init_src4 = __esm({
+var init_src5 = __esm({
   "../gis/src/index.ts"() {
     "use strict";
-    init_metadata_utils();
     init_geoparquet_metadata();
   }
 });
 
 // ../arrow/src/index.ts
-var init_src5 = __esm({
+var init_src6 = __esm({
   "../arrow/src/index.ts"() {
     "use strict";
     init_indexed_arrow_table();
@@ -5154,10 +5222,11 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
   if (!extensionName) {
     return null;
   }
-  if (normalizedEncoding === "wkb" && field.type !== "binary") {
+  const isBinaryGeospatialEncoding = normalizedEncoding === "wkb" || normalizedEncoding === "geometry" || normalizedEncoding === "geography";
+  if (isBinaryGeospatialEncoding && !isBinaryFieldType(field.type)) {
     return null;
   }
-  if (normalizedEncoding !== "wkb" && (field.type === "binary" || field.type === "utf8")) {
+  if (!isBinaryGeospatialEncoding && (isBinaryFieldType(field.type) || isStringFieldType(field.type))) {
     return null;
   }
   const extensionMetadata = {};
@@ -5171,18 +5240,34 @@ function getGeoArrowMetadataFromGeoParquetField(field, columnMetadata) {
   if (columnMetadata.edges && columnMetadata.edges !== "planar") {
     extensionMetadata.edges = columnMetadata.edges;
   }
+  if (normalizedEncoding === "geography" && !extensionMetadata.edges) {
+    extensionMetadata.edges = "spherical";
+  }
+  for (const [metadataKey, metadataValue] of Object.entries(columnMetadata)) {
+    if (metadataValue !== void 0 && metadataValue !== null && !["encoding", "geometry_types", "crs", "edges"].includes(metadataKey)) {
+      extensionMetadata[metadataKey] = metadataValue;
+    }
+  }
   return Object.keys(extensionMetadata).length > 0 ? { extensionName, extensionMetadata } : { extensionName };
 }
+function isBinaryFieldType(fieldType) {
+  return typeof fieldType === "string" && (fieldType === "binary" || fieldType === "binary-view");
+}
+function isStringFieldType(fieldType) {
+  return typeof fieldType === "string" && (fieldType === "utf8" || fieldType === "utf8-view");
+}
 var arrow7, GEOARROW_EXTENSION_NAME_KEY, GEOARROW_EXTENSION_METADATA_KEY, GEOPARQUET_TO_GEOARROW_ENCODINGS;
 var init_geospatial_metadata = __esm({
   "../parquet/src/lib/geo/geospatial-metadata.ts"() {
     "use strict";
     arrow7 = __toESM(require("apache-arrow"), 1);
-    init_src4();
+    init_src3();
     GEOARROW_EXTENSION_NAME_KEY = "ARROW:extension:name";
     GEOARROW_EXTENSION_METADATA_KEY = "ARROW:extension:metadata";
     GEOPARQUET_TO_GEOARROW_ENCODINGS = {
       wkb: "geoarrow.wkb",
+      geometry: "geoarrow.wkb",
+      geography: "geoarrow.wkb",
       point: "geoarrow.point",
       linestring: "geoarrow.linestring",
       polygon: "geoarrow.polygon",
@@ -5208,7 +5293,7 @@ async function getSchemaFromParquetReader(reader) {
 var init_get_parquet_schema = __esm({
   "../parquet/src/lib/parsers/get-parquet-schema.ts"() {
     "use strict";
-    init_src4();
+    init_src3();
     init_convert_schema_from_parquet();
     init_geospatial_metadata();
   }
@@ -5320,7 +5405,7 @@ function doLongitudeIntervalsIntersect(firstWest, firstEast, secondWest, secondE
 var init_geoparquet_covering = __esm({
   "../parquet/src/lib/geo/geoparquet-covering.ts"() {
     "use strict";
-    init_src4();
+    init_src3();
   }
 });
 
@@ -12044,7 +12129,7 @@ var init_parse_bson = __esm({
 });
 
 // ../bson/src/index.ts
-var init_src6 = __esm({
+var init_src7 = __esm({
   "../bson/src/index.ts"() {
     "use strict";
     init_bson_writer();
@@ -12453,7 +12538,7 @@ var PARQUET_LOGICAL_TYPES, kMillisPerDay;
 var init_types = __esm({
   "../parquet/src/parquetjs/schema/types.ts"() {
     "use strict";
-    init_src6();
+    init_src7();
     init_binary_utils();
     PARQUET_LOGICAL_TYPES = {
       BOOLEAN: {
@@ -16474,7 +16559,7 @@ var init_default_codecs = __esm({
 });
 
 // ../compression/src/index.ts
-var init_src7 = __esm({
+var init_src8 = __esm({
   "../compression/src/index.ts"() {
     "use strict";
     init_compression();
@@ -16587,7 +16672,7 @@ var PARQUET_COMPRESSION_METHODS, ParquetSnappyDecompressor;
 var init_compression2 = __esm({
   "../parquet/src/parquetjs/compression.ts"() {
     "use strict";
-    init_src7();
+    init_src8();
     init_snappy_decompressor_hysnappy();
     init_src2();
     init_binary_utils();
@@ -20676,6 +20761,201 @@ var init_decoders = __esm({
   }
 });
 
+// ../parquet/src/parquetjs/parser/encoded-pages.ts
+async function scanParquetEncodedPages(buffer, context) {
+  const pages = [];
+  let offset = 0;
+  let dataPageOrdinal = 0;
+  while (offset < buffer.byteLength) {
+    const { pageHeader, length: headerByteLength } = decodePageHeader(buffer, offset);
+    const bodyOffset = offset + headerByteLength;
+    const bodyEnd = bodyOffset + pageHeader.compressed_page_size;
+    if (pageHeader.compressed_page_size < 0 || pageHeader.uncompressed_page_size < 0 || bodyEnd > buffer.byteLength) {
+      throw new Error("Parquet page extends beyond the available column-chunk buffer");
+    }
+    const compressedBody = buffer.subarray(bodyOffset, bodyEnd);
+    verifyEncodedPageChecksum(compressedBody, pageHeader, context.verifyPageChecksums);
+    const pageType = getThriftEnum(PageType, pageHeader.type);
+    if (pageType === "DICTIONARY_PAGE") {
+      pages.push(await createDictionaryPage(compressedBody, pageHeader, context));
+    } else if (pageType === "DATA_PAGE") {
+      pages.push(await createDataPageV1(compressedBody, pageHeader, context, dataPageOrdinal++));
+    } else if (pageType === "DATA_PAGE_V2") {
+      pages.push(await createDataPageV2(compressedBody, pageHeader, context, dataPageOrdinal++));
+    } else {
+      throw new Error(`Unsupported Parquet page type ${pageType}`);
+    }
+    offset = bodyEnd;
+  }
+  return pages;
+}
+async function createDictionaryPage(compressedBody, pageHeader, context) {
+  const dictionaryHeader = pageHeader.dictionary_page_header;
+  if (!dictionaryHeader) {
+    throw new Error("Parquet dictionary page is missing its dictionary header");
+  }
+  const preserveCompression = shouldPreserveCompression(context);
+  const data = preserveCompression ? compressedBody : await decompressPageBody(compressedBody, pageHeader.uncompressed_page_size, context);
+  return Object.freeze({
+    type: "dictionary",
+    pageOrdinal: -1,
+    encoding: getThriftEnum(Encoding, dictionaryHeader.encoding),
+    compression: context.compression,
+    compressionState: preserveCompression ? "compressed" : "decompressed",
+    valueCount: dictionaryHeader.num_values,
+    nonNullValueCount: dictionaryHeader.num_values,
+    data,
+    values: preserveCompression ? void 0 : createSection(0, data.byteLength),
+    compressedByteLength: pageHeader.compressed_page_size,
+    uncompressedByteLength: pageHeader.uncompressed_page_size
+  });
+}
+async function createDataPageV1(compressedBody, pageHeader, context, pageOrdinal) {
+  const dataHeader = pageHeader.data_page_header;
+  if (!dataHeader) {
+    throw new Error("Parquet V1 data page is missing its data header");
+  }
+  const preserveCompression = shouldPreserveCompression(context);
+  const data = preserveCompression ? compressedBody : await decompressPageBody(compressedBody, pageHeader.uncompressed_page_size, context);
+  let repetitionLevels;
+  let definitionLevels;
+  let values;
+  if (!preserveCompression) {
+    let sectionOffset = 0;
+    const repetitionLevelEncoding = getThriftEnum(Encoding, dataHeader.repetition_level_encoding);
+    repetitionLevels = readV1LevelSection(
+      data,
+      sectionOffset,
+      dataHeader.num_values,
+      context.column.rLevelMax,
+      repetitionLevelEncoding
+    );
+    sectionOffset = repetitionLevels.byteOffset + repetitionLevels.byteLength;
+    const definitionLevelEncoding = getThriftEnum(Encoding, dataHeader.definition_level_encoding);
+    definitionLevels = readV1LevelSection(
+      data,
+      sectionOffset,
+      dataHeader.num_values,
+      context.column.dLevelMax,
+      definitionLevelEncoding
+    );
+    sectionOffset = definitionLevels.byteOffset + definitionLevels.byteLength;
+    values = createSection(sectionOffset, data.byteLength - sectionOffset);
+  }
+  return Object.freeze({
+    type: "data-v1",
+    pageOrdinal,
+    encoding: getThriftEnum(Encoding, dataHeader.encoding),
+    repetitionLevelEncoding: getThriftEnum(Encoding, dataHeader.repetition_level_encoding),
+    definitionLevelEncoding: getThriftEnum(Encoding, dataHeader.definition_level_encoding),
+    compression: context.compression,
+    compressionState: preserveCompression ? "compressed" : "decompressed",
+    valueCount: dataHeader.num_values,
+    data,
+    repetitionLevels,
+    definitionLevels,
+    values,
+    compressedByteLength: pageHeader.compressed_page_size,
+    uncompressedByteLength: pageHeader.uncompressed_page_size
+  });
+}
+async function createDataPageV2(compressedBody, pageHeader, context, pageOrdinal) {
+  const dataHeader = pageHeader.data_page_header_v2;
+  if (!dataHeader) {
+    throw new Error("Parquet V2 data page is missing its data header");
+  }
+  const repetitionByteLength = dataHeader.repetition_levels_byte_length;
+  const definitionByteLength = dataHeader.definition_levels_byte_length;
+  const levelsByteLength = repetitionByteLength + definitionByteLength;
+  if (levelsByteLength > compressedBody.byteLength) {
+    throw new Error("Parquet V2 level sections extend beyond the page body");
+  }
+  const valuesAreCompressed = dataHeader.is_compressed !== false && context.compression !== "UNCOMPRESSED";
+  const preserveCompression = valuesAreCompressed && shouldPreserveCompression(context);
+  let data = compressedBody;
+  if (valuesAreCompressed && !preserveCompression) {
+    const uncompressedValuesByteLength = pageHeader.uncompressed_page_size - levelsByteLength;
+    if (uncompressedValuesByteLength < 0) {
+      throw new Error("Invalid Parquet V2 uncompressed value length");
+    }
+    const values = await context.decompressPage(
+      compressedBody.subarray(levelsByteLength),
+      uncompressedValuesByteLength
+    );
+    data = new Uint8Array(levelsByteLength + values.byteLength);
+    data.set(compressedBody.subarray(0, levelsByteLength));
+    data.set(values, levelsByteLength);
+  }
+  return Object.freeze({
+    type: "data-v2",
+    pageOrdinal,
+    encoding: getThriftEnum(Encoding, dataHeader.encoding),
+    repetitionLevelEncoding: "RLE",
+    definitionLevelEncoding: "RLE",
+    compression: context.compression,
+    compressionState: preserveCompression ? "compressed" : "decompressed",
+    valueCount: dataHeader.num_values,
+    nonNullValueCount: dataHeader.num_values - dataHeader.num_nulls,
+    data,
+    repetitionLevels: createSection(0, repetitionByteLength),
+    definitionLevels: createSection(repetitionByteLength, definitionByteLength),
+    values: createSection(levelsByteLength, data.byteLength - levelsByteLength),
+    compressedByteLength: pageHeader.compressed_page_size,
+    uncompressedByteLength: pageHeader.uncompressed_page_size
+  });
+}
+function shouldPreserveCompression(context) {
+  var _a;
+  return context.compression !== "UNCOMPRESSED" && Boolean((_a = context.preserveCompression) == null ? void 0 : _a.has(context.compression));
+}
+async function decompressPageBody(compressedBody, uncompressedByteLength, context) {
+  if (context.compression === "UNCOMPRESSED") return compressedBody;
+  return await context.decompressPage(compressedBody, uncompressedByteLength);
+}
+function readV1LevelSection(data, offset, valueCount, maxLevel, encoding) {
+  if (maxLevel === 0) return createSection(offset, 0);
+  if (encoding === "RLE") {
+    if (offset + 4 > data.byteLength) {
+      throw new Error("Parquet V1 RLE level stream is missing its length prefix");
+    }
+    const byteLength = readUInt32LE(data, offset);
+    const byteOffset = offset + 4;
+    assertSectionBounds(data, byteOffset, byteLength);
+    return createSection(byteOffset, byteLength);
+  }
+  if (encoding === "BIT_PACKED") {
+    const byteLength = Math.ceil(valueCount / 8) * getBitWidth(maxLevel);
+    assertSectionBounds(data, offset, byteLength);
+    return createSection(offset, byteLength);
+  }
+  throw new Error(`Unsupported Parquet V1 level encoding ${encoding}`);
+}
+function createSection(byteOffset, byteLength) {
+  return Object.freeze({ byteOffset, byteLength });
+}
+function assertSectionBounds(data, byteOffset, byteLength) {
+  if (byteOffset < 0 || byteLength < 0 || byteOffset + byteLength > data.byteLength) {
+    throw new Error("Parquet page section extends beyond the page body");
+  }
+}
+function verifyEncodedPageChecksum(compressedBody, pageHeader, verifyPageChecksums = false) {
+  if (!verifyPageChecksums || pageHeader.crc === void 0) return;
+  const expected = pageHeader.crc >>> 0;
+  const actual = crc32(compressedBody);
+  if (actual !== expected) {
+    throw new Error(`Parquet page checksum mismatch: expected ${expected}, calculated ${actual}`);
+  }
+}
+var init_encoded_pages = __esm({
+  "../parquet/src/parquetjs/parser/encoded-pages.ts"() {
+    "use strict";
+    init_parquet_thrift();
+    init_crc32();
+    init_binary_utils();
+    init_read_utils();
+  }
+});
+
 // ../parquet/src/lib/parquet-encryption.ts
 function readParquetEncryptedModule(buffer, offset = 0) {
   if (offset + 4 > buffer.length) {
@@ -20918,6 +21198,7 @@ var init_parquet_reader = __esm({
     "use strict";
     init_schema();
     init_decoders();
+    init_encoded_pages();
     init_shred();
     init_compression2();
     init_constants();
@@ -21404,6 +21685,50 @@ var init_parquet_reader = __esm({
           columnData: Object.fromEntries(columnEntries)
         };
       }
+      /**
+       * Reads selected column chunks as encoded page descriptors without materializing values.
+       *
+       * Page headers, encryption, checksums, and optional decompression remain reader concerns. Value,
+       * dictionary, and level decoding is deliberately left to the downstream consumer.
+       */
+      async readRowGroupEncodedPages(schema, rowGroup, columnList, preserveCompression, signal, rowGroupOrdinal = 0) {
+        await this.resolveColumnMetadata(rowGroup, rowGroupOrdinal, columnList);
+        const selectedColumnChunks = [];
+        for (let columnOrdinal = 0; columnOrdinal < rowGroup.columns.length; columnOrdinal++) {
+          const columnChunk = rowGroup.columns[columnOrdinal];
+          const path = getColumnChunkPath(columnChunk);
+          if (columnList.length > 0 && (!path || fieldIndexOf(columnList, path) < 0)) continue;
+          const physicalColumnOrdinal = getPhysicalColumnOrdinal(columnChunk, columnOrdinal);
+          const metadata = columnChunk.meta_data || await this.getColumnMetadata(columnChunk, rowGroupOrdinal, physicalColumnOrdinal);
+          if (columnList.length === 0 || fieldIndexOf(columnList, metadata.path_in_schema) >= 0) {
+            selectedColumnChunks.push({
+              columnChunk,
+              metadata,
+              columnOrdinal: physicalColumnOrdinal
+            });
+          }
+        }
+        const columns = [];
+        for (let batchStart = 0; batchStart < selectedColumnChunks.length; batchStart += MAXIMUM_CONCURRENT_COLUMN_READS) {
+          columns.push(
+            ...await Promise.all(
+              selectedColumnChunks.slice(batchStart, batchStart + MAXIMUM_CONCURRENT_COLUMN_READS).map(async ({ columnChunk, metadata, columnOrdinal }) => {
+                const encodedColumn = await this.readColumnChunkEncodedPages(
+                  schema,
+                  columnChunk,
+                  preserveCompression,
+                  signal,
+                  rowGroupOrdinal,
+                  columnOrdinal,
+                  metadata
+                );
+                return encodedColumn;
+              })
+            )
+          );
+        }
+        return Object.freeze(columns);
+      }
       /** Reads independently materializable non-repeated columns for one row range using page indexes. */
       async readRowGroupRange(schema, rowGroup, columnList, rowRange, pageLocations, signal, rowGroupOrdinal = 0) {
         await this.resolveColumnMetadata(rowGroup, rowGroupOrdinal, columnList);
@@ -21596,6 +21921,99 @@ var init_parquet_reader = __esm({
         ) : chunkBuffer.subarray(pagesRelativeOffset, pagesRelativeOffset + pagesSize);
         return await decodeDataPages(pagesBuf, { ...context, dictionary });
       }
+      /** Reads one column chunk as encoded dictionaries, levels, and values for deferred decoding. */
+      async readColumnChunkEncodedPages(schema, columnChunk, preserveCompression, signal, rowGroupOrdinal = 0, columnOrdinal = 0, resolvedMetadata) {
+        if (columnChunk.file_path !== void 0 && columnChunk.file_path !== null) {
+          throw new Error("external references are not supported");
+        }
+        const metadata = resolvedMetadata || columnChunk.meta_data;
+        if (!metadata) throw new Error("Parquet column metadata is missing");
+        const field = schema.findField(metadata.path_in_schema);
+        const physicalType = getThriftEnum(Type, metadata.type);
+        if (physicalType !== field.primitiveType) {
+          throw new Error(`chunk type not matching schema: ${physicalType}`);
+        }
+        const compression = getThriftEnum(CompressionCodec, metadata.codec);
+        const dataPageOffset = Number(metadata.data_page_offset);
+        const dictionaryPageOffset = metadata.dictionary_page_offset;
+        const validDictionaryPageOffset = dictionaryPageOffset !== void 0 && Number(dictionaryPageOffset) > 0 ? Number(dictionaryPageOffset) : void 0;
+        const chunkOffset = Math.min(dataPageOffset, validDictionaryPageOffset ?? dataPageOffset);
+        const chunkEnd = chunkOffset + Number(metadata.total_compressed_size);
+        const chunkByteLength = Math.min(
+          this.file.size - chunkOffset,
+          Math.max(0, chunkEnd - chunkOffset)
+        );
+        const chunkBuffer = await this.readBytes(chunkOffset, chunkByteLength, signal);
+        const dataPageRelativeOffset = dataPageOffset - chunkOffset;
+        const dataPageByteLength = Math.min(
+          chunkBuffer.byteLength - dataPageRelativeOffset,
+          Math.max(0, chunkEnd - dataPageOffset)
+        );
+        const context = {
+          column: field,
+          compression,
+          decompressPage: this.getPageDecompressor(compression),
+          preserveCompression,
+          verifyPageChecksums: this.props.verifyPageChecksums
+        };
+        let dictionary;
+        if (validDictionaryPageOffset !== void 0) {
+          const dictionaryRelativeOffset = validDictionaryPageOffset - chunkOffset;
+          const dictionaryByteLength = Math.min(
+            Math.max(0, dataPageOffset - validDictionaryPageOffset),
+            chunkBuffer.byteLength - dictionaryRelativeOffset,
+            this.props.defaultDictionarySize
+          );
+          const dictionaryBuffer = chunkBuffer.subarray(
+            dictionaryRelativeOffset,
+            dictionaryRelativeOffset + dictionaryByteLength
+          );
+          const plaintextDictionaryBuffer = columnChunk.crypto_metadata ? await this.decryptColumnPages(
+            dictionaryBuffer,
+            {
+              type: physicalType,
+              rLevelMax: field.rLevelMax,
+              dLevelMax: field.dLevelMax,
+              compression,
+              column: field
+            },
+            rowGroupOrdinal,
+            columnOrdinal,
+            this.getColumnKeyMetadata(columnChunk),
+            true
+          ) : dictionaryBuffer;
+          dictionary = (await scanParquetEncodedPages(plaintextDictionaryBuffer, context))[0];
+        }
+        const dataPageBuffer = chunkBuffer.subarray(
+          dataPageRelativeOffset,
+          dataPageRelativeOffset + dataPageByteLength
+        );
+        const plaintextDataPageBuffer = columnChunk.crypto_metadata ? await this.decryptColumnPages(
+          dataPageBuffer,
+          {
+            type: physicalType,
+            rLevelMax: field.rLevelMax,
+            dLevelMax: field.dLevelMax,
+            compression,
+            column: field
+          },
+          rowGroupOrdinal,
+          columnOrdinal,
+          this.getColumnKeyMetadata(columnChunk)
+        ) : dataPageBuffer;
+        const pages = await scanParquetEncodedPages(plaintextDataPageBuffer, context);
+        return Object.freeze({
+          path: Object.freeze([...metadata.path_in_schema]),
+          physicalType,
+          typeLength: field.typeLength,
+          maxRepetitionLevel: field.rLevelMax,
+          maxDefinitionLevel: field.dLevelMax,
+          compression,
+          valueCount: Number(metadata.num_values),
+          dictionary,
+          pages: Object.freeze(pages)
+        });
+      }
       /** Decrypts the length-prefixed page header and data modules in one column chunk. */
       async decryptColumnPages(encryptedPages, context, rowGroupOrdinal, columnOrdinal, keyMetadata, includesDictionary = false, dataPageOrdinalStart = 0) {
         if (!this.encryptionContext || !this.props.keyRetriever) {
@@ -22416,9 +22834,10 @@ var _preloadParquetSourceLoader, _createMetadataDataSource, ParquetSourceLoaderB
 var init_parquet_source_loader = __esm({
   "../parquet/src/parquet-source-loader.ts"() {
     "use strict";
-    init_src5();
+    init_src6();
     init_src2();
-    init_src3();
+    init_src4();
+    init_src5();
     init_get_parquet_schema();
     init_geoparquet_covering();
     init_parquet_bloom_filter();
@@ -22572,6 +22991,73 @@ var init_parquet_source_loader = __esm({
       scan(options = {}) {
         return this.read(options);
       }
+      /**
+       * Reads candidate encoded pages without decoding values or constructing Arrow arrays.
+       *
+       * Footer statistics and Bloom filters prune complete row groups. Any exact predicate or spatial
+       * filter is returned as residual work, and its hidden columns are included in the page batch.
+       */
+      async *readPages(options = {}) {
+        const readOptions = this.getReadOptions({
+          rowGroups: options.rowGroups,
+          columns: options.columns,
+          rowGroupFilter: options.rowGroupFilter,
+          predicate: options.predicate,
+          bbox: options.bbox,
+          geometryColumn: options.geometryColumn,
+          signal: options.signal
+        });
+        const readContext = createReadAbortContext(readOptions.signal);
+        this.activeReadControllers.add(readContext.abortController);
+        try {
+          const initialization = await this.getInitialization(readContext.abortController.signal);
+          await this.getCompressionInitialization();
+          throwIfAborted2(readContext.abortController.signal);
+          const physicalPlan = await this.planRowGroups(
+            initialization,
+            readOptions,
+            readContext.abortController.signal
+          );
+          const projectedColumns = normalizeColumns(readOptions.columns, initialization.schema);
+          const predicateColumns = physicalPlan.predicate ? getParquetPredicateColumns(physicalPlan.predicate) : [];
+          const geoMetadata = readOptions.bbox ? getGeoMetadata2(initialization.schema.metadata) : void 0;
+          const geometryColumn = readOptions.bbox ? readOptions.geometryColumn || (geoMetadata == null ? void 0 : geoMetadata.primary_column) : void 0;
+          const filterColumns = [
+            .../* @__PURE__ */ new Set([...predicateColumns, ...geometryColumn ? [geometryColumn] : []])
+          ];
+          const decodedColumns = projectedColumns.length === 0 ? [] : [.../* @__PURE__ */ new Set([...projectedColumns, ...filterColumns])];
+          const columnList = decodedColumns.map((column) => [column]);
+          const preserveCompression = new Set(options.preserveCompression ?? []);
+          const residualFilter = readOptions.predicate || readOptions.bbox ? Object.freeze({
+            predicate: readOptions.predicate,
+            bbox: readOptions.bbox,
+            geometryColumn
+          }) : void 0;
+          for (const rowGroupIndex of physicalPlan.rowGroupIndices) {
+            throwIfAborted2(readContext.abortController.signal);
+            const columns = await initialization.reader.readRowGroupEncodedPages(
+              initialization.parquetSchema,
+              initialization.fileMetadata.row_groups[rowGroupIndex],
+              columnList,
+              preserveCompression,
+              readContext.abortController.signal,
+              rowGroupIndex
+            );
+            yield Object.freeze({
+              shape: "parquet-encoded-pages",
+              rowGroup: initialization.metadata.rowGroups[rowGroupIndex],
+              projectedColumns: Object.freeze([...projectedColumns]),
+              filterColumns: Object.freeze(filterColumns),
+              columns,
+              residualFilter
+            });
+          }
+        } finally {
+          readContext.abortController.abort();
+          readContext.removeSignalListener();
+          this.activeReadControllers.delete(readContext.abortController);
+        }
+      }
       /** Returns a copy of cumulative transport, decode, conversion, and pruning telemetry. */
       getTelemetry() {
         return Object.freeze({ ...this.telemetry });
@@ -24021,7 +24507,7 @@ var HudiTableSource = class extends DataSource {
   }
   async loadDescriptor(signal) {
     var _a;
-    const text = typeof this.data === "string" ? await (await this.fetch(this.data, { headers: (_a = this.options.hudi) == null ? void 0 : _a.headers, signal })).text() : await this.data.text();
+    const text = typeof this.data === "string" ? await (await this.fetch(this.url, { headers: (_a = this.options.hudi) == null ? void 0 : _a.headers, signal })).text() : await this.data.text();
     let descriptor;
     try {
       descriptor = JSON.parse(text);
@@ -24032,6 +24518,24 @@ var HudiTableSource = class extends DataSource {
     if (!Array.isArray(files)) {
       throw new Error("Hudi snapshot descriptor must contain a files array");
     }
+    for (const [index, file] of files.entries()) {
+      if (!file || typeof file !== "object" || Array.isArray(file)) {
+        throw new Error(`Hudi snapshot file at index ${index} must be an object`);
+      }
+      const value2 = file;
+      if (typeof value2.path !== "string" || !value2.path) {
+        throw new Error(`Hudi snapshot file at index ${index} must contain a non-empty string path`);
+      }
+      if (value2.size !== void 0 && (typeof value2.size !== "number" || !Number.isFinite(value2.size))) {
+        throw new Error(`Hudi snapshot file at index ${index} size must be a finite number`);
+      }
+      if (value2.numRecords !== void 0 && (typeof value2.numRecords !== "number" || !Number.isFinite(value2.numRecords))) {
+        throw new Error(`Hudi snapshot file at index ${index} numRecords must be a finite number`);
+      }
+      if (value2.partitionValues !== void 0 && (!value2.partitionValues || typeof value2.partitionValues !== "object" || Array.isArray(value2.partitionValues))) {
+        throw new Error(`Hudi snapshot file at index ${index} partitionValues must be an object`);
+      }
+    }
     const value = descriptor;
     if (value.tableType && value.tableType !== "COPY_ON_WRITE" && value.tableType !== "MERGE_ON_READ") {
       throw new Error(`Unsupported Hudi table type: ${String(value.tableType)}`);