@gscdump/lakehouse 2.2.0 → 2.3.0

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/src/plan.mjs

@@ -0,0 +1,347 @@
+import { getPhysicalColumns } from "./schema.mjs";
+import { parquetSchema } from "./metadata.mjs";
+import { readColumnIndex, readOffsetIndex } from "./indexes.mjs";
+import { bloomEligibleColumns, readBloomFilter } from "./bloom.mjs";
+import { canSkipRowGroup, filterPageRanges, pathsNeededForFilter } from "./filter.mjs";
+const runLimit = 1 << 21;
+function parquetPlan({ metadata, rowStart = 0, rowEnd = Infinity, columns, filter, filterStrict = true, useOffsetIndex = false, bloomFiltersByGroup, schemaElements, pageRangesByGroup, pageLocationsByGroup }) {
+	if (!metadata) throw new Error("parquetPlan requires metadata");
+	const groups = [];
+	const fetches = [];
+	const indexes = [];
+	const schemaTree = parquetSchema(metadata);
+	const physicalColumns = getPhysicalColumns(schemaTree);
+	const elementsByPath = filter ? {
+		...physicalSchemaElements(schemaTree),
+		...schemaElements
+	} : schemaElements;
+	let groupStart = 0;
+	let rgIdx = 0;
+	for (const rowGroup of metadata.row_groups) {
+		const groupRows = Number(rowGroup.num_rows);
+		const groupEnd = groupStart + groupRows;
+		const bloomFilters = bloomFiltersByGroup?.[rgIdx];
+		if (groupRows > 0 && groupEnd > rowStart && groupStart < rowEnd && !canSkipRowGroup({
+			rowGroup,
+			physicalColumns,
+			filter,
+			strict: filterStrict,
+			bloomFilters,
+			schemaElements: elementsByPath
+		})) {
+			const selectStart = Math.max(rowStart - groupStart, 0);
+			const selectEnd = Math.min(rowEnd - groupStart, groupRows);
+			const pageRanges = pageRangesByGroup?.[rgIdx];
+			const pageLocations = pageLocationsByGroup?.[rgIdx];
+			let subranges = pageRanges ? pageRanges.map(([start, end]) => {
+				return [Math.max(start, selectStart), Math.min(end, selectEnd)];
+			}).filter(([start, end]) => start < end) : [[selectStart, selectEnd]];
+			if (subranges.length > 1) {
+				if (!rowGroup.columns.every((chunk) => {
+					const columnName = chunk.meta_data?.path_in_schema[0];
+					const columnPath = chunk.meta_data?.path_in_schema.join(".");
+					if (columns && columnName && !columns.includes(columnName)) return true;
+					return !!(chunk.offset_index_offset && chunk.offset_index_length) || !!(columnPath && pageLocations?.[columnPath]);
+				})) subranges = [[subranges[0][0], subranges[subranges.length - 1][1]]];
+				else subranges = coalesceOverlappingPageRanges(subranges, rowGroup, columns, pageLocations);
+			}
+			if (subranges.length) {
+				const chunks = [];
+				const narrowed = subranges.length > 1 || subranges[0][0] > 0 || subranges[0][1] < groupRows;
+				for (const chunk of rowGroup.columns) {
+					const meta = chunk.meta_data;
+					if (chunk.file_path) throw new Error("parquet file_path not supported");
+					if (!meta) throw new Error("parquet column metadata is undefined");
+					if (!columns || columns.includes(meta.path_in_schema[0])) {
+						const columnOffset = meta.dictionary_page_offset || meta.data_page_offset;
+						const startByte = Number(columnOffset);
+						const endByte = Number(columnOffset + meta.total_compressed_size);
+						const chunkPageLocations = pageLocations?.[meta.path_in_schema.join(".")];
+						if (chunkPageLocations && narrowed) chunks.push({
+							columnMetadata: meta,
+							pageLocations: chunkPageLocations,
+							range: {
+								startByte,
+								endByte
+							}
+						});
+						else if ((useOffsetIndex || pageRanges) && chunk.offset_index_offset && chunk.offset_index_length && narrowed) {
+							const offsetIndexStart = Number(chunk.offset_index_offset);
+							chunks.push({
+								columnMetadata: meta,
+								offsetIndex: {
+									startByte: offsetIndexStart,
+									endByte: offsetIndexStart + chunk.offset_index_length
+								},
+								range: {
+									startByte,
+									endByte
+								}
+							});
+						} else chunks.push({
+							columnMetadata: meta,
+							range: {
+								startByte,
+								endByte
+							}
+						});
+					}
+				}
+				for (const [subStart, subEnd] of subranges) groups.push({
+					chunks,
+					rowGroup,
+					groupStart,
+					groupRows,
+					selectStart: subStart,
+					selectEnd: subEnd
+				});
+				let run;
+				for (const chunk of chunks) if ("pageLocations" in chunk) {} else if ("offsetIndex" in chunk) indexes.push(chunk.offsetIndex);
+				else {
+					const { range } = chunk;
+					if (columns) fetches.push(range);
+					else if (run && range.endByte - run.startByte <= runLimit) run.endByte = range.endByte;
+					else {
+						if (run) fetches.push(run);
+						run = { ...range };
+					}
+				}
+				if (run) fetches.push(run);
+			}
+		}
+		groupStart = groupEnd;
+		rgIdx++;
+	}
+	if (!isFinite(rowEnd)) rowEnd = groupStart;
+	fetches.push(...indexes);
+	return {
+		metadata,
+		rowStart,
+		rowEnd,
+		columns,
+		fetches,
+		groups
+	};
+}
+function coalesceOverlappingPageRanges(ranges, rowGroup, columns, pageLocations) {
+	const selectedPageLayouts = rowGroup.columns.filter((chunk) => !columns || columns.includes(chunk.meta_data?.path_in_schema[0] || "")).map((chunk) => pageLocations?.[chunk.meta_data?.path_in_schema.join(".") || ""]);
+	const merged = [];
+	for (const range of ranges) {
+		const last = merged[merged.length - 1];
+		const overlapsPage = last && selectedPageLayouts.some((pages) => {
+			if (!pages) return true;
+			const lastPages = pagesForRange(last, pages, Number(rowGroup.num_rows));
+			const rangePages = pagesForRange(range, pages, Number(rowGroup.num_rows));
+			return lastPages[0] <= rangePages[1] && rangePages[0] <= lastPages[1];
+		});
+		if (last && overlapsPage) last[1] = range[1];
+		else merged.push([...range]);
+	}
+	return merged;
+}
+function pagesForRange([rangeStart, rangeEnd], pages, groupRows) {
+	let first = Infinity;
+	let last = -Infinity;
+	for (let i = 0; i < pages.length; i++) {
+		const pageStart = Number(pages[i].first_row_index);
+		if ((i + 1 < pages.length ? Number(pages[i + 1].first_row_index) : groupRows) > rangeStart && pageStart < rangeEnd) {
+			first = Math.min(first, i);
+			last = i;
+		}
+	}
+	return [first, last];
+}
+async function prefetchBloomFilters({ file, metadata, filter, filterStrict = true }) {
+	const result = metadata.row_groups.map(() => ({}));
+	const eligibleCols = bloomEligibleColumns(filter);
+	if (eligibleCols.size === 0) return result;
+	const physicalColumns = getPhysicalColumns(parquetSchema(metadata));
+	const tasks = [];
+	metadata.row_groups.forEach((rowGroup, rgIdx) => {
+		if (canSkipRowGroup({
+			rowGroup,
+			physicalColumns,
+			filter,
+			strict: filterStrict
+		})) return;
+		for (const colName of eligibleCols) {
+			const columnIdx = physicalColumns.indexOf(colName);
+			if (columnIdx === -1) continue;
+			const meta = rowGroup.columns[columnIdx]?.meta_data;
+			if (!meta?.bloom_filter_offset || !meta.bloom_filter_length) continue;
+			const start = Number(meta.bloom_filter_offset);
+			const end = start + meta.bloom_filter_length;
+			tasks.push((async () => {
+				const buffer = await file.slice(start, end);
+				const bloom = readBloomFilter({
+					view: new DataView(buffer),
+					offset: 0
+				});
+				if (bloom) result[rgIdx][colName] = bloom;
+			})());
+		}
+	});
+	if (tasks.length) await Promise.all(tasks);
+	return result;
+}
+async function prefetchPageIndexes({ file, metadata, filter, filterStrict = true, rowStart = 0, rowEnd = Infinity, columns, bloomFiltersByGroup, schemaElements, parsers }) {
+	const pageRangesByGroup = metadata.row_groups.map(() => void 0);
+	const pageLocationsByGroup = metadata.row_groups.map(() => ({}));
+	if (filter && "$nor" in filter && Array.isArray(filter.$nor)) return {
+		pageRangesByGroup,
+		pageLocationsByGroup
+	};
+	const filterColumns = pathsNeededForFilter(filter);
+	if (!filterColumns.length) return {
+		pageRangesByGroup,
+		pageLocationsByGroup
+	};
+	const schemaTree = parquetSchema(metadata);
+	const physicalColumns = getPhysicalColumns(schemaTree);
+	const elementsByPath = {
+		...physicalSchemaElements(schemaTree),
+		...schemaElements
+	};
+	const indexRanges = [];
+	const indexTasks = [];
+	const candidateGroups = [];
+	let groupStart = 0;
+	metadata.row_groups.forEach((rowGroup, rgIdx) => {
+		const groupRows = Number(rowGroup.num_rows);
+		const groupEnd = groupStart + groupRows;
+		const overlaps = groupRows > 0 && groupEnd > rowStart && groupStart < rowEnd;
+		groupStart = groupEnd;
+		if (!overlaps) return;
+		if (canSkipRowGroup({
+			rowGroup,
+			physicalColumns,
+			filter,
+			strict: filterStrict,
+			bloomFilters: bloomFiltersByGroup?.[rgIdx],
+			schemaElements: elementsByPath
+		})) return;
+		const columnPages = {};
+		let columnTaskCount = 0;
+		const scheduledOffsetPaths = /* @__PURE__ */ new Set();
+		let hasFilterIndex = false;
+		for (const columnName of filterColumns) {
+			const columnIdx = physicalColumns.indexOf(columnName);
+			if (columnIdx === -1) continue;
+			const chunk = rowGroup.columns[columnIdx];
+			if (!chunk?.meta_data) continue;
+			if (!chunk.column_index_offset || !chunk.column_index_length) continue;
+			if (!chunk.offset_index_offset || !chunk.offset_index_length) continue;
+			const element = elementsByPath[columnName];
+			if (!element) continue;
+			hasFilterIndex = true;
+			scheduledOffsetPaths.add(columnName);
+			const columnIndexStart = Number(chunk.column_index_offset);
+			const offsetIndexStart = Number(chunk.offset_index_offset);
+			const columnIndexEnd = columnIndexStart + chunk.column_index_length;
+			const offsetIndexEnd = offsetIndexStart + chunk.offset_index_length;
+			indexRanges.push({
+				startByte: columnIndexStart,
+				endByte: columnIndexEnd
+			}, {
+				startByte: offsetIndexStart,
+				endByte: offsetIndexEnd
+			});
+			columnTaskCount++;
+			indexTasks.push(async (prefetchedFile) => {
+				const [columnIndexBuffer, offsetIndexBuffer] = await Promise.all([prefetchedFile.slice(columnIndexStart, columnIndexEnd), prefetchedFile.slice(offsetIndexStart, offsetIndexEnd)]);
+				const columnIndex = readColumnIndex({
+					view: new DataView(columnIndexBuffer),
+					offset: 0
+				}, element, parsers);
+				const offsetIndex = readOffsetIndex({
+					view: new DataView(offsetIndexBuffer),
+					offset: 0
+				});
+				pageLocationsByGroup[rgIdx][columnName] = offsetIndex.page_locations;
+				columnPages[columnName] = {
+					minValues: columnIndex.min_values,
+					maxValues: columnIndex.max_values,
+					nullPages: columnIndex.null_pages,
+					nullCounts: columnIndex.null_counts,
+					pageStarts: offsetIndex.page_locations.map((page) => Number(page.first_row_index)),
+					element
+				};
+			});
+		}
+		if (hasFilterIndex) for (const chunk of rowGroup.columns) {
+			const meta = chunk.meta_data;
+			if (!meta) continue;
+			const columnName = meta.path_in_schema[0];
+			const columnPath = meta.path_in_schema.join(".");
+			if (columns && !columns.includes(columnName)) continue;
+			if (scheduledOffsetPaths.has(columnPath)) continue;
+			if (!chunk.offset_index_offset || !chunk.offset_index_length) continue;
+			scheduledOffsetPaths.add(columnPath);
+			const offsetIndexStart = Number(chunk.offset_index_offset);
+			const offsetIndexEnd = offsetIndexStart + chunk.offset_index_length;
+			indexRanges.push({
+				startByte: offsetIndexStart,
+				endByte: offsetIndexEnd
+			});
+			columnTaskCount++;
+			indexTasks.push(async (prefetchedFile) => {
+				const offsetIndexBuffer = await prefetchedFile.slice(offsetIndexStart, offsetIndexEnd);
+				const offsetIndex = readOffsetIndex({
+					view: new DataView(offsetIndexBuffer),
+					offset: 0
+				});
+				pageLocationsByGroup[rgIdx][columnPath] = offsetIndex.page_locations;
+			});
+		}
+		if (columnTaskCount) candidateGroups.push({
+			rgIdx,
+			groupRows,
+			columnPages
+		});
+	});
+	if (indexTasks.length) {
+		const prefetchedFile = prefetchAsyncBuffer(file, { fetches: coalesceByteRanges(indexRanges) });
+		await Promise.all(indexTasks.map((task) => task(prefetchedFile)));
+		for (const { rgIdx, groupRows, columnPages } of candidateGroups) pageRangesByGroup[rgIdx] = filterPageRanges(filter, columnPages, groupRows, filterStrict);
+	}
+	return {
+		pageRangesByGroup,
+		pageLocationsByGroup
+	};
+}
+function coalesceByteRanges(ranges) {
+	const sorted = ranges.map((range) => ({ ...range })).sort((a, b) => a.startByte - b.startByte || a.endByte - b.endByte);
+	const merged = [];
+	for (const range of sorted) {
+		const last = merged[merged.length - 1];
+		if (last && range.startByte <= last.endByte) last.endByte = Math.max(last.endByte, range.endByte);
+		else merged.push(range);
+	}
+	return merged;
+}
+function physicalSchemaElements(schemaTree) {
+	const elements = {};
+	function traverse(node) {
+		if (node.children.length) for (const child of node.children) traverse(child);
+		else elements[node.path.join(".")] = node.element;
+	}
+	traverse(schemaTree);
+	return elements;
+}
+function prefetchAsyncBuffer(file, { fetches }) {
+	const promises = fetches.map(({ startByte, endByte }) => file.slice(startByte, endByte));
+	return {
+		byteLength: file.byteLength,
+		slice(start, end = file.byteLength) {
+			const index = fetches.findIndex(({ startByte, endByte }) => startByte <= start && end <= endByte);
+			if (index < 0) return file.slice(start, end);
+			if (fetches[index].startByte !== start || fetches[index].endByte !== end) {
+				const startOffset = start - fetches[index].startByte;
+				const endOffset = end - fetches[index].startByte;
+				if (promises[index] instanceof Promise) return promises[index].then((buffer) => buffer.slice(startOffset, endOffset));
+				else return promises[index].slice(startOffset, endOffset);
+			} else return promises[index];
+		}
+	};
+}
+export { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters, prefetchPageIndexes };

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.26.2 → hyparquet@1.28.2}/node_modules/hyparquet/src/read.mjs

@@ -1,7 +1,7 @@
 import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
 import { concat } from "./utils.mjs";
 import { columnsNeededForFilter, matchFilter } from "./filter.mjs";
-import { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters } from "./plan.mjs";
+import { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters, prefetchPageIndexes } from "./plan.mjs";
 import { assembleAsync, asyncGroupToRows, readRowGroup } from "./rowgroup.mjs";
 async function parquetRead(options) {
 	options.metadata ??= await parquetMetadataAsync(options.file, options);
@@ -27,6 +27,7 @@ async function parquetRead(options) {
 		columns: readColumns
 	} : options;
 	readOptions = await withBloomFilters(readOptions);
+	readOptions = await withPageIndexes(readOptions);
 	const asyncGroups = parquetReadAsync(readOptions);
 	if (!onComplete && !onChunk) {
 		await awaitAllColumns(asyncGroups);
@@ -50,8 +51,8 @@ async function parquetRead(options) {
 		await awaitAllColumns(assembled);
 		const rows = [];
 		for (const asyncGroup of assembled) {
-			const selectStart = Math.max(rowStart - asyncGroup.groupStart, 0);
-			const selectEnd = Math.min((rowEnd ?? Infinity) - asyncGroup.groupStart, asyncGroup.groupRows);
+			const selectStart = asyncGroup.selectStart ?? Math.max(rowStart - asyncGroup.groupStart, 0);
+			const selectEnd = asyncGroup.selectEnd ?? Math.min((rowEnd ?? Infinity) - asyncGroup.groupStart, asyncGroup.groupRows);
 			const groupData = rowFormat === "object" ? await asyncGroupToRows(asyncGroup, selectStart, selectEnd, readColumns, "object") : await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns, "array");
 			if (filter) {
 				for (const row of groupData) if (matchFilter(row, filter, filterStrict)) {
@@ -94,6 +95,27 @@ async function withBloomFilters(options) {
 		schemaElements
 	};
 }
+async function withPageIndexes(options) {
+	if (!options.usePageIndex) return options;
+	if (!options.filter || !options.metadata) return options;
+	const { pageRangesByGroup, pageLocationsByGroup } = await prefetchPageIndexes({
+		file: options.file,
+		metadata: options.metadata,
+		filter: options.filter,
+		filterStrict: options.filterStrict,
+		rowStart: options.rowStart,
+		rowEnd: options.rowEnd,
+		columns: options.columns,
+		bloomFiltersByGroup: options.bloomFiltersByGroup,
+		schemaElements: options.schemaElements,
+		parsers: options.parsers
+	});
+	return {
+		...options,
+		pageRangesByGroup,
+		pageLocationsByGroup
+	};
+}
 function parquetReadObjects(options) {
 	return new Promise((onComplete, reject) => {
 		parquetRead({

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.26.2 → hyparquet@1.28.2}/node_modules/hyparquet/src/rowgroup.mjs

@@ -7,7 +7,7 @@ import { readColumn } from "./column.mjs";
 function readRowGroup(options, { metadata }, groupPlan) {
 	const asyncColumns = [];
 	for (const chunk of groupPlan.chunks) {
-		const { data_page_offset, dictionary_page_offset, path_in_schema: pathInSchema } = chunk.columnMetadata;
+		const { path_in_schema: pathInSchema } = chunk.columnMetadata;
 		const schemaPath = getSchemaPath(metadata.schema, pathInSchema);
 		const columnDecoder = {
 			pathInSchema,
@@ -20,66 +20,83 @@ function readRowGroup(options, { metadata }, groupPlan) {
 			...options,
 			...chunk.columnMetadata
 		};
-		let { startByte, endByte } = chunk.range;
-		if (!("offsetIndex" in chunk)) {
-			asyncColumns.push({
-				pathInSchema,
-				data: Promise.resolve(options.file.slice(startByte, endByte)).then((buffer) => {
-					const reader = {
-						view: new DataView(buffer),
-						offset: 0
-					};
-					return readColumn(reader, groupPlan, columnDecoder, options.onPage);
-				})
-			});
-			continue;
-		}
-		asyncColumns.push({
+		const { startByte, endByte } = chunk.range;
+		if ("pageLocations" in chunk) asyncColumns.push({
+			pathInSchema,
+			data: readSelectedPages(options, groupPlan, chunk, chunk.pageLocations, columnDecoder)
+		});
+		else if ("offsetIndex" in chunk) asyncColumns.push({
 			pathInSchema,
-			data: Promise.resolve(options.file.slice(chunk.offsetIndex.startByte, chunk.offsetIndex.endByte)).then(async (arrayBuffer) => {
-				const { selectStart, selectEnd } = groupPlan;
+			data: Promise.resolve(options.file.slice(chunk.offsetIndex.startByte, chunk.offsetIndex.endByte)).then((arrayBuffer) => {
 				const pages = readOffsetIndex({
 					view: new DataView(arrayBuffer),
 					offset: 0
 				}).page_locations;
-				let skipped = -1;
-				const hasDict = dictionary_page_offset || data_page_offset < pages[0].offset;
-				for (let i = 0; i < pages.length; i++) {
-					const page = pages[i];
-					const pageStart = Number(page.first_row_index);
-					const pageEnd = i + 1 < pages.length ? Number(pages[i + 1].first_row_index) : groupPlan.groupRows;
-					if (skipped < 0 && !hasDict && pageEnd > selectStart) {
-						startByte = Number(page.offset);
-						skipped = pageStart;
-					}
-					if (pageStart < selectEnd) endByte = Number(page.offset) + page.compressed_page_size;
-				}
-				if (skipped < 0) skipped = 0;
-				const buffer = await options.file.slice(startByte, endByte);
+				return readSelectedPages(options, groupPlan, chunk, pages, columnDecoder);
+			})
+		});
+		else asyncColumns.push({
+			pathInSchema,
+			data: Promise.resolve(options.file.slice(startByte, endByte)).then((buffer) => {
 				const reader = {
 					view: new DataView(buffer),
 					offset: 0
 				};
-				const adjustedGroupPlan = skipped ? {
-					...groupPlan,
-					groupStart: groupPlan.groupStart + skipped,
-					selectStart: groupPlan.selectStart - skipped,
-					selectEnd: groupPlan.selectEnd - skipped
-				} : groupPlan;
-				const { data, skipped: columnSkipped } = readColumn(reader, adjustedGroupPlan, columnDecoder, options.onPage);
-				return {
-					data,
-					skipped: skipped + columnSkipped
-				};
+				return readColumn(reader, groupPlan, columnDecoder, options.onPage);
 			})
 		});
 	}
 	return {
 		groupStart: groupPlan.groupStart,
 		groupRows: groupPlan.groupRows,
+		selectStart: groupPlan.selectStart,
+		selectEnd: groupPlan.selectEnd,
 		asyncColumns
 	};
 }
+async function readSelectedPages(options, groupPlan, chunk, pages, columnDecoder) {
+	const { data_page_offset, dictionary_page_offset } = chunk.columnMetadata;
+	const { selectStart, selectEnd } = groupPlan;
+	let { startByte, endByte } = chunk.range;
+	let skipped = -1;
+	const hasDict = dictionary_page_offset || data_page_offset < pages[0].offset;
+	for (let i = 0; i < pages.length; i++) {
+		const page = pages[i];
+		const pageStart = Number(page.first_row_index);
+		const pageEnd = i + 1 < pages.length ? Number(pages[i + 1].first_row_index) : groupPlan.groupRows;
+		if (skipped < 0 && pageEnd > selectStart) {
+			startByte = Number(page.offset);
+			skipped = pageStart;
+		}
+		if (pageStart < selectEnd) endByte = Number(page.offset) + page.compressed_page_size;
+	}
+	if (skipped < 0) skipped = 0;
+	let view;
+	if (hasDict && skipped) {
+		const dictLength = Number(pages[0].offset) - chunk.range.startByte;
+		const [dictBuffer, dataBuffer] = await Promise.all([options.file.slice(chunk.range.startByte, Number(pages[0].offset)), options.file.slice(startByte, endByte)]);
+		const combined = new Uint8Array(dictLength + dataBuffer.byteLength);
+		combined.set(new Uint8Array(dictBuffer, 0, dictLength));
+		combined.set(new Uint8Array(dataBuffer), dictLength);
+		view = new DataView(combined.buffer);
+	} else if (hasDict) view = new DataView(await options.file.slice(chunk.range.startByte, endByte));
+	else view = new DataView(await options.file.slice(startByte, endByte));
+	const reader = {
+		view,
+		offset: 0
+	};
+	const adjustedGroupPlan = skipped ? {
+		...groupPlan,
+		groupStart: groupPlan.groupStart + skipped,
+		selectStart: groupPlan.selectStart - skipped,
+		selectEnd: groupPlan.selectEnd - skipped
+	} : groupPlan;
+	const { data, skipped: columnSkipped } = readColumn(reader, adjustedGroupPlan, columnDecoder, options.onPage);
+	return {
+		data,
+		skipped: skipped + columnSkipped
+	};
+}
 async function asyncGroupToRows({ asyncColumns }, selectStart, selectEnd, columns, rowFormat) {
 	const asyncPages = await Promise.all(asyncColumns.map((column) => column.data.then(({ skipped, data }) => ({
 		skipped,
@@ -129,19 +146,20 @@ function assembleAsync(asyncRowGroup, schemaTree, parsers) {
 			data: (async () => {
 				const resolved = await Promise.all(childColumns.map((c) => c.data));
 				const subcolumnData = /* @__PURE__ */ new Map();
-				let minLength = Infinity;
+				const flattened = resolved.map(({ data }) => flatten(data));
+				const skipped = Math.max(...resolved.map((result) => result.skipped));
+				const end = Math.min(...resolved.map((result, i) => result.skipped + flattened[i].length));
 				for (let i = 0; i < childColumns.length; i++) {
-					const flat = flatten(resolved[i].data);
-					subcolumnData.set(childColumns[i].pathInSchema.join("."), flat);
-					minLength = Math.min(minLength, flat.length);
+					const start = skipped - resolved[i].skipped;
+					const length = Math.max(0, end - skipped);
+					subcolumnData.set(childColumns[i].pathInSchema.join("."), flattened[i].slice(start, start + length));
 				}
-				for (const [key, value] of subcolumnData) if (value.length > minLength) subcolumnData.set(key, value.slice(0, minLength));
 				assembleNested(subcolumnData, child, parsers);
 				const assembled = subcolumnData.get(child.element.name);
 				if (!assembled) throw new Error("parquet column data not assembled");
 				return {
 					data: [assembled],
-					skipped: 0
+					skipped
 				};
 			})()
 		});

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.26.2 → hyparquet@1.28.2}/node_modules/hyparquet/src/utils.mjs

@@ -11,6 +11,7 @@ function equals(a, b, strict = true) {
 		for (let i = 0; i < a.length; i++) if (a[i] !== b[i]) return false;
 		return true;
 	}
+	if (a instanceof Date || b instanceof Date) return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
 	if (Array.isArray(a) && Array.isArray(b)) {
 		if (a.length !== b.length) return false;
 		for (let i = 0; i < a.length; i++) if (!equals(a[i], b[i], strict)) return false;

package/dist/_virtual/node_modules/.pnpm/{hyparquet@1.26.2 → hyparquet@1.28.2}/node_modules/hyparquet/types/index.d.mts

@@ -1,9 +1,9 @@
 import { AsyncBuffer as AsyncBuffer$1 } from "../src/types.mjs";
-import "./query.mjs";
-import "./snappy.mjs";
 import "./indexes.mjs";
 import "./metadata.mjs";
 import "./read.mjs";
+import "./query.mjs";
+import "./snappy.mjs";
 import "./utils.mjs";
 type AsyncBuffer = AsyncBuffer$1;
 export { AsyncBuffer };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6}/node_modules/icebird/src/avro/avro.read.mjs

@@ -1,4 +1,4 @@
-import { parseDecimal } from "../../../../../hyparquet@1.26.2/node_modules/hyparquet/src/convert.mjs";
+import { parseDecimal } from "../../../../../hyparquet@1.28.2/node_modules/hyparquet/src/convert.mjs";
 import { gunzip } from "../../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/gzip.mjs";
 import "../../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import { readZigZag, readZigZagBigInt } from "./avro.metadata.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6}/node_modules/icebird/src/avro/avro.write.mjs

@@ -1,5 +1,5 @@
-import { ByteWriter } from "../../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
 function avroWrite({ writer, schema, records, blockSize = 512, metadata }) {
 	writer.appendUint32(23749199);
 	const meta = {

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6}/node_modules/icebird/src/fetch.mjs

@@ -1,11 +1,11 @@
-import { parquetMetadataAsync } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs";
-import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/utils.mjs";
-import { parquetReadObjects } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs";
-import "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/node.mjs";
+import { parquetMetadataAsync } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/metadata.mjs";
+import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/utils.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/read.mjs";
+import "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/node.mjs";
 import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
-import { ByteWriter } from "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
 import { avroMetadata } from "./avro/avro.metadata.mjs";
 import { avroRead } from "./avro/avro.read.mjs";
 import { puffinReadDeletionVector } from "./puffin/puffin.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6}/node_modules/icebird/src/index.mjs

@@ -7,7 +7,7 @@ import "./create.mjs";
 import "./read.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
 import "./catalog/file.mjs";
-import "../../../../squirreling@0.15.0/node_modules/squirreling/src/index.mjs";
+import "../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
 import "./sql/icebergDataSource.mjs";
 import "./sql/icebergQuery.mjs";
 export { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6}/node_modules/icebird/src/read.mjs

@@ -1,7 +1,7 @@
-import { parquetMetadataAsync } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs";
-import { cachedAsyncBuffer, concat } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/utils.mjs";
-import { parquetReadObjects } from "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/read.mjs";
-import "../../../../hyparquet@1.26.2/node_modules/hyparquet/src/node.mjs";
+import { parquetMetadataAsync } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/metadata.mjs";
+import { cachedAsyncBuffer, concat } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/utils.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/read.mjs";
+import "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/node.mjs";
 import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import { equalityMatch, sanitize } from "./utils.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057 → icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6}/node_modules/icebird/src/s3.mjs

@@ -1,5 +1,5 @@
-import { ByteWriter } from "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
 import { signRequest } from "./sigv4.mjs";
 function s3SignedResolver({ accessKeyId, secretAccessKey, sessionToken, region, endpoint, pathStyle = false }) {
 	const ep = endpoint ? new URL(endpoint.replace(/\/$/, "") + "/") : void 0;