@gscdump/lakehouse 3.2.0 → 3.3.0

package/dist/vendor/hysnappy-purejs.mjs

@@ -1,4 +1,4 @@
-import { snappyUncompress } from "../_virtual/node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/src/snappy.mjs";
+import { snappyUncompress } from "../_virtual/node_modules/.pnpm/hyparquet@1.29.2/node_modules/hyparquet/src/snappy.mjs";
 function decode(input, outputLength) {
 	const output = new Uint8Array(outputLength);
 	snappyUncompress(input, output);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "3.2.0",
+  "version": "3.3.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",
@@ -71,11 +71,11 @@
     "node": ">=22"
   },
   "devDependencies": {
-    "@types/node": "^26.2.0",
-    "hyparquet": "^1.28.2",
-    "icebird": "^0.8.22",
+    "@types/node": "^26.4.0",
+    "hyparquet": "^1.29.2",
+    "icebird": "^0.8.26",
     "unstorage": "^1.17.5",
-    "vitest": "^4.1.10"
+    "vitest": "^4.1.11"
   },
   "scripts": {
     "dev": "obuild --stub",

package/dist/_virtual/node_modules/.pnpm/hyparquet@1.28.2/node_modules/hyparquet/src/read.mjs

@@ -1,128 +0,0 @@
-import { parquetMetadataAsync, parquetSchema } from "./metadata.mjs";
-import { concat } from "./utils.mjs";
-import { columnsNeededForFilter, matchFilter } from "./filter.mjs";
-import { parquetPlan, prefetchAsyncBuffer, prefetchBloomFilters, prefetchPageIndexes } from "./plan.mjs";
-import { assembleAsync, asyncGroupToRows, readRowGroup } from "./rowgroup.mjs";
-async function parquetRead(options) {
-	options.metadata ??= await parquetMetadataAsync(options.file, options);
-	const { rowStart = 0, rowEnd, columns, onChunk, onComplete, rowFormat, filter, filterStrict = true } = options;
-	if (filter && rowFormat !== "object") throw new Error("parquet filter requires rowFormat: \"object\"");
-	const filterColumns = columnsNeededForFilter(filter);
-	if (filterColumns.length) {
-		const schemaColumns = parquetSchema(options.metadata).children.map((c) => c.element.name);
-		const missingColumns = filterColumns.filter((c) => !schemaColumns.includes(c));
-		if (missingColumns.length) throw new Error(`parquet filter columns not found: ${missingColumns.join(", ")}`);
-	}
-	let readColumns = columns;
-	let requiresProjection = false;
-	if (columns && filter) {
-		const missingFilterColumns = filterColumns.filter((c) => !columns.includes(c));
-		if (missingFilterColumns.length) {
-			readColumns = [...columns, ...missingFilterColumns];
-			requiresProjection = true;
-		}
-	}
-	let readOptions = readColumns !== columns ? {
-		...options,
-		columns: readColumns
-	} : options;
-	readOptions = await withBloomFilters(readOptions);
-	readOptions = await withPageIndexes(readOptions);
-	const asyncGroups = parquetReadAsync(readOptions);
-	if (!onComplete && !onChunk) {
-		await awaitAllColumns(asyncGroups);
-		return;
-	}
-	const schemaTree = parquetSchema(options.metadata);
-	const assembled = asyncGroups.map((arg) => assembleAsync(arg, schemaTree, options.parsers));
-	if (onChunk) for (const asyncGroup of assembled) for (const asyncColumn of asyncGroup.asyncColumns) asyncColumn.data.then(({ data, skipped }) => {
-		let rowStart = asyncGroup.groupStart + skipped;
-		for (const columnData of data) {
-			onChunk({
-				columnName: asyncColumn.pathInSchema[0],
-				columnData,
-				rowStart,
-				rowEnd: rowStart + columnData.length
-			});
-			rowStart += columnData.length;
-		}
-	}, () => {});
-	if (onComplete) {
-		await awaitAllColumns(assembled);
-		const rows = [];
-		for (const asyncGroup of assembled) {
-			const selectStart = asyncGroup.selectStart ?? Math.max(rowStart - asyncGroup.groupStart, 0);
-			const selectEnd = asyncGroup.selectEnd ?? Math.min((rowEnd ?? Infinity) - asyncGroup.groupStart, asyncGroup.groupRows);
-			const groupData = rowFormat === "object" ? await asyncGroupToRows(asyncGroup, selectStart, selectEnd, readColumns, "object") : await asyncGroupToRows(asyncGroup, selectStart, selectEnd, columns, "array");
-			if (filter) {
-				for (const row of groupData) if (matchFilter(row, filter, filterStrict)) {
-					if (requiresProjection && columns) {
-						for (const col of filterColumns) if (!columns.includes(col)) delete row[col];
-					}
-					rows.push(row);
-				}
-			} else concat(rows, groupData);
-		}
-		onComplete(rows);
-	} else await awaitAllColumns(assembled);
-}
-async function awaitAllColumns(asyncGroups) {
-	const all = asyncGroups.flatMap((g) => g.asyncColumns.map((c) => c.data));
-	const failed = (await Promise.allSettled(all)).find((r) => r.status === "rejected");
-	if (failed) throw failed.reason;
-}
-function parquetReadAsync(options) {
-	if (!options.metadata) throw new Error("parquet requires metadata");
-	const plan = parquetPlan(options);
-	options.file = prefetchAsyncBuffer(options.file, plan);
-	return plan.groups.map((groupPlan) => readRowGroup(options, plan, groupPlan));
-}
-async function withBloomFilters(options) {
-	if (!options.useBloomFilters) return options;
-	if (!options.filter || !options.metadata) return options;
-	const schemaTree = parquetSchema(options.metadata);
-	const schemaElements = {};
-	for (const child of schemaTree.children) schemaElements[child.element.name] = child.element;
-	const bloomFiltersByGroup = await prefetchBloomFilters({
-		file: options.file,
-		metadata: options.metadata,
-		filter: options.filter,
-		filterStrict: options.filterStrict
-	});
-	return {
-		...options,
-		bloomFiltersByGroup,
-		schemaElements
-	};
-}
-async function withPageIndexes(options) {
-	if (!options.usePageIndex) return options;
-	if (!options.filter || !options.metadata) return options;
-	const { pageRangesByGroup, pageLocationsByGroup } = await prefetchPageIndexes({
-		file: options.file,
-		metadata: options.metadata,
-		filter: options.filter,
-		filterStrict: options.filterStrict,
-		rowStart: options.rowStart,
-		rowEnd: options.rowEnd,
-		columns: options.columns,
-		bloomFiltersByGroup: options.bloomFiltersByGroup,
-		schemaElements: options.schemaElements,
-		parsers: options.parsers
-	});
-	return {
-		...options,
-		pageRangesByGroup,
-		pageLocationsByGroup
-	};
-}
-function parquetReadObjects(options) {
-	return new Promise((onComplete, reject) => {
-		parquetRead({
-			...options,
-			rowFormat: "object",
-			onComplete
-		}).catch(reject);
-	});
-}
-export { parquetRead, parquetReadAsync, parquetReadObjects };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/src/write/rewrite.mjs

@@ -1,174 +0,0 @@
-import { uuid4 } from "../utils.mjs";
-import { groupByPartition } from "./partition.mjs";
-import { computeColumnStats } from "./stats.mjs";
-import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot, loadPriorManifests } from "./snapshot.mjs";
-import { writeDataManifest } from "./manifest.mjs";
-import { writeParquet } from "./parquet.mjs";
-import { buildSortComparator } from "./sort.mjs";
-import { checkWriteFormat, newSnapshotId, resolveParquetCodec } from "./stage.mjs";
-import { icebergRead } from "../read.mjs";
-async function icebergStageRewrite({ tableUrl, metadata, resolver, sortOrderId, partitionSpecId, targetFileRows }) {
-	if (!tableUrl) throw new Error("tableUrl is required");
-	if (!resolver?.writer) throw new Error("resolver.writer is required");
-	const writerFn = resolver.writer;
-	if (metadata["format-version"] !== 2 && metadata["format-version"] !== 3) throw new Error(`unsupported format-version: ${metadata["format-version"]}`);
-	const formatVersion = metadata["format-version"];
-	if (targetFileRows !== void 0 && !(targetFileRows > 0)) throw new Error("targetFileRows must be a positive number");
-	if (!currentSnapshot(metadata)) throw new Error("no current snapshot to rewrite");
-	const schema = metadata.schemas.find((s) => s["schema-id"] === metadata["current-schema-id"]);
-	if (!schema) throw new Error("current schema not found in metadata");
-	const specId = partitionSpecId ?? metadata["default-spec-id"];
-	const partitionSpec = metadata["partition-specs"].find((s) => s["spec-id"] === specId);
-	if (!partitionSpec) throw new Error(`partition spec ${specId} not found in metadata`);
-	const orderId = sortOrderId ?? metadata["default-sort-order-id"] ?? 0;
-	const sortOrder = (metadata["sort-orders"] ?? []).find((o) => o["order-id"] === orderId);
-	if (sortOrderId !== void 0 && !sortOrder) throw new Error(`sort order ${sortOrderId} not found in metadata`);
-	const comparator = buildSortComparator(sortOrder, schema);
-	const appliedSortOrderId = comparator ? orderId : 0;
-	checkWriteFormat(metadata.properties?.["write.format.default"]);
-	const codec = resolveParquetCodec(metadata.properties?.["write.parquet.compression-codec"]);
-	const liveRows = await icebergRead({
-		tableUrl,
-		metadata,
-		resolver
-	});
-	const sortedRows = comparator ? [...liveRows].sort(comparator) : liveRows;
-	const rowLineage = formatVersion >= 3;
-	const allLineage = rowLineage && liveRows.length > 0 && liveRows.every((r) => r._row_id != null && r._last_updated_sequence_number != null);
-	const minRowId = allLineage ? liveRows.reduce((min, r) => r._row_id < min ? r._row_id : min, liveRows[0]._row_id) : void 0;
-	const groups = partitionSpec.fields.length ? groupByPartition(sortedRows, schema, partitionSpec) : [{
-		partition: {},
-		records: sortedRows
-	}];
-	const snapshotId = newSnapshotId(metadata);
-	const manifestUuid = uuid4();
-	const writeSchema = rowLineage ? {
-		...schema,
-		fields: [
-			...schema.fields,
-			{
-				id: 2147483540,
-				name: "_row_id",
-				required: false,
-				type: "long"
-			},
-			{
-				id: 2147483539,
-				name: "_last_updated_sequence_number",
-				required: false,
-				type: "long"
-			}
-		]
-	} : schema;
-	const writtenDataFiles = [];
-	for (const group of groups) {
-		const chunks = targetFileRows ? chunkRecords(group.records, targetFileRows) : [group.records];
-		for (const chunk of chunks) {
-			if (chunk.length === 0) continue;
-			const dataPath = `${tableUrl}/data/${uuid4()}.parquet`;
-			const dataWriter = writerFn(dataPath);
-			await writeParquet({
-				writer: dataWriter,
-				schema: writeSchema,
-				records: chunk,
-				codec
-			});
-			const stats = computeColumnStats(chunk, schema);
-			writtenDataFiles.push({
-				partition: group.partition,
-				dataFile: {
-					content: 0,
-					file_path: dataPath,
-					file_format: "parquet",
-					partition: group.partition,
-					record_count: BigInt(chunk.length),
-					file_size_in_bytes: BigInt(dataWriter.offset),
-					value_counts: stats.value_counts,
-					null_value_counts: stats.null_value_counts,
-					nan_value_counts: stats.nan_value_counts,
-					lower_bounds: stats.lower_bounds,
-					upper_bounds: stats.upper_bounds,
-					sort_order_id: appliedSortOrderId
-				},
-				path: dataPath
-			});
-		}
-	}
-	const manifestPath = `${tableUrl}/metadata/${manifestUuid}-m0.avro`;
-	const manifestWriter = writerFn(manifestPath);
-	await writeDataManifest({
-		writer: manifestWriter,
-		schema,
-		partitionSpec,
-		snapshotId,
-		dataFiles: writtenDataFiles.map((f) => f.dataFile),
-		formatVersion
-	});
-	const manifestLength = BigInt(manifestWriter.offset);
-	const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
-	const timestampMs = Date.now();
-	const addedRowCount = writtenDataFiles.reduce((sum, f) => sum + f.dataFile.record_count, 0n);
-	const addedFilesSize = writtenDataFiles.reduce((sum, f) => sum + f.dataFile.file_size_in_bytes, 0n);
-	const partitions = buildPartitionSummaries(writtenDataFiles.map((f) => f.dataFile.partition), schema, partitionSpec);
-	const newManifest = {
-		manifest_path: manifestPath,
-		manifest_length: manifestLength,
-		partition_spec_id: partitionSpec["spec-id"],
-		content: 0,
-		sequence_number: sequenceNumber,
-		min_sequence_number: sequenceNumber,
-		added_snapshot_id: snapshotId,
-		added_files_count: writtenDataFiles.length,
-		existing_files_count: 0,
-		deleted_files_count: 0,
-		added_rows_count: addedRowCount,
-		existing_rows_count: 0n,
-		deleted_rows_count: 0n,
-		partitions
-	};
-	if (allLineage) newManifest.first_row_id = minRowId;
-	const priorManifests = await loadPriorManifests(metadata, resolver);
-	const skipPriorManifestPaths = new Set(priorManifests.map((m) => m.manifest_path));
-	let deletedDataFiles = 0;
-	let deletedRecords = 0n;
-	for (const m of priorManifests) {
-		if (m.content !== 0) continue;
-		deletedDataFiles += (m.added_files_count ?? 0) + (m.existing_files_count ?? 0);
-		deletedRecords += BigInt(m.added_rows_count ?? 0) + BigInt(m.existing_rows_count ?? 0);
-	}
-	const summary = {
-		operation: "replace",
-		"added-data-files": String(writtenDataFiles.length),
-		"added-records": String(addedRowCount),
-		"added-files-size": String(addedFilesSize),
-		"deleted-data-files": String(deletedDataFiles),
-		"deleted-records": String(deletedRecords),
-		"total-records": String(addedRowCount),
-		"total-files-size": String(addedFilesSize),
-		"total-data-files": String(writtenDataFiles.length),
-		"total-delete-files": "0",
-		"total-position-deletes": "0",
-		"total-equality-deletes": "0"
-	};
-	return await buildSnapshotUpdate({
-		tableUrl,
-		metadata,
-		resolver,
-		snapshotId,
-		sequenceNumber,
-		manifestUuid,
-		timestampMs,
-		formatVersion,
-		newManifests: [newManifest],
-		summary,
-		writtenFiles: [...writtenDataFiles.map((f) => f.path), manifestPath],
-		priorManifests,
-		skipPriorManifestPaths
-	});
-}
-function chunkRecords(arr, size) {
-	const out = [];
-	for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
-	return out;
-}
-export { icebergStageRewrite };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/types/read.d.mts

@@ -1,2 +0,0 @@
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/types/node.mjs";
-import "../src/types.mjs";

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/types/sql/icebergDataSource.d.mts

@@ -1,2 +0,0 @@
-import "../../src/types.mjs";
-import "../../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6/node_modules/icebird/types/sql/icebergQuery.d.mts

@@ -1,2 +0,0 @@
-import "../../src/types.mjs";
-import "../../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";

package/dist/_virtual/node_modules/.pnpm/squirreling@0.15.3/node_modules/squirreling/src/expression/binary.mjs

@@ -1 +0,0 @@
-export {};

package/dist/_virtual/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/execute/join.mjs

@@ -1,5 +1,5 @@
-import "./utils.mjs";
 import "./yield.mjs";
+import "./utils.mjs";
 import "../expression/evaluate.mjs";
 import "./execute.mjs";
 export {};

package/dist/_virtual/node_modules/.pnpm/{squirreling@0.15.3 → squirreling@0.16.2}/node_modules/squirreling/src/execute/window.mjs

@@ -1,5 +1,5 @@
-import "./utils.mjs";
 import "./yield.mjs";
+import "./utils.mjs";
 import "../expression/evaluate.mjs";
 import "./execute.mjs";
 export {};