@gscdump/lakehouse 1.4.0 → 1.4.2

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/stage.mjs

@@ -0,0 +1,354 @@
+import { uuid4 } from "../utils.mjs";
+import { validateSchemaForVersion } from "../schema.mjs";
+import { groupByPartition } from "./partition.mjs";
+import { applyUpdates } from "./commit.mjs";
+import { writeDataManifest } from "./manifest.mjs";
+import { computeColumnStats } from "./stats.mjs";
+import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot } from "./snapshot.mjs";
+import { writeParquet } from "./parquet.mjs";
+import { buildSortComparator } from "./sort.mjs";
+async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderId, snapshotId }) {
+	if (!tableUrl) throw new Error("tableUrl is required");
+	if (!resolver?.writer) throw new Error("resolver.writer is required");
+	const writerFn = resolver.writer;
+	if (metadata["format-version"] !== 2 && metadata["format-version"] !== 3) throw new Error(`unsupported format-version: ${metadata["format-version"]}`);
+	const formatVersion = metadata["format-version"];
+	const partitionSpec = metadata["partition-specs"].find((s) => s["spec-id"] === metadata["default-spec-id"]);
+	if (!partitionSpec) throw new Error("default partition spec not found in metadata");
+	const schema = metadata.schemas.find((s) => s["schema-id"] === metadata["current-schema-id"]);
+	if (!schema) throw new Error("current schema not found in metadata");
+	validateSchemaForVersion(schema, formatVersion);
+	const appendSnapshotId = snapshotId ?? newSnapshotId(metadata);
+	const manifestUuid = uuid4();
+	checkWriteFormat(metadata.properties?.["write.format.default"]);
+	const codec = resolveParquetCodec(metadata.properties?.["write.parquet.compression-codec"]);
+	const orderId = sortOrderId ?? metadata["default-sort-order-id"] ?? 0;
+	const sortOrder = (metadata["sort-orders"] ?? []).find((o) => o["order-id"] === orderId);
+	if (sortOrderId !== void 0 && !sortOrder) throw new Error(`sort order ${sortOrderId} not found in metadata`);
+	const comparator = buildSortComparator(sortOrder, schema);
+	const appliedSortOrderId = comparator ? orderId : 0;
+	const groups = partitionSpec.fields.length ? groupByPartition(records, schema, partitionSpec) : [{
+		partition: {},
+		records
+	}];
+	const writtenDataFiles = await Promise.all(groups.map(async (group) => {
+		const sortedRecords = comparator ? [...group.records].sort(comparator) : group.records;
+		const dataPath = `${tableUrl}/data/${uuid4()}.parquet`;
+		const dataWriter = writerFn(dataPath);
+		await writeParquet({
+			writer: dataWriter,
+			schema,
+			records: sortedRecords,
+			codec
+		});
+		const stats = computeColumnStats(sortedRecords, schema);
+		return {
+			partition: group.partition,
+			records: sortedRecords,
+			dataFile: {
+				content: 0,
+				file_path: dataPath,
+				file_format: "parquet",
+				partition: group.partition,
+				record_count: BigInt(sortedRecords.length),
+				file_size_in_bytes: BigInt(dataWriter.offset),
+				value_counts: stats.value_counts,
+				null_value_counts: stats.null_value_counts,
+				nan_value_counts: stats.nan_value_counts,
+				lower_bounds: stats.lower_bounds,
+				upper_bounds: stats.upper_bounds,
+				sort_order_id: appliedSortOrderId
+			},
+			path: dataPath
+		};
+	}));
+	const manifestPath = `${tableUrl}/metadata/${manifestUuid}-m0.avro`;
+	const manifestWriter = writerFn(manifestPath);
+	await writeDataManifest({
+		writer: manifestWriter,
+		schema,
+		partitionSpec,
+		snapshotId: appendSnapshotId,
+		dataFiles: writtenDataFiles.map((f) => f.dataFile),
+		formatVersion
+	});
+	const manifestLength = BigInt(manifestWriter.offset);
+	const addedRowCount = writtenDataFiles.reduce((sum, f) => sum + BigInt(f.records.length), 0n);
+	const addedFilesSize = writtenDataFiles.reduce((sum, f) => sum + f.dataFile.file_size_in_bytes, 0n);
+	const partitions = buildPartitionSummaries(writtenDataFiles.map((f) => f.dataFile.partition), schema, partitionSpec);
+	return {
+		snapshotId: appendSnapshotId,
+		manifestUuid,
+		formatVersion,
+		manifestPath,
+		manifestLength,
+		partitionSpecId: partitionSpec["spec-id"],
+		partitions,
+		addedDataFilesCount: writtenDataFiles.length,
+		addedRowCount,
+		addedFilesSize,
+		recordsCount: records.length,
+		writtenFiles: [...writtenDataFiles.map((f) => f.path), manifestPath]
+	};
+}
+async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver, snapshotProperties }) {
+	if (!tableUrl) throw new Error("tableUrl is required");
+	if (!resolver?.writer) throw new Error("resolver.writer is required");
+	const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
+	const timestampMs = Date.now();
+	const newManifest = {
+		manifest_path: prepared.manifestPath,
+		manifest_length: prepared.manifestLength,
+		partition_spec_id: prepared.partitionSpecId,
+		content: 0,
+		sequence_number: sequenceNumber,
+		min_sequence_number: sequenceNumber,
+		added_snapshot_id: prepared.snapshotId,
+		added_files_count: prepared.addedDataFilesCount,
+		existing_files_count: 0,
+		deleted_files_count: 0,
+		added_rows_count: prepared.addedRowCount,
+		existing_rows_count: 0n,
+		deleted_rows_count: 0n,
+		partitions: prepared.partitions
+	};
+	const prevSummary = currentSnapshot(metadata)?.summary;
+	const prevTotals = {
+		records: BigInt(prevSummary?.["total-records"] ?? "0"),
+		size: BigInt(prevSummary?.["total-files-size"] ?? "0"),
+		files: BigInt(prevSummary?.["total-data-files"] ?? "0")
+	};
+	const summary = {
+		operation: "append",
+		"added-data-files": String(prepared.addedDataFilesCount),
+		"added-records": String(prepared.recordsCount),
+		"added-files-size": String(prepared.addedFilesSize),
+		"changed-partition-count": String(prepared.addedDataFilesCount),
+		"total-records": String(prevTotals.records + BigInt(prepared.recordsCount)),
+		"total-files-size": String(prevTotals.size + prepared.addedFilesSize),
+		"total-data-files": String(prevTotals.files + BigInt(prepared.addedDataFilesCount)),
+		"total-delete-files": "0",
+		"total-position-deletes": "0",
+		"total-equality-deletes": "0",
+		...snapshotProperties ?? {}
+	};
+	return await buildSnapshotUpdate({
+		tableUrl,
+		metadata,
+		resolver,
+		snapshotId: prepared.snapshotId,
+		sequenceNumber,
+		manifestUuid: prepared.manifestUuid,
+		timestampMs,
+		formatVersion: prepared.formatVersion,
+		newManifests: [newManifest],
+		summary,
+		writtenFiles: []
+	});
+}
+async function stageSnapshotForAppendBatches({ tableUrl, metadata, prepared, resolver, snapshotProperties }) {
+	if (!tableUrl) throw new Error("tableUrl is required");
+	if (!resolver?.writer) throw new Error("resolver.writer is required");
+	if (prepared.length === 0) throw new Error("prepared batches must not be empty");
+	const first = prepared[0];
+	for (const batch of prepared) {
+		if (batch.snapshotId !== first.snapshotId) throw new Error("prepared batches must share one snapshot id");
+		if (batch.formatVersion !== first.formatVersion) throw new Error("prepared batches must share one format version");
+	}
+	const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
+	const timestampMs = Date.now();
+	const newManifests = prepared.map((batch) => ({
+		manifest_path: batch.manifestPath,
+		manifest_length: batch.manifestLength,
+		partition_spec_id: batch.partitionSpecId,
+		content: 0,
+		sequence_number: sequenceNumber,
+		min_sequence_number: sequenceNumber,
+		added_snapshot_id: first.snapshotId,
+		added_files_count: batch.addedDataFilesCount,
+		existing_files_count: 0,
+		deleted_files_count: 0,
+		added_rows_count: batch.addedRowCount,
+		existing_rows_count: 0n,
+		deleted_rows_count: 0n,
+		partitions: batch.partitions
+	}));
+	const addedDataFilesCount = prepared.reduce((sum, batch) => sum + batch.addedDataFilesCount, 0);
+	const recordsCount = prepared.reduce((sum, batch) => sum + batch.recordsCount, 0);
+	const addedFilesSize = prepared.reduce((sum, batch) => sum + batch.addedFilesSize, 0n);
+	const prevSummary = currentSnapshot(metadata)?.summary;
+	const prevTotals = {
+		records: BigInt(prevSummary?.["total-records"] ?? "0"),
+		size: BigInt(prevSummary?.["total-files-size"] ?? "0"),
+		files: BigInt(prevSummary?.["total-data-files"] ?? "0")
+	};
+	const summary = {
+		operation: "append",
+		"added-data-files": String(addedDataFilesCount),
+		"added-records": String(recordsCount),
+		"added-files-size": String(addedFilesSize),
+		"changed-partition-count": String(addedDataFilesCount),
+		"total-records": String(prevTotals.records + BigInt(recordsCount)),
+		"total-files-size": String(prevTotals.size + addedFilesSize),
+		"total-data-files": String(prevTotals.files + BigInt(addedDataFilesCount)),
+		"total-delete-files": "0",
+		"total-position-deletes": "0",
+		"total-equality-deletes": "0",
+		...snapshotProperties ?? {}
+	};
+	return await buildSnapshotUpdate({
+		tableUrl,
+		metadata,
+		resolver,
+		snapshotId: first.snapshotId,
+		sequenceNumber,
+		manifestUuid: first.manifestUuid,
+		timestampMs,
+		formatVersion: first.formatVersion,
+		newManifests,
+		summary,
+		writtenFiles: []
+	});
+}
+async function icebergStageAppend({ tableUrl, metadata, records, resolver, sortOrderId }) {
+	const prepared = await prepareAppend({
+		tableUrl,
+		metadata,
+		records,
+		resolver,
+		sortOrderId
+	});
+	const staged = await stageSnapshotForAppend({
+		tableUrl,
+		metadata,
+		prepared,
+		resolver
+	});
+	return {
+		...staged,
+		writtenFiles: [...prepared.writtenFiles, ...staged.writtenFiles]
+	};
+}
+function icebergStageSetRef({ metadata, ref, snapshotId, type = "branch", minSnapshotsToKeep, maxSnapshotAgeMs, maxRefAgeMs }) {
+	if (!ref) throw new Error("ref is required");
+	if (type !== "branch" && type !== "tag") throw new Error(`unknown ref type: ${type}`);
+	if (type === "tag" && (minSnapshotsToKeep !== void 0 || maxSnapshotAgeMs !== void 0)) throw new Error("tags do not support min-snapshots-to-keep or max-snapshot-age-ms");
+	const targetId = BigInt(snapshotId);
+	const snapshot = metadata.snapshots?.find((s) => BigInt(s["snapshot-id"]) === targetId);
+	if (!snapshot) throw new Error(`snapshot ${snapshotId} not found in metadata.snapshots`);
+	const existingRef = metadata.refs?.[ref];
+	let currentSnapshotId = existingRef?.["snapshot-id"] ?? null;
+	if (currentSnapshotId === null && ref === "main") currentSnapshotId = metadata["current-snapshot-id"] ?? null;
+	if (existingRef && existingRef.type !== type) throw new Error(`ref ${ref} is a ${existingRef.type}, cannot set as ${type}`);
+	const requirements = [{
+		type: "assert-table-uuid",
+		uuid: metadata["table-uuid"]
+	}, {
+		type: "assert-ref-snapshot-id",
+		ref,
+		"snapshot-id": currentSnapshotId
+	}];
+	const update = {
+		action: "set-snapshot-ref",
+		"ref-name": ref,
+		type,
+		"snapshot-id": snapshotId
+	};
+	if (minSnapshotsToKeep !== void 0) update["min-snapshots-to-keep"] = minSnapshotsToKeep;
+	if (maxSnapshotAgeMs !== void 0) update["max-snapshot-age-ms"] = maxSnapshotAgeMs;
+	if (maxRefAgeMs !== void 0) update["max-ref-age-ms"] = maxRefAgeMs;
+	return {
+		snapshot,
+		requirements,
+		updates: [update],
+		writtenFiles: []
+	};
+}
+function icebergStageExpireSnapshots({ metadata, snapshotIds }) {
+	if (!Array.isArray(snapshotIds) || snapshotIds.length === 0) throw new Error("snapshotIds must be a non-empty array");
+	const removeIds = new Set(snapshotIds.map(BigInt));
+	const snapshots = metadata.snapshots ?? [];
+	for (const id of removeIds) if (!snapshots.some((s) => BigInt(s["snapshot-id"]) === id)) throw new Error(`snapshot ${id} not found in metadata.snapshots`);
+	const refs = metadata.refs ?? {};
+	for (const [name, ref] of Object.entries(refs)) if (removeIds.has(BigInt(ref["snapshot-id"]))) throw new Error(`snapshot ${ref["snapshot-id"]} is referenced by ${ref.type} ${name}`);
+	const currentId = metadata["current-snapshot-id"];
+	if (currentId !== void 0 && currentId !== null && removeIds.has(BigInt(currentId)) && !refs.main) throw new Error(`snapshot ${currentId} is the current snapshot`);
+	const requirements = [{
+		type: "assert-table-uuid",
+		uuid: metadata["table-uuid"]
+	}, {
+		type: "assert-ref-snapshot-id",
+		ref: "main",
+		"snapshot-id": refs.main?.["snapshot-id"] ?? currentId ?? null
+	}];
+	const update = {
+		action: "remove-snapshots",
+		"snapshot-ids": snapshotIds
+	};
+	const tip = currentSnapshot(metadata) ?? snapshots[0];
+	if (!tip) throw new Error("cannot expire snapshots from a table with no snapshots");
+	return {
+		snapshot: tip,
+		requirements,
+		updates: [update],
+		writtenFiles: []
+	};
+}
+function icebergStageUpdateSchema({ metadata, schema }) {
+	if (!schema || schema.type !== "struct" || !Array.isArray(schema.fields)) throw new Error("schema must be a struct with a fields array");
+	const requirements = [
+		{
+			type: "assert-table-uuid",
+			uuid: metadata["table-uuid"]
+		},
+		{
+			type: "assert-current-schema-id",
+			"current-schema-id": metadata["current-schema-id"]
+		},
+		{
+			type: "assert-last-assigned-field-id",
+			"last-assigned-field-id": metadata["last-column-id"]
+		}
+	];
+	const updates = [{
+		action: "add-schema",
+		schema: {
+			...schema,
+			"schema-id": -1
+		}
+	}, {
+		action: "set-current-schema",
+		"schema-id": -1
+	}];
+	applyUpdates(metadata, updates);
+	return {
+		requirements,
+		updates,
+		writtenFiles: []
+	};
+}
+function checkWriteFormat(value) {
+	if (value === void 0) return;
+	if (value.toLowerCase() !== "parquet") throw new Error(`unsupported write.format.default: ${value}`);
+}
+function resolveParquetCodec(value) {
+	if (value === void 0) return void 0;
+	switch (value.toLowerCase()) {
+		case "snappy": return "SNAPPY";
+		case "none":
+		case "uncompressed": return "UNCOMPRESSED";
+		default: throw new Error(`unsupported write.parquet.compression-codec: ${value}`);
+	}
+}
+function newSnapshotId(metadata) {
+	const used = new Set((metadata?.snapshots ?? []).map((s) => BigInt(s["snapshot-id"])));
+	const arr = /* @__PURE__ */ new BigInt64Array(1);
+	for (let attempt = 0; attempt < 32; attempt++) {
+		globalThis.crypto.getRandomValues(arr);
+		const masked = arr[0] & 9007199254740991n;
+		const id = masked === 0n ? 1n : masked;
+		if (!used.has(id)) return id;
+	}
+	throw new Error("newSnapshotId: failed to find an unused id after 32 attempts");
+}
+export { checkWriteFormat, icebergStageAppend, icebergStageExpireSnapshots, icebergStageSetRef, icebergStageUpdateSchema, newSnapshotId, prepareAppend, resolveParquetCodec, stageSnapshotForAppend, stageSnapshotForAppendBatches };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/stats.mjs

@@ -0,0 +1,153 @@
+import { typeName } from "../schema.mjs";
+import { computeGeoBounds, isGeoType } from "./geospatial.mjs";
+import { compare, serializeValue } from "./serde.mjs";
+const TRUNCATE_LIMIT = 16;
+function computeColumnStats(records, schema) {
+	const value_counts = {};
+	const null_value_counts = {};
+	const nan_value_counts = {};
+	const lower_bounds = {};
+	const upper_bounds = {};
+	for (const field of schema.fields) {
+		const type = typeName(field.type);
+		if (type === "unknown") continue;
+		if (type === "list" || type === "map" || type === "struct") continue;
+		if (isGeoType(type)) {
+			const { value_count, null_count, lower, upper } = computeGeoBounds(records, field);
+			value_counts[field.id] = value_count;
+			null_value_counts[field.id] = null_count;
+			if (lower) lower_bounds[field.id] = lower;
+			if (upper) upper_bounds[field.id] = upper;
+			continue;
+		}
+		let nulls = 0n;
+		let nans = 0n;
+		let min;
+		let max;
+		const isFloat = type === "float" || type === "double";
+		const trackBounds = hasComparableBounds(field.type);
+		const writeDefault = field["write-default"];
+		for (const record of records) {
+			let v = record[field.name];
+			if (v === void 0 && writeDefault !== void 0) v = writeDefault;
+			if (v === null || v === void 0) {
+				nulls++;
+				continue;
+			}
+			if (isFloat && Number.isNaN(v)) {
+				nans++;
+				continue;
+			}
+			if (trackBounds) {
+				if (min === void 0 || compare(v, min, field.type) < 0) min = v;
+				if (max === void 0 || compare(v, max, field.type) > 0) max = v;
+			}
+		}
+		value_counts[field.id] = BigInt(records.length);
+		null_value_counts[field.id] = nulls;
+		if (isFloat) nan_value_counts[field.id] = nans;
+		if (min !== void 0) {
+			const lo = serializeValue(truncateLower(min, field.type), field.type);
+			if (lo) lower_bounds[field.id] = lo;
+		}
+		if (max !== void 0) {
+			const truncated = truncateUpper(max, field.type);
+			if (truncated !== void 0) {
+				const hi = serializeValue(truncated, field.type);
+				if (hi) upper_bounds[field.id] = hi;
+			}
+		}
+	}
+	return {
+		value_counts,
+		null_value_counts,
+		nan_value_counts,
+		lower_bounds,
+		upper_bounds
+	};
+}
+function hasComparableBounds(type) {
+	const name = typeName(type);
+	if (isGeoType(name)) return false;
+	return name !== "unknown" && name !== "variant";
+}
+function computeFieldSummary(values, type) {
+	const name = typeName(type);
+	const isFloat = name === "float" || name === "double";
+	const trackBounds = hasComparableBounds(type);
+	let containsNull = false;
+	let containsNan = false;
+	let min;
+	let max;
+	for (const v of values) {
+		if (v === null || v === void 0) {
+			containsNull = true;
+			continue;
+		}
+		if (isFloat && Number.isNaN(v)) {
+			containsNan = true;
+			continue;
+		}
+		if (trackBounds) {
+			if (min === void 0 || compare(v, min, type) < 0) min = v;
+			if (max === void 0 || compare(v, max, type) > 0) max = v;
+		}
+	}
+	const summary = { contains_null: containsNull };
+	if (isFloat) summary.contains_nan = containsNan;
+	if (min !== void 0) {
+		const lo = serializeValue(truncateLower(min, type), type);
+		if (lo) summary.lower_bound = lo;
+	}
+	if (max !== void 0) {
+		const truncated = truncateUpper(max, type);
+		if (truncated !== void 0) {
+			const hi = serializeValue(truncated, type);
+			if (hi) summary.upper_bound = hi;
+		}
+	}
+	return summary;
+}
+function truncateLower(value, type) {
+	const name = typeName(type);
+	if (name === "string" && typeof value === "string") {
+		const cps = Array.from(value);
+		if (cps.length <= TRUNCATE_LIMIT) return value;
+		return cps.slice(0, TRUNCATE_LIMIT).join("");
+	}
+	if ((name === "binary" || name.startsWith("fixed[")) && value instanceof Uint8Array) {
+		if (value.length <= TRUNCATE_LIMIT) return value;
+		return value.slice(0, TRUNCATE_LIMIT);
+	}
+	return value;
+}
+function truncateUpper(value, type) {
+	const name = typeName(type);
+	if (name === "string" && typeof value === "string") {
+		const cps = Array.from(value);
+		if (cps.length <= TRUNCATE_LIMIT) return value;
+		const prefix = cps.slice(0, TRUNCATE_LIMIT);
+		while (prefix.length > 0) {
+			const cp = prefix[prefix.length - 1].codePointAt(0);
+			const next = cp + 1 === 55296 ? 57344 : cp + 1;
+			if (next <= 1114111) {
+				prefix[prefix.length - 1] = String.fromCodePoint(next);
+				return prefix.join("");
+			}
+			prefix.pop();
+		}
+		return;
+	}
+	if ((name === "binary" || name.startsWith("fixed[")) && value instanceof Uint8Array) {
+		if (value.length <= TRUNCATE_LIMIT) return value;
+		const prefix = value.slice(0, TRUNCATE_LIMIT);
+		for (let i = prefix.length - 1; i >= 0; i--) if (prefix[i] < 255) {
+			const out = prefix.slice(0, i + 1);
+			out[i]++;
+			return out;
+		}
+		return;
+	}
+	return value;
+}
+export { computeColumnStats, computeFieldSummary };