@gscdump/lakehouse 1.4.0 → 1.4.1

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/conversions.mjs

@@ -0,0 +1,59 @@
+function parseDecimalType(type) {
+	const m = /^decimal\((\d+),\s*(\d+)\)$/.exec(type);
+	if (!m) return void 0;
+	return {
+		precision: parseInt(m[1], 10),
+		scale: parseInt(m[2], 10)
+	};
+}
+function decimalRequiredBytes(precision) {
+	const limit = 10n ** BigInt(precision);
+	let n = 1;
+	let bound = 128n;
+	while (limit > bound) {
+		n++;
+		bound <<= 8n;
+	}
+	return n;
+}
+function decimalToFixedBytes(value, precision, scale, label) {
+	const size = decimalRequiredBytes(precision);
+	if (value instanceof Uint8Array) {
+		if (value.length !== size) throw new Error(`expected ${label}`);
+		return value;
+	}
+	if (typeof value !== "number" && typeof value !== "bigint") throw new Error(`expected ${label}`);
+	const factor = 10n ** BigInt(scale);
+	const unscaled = typeof value === "bigint" ? value * factor : BigInt(Math.round(value * Number(factor)));
+	const limit = 10n ** BigInt(precision);
+	if (unscaled >= limit || unscaled <= -limit) throw new Error(`${label} exceeds precision ${precision}`);
+	return bigintToFixedBytes(unscaled, size, label);
+}
+function toUint8Array(value) {
+	return value instanceof Uint8Array ? value : new Uint8Array(value);
+}
+function uuidToBytes(value, label) {
+	if (value instanceof Uint8Array) {
+		if (value.length !== 16) throw new Error(`expected ${label}`);
+		return value;
+	}
+	if (typeof value !== "string") throw new Error(`expected ${label}`);
+	const hex = value.toLowerCase().replace(/-/g, "");
+	if (!/^[0-9a-f]{32}$/.test(hex)) throw new Error(`expected ${label}`);
+	const bytes = /* @__PURE__ */ new Uint8Array(16);
+	for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
+	return bytes;
+}
+function bigintToFixedBytes(value, size, label) {
+	const bytes = new Uint8Array(size);
+	let v = value;
+	for (let i = size - 1; i >= 0; i--) {
+		bytes[i] = Number(v & 255n);
+		v >>= 8n;
+	}
+	const negative = value < 0n;
+	const signBitSet = (bytes[0] & 128) !== 0;
+	if (!negative && (v !== 0n || signBitSet) || negative && (v !== -1n || !signBitSet)) throw new Error(`${label} does not fit in ${size} bytes`);
+	return bytes;
+}
+export { decimalRequiredBytes, decimalToFixedBytes, parseDecimalType, toUint8Array, uuidToBytes };

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/delete-file.mjs

@@ -0,0 +1,3 @@
+import "./stats.mjs";
+import "./parquet.mjs";
+export {};

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/geospatial.mjs

@@ -0,0 +1,72 @@
+function isGeoType(name) {
+	return name.startsWith("geometry") || name.startsWith("geography");
+}
+function computeGeoBounds(records, field) {
+	let partial;
+	let nulls = 0n;
+	const writeDefault = field["write-default"];
+	for (const record of records) {
+		let v = record[field.name];
+		if (v === void 0 && writeDefault !== void 0) v = writeDefault;
+		if (v === null || v === void 0) {
+			nulls++;
+			continue;
+		}
+		if (typeof v !== "object") throw new Error("geospatial column expects GeoJSON geometries");
+		partial = extendBoundsFromGeometry(partial, v);
+	}
+	const result = {
+		value_count: BigInt(records.length),
+		null_count: nulls
+	};
+	const { xmin, ymin, xmax, ymax, zmin, zmax, mmin, mmax } = partial ?? {};
+	if (xmin === void 0 || ymin === void 0 || xmax === void 0 || ymax === void 0) return result;
+	const hasZ = zmin !== void 0;
+	const hasM = mmin !== void 0;
+	return {
+		...result,
+		lower: encodeGeoPoint(xmin, ymin, zmin, mmin, hasZ, hasM),
+		upper: encodeGeoPoint(xmax, ymax, zmax, mmax, hasZ, hasM)
+	};
+}
+function extendBoundsFromGeometry(bbox, geometry) {
+	if (geometry.type === "GeometryCollection") {
+		for (const child of geometry.geometries || []) bbox = extendBoundsFromGeometry(bbox, child);
+		return bbox;
+	}
+	return extendBoundsFromCoordinates(bbox, geometry.coordinates);
+}
+function extendBoundsFromCoordinates(bbox, coordinates) {
+	if (typeof coordinates[0] === "number") {
+		bbox = updateAxis(bbox, "xmin", "xmax", coordinates[0]);
+		bbox = updateAxis(bbox, "ymin", "ymax", coordinates[1]);
+		if (coordinates.length > 2) bbox = updateAxis(bbox, "zmin", "zmax", coordinates[2]);
+		if (coordinates.length > 3) bbox = updateAxis(bbox, "mmin", "mmax", coordinates[3]);
+		return bbox;
+	}
+	for (const child of coordinates) bbox = extendBoundsFromCoordinates(bbox, child);
+	return bbox;
+}
+function updateAxis(bbox, minKey, maxKey, value) {
+	if (value === void 0 || !Number.isFinite(value)) return bbox;
+	if (!bbox) bbox = {};
+	const min = bbox[minKey];
+	const max = bbox[maxKey];
+	if (min === void 0 || value < min) bbox[minKey] = value;
+	if (max === void 0 || value > max) bbox[maxKey] = value;
+	return bbox;
+}
+function encodeGeoPoint(x, y, z, m, hasZ, hasM) {
+	const len = !hasZ && !hasM ? 16 : hasZ && !hasM ? 24 : 32;
+	const buf = new ArrayBuffer(len);
+	const view = new DataView(buf);
+	view.setFloat64(0, x, true);
+	view.setFloat64(8, y, true);
+	if (len === 24) view.setFloat64(16, z, true);
+	else if (len === 32) {
+		view.setFloat64(16, hasZ ? z : NaN, true);
+		view.setFloat64(24, m, true);
+	}
+	return new Uint8Array(buf);
+}
+export { computeGeoBounds, isGeoType };

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/manifest-list.mjs

@@ -0,0 +1,152 @@
+import { avroWrite } from "../avro/avro.write.mjs";
+function manifestFileSchema(formatVersion) {
+	const fields = [
+		{
+			name: "manifest_path",
+			type: "string",
+			"field-id": 500
+		},
+		{
+			name: "manifest_length",
+			type: "long",
+			"field-id": 501
+		},
+		{
+			name: "partition_spec_id",
+			type: "int",
+			"field-id": 502
+		},
+		{
+			name: "content",
+			type: "int",
+			"field-id": 517
+		},
+		{
+			name: "sequence_number",
+			type: "long",
+			"field-id": 515
+		},
+		{
+			name: "min_sequence_number",
+			type: "long",
+			"field-id": 516
+		},
+		{
+			name: "added_snapshot_id",
+			type: "long",
+			"field-id": 503
+		},
+		{
+			name: "added_files_count",
+			type: "int",
+			"field-id": 504
+		},
+		{
+			name: "existing_files_count",
+			type: "int",
+			"field-id": 505
+		},
+		{
+			name: "deleted_files_count",
+			type: "int",
+			"field-id": 506
+		},
+		{
+			name: "added_rows_count",
+			type: "long",
+			"field-id": 512
+		},
+		{
+			name: "existing_rows_count",
+			type: "long",
+			"field-id": 513
+		},
+		{
+			name: "deleted_rows_count",
+			type: "long",
+			"field-id": 514
+		},
+		{
+			name: "partitions",
+			type: ["null", {
+				type: "array",
+				"element-id": 508,
+				items: {
+					type: "record",
+					name: "r508",
+					fields: [
+						{
+							name: "contains_null",
+							type: "boolean",
+							"field-id": 509
+						},
+						{
+							name: "contains_nan",
+							type: ["null", "boolean"],
+							default: null,
+							"field-id": 518
+						},
+						{
+							name: "lower_bound",
+							type: ["null", "bytes"],
+							default: null,
+							"field-id": 510
+						},
+						{
+							name: "upper_bound",
+							type: ["null", "bytes"],
+							default: null,
+							"field-id": 511
+						}
+					]
+				}
+			}],
+			default: null,
+			"field-id": 507
+		}
+	];
+	if (formatVersion >= 3) fields.push({
+		name: "first_row_id",
+		type: ["null", "long"],
+		default: null,
+		"field-id": 520
+	});
+	return {
+		type: "record",
+		name: "manifest_file",
+		fields
+	};
+}
+function writeManifestList({ writer, snapshotId, sequenceNumber, manifests, formatVersion = 2 }) {
+	const records = manifests.map((m) => {
+		const record = {
+			manifest_path: m.manifest_path,
+			manifest_length: m.manifest_length,
+			partition_spec_id: m.partition_spec_id,
+			content: m.content,
+			sequence_number: m.sequence_number ?? sequenceNumber,
+			min_sequence_number: m.min_sequence_number ?? sequenceNumber,
+			added_snapshot_id: m.added_snapshot_id,
+			added_files_count: m.added_files_count,
+			existing_files_count: m.existing_files_count,
+			deleted_files_count: m.deleted_files_count,
+			added_rows_count: m.added_rows_count,
+			existing_rows_count: m.existing_rows_count,
+			deleted_rows_count: m.deleted_rows_count,
+			partitions: m.partitions ?? null
+		};
+		if (formatVersion >= 3) record.first_row_id = m.content === 0 ? m.first_row_id ?? null : null;
+		return record;
+	});
+	return avroWrite({
+		writer,
+		schema: manifestFileSchema(formatVersion),
+		records,
+		metadata: {
+			"format-version": String(formatVersion),
+			"snapshot-id": String(snapshotId),
+			"sequence-number": String(sequenceNumber)
+		}
+	});
+}
+export { writeManifestList };

package/dist/node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/manifest.mjs

@@ -0,0 +1,212 @@
+import { partitionAvroSchema, partitionSpecJson, partitionToAvroRecord } from "./partition.mjs";
+import { avroWrite } from "../avro/avro.write.mjs";
+function manifestEntrySchema(schema, partitionSpec, formatVersion, manifestContent = 0) {
+	const dataFileFields = [
+		{
+			name: "content",
+			type: "int",
+			"field-id": 134
+		},
+		{
+			name: "file_path",
+			type: "string",
+			"field-id": 100
+		},
+		{
+			name: "file_format",
+			type: "string",
+			"field-id": 101
+		},
+		{
+			name: "partition",
+			"field-id": 102,
+			type: partitionAvroSchema(schema, partitionSpec)
+		},
+		{
+			name: "record_count",
+			type: "long",
+			"field-id": 103
+		},
+		{
+			name: "file_size_in_bytes",
+			type: "long",
+			"field-id": 104
+		},
+		mapField("column_sizes", 108, "k117_v118", 117, 118, "long"),
+		mapField("value_counts", 109, "k119_v120", 119, 120, "long"),
+		mapField("null_value_counts", 110, "k121_v122", 121, 122, "long"),
+		mapField("nan_value_counts", 137, "k138_v139", 138, 139, "long"),
+		mapField("lower_bounds", 125, "k126_v127", 126, 127, "bytes"),
+		mapField("upper_bounds", 128, "k129_v130", 129, 130, "bytes"),
+		{
+			name: "sort_order_id",
+			type: ["null", "int"],
+			default: null,
+			"field-id": 140
+		}
+	];
+	if (manifestContent === 1) {
+		dataFileFields.push({
+			name: "equality_ids",
+			"field-id": 135,
+			default: null,
+			type: ["null", {
+				type: "array",
+				items: "int",
+				"element-id": 136
+			}]
+		});
+		dataFileFields.push({
+			name: "referenced_data_file",
+			type: ["null", "string"],
+			default: null,
+			"field-id": 143
+		});
+		if (formatVersion >= 3) {
+			dataFileFields.push({
+				name: "content_offset",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 144
+			});
+			dataFileFields.push({
+				name: "content_size_in_bytes",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 145
+			});
+		}
+	}
+	if (formatVersion >= 3) dataFileFields.push({
+		name: "first_row_id",
+		type: ["null", "long"],
+		default: null,
+		"field-id": 142
+	});
+	return {
+		type: "record",
+		name: "manifest_entry",
+		fields: [
+			{
+				name: "status",
+				type: "int",
+				"field-id": 0
+			},
+			{
+				name: "snapshot_id",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 1
+			},
+			{
+				name: "sequence_number",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 3
+			},
+			{
+				name: "file_sequence_number",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 4
+			},
+			{
+				name: "data_file",
+				"field-id": 2,
+				type: {
+					type: "record",
+					name: "r2",
+					fields: dataFileFields
+				}
+			}
+		]
+	};
+}
+function mapField(name, fieldId, recName, keyId, valueId, valueType) {
+	return {
+		name,
+		"field-id": fieldId,
+		default: null,
+		type: ["null", {
+			type: "array",
+			logicalType: "map",
+			items: {
+				type: "record",
+				name: recName,
+				fields: [{
+					name: "key",
+					type: "int",
+					"field-id": keyId
+				}, {
+					name: "value",
+					type: valueType,
+					"field-id": valueId
+				}]
+			}
+		}]
+	};
+}
+function icebergSchemaJson(schema) {
+	return JSON.stringify(schema);
+}
+function encodeMap(m) {
+	if (!m) return null;
+	const entries = Object.entries(m);
+	if (!entries.length) return null;
+	return entries.map(([k, value]) => ({
+		key: Number(k),
+		value
+	}));
+}
+function writeDataManifest({ writer, schema, partitionSpec, snapshotId, dataFiles, formatVersion = 2 }) {
+	const records = dataFiles.map((dataFile) => {
+		if (dataFile.content !== 0) throw new Error(`writeDataManifest expects data files (content=0), got content=${dataFile.content}`);
+		return manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, formatVersion, 0);
+	});
+	return avroWrite({
+		writer,
+		schema: manifestEntrySchema(schema, partitionSpec, formatVersion, 0),
+		records,
+		metadata: {
+			"format-version": String(formatVersion),
+			content: "data",
+			schema: icebergSchemaJson(schema),
+			"partition-spec": partitionSpecJson(partitionSpec),
+			"partition-spec-id": String(partitionSpec["spec-id"])
+		}
+	});
+}
+function manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, formatVersion, manifestContent) {
+	const dataFileRecord = {
+		content: dataFile.content,
+		file_path: dataFile.file_path,
+		file_format: dataFile.file_format.toUpperCase(),
+		partition: partitionToAvroRecord(dataFile.partition ?? {}, schema, partitionSpec),
+		record_count: dataFile.record_count,
+		file_size_in_bytes: dataFile.file_size_in_bytes,
+		column_sizes: encodeMap(dataFile.column_sizes),
+		value_counts: encodeMap(dataFile.value_counts),
+		null_value_counts: encodeMap(dataFile.null_value_counts),
+		nan_value_counts: encodeMap(dataFile.nan_value_counts),
+		lower_bounds: encodeMap(dataFile.lower_bounds),
+		upper_bounds: encodeMap(dataFile.upper_bounds),
+		sort_order_id: dataFile.content === 1 ? null : dataFile.sort_order_id ?? 0
+	};
+	if (manifestContent === 1) {
+		dataFileRecord.equality_ids = dataFile.equality_ids?.length ? dataFile.equality_ids : null;
+		dataFileRecord.referenced_data_file = dataFile.referenced_data_file ?? null;
+		if (formatVersion >= 3) {
+			dataFileRecord.content_offset = dataFile.content_offset ?? null;
+			dataFileRecord.content_size_in_bytes = dataFile.content_size_in_bytes ?? null;
+		}
+	}
+	if (formatVersion >= 3) dataFileRecord.first_row_id = dataFile.first_row_id ?? null;
+	return {
+		status: 1,
+		snapshot_id: snapshotId,
+		sequence_number: null,
+		file_sequence_number: null,
+		data_file: dataFileRecord
+	};
+}
+export { writeDataManifest };