@gscdump/lakehouse 1.4.0 → 1.4.2

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/manifest-list.mjs

@@ -0,0 +1,152 @@
+import { avroWrite } from "../avro/avro.write.mjs";
+function manifestFileSchema(formatVersion) {
+	const fields = [
+		{
+			name: "manifest_path",
+			type: "string",
+			"field-id": 500
+		},
+		{
+			name: "manifest_length",
+			type: "long",
+			"field-id": 501
+		},
+		{
+			name: "partition_spec_id",
+			type: "int",
+			"field-id": 502
+		},
+		{
+			name: "content",
+			type: "int",
+			"field-id": 517
+		},
+		{
+			name: "sequence_number",
+			type: "long",
+			"field-id": 515
+		},
+		{
+			name: "min_sequence_number",
+			type: "long",
+			"field-id": 516
+		},
+		{
+			name: "added_snapshot_id",
+			type: "long",
+			"field-id": 503
+		},
+		{
+			name: "added_files_count",
+			type: "int",
+			"field-id": 504
+		},
+		{
+			name: "existing_files_count",
+			type: "int",
+			"field-id": 505
+		},
+		{
+			name: "deleted_files_count",
+			type: "int",
+			"field-id": 506
+		},
+		{
+			name: "added_rows_count",
+			type: "long",
+			"field-id": 512
+		},
+		{
+			name: "existing_rows_count",
+			type: "long",
+			"field-id": 513
+		},
+		{
+			name: "deleted_rows_count",
+			type: "long",
+			"field-id": 514
+		},
+		{
+			name: "partitions",
+			type: ["null", {
+				type: "array",
+				"element-id": 508,
+				items: {
+					type: "record",
+					name: "r508",
+					fields: [
+						{
+							name: "contains_null",
+							type: "boolean",
+							"field-id": 509
+						},
+						{
+							name: "contains_nan",
+							type: ["null", "boolean"],
+							default: null,
+							"field-id": 518
+						},
+						{
+							name: "lower_bound",
+							type: ["null", "bytes"],
+							default: null,
+							"field-id": 510
+						},
+						{
+							name: "upper_bound",
+							type: ["null", "bytes"],
+							default: null,
+							"field-id": 511
+						}
+					]
+				}
+			}],
+			default: null,
+			"field-id": 507
+		}
+	];
+	if (formatVersion >= 3) fields.push({
+		name: "first_row_id",
+		type: ["null", "long"],
+		default: null,
+		"field-id": 520
+	});
+	return {
+		type: "record",
+		name: "manifest_file",
+		fields
+	};
+}
+function writeManifestList({ writer, snapshotId, sequenceNumber, manifests, formatVersion = 2 }) {
+	const records = manifests.map((m) => {
+		const record = {
+			manifest_path: m.manifest_path,
+			manifest_length: m.manifest_length,
+			partition_spec_id: m.partition_spec_id,
+			content: m.content,
+			sequence_number: m.sequence_number ?? sequenceNumber,
+			min_sequence_number: m.min_sequence_number ?? sequenceNumber,
+			added_snapshot_id: m.added_snapshot_id,
+			added_files_count: m.added_files_count,
+			existing_files_count: m.existing_files_count,
+			deleted_files_count: m.deleted_files_count,
+			added_rows_count: m.added_rows_count,
+			existing_rows_count: m.existing_rows_count,
+			deleted_rows_count: m.deleted_rows_count,
+			partitions: m.partitions ?? null
+		};
+		if (formatVersion >= 3) record.first_row_id = m.content === 0 ? m.first_row_id ?? null : null;
+		return record;
+	});
+	return avroWrite({
+		writer,
+		schema: manifestFileSchema(formatVersion),
+		records,
+		metadata: {
+			"format-version": String(formatVersion),
+			"snapshot-id": String(snapshotId),
+			"sequence-number": String(sequenceNumber)
+		}
+	});
+}
+export { writeManifestList };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/manifest.mjs

@@ -0,0 +1,256 @@
+import { partitionAvroSchema, partitionSpecJson, partitionToAvroRecord } from "./partition.mjs";
+import { avroWrite } from "../avro/avro.write.mjs";
+function manifestEntrySchema(schema, partitionSpec, formatVersion, manifestContent = 0) {
+	const dataFileFields = [
+		{
+			name: "content",
+			type: "int",
+			"field-id": 134
+		},
+		{
+			name: "file_path",
+			type: "string",
+			"field-id": 100
+		},
+		{
+			name: "file_format",
+			type: "string",
+			"field-id": 101
+		},
+		{
+			name: "partition",
+			"field-id": 102,
+			type: partitionAvroSchema(schema, partitionSpec)
+		},
+		{
+			name: "record_count",
+			type: "long",
+			"field-id": 103
+		},
+		{
+			name: "file_size_in_bytes",
+			type: "long",
+			"field-id": 104
+		},
+		mapField("column_sizes", 108, "k117_v118", 117, 118, "long"),
+		mapField("value_counts", 109, "k119_v120", 119, 120, "long"),
+		mapField("null_value_counts", 110, "k121_v122", 121, 122, "long"),
+		mapField("nan_value_counts", 137, "k138_v139", 138, 139, "long"),
+		mapField("lower_bounds", 125, "k126_v127", 126, 127, "bytes"),
+		mapField("upper_bounds", 128, "k129_v130", 129, 130, "bytes"),
+		{
+			name: "sort_order_id",
+			type: ["null", "int"],
+			default: null,
+			"field-id": 140
+		}
+	];
+	if (manifestContent === 1) {
+		dataFileFields.push({
+			name: "equality_ids",
+			"field-id": 135,
+			default: null,
+			type: ["null", {
+				type: "array",
+				items: "int",
+				"element-id": 136
+			}]
+		});
+		dataFileFields.push({
+			name: "referenced_data_file",
+			type: ["null", "string"],
+			default: null,
+			"field-id": 143
+		});
+		if (formatVersion >= 3) {
+			dataFileFields.push({
+				name: "content_offset",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 144
+			});
+			dataFileFields.push({
+				name: "content_size_in_bytes",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 145
+			});
+		}
+	}
+	if (formatVersion >= 3) dataFileFields.push({
+		name: "first_row_id",
+		type: ["null", "long"],
+		default: null,
+		"field-id": 142
+	});
+	return {
+		type: "record",
+		name: "manifest_entry",
+		fields: [
+			{
+				name: "status",
+				type: "int",
+				"field-id": 0
+			},
+			{
+				name: "snapshot_id",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 1
+			},
+			{
+				name: "sequence_number",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 3
+			},
+			{
+				name: "file_sequence_number",
+				type: ["null", "long"],
+				default: null,
+				"field-id": 4
+			},
+			{
+				name: "data_file",
+				"field-id": 2,
+				type: {
+					type: "record",
+					name: "r2",
+					fields: dataFileFields
+				}
+			}
+		]
+	};
+}
+function mapField(name, fieldId, recName, keyId, valueId, valueType) {
+	return {
+		name,
+		"field-id": fieldId,
+		default: null,
+		type: ["null", {
+			type: "array",
+			logicalType: "map",
+			items: {
+				type: "record",
+				name: recName,
+				fields: [{
+					name: "key",
+					type: "int",
+					"field-id": keyId
+				}, {
+					name: "value",
+					type: valueType,
+					"field-id": valueId
+				}]
+			}
+		}]
+	};
+}
+function icebergSchemaJson(schema) {
+	return JSON.stringify(schema);
+}
+function encodeMap(m) {
+	if (!m) return null;
+	const entries = Object.entries(m);
+	if (!entries.length) return null;
+	return entries.map(([k, value]) => ({
+		key: Number(k),
+		value
+	}));
+}
+function writeDataManifest({ writer, schema, partitionSpec, snapshotId, dataFiles, formatVersion = 2 }) {
+	const records = dataFiles.map((dataFile) => {
+		if (dataFile.content !== 0) throw new Error(`writeDataManifest expects data files (content=0), got content=${dataFile.content}`);
+		return manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, formatVersion, 0);
+	});
+	return avroWrite({
+		writer,
+		schema: manifestEntrySchema(schema, partitionSpec, formatVersion, 0),
+		records,
+		metadata: {
+			"format-version": String(formatVersion),
+			content: "data",
+			schema: icebergSchemaJson(schema),
+			"partition-spec": partitionSpecJson(partitionSpec),
+			"partition-spec-id": String(partitionSpec["spec-id"])
+		}
+	});
+}
+function writeDeleteManifest({ writer, schema, partitionSpec, snapshotId, deleteFiles, formatVersion = 2 }) {
+	const records = deleteFiles.map((deleteFile) => {
+		if (deleteFile.content !== 1 && deleteFile.content !== 2) throw new Error(`writeDeleteManifest expects delete files (content=1 or 2), got content=${deleteFile.content}`);
+		if (deleteFile.content === 2 && !deleteFile.equality_ids?.length) throw new Error("equality delete file missing equality_ids");
+		return manifestEntryRecord(deleteFile, schema, partitionSpec, snapshotId, formatVersion, 1);
+	});
+	return avroWrite({
+		writer,
+		schema: manifestEntrySchema(schema, partitionSpec, formatVersion, 1),
+		records,
+		metadata: {
+			"format-version": String(formatVersion),
+			content: "deletes",
+			schema: icebergSchemaJson(schema),
+			"partition-spec": partitionSpecJson(partitionSpec),
+			"partition-spec-id": String(partitionSpec["spec-id"])
+		}
+	});
+}
+function writeExistingDeleteManifest({ writer, schema, partitionSpec, entries, formatVersion = 2 }) {
+	const records = entries.map((entry) => {
+		const deleteFile = entry.data_file;
+		if (deleteFile.content !== 1 && deleteFile.content !== 2) throw new Error(`writeExistingDeleteManifest expects delete files (content=1 or 2), got content=${deleteFile.content}`);
+		const record = manifestEntryRecord(deleteFile, schema, partitionSpec, 0n, formatVersion, 1);
+		record.status = 0;
+		record.snapshot_id = entry.snapshot_id ?? null;
+		record.sequence_number = entry.sequence_number ?? null;
+		record.file_sequence_number = entry.file_sequence_number ?? null;
+		if (record.sequence_number == null || record.file_sequence_number == null) throw new Error("existing delete manifest entry missing sequence numbers");
+		return record;
+	});
+	return avroWrite({
+		writer,
+		schema: manifestEntrySchema(schema, partitionSpec, formatVersion, 1),
+		records,
+		metadata: {
+			"format-version": String(formatVersion),
+			content: "deletes",
+			schema: icebergSchemaJson(schema),
+			"partition-spec": partitionSpecJson(partitionSpec),
+			"partition-spec-id": String(partitionSpec["spec-id"])
+		}
+	});
+}
+function manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, formatVersion, manifestContent) {
+	const dataFileRecord = {
+		content: dataFile.content,
+		file_path: dataFile.file_path,
+		file_format: dataFile.file_format.toUpperCase(),
+		partition: partitionToAvroRecord(dataFile.partition ?? {}, schema, partitionSpec),
+		record_count: dataFile.record_count,
+		file_size_in_bytes: dataFile.file_size_in_bytes,
+		column_sizes: encodeMap(dataFile.column_sizes),
+		value_counts: encodeMap(dataFile.value_counts),
+		null_value_counts: encodeMap(dataFile.null_value_counts),
+		nan_value_counts: encodeMap(dataFile.nan_value_counts),
+		lower_bounds: encodeMap(dataFile.lower_bounds),
+		upper_bounds: encodeMap(dataFile.upper_bounds),
+		sort_order_id: dataFile.content === 1 ? null : dataFile.sort_order_id ?? 0
+	};
+	if (manifestContent === 1) {
+		dataFileRecord.equality_ids = dataFile.equality_ids?.length ? dataFile.equality_ids : null;
+		dataFileRecord.referenced_data_file = dataFile.referenced_data_file ?? null;
+		if (formatVersion >= 3) {
+			dataFileRecord.content_offset = dataFile.content_offset ?? null;
+			dataFileRecord.content_size_in_bytes = dataFile.content_size_in_bytes ?? null;
+		}
+	}
+	if (formatVersion >= 3) dataFileRecord.first_row_id = dataFile.first_row_id ?? null;
+	return {
+		status: 1,
+		snapshot_id: snapshotId,
+		sequence_number: null,
+		file_sequence_number: null,
+		data_file: dataFileRecord
+	};
+}
+export { writeDataManifest, writeDeleteManifest, writeExistingDeleteManifest };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/parquet.mjs

@@ -0,0 +1,305 @@
+import { parquetWrite } from "../../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/write.mjs";
+import "../../../../../hyparquet-writer@0.16.1_patch_hash=7a5e451ac7634d7546ac56bcc5eadacf7715a6241640c2b5a5bc0cfc9763b8ec/node_modules/hyparquet-writer/src/node.mjs";
+import { sanitize } from "../utils.mjs";
+import { typeName } from "../schema.mjs";
+import { decimalRequiredBytes, parseDecimalType } from "./conversions.mjs";
+function writeParquet({ writer, schema, records, codec }) {
+	const columnData = [];
+	const parquetFields = [];
+	let rootChildren = 0;
+	for (const field of schema.fields) {
+		const name = sanitize(field.name);
+		const fieldElements = icebergTypeToParquetFields(name, field.type, field.required, field.id);
+		if (!fieldElements.length) continue;
+		columnData.push({
+			name,
+			data: extractColumn(records, field)
+		});
+		parquetFields.push(...fieldElements);
+		rootChildren++;
+	}
+	return parquetWrite({
+		writer,
+		columnData,
+		schema: [{
+			name: "root",
+			num_children: rootChildren
+		}, ...parquetFields],
+		kvMetadata: [{
+			key: "iceberg.schema",
+			value: JSON.stringify(schema)
+		}],
+		codec
+	});
+}
+function extractColumn(records, field) {
+	const out = new Array(records.length);
+	for (let i = 0; i < records.length; i++) out[i] = materializeFieldValue(records[i][field.name], field);
+	return out;
+}
+function materializeFieldValue(value, field) {
+	const writeDefault = field["write-default"];
+	return materializeNestedDefaults(value !== void 0 ? value : writeDefault !== void 0 ? writeDefault : null, field.type);
+}
+function materializeNestedDefaults(value, type) {
+	if (value === null || value === void 0 || typeof type !== "object") return value;
+	if (type.type === "struct") {
+		if (typeof value !== "object" || Array.isArray(value)) return value;
+		const out = { ...value };
+		for (const child of type.fields) out[child.name] = materializeFieldValue(value[child.name], child);
+		return out;
+	}
+	if (type.type === "list") {
+		if (!Array.isArray(value)) return value;
+		return value.map((v) => materializeNestedDefaults(v, type.element));
+	}
+	if (type.type === "map") return materializeMapDefaults(value, type);
+	return value;
+}
+function materializeMapDefaults(value, type) {
+	if (typeof type.key !== "object" && typeof type.value !== "object") return value;
+	if (value instanceof Map) return Array.from(value.entries(), ([key, entryValue]) => ({
+		key: materializeNestedDefaults(key, type.key),
+		value: materializeNestedDefaults(entryValue, type.value)
+	}));
+	if (Array.isArray(value)) return value.map((entry) => {
+		if (entry && typeof entry === "object" && "key" in entry && "value" in entry) return {
+			key: materializeNestedDefaults(entry.key, type.key),
+			value: materializeNestedDefaults(entry.value, type.value)
+		};
+		if (Array.isArray(entry) && entry.length === 2) return {
+			key: materializeNestedDefaults(entry[0], type.key),
+			value: materializeNestedDefaults(entry[1], type.value)
+		};
+		return entry;
+	});
+	if (typeof value === "object") return Object.fromEntries(Object.entries(value).map(([key, entryValue]) => [key, materializeNestedDefaults(entryValue, type.value)]));
+	return value;
+}
+function icebergTypeToParquetFields(name, type, required, fieldId) {
+	const repetition_type = required ? "REQUIRED" : "OPTIONAL";
+	if (typeof type === "object") {
+		if (type.type === "list") {
+			const elementFields = icebergTypeToParquetFields("element", type.element, type["element-required"], type["element-id"]);
+			if (!elementFields.length) throw new Error(`unsupported iceberg list element type: ${typeName(type.element)}`);
+			return [
+				{
+					name,
+					converted_type: "LIST",
+					logical_type: { type: "LIST" },
+					repetition_type,
+					num_children: 1,
+					field_id: fieldId
+				},
+				{
+					name: "list",
+					repetition_type: "REPEATED",
+					num_children: 1
+				},
+				...elementFields
+			];
+		}
+		if (type.type === "struct") {
+			const allChildren = [];
+			let directChildren = 0;
+			for (const child of type.fields) {
+				const sub = icebergTypeToParquetFields(child.name, child.type, child.required, child.id);
+				if (!sub.length) continue;
+				allChildren.push(...sub);
+				directChildren++;
+			}
+			if (!directChildren) throw new Error(`struct ${name} has no writable children`);
+			return [{
+				name,
+				repetition_type,
+				num_children: directChildren,
+				field_id: fieldId
+			}, ...allChildren];
+		}
+		if (type.type === "map") {
+			if (type.key !== "string" && type.key !== "int") throw new Error(`unsupported iceberg map key type: ${typeName(type.key)}`);
+			const keyFields = icebergTypeToParquetFields("key", type.key, true, type["key-id"]);
+			const valueFields = icebergTypeToParquetFields("value", type.value, type["value-required"], type["value-id"]);
+			if (!keyFields.length) throw new Error(`unsupported iceberg map key type: ${typeName(type.key)}`);
+			if (!valueFields.length) throw new Error(`unsupported iceberg map value type: ${typeName(type.value)}`);
+			return [
+				{
+					name,
+					converted_type: "MAP",
+					logical_type: { type: "MAP" },
+					repetition_type,
+					num_children: 1,
+					field_id: fieldId
+				},
+				{
+					name: "key_value",
+					repetition_type: "REPEATED",
+					num_children: 2
+				},
+				...keyFields,
+				...valueFields
+			];
+		}
+		throw new Error(`unsupported iceberg nested type: ${JSON.stringify(type)}`);
+	}
+	if (type.startsWith("geometry")) return [{
+		name,
+		type: "BYTE_ARRAY",
+		logical_type: { type: "GEOMETRY" },
+		repetition_type,
+		field_id: fieldId
+	}];
+	if (type.startsWith("geography")) return [{
+		name,
+		type: "BYTE_ARRAY",
+		logical_type: { type: "GEOGRAPHY" },
+		repetition_type,
+		field_id: fieldId
+	}];
+	const decimal = parseDecimalType(type);
+	if (decimal) {
+		const { precision, scale } = decimal;
+		return [{
+			name,
+			type: "FIXED_LEN_BYTE_ARRAY",
+			type_length: decimalRequiredBytes(precision),
+			converted_type: "DECIMAL",
+			logical_type: {
+				type: "DECIMAL",
+				precision,
+				scale
+			},
+			precision,
+			scale,
+			repetition_type,
+			field_id: fieldId
+		}];
+	}
+	const fixedLen = parseFixedType(type);
+	if (fixedLen !== void 0) return [{
+		name,
+		type: "FIXED_LEN_BYTE_ARRAY",
+		type_length: fixedLen,
+		repetition_type,
+		field_id: fieldId
+	}];
+	switch (type) {
+		case "unknown":
+			if (required) throw new Error("unsupported required iceberg type: unknown");
+			return [];
+		case "variant": return [
+			{
+				name,
+				repetition_type,
+				num_children: 2,
+				logical_type: { type: "VARIANT" },
+				field_id: fieldId
+			},
+			{
+				name: "metadata",
+				type: "BYTE_ARRAY",
+				repetition_type: "REQUIRED"
+			},
+			{
+				name: "value",
+				type: "BYTE_ARRAY",
+				repetition_type: "OPTIONAL"
+			}
+		];
+		case "boolean": return [{
+			name,
+			type: "BOOLEAN",
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "int": return [{
+			name,
+			type: "INT32",
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "long": return [{
+			name,
+			type: "INT64",
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "float": return [{
+			name,
+			type: "FLOAT",
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "double": return [{
+			name,
+			type: "DOUBLE",
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "string": return [{
+			name,
+			type: "BYTE_ARRAY",
+			converted_type: "UTF8",
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "binary": return [{
+			name,
+			type: "BYTE_ARRAY",
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "uuid": return [{
+			name,
+			type: "FIXED_LEN_BYTE_ARRAY",
+			type_length: 16,
+			logical_type: { type: "UUID" },
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "date": return [{
+			name,
+			type: "INT32",
+			converted_type: "DATE",
+			logical_type: { type: "DATE" },
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "time": return [{
+			name,
+			type: "INT64",
+			converted_type: "TIME_MICROS",
+			logical_type: {
+				type: "TIME",
+				isAdjustedToUTC: false,
+				unit: "MICROS"
+			},
+			repetition_type,
+			field_id: fieldId
+		}];
+		case "timestamp": return [timestampField(name, repetition_type, false, "MICROS", fieldId)];
+		case "timestamptz": return [timestampField(name, repetition_type, true, "MICROS", fieldId)];
+		case "timestamp_ns": return [timestampField(name, repetition_type, false, "NANOS", fieldId)];
+		case "timestamptz_ns": return [timestampField(name, repetition_type, true, "NANOS", fieldId)];
+		default: throw new Error(`unsupported iceberg type: ${type}`);
+	}
+}
+function parseFixedType(type) {
+	const m = /^fixed\[(\d+)\]$/.exec(type);
+	if (!m) return void 0;
+	return parseInt(m[1], 10);
+}
+function timestampField(name, repetition_type, isAdjustedToUTC, unit, field_id) {
+	return {
+		name,
+		type: "INT64",
+		logical_type: {
+			type: "TIMESTAMP",
+			isAdjustedToUTC,
+			unit
+		},
+		repetition_type,
+		field_id
+	};
+}
+export { writeParquet };