@gscdump/lakehouse 3.4.0 → 3.4.2

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/index.mjs

@@ -4,7 +4,7 @@ import { icebergManifests } from "./manifest.mjs";
 import { s3SignedResolver } from "./s3.mjs";
 import "./metadata.mjs";
 import "./create.mjs";
-import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
+import "../../../../squirreling@0.16.3/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
 import "./catalog/file.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/manifest.mjs

@@ -1,4 +1,7 @@
+import { avroMetadata } from "./avro/avro.metadata.mjs";
+import { avroRead } from "./avro/avro.read.mjs";
 import { fetchAvroRecords, urlResolver } from "./fetch.mjs";
+const MANIFEST_FETCH_CONCURRENCY = 8;
 async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }) {
 	resolver ??= urlResolver();
 	const rawTarget = snapshotId ?? metadata["current-snapshot-id"];
@@ -10,20 +13,60 @@ async function icebergManifests({ metadata, resolver, snapshotId, partitionFilte
 	if (snapshot["manifest-list"]) {
 		const manifestListUrl = snapshot["manifest-list"];
 		manifests = await fetchAvroRecords(manifestListUrl, resolver);
-	} else if (snapshot.manifests) manifests = snapshot.manifests;
+	} else if (snapshot.manifests) manifests = await resolveInlineManifests(snapshot, resolver);
 	else throw new Error("No manifest information found in snapshot");
 	if (partitionFilter) manifests = manifests.filter((manifest) => {
-		let keep = true;
 		try {
-			keep = partitionFilter(manifest.partitions, manifest.partition_spec_id ?? 0, manifest) !== false;
+			return partitionFilter(manifest.partitions, manifest.partition_spec_id ?? 0, manifest) !== false;
 		} catch {
-			keep = true;
+			return true;
 		}
-		return keep;
 	});
 	return await fetchManifests(manifests, resolver);
 }
-const MANIFEST_FETCH_CONCURRENCY = 8;
+async function resolveInlineManifests(snapshot, resolver) {
+	return await mapWithConcurrency(snapshot.manifests ?? [], async (manifestPath) => {
+		const ab = await resolver.reader(manifestPath);
+		const buffer = await ab.slice(0, ab.byteLength);
+		const reader = {
+			view: new DataView(buffer),
+			offset: 0
+		};
+		const { metadata, syncMarker } = await avroMetadata(reader);
+		const entries = await avroRead({
+			reader,
+			metadata,
+			syncMarker
+		});
+		const counts = [
+			0,
+			0,
+			0
+		];
+		const rows = [
+			0n,
+			0n,
+			0n
+		];
+		for (const entry of entries) {
+			counts[entry.status]++;
+			rows[entry.status] += BigInt(entry.data_file.record_count);
+		}
+		return {
+			manifest_path: manifestPath,
+			manifest_length: BigInt(ab.byteLength),
+			partition_spec_id: Number(metadata["partition-spec-id"] ?? 0),
+			content: 0,
+			added_snapshot_id: BigInt(snapshot["snapshot-id"]),
+			added_files_count: counts[1],
+			existing_files_count: counts[0],
+			deleted_files_count: counts[2],
+			added_rows_count: rows[1],
+			existing_rows_count: rows[0],
+			deleted_rows_count: rows[2]
+		};
+	});
+}
 async function fetchOneManifest(manifest, resolver) {
 	const url = manifest.manifest_path;
 	const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
@@ -43,17 +86,20 @@ async function fetchOneManifest(manifest, resolver) {
 	};
 }
 async function fetchManifests(manifests, resolver) {
-	const results = new Array(manifests.length);
+	return await mapWithConcurrency(manifests, (manifest) => fetchOneManifest(manifest, resolver));
+}
+async function mapWithConcurrency(items, fn) {
+	const results = new Array(items.length);
 	let next = 0;
 	async function worker() {
-		while (next < manifests.length) {
-			const i = next++;
-			results[i] = await fetchOneManifest(manifests[i], resolver);
+		while (next < items.length) {
+			const index = next++;
+			results[index] = await fn(items[index]);
 		}
 	}
-	const poolSize = Math.min(MANIFEST_FETCH_CONCURRENCY, manifests.length);
+	const poolSize = Math.min(MANIFEST_FETCH_CONCURRENCY, items.length);
 	const workers = [];
-	for (let w = 0; w < poolSize; w++) workers.push(worker());
+	for (let index = 0; index < poolSize; index++) workers.push(worker());
 	await Promise.all(workers);
 	return results;
 }
@@ -82,4 +128,4 @@ function splitManifestEntries(manifests) {
 		deleteEntries
 	};
 }
-export { icebergManifests, splitManifestEntries };
+export { icebergManifests, resolveInlineManifests, splitManifestEntries };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/read.mjs

@@ -1,16 +1,18 @@
-import { cachedAsyncBuffer } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/utils.mjs";
-import { columnsNeededForFilter, matchFilter } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/filter.mjs";
-import { parquetScan } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/scan.mjs";
-import { parquetReadObjects } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/read.mjs";
-import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/node.mjs";
+import { isListLike, isMapLike } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/schema.mjs";
+import { parquetSchema } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/metadata.mjs";
+import { cachedAsyncBuffer } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/utils.mjs";
+import { columnsNeededForFilter, matchFilter } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/filter.mjs";
+import { parquetScan } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/scan.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/read.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/src/node.mjs";
 import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import { equalityMatch, sanitize } from "./utils.mjs";
+import { applicablePositionDeletes, deleteFileAppliesToDataEntry } from "./delete.mjs";
 import { readParquetMetadata } from "./fetch.mjs";
 import "./manifest.mjs";
 import "./metadata.mjs";
-import { deleteFileAppliesToDataEntry } from "./delete.mjs";
-import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
+import "../../../../squirreling@0.16.3/node_modules/squirreling/src/index.mjs";
 async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, metadata, resolver, rowLineage, positionDeletesMap = /* @__PURE__ */ new Map(), equalityDeleteGroups = [], wantedColumns, rowGroupConcurrency = 1, filter, signal }) {
 	const { data_file, sequence_number, partition_spec_id } = dataEntry;
 	if (sequence_number === void 0) throw new Error("sequence number not found, check v2 inheritance logic");
@@ -19,25 +21,22 @@ async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, meta
 	const resolved = await resolver.reader(data_file.file_path, Number(data_file.file_size_in_bytes));
 	const asyncBuffer = cachedAsyncBuffer(resolved);
 	const parquetMetadata = await readParquetMetadata(asyncBuffer);
-	const kv = parquetMetadata.key_value_metadata?.find((k) => k.key === "iceberg.schema");
-	let parquetIcebergSchema;
-	if (kv?.value) parquetIcebergSchema = JSON.parse(kv.value);
-	else if (parquetMetadata.schema.some((s) => s.field_id !== void 0)) parquetIcebergSchema = parquetSchemaToIceberg(parquetMetadata.schema);
-	else parquetIcebergSchema = schema;
+	const parquetIcebergSchema = fileIcebergSchema(parquetMetadata, schema, tableNameMappings(metadata));
 	const parquetColumnNames = [];
+	const projectors = [];
 	for (const field of schema.fields) {
 		const parquetField = parquetIcebergSchema.fields.find((f) => f.id === field.id);
-		if (parquetField && field.type !== "unknown") parquetColumnNames.push(sanitize(parquetField.name));
-		else parquetColumnNames.push(void 0);
+		if (parquetField && field.type !== "unknown") {
+			parquetColumnNames.push(sanitize(parquetField.name));
+			projectors.push(nestedProjector(field.type, parquetField.type));
+		} else {
+			parquetColumnNames.push(void 0);
+			projectors.push(void 0);
+		}
 	}
 	const lineageColumns = rowLineage ? rowLineageColumnNames(parquetIcebergSchema) : {};
 	const dataColumnNamesById = columnNamesById(parquetIcebergSchema);
-	const positionDeleteGroups = positionDeletesMap.get(data_file.file_path);
-	const positionDeletes = /* @__PURE__ */ new Set();
-	if (positionDeleteGroups) for (const group of positionDeleteGroups) {
-		if (!deleteFileAppliesToDataEntry(dataEntry, group.deleteEntry, metadata, "position")) continue;
-		for (const pos of group.positions) positionDeletes.add(pos);
-	}
+	const positionDeletes = applicablePositionDeletes(dataEntry, positionDeletesMap.get(data_file.file_path), metadata);
 	const applicableEqualityGroups = equalityDeleteGroups.filter((group) => deleteFileAppliesToDataEntry(dataEntry, group.deleteEntry, metadata, "equality"));
 	const wantedSet = wantedColumns ? new Set(wantedColumns) : null;
 	const wantsLineageOutput = !wantedSet || wantedSet.has("_row_id") || wantedSet.has("_last_updated_sequence_number");
@@ -104,18 +103,14 @@ async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, meta
 				const field = schema.fields[i];
 				if (wantedSet && !wantedSet.has(field.name)) continue;
 				const parquetColumnName = parquetColumnNames[i];
-				if (parquetColumnName) mapped[field.name] = row[parquetColumnName];
-				else {
+				if (parquetColumnName) {
+					const project = projectors[i];
+					const value = row[parquetColumnName];
+					mapped[field.name] = project ? project(value) : value;
+				} else {
 					const partitionField = partitionSpec?.fields.find((pf) => pf["source-id"] === field.id && pf.transform === "identity");
-					let nameMapping;
-					if (metadata.properties?.["schema.name-mapping.default"]) nameMapping = nameMappingById(JSON.parse(metadata.properties["schema.name-mapping.default"]), field.id);
 					if (partitionField && Object.hasOwn(data_file.partition, partitionField.name)) mapped[field.name] = data_file.partition[partitionField.name] ?? null;
-					else if (nameMapping) {
-						for (const name of nameMapping.names) if (parquetColumnNames.indexOf(name) !== -1) {
-							mapped[field.name] = row[name];
-							break;
-						}
-					} else if (field["initial-default"] !== void 0) mapped[field.name] = field["initial-default"];
+					else if (field["initial-default"] !== void 0) mapped[field.name] = field["initial-default"];
 					else mapped[field.name] = null;
 				}
 			}
@@ -206,15 +201,6 @@ function normalizeRowGroupConcurrency(value) {
 	if (!Number.isFinite(normalized) || normalized < 1) throw new Error("rowGroupConcurrency must be at least 1");
 	return normalized;
 }
-function nameMappingById(mappings, fieldId) {
-	for (const m of mappings) {
-		if (m["field-id"] === fieldId) return m;
-		if (m.fields) {
-			const hit = nameMappingById(m.fields, fieldId);
-			if (hit) return hit;
-		}
-	}
-}
 function rowLineageColumnNames(parquetIcebergSchema) {
 	return {
 		rowId: columnNameByFieldId(parquetIcebergSchema, 2147483540),
@@ -225,22 +211,130 @@ function columnNameByFieldId(schema, fieldId) {
 	const field = schema.fields.find((f) => f.id === fieldId);
 	return field ? sanitize(field.name) : void 0;
 }
-function parquetSchemaToIceberg(parquetSchema) {
+function tableNameMappings(metadata) {
+	const json = metadata.properties?.["schema.name-mapping.default"];
+	return json ? JSON.parse(json) : void 0;
+}
+function fileIcebergSchema(parquetMetadata, schema, nameMappings) {
+	const kv = parquetMetadata.key_value_metadata?.find((k) => k.key === "iceberg.schema");
+	if (kv?.value) return JSON.parse(kv.value);
+	const hasIds = parquetMetadata.schema.some((s) => s.field_id !== void 0);
+	const mappings = hasIds ? void 0 : nameMappings;
+	const fallbackFields = hasIds || nameMappings ? void 0 : schema.fields;
+	return {
+		type: "struct",
+		"schema-id": schema["schema-id"],
+		fields: parquetFieldsToIceberg(parquetSchema(parquetMetadata).children, mappings, fallbackFields)
+	};
+}
+function parquetFieldsToIceberg(nodes, mappings, fallbackFields) {
 	const fields = [];
-	for (const elem of parquetSchema) {
-		if (elem.field_id === void 0) continue;
+	for (const node of nodes) {
+		const { name, field_id } = node.element;
+		const mapping = mappings?.find((m) => m.names.includes(name));
+		const fallback = fallbackFields?.find((f) => f.name === name);
+		const id = field_id ?? mapping?.["field-id"] ?? fallback?.id;
+		if (id === void 0) continue;
 		fields.push({
-			id: elem.field_id,
-			name: elem.name,
+			id,
+			name,
 			required: false,
-			type: "unknown"
+			type: parquetTypeToIceberg(node, mapping?.fields, fallback?.type)
 		});
 	}
-	return {
-		type: "struct",
-		"schema-id": 0,
-		fields
-	};
+	return fields;
+}
+function parquetTypeToIceberg(node, mappings, fallback) {
+	if (isListLike(node)) {
+		const element = node.children[0].children[0];
+		const mapping = mappings?.find((m) => m.names.includes("element"));
+		const fallbackList = typeof fallback === "object" && fallback.type === "list" ? fallback : void 0;
+		return {
+			type: "list",
+			"element-id": element.element.field_id ?? mapping?.["field-id"] ?? fallbackList?.["element-id"] ?? -1,
+			"element-required": false,
+			element: parquetTypeToIceberg(element, mapping?.fields, fallbackList?.element)
+		};
+	}
+	if (isMapLike(node)) {
+		const [key, value] = ["key", "value"].map((childName) => {
+			const child = node.children[0].children.find((c) => c.element.name === childName);
+			if (!child) throw new Error(`parquet map column ${node.element.name} missing ${childName}`);
+			return child;
+		});
+		const keyMapping = mappings?.find((m) => m.names.includes("key"));
+		const valueMapping = mappings?.find((m) => m.names.includes("value"));
+		const fallbackMap = typeof fallback === "object" && fallback.type === "map" ? fallback : void 0;
+		return {
+			type: "map",
+			"key-id": key.element.field_id ?? keyMapping?.["field-id"] ?? fallbackMap?.["key-id"] ?? -1,
+			key: parquetTypeToIceberg(key, keyMapping?.fields, fallbackMap?.key),
+			"value-id": value.element.field_id ?? valueMapping?.["field-id"] ?? fallbackMap?.["value-id"] ?? -1,
+			"value-required": false,
+			value: parquetTypeToIceberg(value, valueMapping?.fields, fallbackMap?.value)
+		};
+	}
+	if (node.children.length) {
+		const fallbackStruct = typeof fallback === "object" && fallback.type === "struct" ? fallback : void 0;
+		return {
+			type: "struct",
+			"schema-id": 0,
+			fields: parquetFieldsToIceberg(node.children, mappings, fallbackStruct?.fields)
+		};
+	}
+	return "unknown";
+}
+function nestedProjector(type, fileType) {
+	if (typeof type === "string" || typeof fileType === "string") return void 0;
+	if (type.type === "struct" && fileType.type === "struct") {
+		const plan = [];
+		let identity = type.fields.length === fileType.fields.length;
+		for (const field of type.fields) {
+			const fileField = fileType.fields.find((f) => f.id === field.id);
+			if (fileField) {
+				const source = sanitize(fileField.name);
+				const project = nestedProjector(field.type, fileField.type);
+				if (source !== field.name || project) identity = false;
+				plan.push({
+					name: field.name,
+					source,
+					project
+				});
+			} else {
+				identity = false;
+				plan.push({
+					name: field.name,
+					missing: field["initial-default"] ?? null
+				});
+			}
+		}
+		if (identity) return void 0;
+		return (value) => {
+			if (value == null) return value;
+			const out = {};
+			for (const { name, source, project, missing } of plan) if (source === void 0) out[name] = missing;
+			else {
+				const child = value[source];
+				out[name] = project ? project(child) : child ?? null;
+			}
+			return out;
+		};
+	}
+	if (type.type === "list" && fileType.type === "list") {
+		const project = nestedProjector(type.element, fileType.element);
+		if (!project) return void 0;
+		return (value) => Array.isArray(value) ? value.map(project) : value;
+	}
+	if (type.type === "map" && fileType.type === "map") {
+		const project = nestedProjector(type.value, fileType.value);
+		if (!project) return void 0;
+		return (value) => {
+			if (value == null || typeof value !== "object") return value;
+			const out = {};
+			for (const key of Object.keys(value)) out[key] = project(value[key]);
+			return out;
+		};
+	}
 }
 function columnNamesById(schema) {
 	const out = {};

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/s3.mjs

@@ -1,5 +1,5 @@
-import { ByteWriter } from "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
-import "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
+import { ByteWriter } from "../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 import { signRequest } from "./sigv4.mjs";
 function s3SignedResolver({ accessKeyId, secretAccessKey, sessionToken, region, endpoint, pathStyle = false }) {
 	const ep = endpoint ? new URL(endpoint.replace(/\/$/, "") + "/") : void 0;

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/sql/icebergDataSource.mjs

@@ -1,7 +1,7 @@
 import "../fetch.mjs";
 import "../manifest.mjs";
 import "../metadata.mjs";
-import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
+import "../../../../../squirreling@0.16.3/node_modules/squirreling/src/index.mjs";
 import "../read.mjs";
 import "../prune.mjs";
 import "./whereFilter.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/sql/icebergQuery.mjs

@@ -1,5 +1,5 @@
 import "../fetch.mjs";
 import "../catalog/loadTable.mjs";
-import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
+import "../../../../../squirreling@0.16.3/node_modules/squirreling/src/index.mjs";
 import "./icebergDataSource.mjs";
 export {};

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/types.d.mts

@@ -1,7 +1,7 @@
-import { AsyncBuffer } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/index.mjs";
-import "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/node.mjs";
-import { Writer } from "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/types/index.mjs";
-import "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/types/node.mjs";
+import { AsyncBuffer } from "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/index.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
+import { Writer } from "../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/index.mjs";
+import "../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/types/node.mjs";
 interface WriterOptions {
   /**
    * If `'*'`, the writer must create the object only if it does not already
@@ -141,7 +141,7 @@ interface Snapshot {
   'sequence-number': number;
   'timestamp-ms': number;
   'manifest-list': string;
-  manifests?: Manifest[];
+  manifests?: string[]; // v1 only: manifest file locations
   summary: {
     // spec: "value of these fields should be of string type"
     operation: string;

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/commit.mjs

@@ -290,26 +290,78 @@ function validateAssignedId(id, kind, path, priorAssignedIds, priorLastColumnId)
 	if (prior.kind !== kind) throw new Error(`add-schema: ${kind} ${path} uses id ${id} previously assigned to ${prior.kind} ${prior.path}`);
 }
 function validateNewRequiredFields(schema, priorLastColumnId) {
-	for (const field of schema.fields) if (field.id > priorLastColumnId && field.required) {
-		if (field["initial-default"] == null) throw new Error(`add-schema: required field ${field.name} (id ${field.id}) needs a non-null initial-default`);
-		if (field["write-default"] == null) throw new Error(`add-schema: required field ${field.name} (id ${field.id}) needs a non-null write-default`);
+	validateNewRequiredFieldsIn(schema.fields, "", priorLastColumnId);
+}
+function validateNewRequiredFieldsIn(fields, prefix, priorLastColumnId) {
+	for (const field of fields) {
+		const path = prefix ? `${prefix}.${field.name}` : field.name;
+		const isNew = field.id > priorLastColumnId;
+		if (isNew && field.required) {
+			if (field["initial-default"] == null) throw new Error(`add-schema: required field ${path} (id ${field.id}) needs a non-null initial-default`);
+			if (field["write-default"] == null) throw new Error(`add-schema: required field ${path} (id ${field.id}) needs a non-null write-default`);
+		}
+		if (isNew && field["initial-default"] == null) continue;
+		validateNewRequiredTypeFields(field.type, path, priorLastColumnId);
 	}
 }
+function validateNewRequiredTypeFields(type, path, priorLastColumnId) {
+	if (typeof type === "string") return;
+	if (type.type === "struct") validateNewRequiredFieldsIn(type.fields, path, priorLastColumnId);
+	else if (type.type === "list") validateNewRequiredTypeFields(type.element, `${path}.element`, priorLastColumnId);
+	else if (type.type === "map") validateNewRequiredTypeFields(type.value, `${path}.value`, priorLastColumnId);
+}
 function validateSchemaEvolution(schemas, newSchema, priorLastColumnId, formatVersion) {
-	for (const field of newSchema.fields) {
-		if (field.id > priorLastColumnId) continue;
-		const prior = latestFieldById(schemas, field.id);
-		if (!prior) continue;
-		if (!canPromoteType(prior.type, field.type, formatVersion)) throw new Error(`add-schema: cannot promote field ${field.name} from ${typeToString(prior.type)} to ${typeToString(field.type)}`);
-		if (!defaultsEqual(prior["initial-default"], field["initial-default"])) throw new Error(`add-schema: initial-default for field ${field.name} cannot change`);
+	const priorTypes = /* @__PURE__ */ new Map();
+	for (const schema of schemas) indexPriorFields(schema.fields, priorTypes);
+	validateFieldsEvolution(newSchema.fields, "", priorTypes, priorLastColumnId, formatVersion);
+}
+function indexPriorFields(fields, index) {
+	for (const field of fields) {
+		index.set(field.id, {
+			type: field.type,
+			"initial-default": field["initial-default"]
+		});
+		indexPriorType(field.type, index);
 	}
 }
-function latestFieldById(schemas, id) {
-	for (let i = schemas.length - 1; i >= 0; i--) {
-		const field = schemas[i].fields.find((f) => f.id === id);
-		if (field) return field;
+function indexPriorType(type, index) {
+	if (typeof type === "string") return;
+	if (type.type === "struct") indexPriorFields(type.fields, index);
+	else if (type.type === "list") {
+		index.set(type["element-id"], { type: type.element });
+		indexPriorType(type.element, index);
+	} else if (type.type === "map") {
+		index.set(type["key-id"], { type: type.key });
+		index.set(type["value-id"], { type: type.value });
+		indexPriorType(type.key, index);
+		indexPriorType(type.value, index);
+	}
+}
+function validateFieldsEvolution(fields, prefix, priorTypes, priorLastColumnId, formatVersion) {
+	for (const field of fields) {
+		const path = prefix ? `${prefix}.${field.name}` : field.name;
+		const prior = field.id > priorLastColumnId ? void 0 : priorTypes.get(field.id);
+		if (prior) {
+			if (!canPromoteType(prior.type, field.type, formatVersion)) throw new Error(`add-schema: cannot promote field ${path} from ${typeToString(prior.type)} to ${typeToString(field.type)}`);
+			if (!defaultsEqual(prior["initial-default"], field["initial-default"])) throw new Error(`add-schema: initial-default for field ${path} cannot change`);
+		}
+		validateTypeEvolution(field.type, path, priorTypes, priorLastColumnId, formatVersion);
 	}
 }
+function validateTypeEvolution(type, path, priorTypes, priorLastColumnId, formatVersion) {
+	if (typeof type === "string") return;
+	if (type.type === "struct") validateFieldsEvolution(type.fields, path, priorTypes, priorLastColumnId, formatVersion);
+	else if (type.type === "list") validateChildEvolution(type["element-id"], type.element, `${path}.element`, priorTypes, priorLastColumnId, formatVersion);
+	else if (type.type === "map") {
+		validateChildEvolution(type["key-id"], type.key, `${path}.key`, priorTypes, priorLastColumnId, formatVersion);
+		validateChildEvolution(type["value-id"], type.value, `${path}.value`, priorTypes, priorLastColumnId, formatVersion);
+	}
+}
+function validateChildEvolution(id, type, path, priorTypes, priorLastColumnId, formatVersion) {
+	const prior = id > priorLastColumnId ? void 0 : priorTypes.get(id);
+	if (prior && !canPromoteType(prior.type, type, formatVersion)) throw new Error(`add-schema: cannot promote ${path} from ${typeToString(prior.type)} to ${typeToString(type)}`);
+	validateTypeEvolution(type, path, priorTypes, priorLastColumnId, formatVersion);
+}
 function canPromoteType(from, to, formatVersion) {
 	if (typesEqual(from, to)) return true;
 	if (typeof from !== "string" || typeof to !== "string") return false;
@@ -321,7 +373,7 @@ function canPromoteType(from, to, formatVersion) {
 }
 function typesEqual(a, b) {
 	if (typeof a === "string" || typeof b === "string") return a === b;
-	return JSON.stringify(a) === JSON.stringify(b);
+	return a.type === b.type;
 }
 function decimalPromotionAllowed(from, to) {
 	const a = parseDecimalType(from);

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/parquet.mjs

@@ -1,5 +1,5 @@
-import { parquetWrite } from "../../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/write.mjs";
-import "../../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/node.mjs";
+import { parquetWrite } from "../../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/write.mjs";
+import "../../../../../hyparquet-writer@0.16.9_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
 import { sanitize } from "../utils.mjs";
 import { typeName } from "../schema.mjs";
 import { decimalRequiredBytes, parseDecimalType } from "./conversions.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/rewrite.mjs

@@ -1,4 +1,4 @@
-import { concat } from "../../../../../hyparquet@1.29.1/node_modules/hyparquet/src/utils.mjs";
+import { concat } from "../../../../../hyparquet@1.29.2/node_modules/hyparquet/src/utils.mjs";
 import { uuid4 } from "../utils.mjs";
 import { fetchDeleteMaps } from "../fetch.mjs";
 import { icebergManifests, splitManifestEntries } from "../manifest.mjs";

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/snapshot.mjs

@@ -1,4 +1,5 @@
 import { fetchAvroRecords } from "../fetch.mjs";
+import { resolveInlineManifests } from "../manifest.mjs";
 import { transformResultType } from "./transform.mjs";
 import { writeManifestList } from "./manifest-list.mjs";
 import { computeFieldSummary } from "./stats.mjs";
@@ -13,8 +14,9 @@ function currentSnapshot(metadata) {
 }
 async function loadPriorManifests(metadata, resolver) {
 	const snap = currentSnapshot(metadata);
-	if (!snap?.["manifest-list"]) return [];
-	return await fetchAvroRecords(snap["manifest-list"], resolver);
+	if (!snap) return [];
+	if (snap["manifest-list"]) return await fetchAvroRecords(snap["manifest-list"], resolver);
+	return await resolveInlineManifests(snap, resolver);
 }
 async function buildSnapshotUpdate({ tableUrl, metadata, resolver, snapshotId, sequenceNumber, manifestUuid, timestampMs, formatVersion, newManifests, summary, writtenFiles, priorManifests, skipPriorManifestPaths }) {
 	const writerFn = resolver.writer;

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/src/write/stage-deletion-vector.mjs

@@ -1,10 +1,10 @@
+import { uuid4 } from "../utils.mjs";
+import { deleteFileAppliesToDataEntry, isDeletionVector } from "../delete.mjs";
 import { writeDeletionVector } from "../puffin/deletion-vector.mjs";
 import { writePuffinFile } from "../puffin/puffin.mjs";
-import { uuid4 } from "../utils.mjs";
 import { fetchDeleteMaps } from "../fetch.mjs";
 import { validateSchemaForVersion } from "../schema.mjs";
 import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot, loadPriorManifests } from "./snapshot.mjs";
-import { deleteFileAppliesToDataEntry } from "../delete.mjs";
 import { writeDeleteManifest, writeExistingDeleteManifest } from "./manifest.mjs";
 import { checkWriteFormat, newSnapshotId } from "./stage.mjs";
 import { findDataFileEntries, loadManifestEntries, partitionTupleKey } from "./stage-position-delete.mjs";
@@ -61,6 +61,36 @@ async function icebergStageDeletionVector({ tableUrl, metadata, deletes, resolve
 			}
 			targets.add(targetPath);
 		}
+		let expandedTargets = true;
+		while (expandedTargets) {
+			expandedTargets = false;
+			for (const [entry, coveredTargets] of positionDeleteTargets) {
+				if (isDeletionVector(entry) || obsoletePositionDeleteEntries.has(entry)) continue;
+				let appliesToTarget = false;
+				for (const targetPath of coveredTargets) {
+					const info = byFile.get(targetPath);
+					if (info && deleteFileAppliesToDataEntry(info.dataEntry, entry, metadata, "position")) {
+						appliesToTarget = true;
+						break;
+					}
+				}
+				if (!appliesToTarget) continue;
+				obsoletePositionDeleteEntries.add(entry);
+				for (const targetPath of coveredTargets) {
+					const found = dataFileMap.get(targetPath);
+					if (!found || !deleteFileAppliesToDataEntry(found.entry, entry, metadata, "position")) continue;
+					if (byFile.has(targetPath)) continue;
+					byFile.set(targetPath, {
+						positions: /* @__PURE__ */ new Set(),
+						partition: found.partition,
+						partitionSpecId: found.partitionSpecId,
+						dataEntry: found.entry
+					});
+					targetPaths.add(targetPath);
+					expandedTargets = true;
+				}
+			}
+		}
 		for (const [targetPath, info] of byFile) {
 			const groups = positionDeletesMap.get(targetPath) ?? [];
 			for (const group of groups) {
@@ -68,20 +98,6 @@ async function icebergStageDeletionVector({ tableUrl, metadata, deletes, resolve
 				for (const pos of group.positions) info.positions.add(pos);
 			}
 		}
-		for (const [entry, coveredTargets] of positionDeleteTargets) {
-			if (isDeletionVectorForTarget(entry, targetPaths)) continue;
-			let allTargetsCovered = true;
-			let appliesToTarget = false;
-			for (const targetPath of coveredTargets) {
-				if (!targetPaths.has(targetPath)) {
-					allTargetsCovered = false;
-					break;
-				}
-				const info = byFile.get(targetPath);
-				if (info && deleteFileAppliesToDataEntry(info.dataEntry, entry, metadata, "position")) appliesToTarget = true;
-			}
-			if (allTargetsCovered && appliesToTarget) obsoletePositionDeleteEntries.add(entry);
-		}
 	}
 	const skipPriorManifestPaths = /* @__PURE__ */ new Set();
 	const replacementManifests = [];

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/fetch.d.mts

@@ -1,4 +1,4 @@
-import "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/node.mjs";
+import "../../../../hyparquet@1.29.2/node_modules/hyparquet/types/node.mjs";
 import { Resolver } from "../src/types.mjs";
 /**
  * Wrap a `Resolver` so reads of the same path share one HTTP fetch and

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244 → icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7}/node_modules/icebird/types/index.d.mts

@@ -4,7 +4,7 @@ import { cachingResolver } from "./fetch.mjs";
 import "./create.mjs";
 import "./catalog/file.mjs";
 import { s3SignedResolver } from "./s3.mjs";
-import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
+import "../../../../squirreling@0.16.3/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";
 import "./metadata.mjs";
 import { icebergManifests } from "./manifest.mjs";