@gscdump/lakehouse 3.2.0 → 3.4.0

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/fetch.mjs

@@ -1,7 +1,7 @@
-import { parquetMetadataAsync } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/metadata.mjs";
-import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/utils.mjs";
-import { parquetReadObjects } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/read.mjs";
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/node.mjs";
+import { parquetMetadataAsync } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/metadata.mjs";
+import { asyncBufferFromUrl, cachedAsyncBuffer } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/utils.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/read.mjs";
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/node.mjs";
 import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import { ByteWriter } from "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/src/bytewriter.mjs";
@@ -10,6 +10,12 @@ import { avroMetadata } from "./avro/avro.metadata.mjs";
 import { avroRead } from "./avro/avro.read.mjs";
 import { puffinReadDeletionVector } from "./puffin/puffin.mjs";
 import { sanitize } from "./utils.mjs";
+const SMALL_PARQUET_FILE_SIZE = 8388608;
+const SMALL_PARQUET_FOOTER_SIZE = 131072;
+function readParquetMetadata(file) {
+	if (file.byteLength >= SMALL_PARQUET_FILE_SIZE) return parquetMetadataAsync(file);
+	return parquetMetadataAsync(file, { initialFetchSize: SMALL_PARQUET_FOOTER_SIZE });
+}
 function translateS3Url(url) {
 	if (url.startsWith("s3a://") || url.startsWith("s3://")) {
 		const rest = url.slice(url.indexOf("://") + 3);
@@ -153,8 +159,10 @@ async function fetchDeleteMaps(deleteEntries, resolver) {
 				for (const pos of positions) set.add(pos);
 				addPositionDeleteGroup(positionDeletesMap, referenced_data_file, deleteEntry, set);
 			} else {
+				const metadata = await readParquetMetadata(file);
 				const deleteRows = await parquetReadObjects({
 					file,
+					metadata,
 					compressors
 				});
 				const positionsByPath = /* @__PURE__ */ new Map();
@@ -176,7 +184,7 @@ async function fetchDeleteMaps(deleteEntries, resolver) {
 			const equalityIds = deleteEntry.data_file.equality_ids;
 			if (sequence_number === void 0) throw new Error("equality delete missing sequence number");
 			if (!equalityIds?.length) throw new Error("equality delete missing equality_ids");
-			const metadata = await parquetMetadataAsync(file);
+			const metadata = await readParquetMetadata(file);
 			const columnNamesById = equalityColumnNamesById(metadata, equalityIds);
 			const columns = equalityIds.map((id) => columnNamesById[id]);
 			const deleteRows = await parquetReadObjects({
@@ -261,4 +269,4 @@ async function fetchAvroRecords(url, resolver, byteLength) {
 		syncMarker
 	});
 }
-export { cachingResolver, fetchAvroRecords, fetchDeleteMaps, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };
+export { cachingResolver, fetchAvroRecords, fetchDeleteMaps, readParquetMetadata, resolveText, s3Lister, s3ParseUrl, translateS3Url, urlResolver };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/index.mjs

@@ -4,10 +4,10 @@ import { icebergManifests } from "./manifest.mjs";
 import { s3SignedResolver } from "./s3.mjs";
 import "./metadata.mjs";
 import "./create.mjs";
+import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
 import "./catalog/file.mjs";
-import "../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
 import "./sql/icebergDataSource.mjs";
 import "./sql/icebergQuery.mjs";
 export { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/manifest.mjs

@@ -29,6 +29,7 @@ async function fetchOneManifest(manifest, resolver) {
 	const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
 	for (const entry of entries) {
 		entry.partition_spec_id = manifest.partition_spec_id ?? 0;
+		if (entry.snapshot_id == null) entry.snapshot_id = manifest.added_snapshot_id;
 		if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number ?? 0n;
 		if (entry.status === 1) {
 			if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number;

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/metadata.mjs

@@ -40,14 +40,6 @@ function icebergLatestVersion({ tableUrl, resolver, lister }) {
 		throw new Error(`failed to determine latest iceberg version of ${tableUrl}: ${err.message}`);
 	});
 }
-async function icebergMetadata({ tableUrl, metadataFileName, resolver, lister }) {
-	return (await resolveMetadata({
-		tableUrl,
-		metadataFileName,
-		resolver,
-		lister
-	})).metadata;
-}
 async function resolveMetadata({ tableUrl, metadataFileName, resolver, lister }) {
 	resolver ??= urlResolver();
 	lister ??= s3Lister();
@@ -161,4 +153,4 @@ function metadataFilePreference(file, paddedVersion) {
 	if (file.startsWith(`${paddedVersion}-`) && file.endsWith(".metadata.json.gz")) return 5;
 	return 6;
 }
-export { icebergLatestVersion, icebergMetadata, loadLatestFileCatalogMetadata, resolveMetadata };
+export { icebergLatestVersion, loadLatestFileCatalogMetadata, resolveMetadata };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/read.mjs

@@ -1,91 +1,24 @@
-import { parquetMetadataAsync } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/metadata.mjs";
-import { cachedAsyncBuffer, concat } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/utils.mjs";
-import { parquetReadObjects } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/read.mjs";
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/src/node.mjs";
+import { cachedAsyncBuffer } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/utils.mjs";
+import { columnsNeededForFilter, matchFilter } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/filter.mjs";
+import { parquetScan } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/scan.mjs";
+import { parquetReadObjects } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/read.mjs";
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/src/node.mjs";
 import { compressors } from "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs";
 import "../../../../hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/index.mjs";
 import { equalityMatch, sanitize } from "./utils.mjs";
-import { fetchDeleteMaps, urlResolver } from "./fetch.mjs";
-import { icebergManifests, splitManifestEntries } from "./manifest.mjs";
-import { icebergMetadata } from "./metadata.mjs";
+import { readParquetMetadata } from "./fetch.mjs";
+import "./manifest.mjs";
+import "./metadata.mjs";
 import { deleteFileAppliesToDataEntry } from "./delete.mjs";
-const DEFAULT_ROW_GROUP_CONCURRENCY = 4;
-async function icebergRead({ tableUrl, rowStart = 0, rowEnd = Infinity, metadataFileName, metadata, snapshotId, resolver, lister, rowGroupConcurrency = DEFAULT_ROW_GROUP_CONCURRENCY }) {
-	if (!tableUrl) throw new Error("tableUrl is required");
-	if (rowStart > rowEnd) throw new Error("rowStart must be less than rowEnd");
-	if (rowStart < 0) throw new Error("rowStart must be positive");
-	rowGroupConcurrency = normalizeRowGroupConcurrency(rowGroupConcurrency);
-	resolver ??= urlResolver();
-	metadata ??= await icebergMetadata({
-		tableUrl,
-		metadataFileName,
-		resolver,
-		lister
-	});
-	const manifestList = await icebergManifests({
-		metadata,
-		resolver,
-		snapshotId
-	});
-	const currentSchemaId = metadata["current-schema-id"];
-	const schema = metadata.schemas.find((s) => s["schema-id"] === currentSchemaId);
-	if (!schema) throw new Error("current schema not found in metadata");
-	const rowLineage = metadata["format-version"] >= 3;
-	const { dataEntries, deleteEntries } = splitManifestEntries(manifestList);
-	if (dataEntries.length === 0) throw new Error("No data manifest files found for current snapshot");
-	const hasDeletes = deleteEntries.length > 0;
-	const deleteMaps = fetchDeleteMaps(deleteEntries, resolver);
-	const totalRowsToRead = rowEnd === Infinity ? Infinity : rowEnd - rowStart;
-	let fileIndex = 0;
-	let skipRows = rowStart;
-	if (hasDeletes) skipRows = 0;
-	else while (fileIndex < dataEntries.length && skipRows >= dataEntries[fileIndex].data_file.record_count) {
-		skipRows -= Number(dataEntries[fileIndex].data_file.record_count);
-		fileIndex++;
-	}
-	const fileReads = [];
-	let rowsRemaining = hasDeletes ? Infinity : totalRowsToRead;
-	for (let i = fileIndex; i < dataEntries.length && rowsRemaining > 0; i++) {
-		const recordCount = Number(dataEntries[i].data_file.record_count);
-		const fileRowStart = i === fileIndex ? skipRows : 0;
-		const availableRows = recordCount - fileRowStart;
-		if (availableRows <= 0) continue;
-		const rowsToRead = rowsRemaining === Infinity ? availableRows : Math.min(rowsRemaining, availableRows);
-		fileReads.push({
-			entry: dataEntries[i],
-			fileRowStart,
-			fileRowEnd: fileRowStart + rowsToRead
-		});
-		if (rowsRemaining !== Infinity) rowsRemaining -= rowsToRead;
-	}
-	const { positionDeletesMap, equalityDeleteGroups } = await deleteMaps;
-	const rows = (await Promise.all(fileReads.map(async ({ entry, fileRowStart, fileRowEnd }) => {
-		const rows = [];
-		for await (const batch of readDataFile({
-			dataEntry: entry,
-			fileRowStart,
-			fileRowEnd,
-			schema,
-			metadata,
-			resolver,
-			rowLineage,
-			positionDeletesMap,
-			equalityDeleteGroups,
-			rowGroupConcurrency
-		})) concat(rows, batch);
-		return rows;
-	}))).flat();
-	if (!hasDeletes) return rows;
-	return rows.slice(rowStart, rowEnd === Infinity ? void 0 : rowEnd);
-}
-async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, metadata, resolver, rowLineage, positionDeletesMap, equalityDeleteGroups, wantedColumns, rowGroupConcurrency = 1, filter, signal }) {
+import "../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
+async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, metadata, resolver, rowLineage, positionDeletesMap = /* @__PURE__ */ new Map(), equalityDeleteGroups = [], wantedColumns, rowGroupConcurrency = 1, filter, signal }) {
 	const { data_file, sequence_number, partition_spec_id } = dataEntry;
 	if (sequence_number === void 0) throw new Error("sequence number not found, check v2 inheritance logic");
 	const sequenceNumber = sequence_number;
 	const partitionSpec = metadata["partition-specs"].find((s) => s["spec-id"] === partition_spec_id);
 	const resolved = await resolver.reader(data_file.file_path, Number(data_file.file_size_in_bytes));
 	const asyncBuffer = cachedAsyncBuffer(resolved);
-	const parquetMetadata = await parquetMetadataAsync(asyncBuffer);
+	const parquetMetadata = await readParquetMetadata(asyncBuffer);
 	const kv = parquetMetadata.key_value_metadata?.find((k) => k.key === "iceberg.schema");
 	let parquetIcebergSchema;
 	if (kv?.value) parquetIcebergSchema = JSON.parse(kv.value);
@@ -130,23 +63,40 @@ async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, meta
 		if (parquetName) icebergToParquet[schema.fields[i].name] = parquetName;
 	}
 	const parquetFilter = filter ? remapFilterColumns(filter, icebergToParquet) : void 0;
-	async function readRowGroupRange({ readStart, readEnd }) {
+	const readColumns = [...parquetColumns];
+	if (parquetFilter) {
+		for (const column of columnsNeededForFilter(parquetFilter)) if (!readColumns.includes(column)) readColumns.push(column);
+	}
+	const scan = await parquetScan({
+		file: asyncBuffer,
+		metadata: parquetMetadata,
+		columns: readColumns,
+		pruningFilter: parquetFilter,
+		rowStart: fileRowStart,
+		rowEnd: fileRowEnd,
+		compressors,
+		filterStrict: false,
+		useBloomFilters: true,
+		usePageIndex: true,
+		utf8: false
+	});
+	if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
+	async function readCandidateRange({ readStart, readEnd }) {
 		const rows = await parquetReadObjects({
 			file: asyncBuffer,
 			metadata: parquetMetadata,
-			columns: parquetColumns,
+			columns: readColumns,
 			rowStart: readStart,
 			rowEnd: readEnd,
 			compressors,
-			filter: parquetFilter,
-			filterStrict: false,
-			useBloomFilters: true,
+			useOffsetIndex: true,
 			utf8: false
 		});
 		const batch = [];
 		for (let idx = 0; idx < rows.length; idx++) {
 			const row = rows[idx];
 			const pos = BigInt(readStart + idx);
+			if (parquetFilter && !matchFilter(row, parquetFilter, false)) continue;
 			if (positionDeletes.has(pos)) continue;
 			if (applicableEqualityGroups.some((group) => group.rows.some((predicate) => equalityMatch(row, predicate, dataColumnNamesById)))) continue;
 			const mapped = {};
@@ -185,30 +135,15 @@ async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, meta
 		}
 		return batch;
 	}
-	const rowGroupRanges = [];
-	let groupStart = 0;
-	for (const rowGroup of parquetMetadata.row_groups) {
-		if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
-		const rowCount = Number(rowGroup.num_rows);
-		const groupEnd = groupStart + rowCount;
-		if (groupEnd <= fileRowStart) {
-			groupStart = groupEnd;
-			continue;
-		}
-		if (groupStart >= fileRowEnd) break;
-		const readStart = Math.max(groupStart, fileRowStart);
-		const readEnd = Math.min(groupEnd, fileRowEnd);
-		rowGroupRanges.push({
-			readStart,
-			readEnd
-		});
-		groupStart = groupEnd;
-	}
+	const candidateRanges = scan.ranges.map(({ rowStart, rowEnd }) => ({
+		readStart: rowStart,
+		readEnd: rowEnd
+	}));
 	const concurrency = normalizeRowGroupConcurrency(rowGroupConcurrency);
-	if (concurrency === 1 || rowGroupRanges.length <= 1) {
-		for (const range of rowGroupRanges) {
+	if (concurrency === 1 || candidateRanges.length <= 1) {
+		for (const range of candidateRanges) {
 			if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
-			const batch = await readRowGroupRange(range);
+			const batch = await readCandidateRange(range);
 			if (batch.length > 0) yield batch;
 		}
 		return;
@@ -217,21 +152,21 @@ async function* readDataFile({ dataEntry, fileRowStart, fileRowEnd, schema, meta
 	let nextStart = 0;
 	let nextYield = 0;
 	function startTask(index) {
-		const task = readRowGroupRange(rowGroupRanges[index]).then((batch) => ({ batch }), (error) => ({ error }));
+		const task = readCandidateRange(candidateRanges[index]).then((batch) => ({ batch }), (error) => ({ error }));
 		tasks.set(index, task);
 	}
-	while (nextStart < rowGroupRanges.length && tasks.size < concurrency) {
+	while (nextStart < candidateRanges.length && tasks.size < concurrency) {
 		startTask(nextStart);
 		nextStart++;
 	}
-	while (nextYield < rowGroupRanges.length) {
+	while (nextYield < candidateRanges.length) {
 		if (signal?.aborted) throw new DOMException("Aborted", "AbortError");
 		const task = tasks.get(nextYield);
 		if (!task) throw new Error("internal row group scheduling error");
 		const result = await task;
 		tasks.delete(nextYield);
 		if (result.error) throw result.error;
-		while (nextStart < rowGroupRanges.length && tasks.size < concurrency) {
+		while (nextStart < candidateRanges.length && tasks.size < concurrency) {
 			startTask(nextStart);
 			nextStart++;
 		}
@@ -322,4 +257,4 @@ function applyRowLineage(out, { row, pos, firstRowId, sequenceNumber, rowIdColum
 	else if (firstRowId != null) out._last_updated_sequence_number = sequenceNumber;
 	else out._last_updated_sequence_number = null;
 }
-export { icebergRead, readDataFile };
+export { readDataFile };

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/sql/icebergDataSource.mjs

@@ -1,8 +1,8 @@
 import "../fetch.mjs";
 import "../manifest.mjs";
 import "../metadata.mjs";
+import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
 import "../read.mjs";
-import "../../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
 import "../prune.mjs";
 import "./whereFilter.mjs";
 export {};

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/sql/icebergQuery.mjs

@@ -1,5 +1,5 @@
 import "../fetch.mjs";
 import "../catalog/loadTable.mjs";
-import "../../../../../squirreling@0.15.3/node_modules/squirreling/src/index.mjs";
+import "../../../../../squirreling@0.16.2/node_modules/squirreling/src/index.mjs";
 import "./icebergDataSource.mjs";
 export {};

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/types.d.mts

@@ -1,5 +1,5 @@
-import { AsyncBuffer } from "../../../../hyparquet@1.28.2/node_modules/hyparquet/types/index.mjs";
-import "../../../../hyparquet@1.28.2/node_modules/hyparquet/types/node.mjs";
+import { AsyncBuffer } from "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/index.mjs";
+import "../../../../hyparquet@1.29.1/node_modules/hyparquet/types/node.mjs";
 import { Writer } from "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/types/index.mjs";
 import "../../../../hyparquet-writer@0.16.6_patch_hash=549d1fd1fb9614da77df9123bf6978a8fccfe1d9bb8882863ce6159f666e821a/node_modules/hyparquet-writer/types/node.mjs";
 interface WriterOptions {

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.22_patch_hash=4fa7fd5702c4ce432fb2902c6abe07cc55565e87f6375b96ece63a10999712e6 → icebird@0.8.26_patch_hash=75eacba5b0363ee2a40baa0ef77c7fb20d06389025186ca95b514a927cad0244}/node_modules/icebird/src/write/manifest.mjs

@@ -38,6 +38,16 @@ function manifestEntrySchema(schema, partitionSpec, formatVersion, manifestConte
 		mapField("nan_value_counts", 137, "k138_v139", 138, 139, "long"),
 		mapField("lower_bounds", 125, "k126_v127", 126, 127, "bytes"),
 		mapField("upper_bounds", 128, "k129_v130", 129, 130, "bytes"),
+		{
+			name: "split_offsets",
+			type: ["null", {
+				type: "array",
+				items: "long",
+				"element-id": 133
+			}],
+			default: null,
+			"field-id": 132
+		},
 		{
 			name: "sort_order_id",
 			type: ["null", "int"],
@@ -151,6 +161,7 @@ function icebergSchemaJson(schema) {
 }
 function encodeMap(m) {
 	if (!m) return null;
+	if (Array.isArray(m)) return m.length ? m : null;
 	const entries = Object.entries(m);
 	if (!entries.length) return null;
 	return entries.map(([k, value]) => ({
@@ -169,6 +180,7 @@ function writeDataManifest({ writer, schema, partitionSpec, snapshotId, dataFile
 		records,
 		metadata: {
 			"format-version": String(formatVersion),
+			"schema-id": String(schema["schema-id"]),
 			content: "data",
 			schema: icebergSchemaJson(schema),
 			"partition-spec": partitionSpecJson(partitionSpec),
@@ -188,6 +200,7 @@ function writeDeleteManifest({ writer, schema, partitionSpec, snapshotId, delete
 		records,
 		metadata: {
 			"format-version": String(formatVersion),
+			"schema-id": String(schema["schema-id"]),
 			content: "deletes",
 			schema: icebergSchemaJson(schema),
 			"partition-spec": partitionSpecJson(partitionSpec),
@@ -195,6 +208,35 @@ function writeDeleteManifest({ writer, schema, partitionSpec, snapshotId, delete
 		}
 	});
 }
+function writeExistingDataManifest({ writer, schema, partitionSpec, entries, formatVersion = 2 }) {
+	const records = entries.map((entry) => {
+		const dataFile = entry.data_file;
+		if (entry.status === 2) throw new Error("writeExistingDataManifest cannot rewrite deleted entries as existing");
+		if (dataFile.content !== 0) throw new Error(`writeExistingDataManifest expects data files (content=0), got content=${dataFile.content}`);
+		if (entry.partition_spec_id !== partitionSpec["spec-id"]) throw new Error(`existing data entry partition spec ${entry.partition_spec_id} does not match ${partitionSpec["spec-id"]}`);
+		const record = manifestEntryRecord(dataFile, schema, partitionSpec, 0n, formatVersion, 0);
+		record.status = 0;
+		record.snapshot_id = entry.snapshot_id ?? null;
+		record.sequence_number = entry.sequence_number ?? null;
+		record.file_sequence_number = entry.file_sequence_number ?? null;
+		if (record.snapshot_id == null) throw new Error("existing data manifest entry missing snapshot id");
+		if (record.sequence_number == null || record.file_sequence_number == null) throw new Error("existing data manifest entry missing sequence numbers");
+		return record;
+	});
+	return avroWrite({
+		writer,
+		schema: manifestEntrySchema(schema, partitionSpec, formatVersion, 0),
+		records,
+		metadata: {
+			"format-version": String(formatVersion),
+			"schema-id": String(schema["schema-id"]),
+			content: "data",
+			schema: icebergSchemaJson(schema),
+			"partition-spec": partitionSpecJson(partitionSpec),
+			"partition-spec-id": String(partitionSpec["spec-id"])
+		}
+	});
+}
 function writeExistingDeleteManifest({ writer, schema, partitionSpec, entries, formatVersion = 2 }) {
 	const records = entries.map((entry) => {
 		const deleteFile = entry.data_file;
@@ -213,6 +255,7 @@ function writeExistingDeleteManifest({ writer, schema, partitionSpec, entries, f
 		records,
 		metadata: {
 			"format-version": String(formatVersion),
+			"schema-id": String(schema["schema-id"]),
 			content: "deletes",
 			schema: icebergSchemaJson(schema),
 			"partition-spec": partitionSpecJson(partitionSpec),
@@ -234,6 +277,7 @@ function manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, format
 		nan_value_counts: encodeMap(dataFile.nan_value_counts),
 		lower_bounds: encodeMap(dataFile.lower_bounds),
 		upper_bounds: encodeMap(dataFile.upper_bounds),
+		split_offsets: dataFile.split_offsets?.length ? dataFile.split_offsets : null,
 		sort_order_id: dataFile.content === 1 ? null : dataFile.sort_order_id ?? 0
 	};
 	if (manifestContent === 1) {
@@ -253,4 +297,4 @@ function manifestEntryRecord(dataFile, schema, partitionSpec, snapshotId, format
 		data_file: dataFileRecord
 	};
 }
-export { writeDataManifest, writeDeleteManifest, writeExistingDeleteManifest };
+export { writeDataManifest, writeDeleteManifest, writeExistingDataManifest, writeExistingDeleteManifest };