@gscdump/lakehouse 7.1.1 → 7.2.0

package/README.md

@@ -18,8 +18,8 @@ Node.js 22 or newer is required.
 - `@gscdump/lakehouse/bigint`: bigint-safe JSON helpers without catalog or
   Node dependencies.
 - `@gscdump/lakehouse/schema`: lightweight shared schema constants.
-- `@gscdump/lakehouse/maintenance`: stable commit classification and orphan
-  cleanup workflows.
+- `@gscdump/lakehouse/maintenance`: stable commit classification, manifest
+  merging, and orphan cleanup workflows.
 - `@gscdump/lakehouse/provisioning`: adopt, allocate, or provision R2 Data
   Catalog resources.
 - `@gscdump/lakehouse/unsafe-raw`: explicit escape hatch for package adapters
@@ -66,6 +66,34 @@ Older writers cannot recognize the new content ID format.
 
 Batch appends already require an explicit `appendId` and retain that behavior.
 
+## Manifest merging
+
+Use `mergeIcebergManifests(connection, table, options)` from `@gscdump/lakehouse/maintenance`
+to reduce current metadata reads for append-only v2 tables.
+The connection supplies the exact catalog and namespace.
+The operation preserves DATA files, their metadata and sequence numbers, and historical append snapshots and manifests.
+It groups entries by partition specification and writes encoded manifests up to four MiB.
+It bounds aggregate input and staged output to eight MiB each, with ten thousand live entries.
+Options can lower these limits.
+
+`Merged` reports the snapshot ID, manifest counts, and file count.
+`Unchanged` avoids writes when merging cannot reduce the manifest count.
+`Conflict` leaves a concurrent UUID or main-snapshot change intact.
+Reload the current head before a later attempt.
+`Err` reports unsupported formats, deletes, invalid inventory, exceeded limits, or changed external authority.
+Infrastructure errors propagate.
+Non-null DATA metadata fields unsupported by the v2 writer, including encryption metadata, produce `invalid-inventory`.
+
+Supply `confirmAuthority` to recheck external authority immediately before the catalog commit.
+Returning false prevents that commit; callback exceptions propagate.
+This callback does not create an atomic transaction with an external database.
+Supply `cache` to invalidate the captured namespace and table's cached snapshot pointer after success.
+
+Failed or uncertain commits retain staged metadata.
+An uncertain response may follow a successful commit, so deleting staged files could break the table.
+The operation never expires historical snapshots or deletes historical manifests.
+Keep those objects while authenticated receipts or append retries still need them.
+
 ## License
 
 [MIT](../../LICENSE)

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/manifest.mjs

@@ -1,5 +1,5 @@
-import { partitionAvroSchema, partitionSpecJson, partitionToAvroRecord } from "./partition.mjs";
 import { avroWrite } from "../avro/avro.write.mjs";
+import { partitionAvroSchema, partitionSpecJson, partitionToAvroRecord } from "./partition.mjs";
 function manifestEntrySchema(schema, partitionSpec, formatVersion, manifestContent = 0) {
 	const dataFileFields = [
 		{

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/rewrite.mjs

@@ -3,9 +3,9 @@ import { uuid4 } from "../utils.mjs";
 import { fetchDeleteMaps } from "../fetch.mjs";
 import { icebergManifests, splitManifestEntries } from "../manifest.mjs";
 import { groupByPartition } from "./partition.mjs";
+import { writeDataManifest, writeExistingDataManifest } from "./manifest.mjs";
 import { computeColumnStats } from "./stats.mjs";
 import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot, loadPriorManifests } from "./snapshot.mjs";
-import { writeDataManifest, writeExistingDataManifest } from "./manifest.mjs";
 import { writeParquet } from "./parquet.mjs";
 import { buildSortComparator } from "./sort.mjs";
 import { checkWriteFormat, newSnapshotId, resolveParquetCodec } from "./stage.mjs";

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/stage-deletion-vector.mjs

@@ -4,8 +4,8 @@ import { writeDeletionVector } from "../puffin/deletion-vector.mjs";
 import { writePuffinFile } from "../puffin/puffin.mjs";
 import { fetchDeleteMaps } from "../fetch.mjs";
 import { validateSchemaForVersion } from "../schema.mjs";
-import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot, loadPriorManifests } from "./snapshot.mjs";
 import { writeDeleteManifest, writeExistingDeleteManifest } from "./manifest.mjs";
+import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot, loadPriorManifests } from "./snapshot.mjs";
 import { checkWriteFormat, newSnapshotId } from "./stage.mjs";
 import { findDataFileEntries, loadManifestEntries, partitionTupleKey } from "./stage-position-delete.mjs";
 async function icebergStageDeletionVector({ tableUrl, metadata, deletes, resolver }) {

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/stage-position-delete.mjs

@@ -1,8 +1,8 @@
 import { bytesToHex, uuid4 } from "../utils.mjs";
 import { fetchAvroRecords } from "../fetch.mjs";
 import { validateSchemaForVersion } from "../schema.mjs";
-import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot } from "./snapshot.mjs";
 import { writeDeleteManifest } from "./manifest.mjs";
+import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot } from "./snapshot.mjs";
 import { checkWriteFormat, newSnapshotId, resolveParquetCodec } from "./stage.mjs";
 import { writePositionDeleteFile } from "./delete-file.mjs";
 async function icebergStagePositionDelete({ tableUrl, metadata, deletes, resolver }) {

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/stage.mjs

@@ -1,10 +1,10 @@
 import { uuid4 } from "../utils.mjs";
 import { validateSchemaForVersion } from "../schema.mjs";
 import { groupByPartition } from "./partition.mjs";
+import { writeDataManifest } from "./manifest.mjs";
 import { computeColumnStats } from "./stats.mjs";
 import { buildPartitionSummaries, buildSnapshotUpdate, currentSnapshot, normalizeCurrentSnapshotId } from "./snapshot.mjs";
 import { applyUpdates } from "./commit.mjs";
-import { writeDataManifest } from "./manifest.mjs";
 import { writeParquet } from "./parquet.mjs";
 import { buildSortComparator } from "./sort.mjs";
 async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderId, snapshotId }) {

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs

@@ -1,9 +1,9 @@
 import { restCatalogCreateTable, restCatalogDropTable, restCatalogLoadTable, restCatalogUpdateTable } from "../catalog/rest.mjs";
+import { applyUpdates, fileCatalogCommit } from "./commit.mjs";
+import { icebergStageAppend, icebergStageExpireSnapshots, icebergStageSetRef, icebergStageUpdateSchema, prepareAppend, stageSnapshotForAppend, stageSnapshotForAppendBatches } from "./stage.mjs";
 import { loadLatestFileCatalogMetadata } from "../metadata.mjs";
 import { loadTable } from "../catalog/loadTable.mjs";
 import { icebergCreate } from "../create.mjs";
-import { applyUpdates, fileCatalogCommit } from "./commit.mjs";
-import { icebergStageAppend, icebergStageExpireSnapshots, icebergStageSetRef, icebergStageUpdateSchema, prepareAppend, stageSnapshotForAppend, stageSnapshotForAppendBatches } from "./stage.mjs";
 import { icebergStagePositionDelete } from "./stage-position-delete.mjs";
 import { icebergStageDeletionVector } from "./stage-deletion-vector.mjs";
 import { icebergStageRewrite } from "./rewrite.mjs";

package/dist/maintenance.d.mts

@@ -1,3 +1,4 @@
 import { isCommitRateLimited, isCommitServerError, isCommitTransient } from "./catalog.mjs";
+import { MergeIcebergManifestsOptions, MergeIcebergManifestsResult, mergeIcebergManifests } from "./manifest-merge.mjs";
 import { SweepListPage, SweepListedObject, SweepStorageClient, SweepUncommittedOrphansOptions, SweepUncommittedOrphansResult, sweepUncommittedOrphans } from "./orphan-sweep.mjs";
-export { type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, isCommitRateLimited, isCommitServerError, isCommitTransient, sweepUncommittedOrphans };
+export { type MergeIcebergManifestsOptions, type MergeIcebergManifestsResult, type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, isCommitRateLimited, isCommitServerError, isCommitTransient, mergeIcebergManifests, sweepUncommittedOrphans };

package/dist/maintenance.mjs

@@ -1,3 +1,4 @@
 import { isCommitRateLimited, isCommitServerError, isCommitTransient } from "./catalog.mjs";
+import { mergeIcebergManifests } from "./manifest-merge.mjs";
 import { sweepUncommittedOrphans } from "./orphan-sweep.mjs";
-export { isCommitRateLimited, isCommitServerError, isCommitTransient, sweepUncommittedOrphans };
+export { isCommitRateLimited, isCommitServerError, isCommitTransient, mergeIcebergManifests, sweepUncommittedOrphans };

package/dist/manifest-merge.d.mts

@@ -0,0 +1,34 @@
+import { CatalogCache } from "./catalog-cache.mjs";
+import { IcebergConnection } from "./catalog.mjs";
+export interface MergeIcebergManifestsOptions {
+  /** Lower the four-MiB encoded manifest limit. */
+  maxManifestBytes?: number;
+  /** Lower the eight-MiB aggregate metadata input/output limit. */
+  maxInputBytes?: number;
+  /** Lower the ten-thousand live-entry limit. */
+  maxEntries?: number;
+  /** Invalidate this reader cache after a successful commit. */
+  cache?: CatalogCache;
+  /** Recheck external authority immediately before the catalog CAS. Exceptions propagate. */
+  confirmAuthority?: () => Promise<boolean>;
+}
+export type MergeIcebergManifestsResult = {
+  _tag: 'Merged';
+  snapshotId: string;
+  manifestsBefore: number;
+  manifestsAfter: number;
+  files: number;
+} | {
+  _tag: 'Unchanged';
+} | {
+  _tag: 'Conflict';
+} | {
+  _tag: 'Err';
+  reason: 'unsupported-format' | 'unsupported-deletes' | 'invalid-inventory' | 'metadata-limit' | 'authority-changed';
+};
+/**
+ * Merge current append-only v2 manifests without reading or rewriting DATA files.
+ * Historical snapshots and their immutable manifests remain available to append retries.
+ * Conflicts leave staged metadata unreferenced. Never delete it after an ambiguous commit response.
+ */
+export declare function mergeIcebergManifests(conn: IcebergConnection, table: string, options?: MergeIcebergManifestsOptions): Promise<MergeIcebergManifestsResult>;

package/dist/manifest-merge.mjs

@@ -0,0 +1,245 @@
+import { restCatalogLoadTable, restCatalogUpdateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
+import { ByteWriter } from "./_virtual/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
+import "./_virtual/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
+import { uuid4 } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/utils.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
+import { invalidateSnapshotRef } from "./catalog.mjs";
+import { writeExistingDataManifest } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/manifest.mjs";
+import { buildPartitionSummaries, buildSnapshotUpdate, loadPriorManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/snapshot.mjs";
+import { newSnapshotId } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/stage.mjs";
+const MAX_MANIFEST_BYTES = 4194304;
+const MAX_INPUT_BYTES = 8388608;
+const MAX_ENTRIES = 1e4;
+const PRESERVED_DATA_FIELDS = /* @__PURE__ */ new Set([
+	"content",
+	"file_path",
+	"file_format",
+	"partition",
+	"record_count",
+	"file_size_in_bytes",
+	"column_sizes",
+	"value_counts",
+	"null_value_counts",
+	"nan_value_counts",
+	"lower_bounds",
+	"upper_bounds",
+	"split_offsets",
+	"sort_order_id"
+]);
+async function mergeIcebergManifests(conn, table, options = {}) {
+	const limit = (value, ceiling) => {
+		if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0 || value > ceiling)) throw new RangeError(`Maintenance limit must be between 1 and ${ceiling}`);
+		return value ?? ceiling;
+	};
+	const maxManifestBytes = limit(options.maxManifestBytes, MAX_MANIFEST_BYTES);
+	const maxInputBytes = limit(options.maxInputBytes, MAX_INPUT_BYTES);
+	const maxEntries = limit(options.maxEntries, MAX_ENTRIES);
+	if (!conn.resolver.writer) throw new Error("Manifest merging requires a metadata writer");
+	const { metadata } = await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table
+	});
+	if (metadata["format-version"] !== 2) return {
+		_tag: "Err",
+		reason: "unsupported-format"
+	};
+	if (metadata["current-snapshot-id"] == null || metadata["current-snapshot-id"] < 0) return { _tag: "Unchanged" };
+	const overflow = /* @__PURE__ */ new Error("Metadata merge byte limit exceeded");
+	let inputBytes = 0;
+	const resolver = {
+		...conn.resolver,
+		reader: async (path) => {
+			const file = await conn.resolver.reader(path);
+			if (!Number.isSafeInteger(file.byteLength) || file.byteLength < 0 || file.byteLength > MAX_MANIFEST_BYTES || inputBytes + file.byteLength > maxInputBytes) throw overflow;
+			inputBytes += file.byteLength;
+			return {
+				byteLength: file.byteLength,
+				slice: async (start, end) => {
+					const bytes = await file.slice(start, end);
+					if (bytes.byteLength !== (end ?? file.byteLength) - start) throw new Error("Metadata object length changed while reading");
+					return bytes;
+				}
+			};
+		}
+	};
+	const lists = /* @__PURE__ */ new Map();
+	const prior = await loadPriorManifests(metadata, resolver).catch((error) => {
+		if (error === overflow) return null;
+		throw error;
+	});
+	if (!prior) return {
+		_tag: "Err",
+		reason: "metadata-limit"
+	};
+	if (prior.some((manifest) => manifest.content !== 0 || manifest.deleted_files_count > 0)) return {
+		_tag: "Err",
+		reason: "unsupported-deletes"
+	};
+	if (prior.length < 2) return { _tag: "Unchanged" };
+	const snapshot = metadata.snapshots?.find((s) => BigInt(s["snapshot-id"]) === BigInt(metadata["current-snapshot-id"]));
+	if (snapshot?.["manifest-list"]) lists.set(snapshot["manifest-list"], Promise.resolve(prior));
+	const inventory = await icebergManifests({
+		metadata,
+		resolver,
+		manifestCache: {
+			lists,
+			entries: /* @__PURE__ */ new Map()
+		}
+	}).catch((error) => {
+		if (error === overflow) return null;
+		throw error;
+	});
+	if (!inventory) return {
+		_tag: "Err",
+		reason: "metadata-limit"
+	};
+	const groups = /* @__PURE__ */ new Map();
+	const paths = /* @__PURE__ */ new Set();
+	for (const manifest of inventory) for (const entry of manifest.entries) {
+		if (entry.status === 2 || entry.data_file.content !== 0) return {
+			_tag: "Err",
+			reason: "unsupported-deletes"
+		};
+		if (Object.entries(entry.data_file).some(([field, value]) => value != null && !PRESERVED_DATA_FIELDS.has(field))) return {
+			_tag: "Err",
+			reason: "invalid-inventory"
+		};
+		if (entry.snapshot_id == null || entry.sequence_number == null || entry.file_sequence_number == null || entry.partition_spec_id == null || paths.has(entry.data_file.file_path) || !metadata["partition-specs"].some((spec) => spec["spec-id"] === entry.partition_spec_id)) return {
+			_tag: "Err",
+			reason: "invalid-inventory"
+		};
+		paths.add(entry.data_file.file_path);
+		if (paths.size > maxEntries) return {
+			_tag: "Err",
+			reason: "metadata-limit"
+		};
+		const group = groups.get(entry.partition_spec_id) ?? [];
+		group.push(entry);
+		groups.set(entry.partition_spec_id, group);
+	}
+	const schema = metadata.schemas.find((schema) => schema["schema-id"] === metadata["current-schema-id"]);
+	if (!schema) return {
+		_tag: "Err",
+		reason: "invalid-inventory"
+	};
+	const planned = [];
+	let outputBytes = 0;
+	function boundedWriter() {
+		const writer = new ByteWriter(Math.min(1024, maxManifestBytes));
+		const ensure = writer.ensure.bind(writer);
+		writer.ensure = (size) => {
+			if (writer.offset + size > maxManifestBytes) throw overflow;
+			ensure(size);
+		};
+		return writer;
+	}
+	async function encode(entries, specId) {
+		const writer = boundedWriter();
+		const spec = metadata["partition-specs"].find((spec) => spec["spec-id"] === specId);
+		if (!await Promise.resolve().then(() => writeExistingDataManifest({
+			writer,
+			schema,
+			partitionSpec: spec,
+			entries,
+			formatVersion: 2
+		})).then(() => true, (error) => {
+			if (error === overflow) return false;
+			throw error;
+		})) {
+			if (entries.length < 2) return false;
+			const middle = Math.floor(entries.length / 2);
+			return await encode(entries.slice(0, middle), specId) && await encode(entries.slice(middle), specId);
+		}
+		if (outputBytes + writer.offset > maxInputBytes) return false;
+		outputBytes += writer.offset;
+		planned.push({
+			bytes: new Uint8Array(writer.getBuffer()),
+			entries,
+			specId
+		});
+		return true;
+	}
+	for (const [specId, entries] of groups) if (!await encode(entries, specId)) return {
+		_tag: "Err",
+		reason: "metadata-limit"
+	};
+	if (planned.length >= prior.length || planned.length === 0) return { _tag: "Unchanged" };
+	const snapshotId = newSnapshotId(metadata);
+	const sequenceNumber = BigInt(metadata["last-sequence-number"]) + 1n;
+	const manifestUuid = uuid4();
+	const manifests = [];
+	for (const [index, plan] of planned.entries()) {
+		const path = `${metadata.location}/metadata/${manifestUuid}-merge-${index}.avro`;
+		const writer = conn.resolver.writer(path);
+		writer.appendBytes(plan.bytes);
+		await writer.finish();
+		const spec = metadata["partition-specs"].find((spec) => spec["spec-id"] === plan.specId);
+		manifests.push({
+			manifest_path: path,
+			manifest_length: BigInt(plan.bytes.byteLength),
+			partition_spec_id: plan.specId,
+			content: 0,
+			sequence_number: sequenceNumber,
+			min_sequence_number: plan.entries.reduce((min, entry) => entry.sequence_number < min ? entry.sequence_number : min, sequenceNumber),
+			added_snapshot_id: snapshotId,
+			added_files_count: 0,
+			existing_files_count: plan.entries.length,
+			deleted_files_count: 0,
+			added_rows_count: 0n,
+			existing_rows_count: plan.entries.reduce((sum, entry) => sum + entry.data_file.record_count, 0n),
+			deleted_rows_count: 0n,
+			partitions: buildPartitionSummaries(plan.entries.map((entry) => entry.data_file.partition), schema, spec)
+		});
+	}
+	const listWriter = boundedWriter();
+	const staged = await buildSnapshotUpdate({
+		tableUrl: metadata.location,
+		metadata,
+		resolver: {
+			...resolver,
+			writer: () => listWriter
+		},
+		snapshotId,
+		sequenceNumber,
+		manifestUuid,
+		timestampMs: Date.now(),
+		formatVersion: 2,
+		priorManifests: prior,
+		skipPriorManifestPaths: new Set(prior.map((manifest) => manifest.manifest_path)),
+		newManifests: manifests,
+		summary: { operation: "replace" },
+		writtenFiles: manifests.map((manifest) => manifest.manifest_path)
+	}).catch((error) => {
+		if (error === overflow) return null;
+		throw error;
+	});
+	if (!staged || outputBytes + listWriter.offset > maxInputBytes) return {
+		_tag: "Err",
+		reason: "metadata-limit"
+	};
+	const persistedList = conn.resolver.writer(staged.snapshot["manifest-list"]);
+	persistedList.appendBytes(listWriter.getBytes());
+	await persistedList.finish();
+	if (options.confirmAuthority && !await options.confirmAuthority()) return {
+		_tag: "Err",
+		reason: "authority-changed"
+	};
+	if (!await restCatalogUpdateTable(conn.catalog, {
+		namespace: conn.namespace,
+		table,
+		requirements: staged.requirements,
+		updates: staged.updates
+	}).then(() => true, (error) => {
+		if (typeof error === "object" && error !== null && "status" in error && (error.status === 409 || error.status === 412)) return false;
+		throw error;
+	})) return { _tag: "Conflict" };
+	if (options.cache) await invalidateSnapshotRef(options.cache, conn.namespace, table, conn.cacheScope);
+	return {
+		_tag: "Merged",
+		snapshotId: String(snapshotId),
+		manifestsBefore: prior.length,
+		manifestsAfter: manifests.length,
+		files: paths.size
+	};
+}
+export { mergeIcebergManifests };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "7.1.1",
+  "version": "7.2.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",