@gscdump/lakehouse 5.2.0 → 5.6.0

package/dist/catalog.d.mts

@@ -108,6 +108,12 @@ export declare function connectIcebergCatalog(config: IcebergCatalogConfig, opts
  * exact keys readers populate — see {@link invalidateSnapshotRef}.
  */
 export declare function catalogCacheScope(config: Pick<IcebergCatalogConfig, 'catalogUri' | 'warehouse'>): string;
+/** An existing namespace cannot prove this caller created or owns its contents. */
+export declare function createIcebergNamespace(conn: IcebergConnection): Promise<{
+  _tag: 'Created';
+} | {
+  _tag: 'Exists';
+}>;
 /**
  * Ensure the catalog namespace exists. Idempotent — an "already exists"
  * response from the REST catalog is swallowed.

package/dist/catalog.mjs

@@ -163,6 +163,12 @@ function isNamespaceAlreadyExistsError(err) {
 	const msg = (err instanceof Error ? err.message : String(err)).toLowerCase();
 	return msg.includes("already exists") || msg.includes("409") || msg.includes("conflict");
 }
+async function createIcebergNamespace(conn) {
+	return restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).then(() => ({ _tag: "Created" })).catch((error) => {
+		if (error && typeof error === "object" && "status" in error && error.status === 409) return { _tag: "Exists" };
+		throw error;
+	});
+}
 async function ensureIcebergNamespace(conn) {
 	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch((err) => {
 		if (!isNamespaceAlreadyExistsError(err)) throw err;
@@ -459,6 +465,144 @@ async function loadSnapshotIdUnshared(conn, namespace, table, cache, now) {
 		metadata
 	};
 }
+async function resolveIcebergAppendFiles(conn, table, appendId, matches) {
+	return (await createIcebergAppendFileResolver(conn, table))(appendId, matches);
+}
+async function createIcebergAppendFileResolver(conn, table, dateColumn) {
+	const { metadata, metadataLocation } = await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table
+	});
+	const dateFieldId = dateColumnFieldId(metadata, dateColumn);
+	const manifestCache = {
+		lists: /* @__PURE__ */ new Map(),
+		entries: /* @__PURE__ */ new Map()
+	};
+	const immutableBytes = /* @__PURE__ */ new Map();
+	const maxRetainedBytes = 4194304;
+	const budgetFailure = { _tag: "ManifestBudgetExceeded" };
+	let retainedBytes = 0;
+	let unavailable = false;
+	const resolver = {
+		...conn.resolver,
+		reader: async (url) => {
+			let bytes = immutableBytes.get(url);
+			if (!bytes) {
+				bytes = Promise.resolve(conn.resolver.reader(url)).then(async (buffer) => {
+					if (!Number.isSafeInteger(buffer.byteLength) || buffer.byteLength < 0 || retainedBytes + buffer.byteLength > maxRetainedBytes) {
+						unavailable = true;
+						throw budgetFailure;
+					}
+					retainedBytes += buffer.byteLength;
+					const content = await buffer.slice(0, buffer.byteLength);
+					if (content.byteLength !== buffer.byteLength) {
+						unavailable = true;
+						throw budgetFailure;
+					}
+					return content;
+				});
+				immutableBytes.set(url, bytes);
+			}
+			const buffer = await bytes;
+			return {
+				byteLength: buffer.byteLength,
+				slice: async (start, end) => buffer.slice(start, end)
+			};
+		}
+	};
+	const budgetUnavailable = (error) => {
+		if (error !== budgetFailure) throw error;
+		return [];
+	};
+	const current = metadata["current-snapshot-id"] == null ? [] : await icebergManifests({
+		metadata,
+		resolver,
+		manifestCache
+	}).catch(budgetUnavailable);
+	const currentFiles = /* @__PURE__ */ new Map();
+	for (const manifest of current) for (const entry of manifest.entries) if (entry.status !== 2 && entry.data_file.content === 0) currentFiles.set(entry.data_file.file_path, entry.data_file);
+	const resolve = async (appendId, matches) => {
+		if (unavailable) return {
+			_tag: "Err",
+			reason: "append-unavailable"
+		};
+		const snapshots = metadata.snapshots?.filter((snapshot) => snapshot.summary?.[APPEND_ID_SUMMARY_KEY] === appendId) ?? [];
+		if (!appendId || snapshots.length !== 1) return {
+			_tag: "Err",
+			reason: "append-unavailable"
+		};
+		if (snapshots[0].summary?.operation !== "append") return {
+			_tag: "Err",
+			reason: "append-unavailable"
+		};
+		const snapshotId = snapshots[0]["snapshot-id"];
+		const manifests = await icebergManifests({
+			metadata,
+			resolver,
+			snapshotId,
+			manifestCache
+		}).catch(budgetUnavailable);
+		if (unavailable) return {
+			_tag: "Err",
+			reason: "append-unavailable"
+		};
+		const files = [];
+		for (const manifest of manifests) for (const entry of manifest.entries) {
+			if (entry.status === 2 || entry.data_file.content !== 0) return {
+				_tag: "Err",
+				reason: "append-unavailable"
+			};
+			if (entry.status !== 1 || entry.snapshot_id == null || String(entry.snapshot_id) !== String(snapshotId)) continue;
+			const file = toListedFile(entry, matches, void 0, /* @__PURE__ */ new Set(), null);
+			if (file) {
+				const member = currentFiles.get(file.filePath);
+				if (!member || Number(member.record_count) !== file.rowCount || Number(member.file_size_in_bytes) !== file.bytes) return {
+					_tag: "Err",
+					reason: "append-unavailable"
+				};
+				files.push(file);
+			}
+		}
+		if (files.length === 0) return {
+			_tag: "Err",
+			reason: "append-unavailable"
+		};
+		return {
+			_tag: "Ok",
+			snapshotId: String(snapshotId),
+			files
+		};
+	};
+	return Object.assign(resolve, {
+		confirmCurrent: async () => {
+			if (unavailable) return false;
+			const fresh = await restCatalogLoadTable(conn.catalog, {
+				namespace: conn.namespace,
+				table
+			});
+			return fresh.metadataLocation === metadataLocation && fresh.metadata["table-uuid"] === metadata["table-uuid"] && String(fresh.metadata["current-snapshot-id"]) === String(metadata["current-snapshot-id"]);
+		},
+		currentFiles: (matches) => {
+			if (unavailable) return {
+				_tag: "Err",
+				reason: "append-unavailable"
+			};
+			const files = [];
+			for (const manifest of current) for (const entry of manifest.entries) {
+				if (entry.status !== 2 && entry.data_file.content !== 0) return {
+					_tag: "Err",
+					reason: "append-unavailable"
+				};
+				const file = toListedFile(entry, matches, void 0, /* @__PURE__ */ new Set(), dateFieldId);
+				if (file) files.push(file);
+			}
+			return {
+				_tag: "Ok",
+				files
+			};
+		}
+	});
+}
 function toListedFile(entry, matches, monthFieldName, wantedMonths, dateFieldId) {
 	if (entry.status === 2) return null;
 	const df = entry.data_file;
@@ -639,4 +783,4 @@ async function resolveIcebergDataFilesInBatch(conn, opts) {
 	}
 	return out;
 }
-export { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, isCommitServerError, isCommitTransient, listIcebergTables, resolveIcebergDataFiles };
+export { catalogCacheScope, connectIcebergCatalog, createIcebergAppendFileResolver, createIcebergNamespace, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, isCommitServerError, isCommitTransient, listIcebergTables, resolveIcebergAppendFiles, resolveIcebergDataFiles };

package/dist/dataset-runtime.mjs

@@ -1,5 +1,23 @@
-import { connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
+import { restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
+import { connectIcebergCatalog, createIcebergAppendFileResolver, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergAppendFiles, resolveIcebergDataFiles } from "./catalog.mjs";
 import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs";
+async function verifyDatasetTable(conn, table, schema, partitionSpec) {
+	const { metadata } = await restCatalogLoadTable(conn.catalog, {
+		namespace: conn.namespace,
+		table
+	});
+	const currentSchema = metadata.schemas?.find((s) => s["schema-id"] === metadata["current-schema-id"]);
+	if (!currentSchema || currentSchema.fields.length !== schema.fields.length || !schema.fields.every((expected) => currentSchema.fields.some((actual) => actual.id === expected.id && actual.name === expected.name && actual.type === expected.type && actual.required === expected.required))) return {
+		_tag: "Err",
+		reason: "schema-mismatch"
+	};
+	const currentSpec = metadata["partition-specs"]?.find((s) => s["spec-id"] === metadata["default-spec-id"]);
+	if (!currentSpec || currentSpec.fields.length !== partitionSpec.fields.length || !partitionSpec.fields.every((expected) => currentSpec.fields.some((actual) => actual["source-id"] === expected["source-id"] && actual["field-id"] === expected["field-id"] && actual.name === expected.name && actual.transform === expected.transform))) return {
+		_tag: "Err",
+		reason: "partition-mismatch"
+	};
+	return { _tag: "Ok" };
+}
 async function createDatasetTable(conn, table, schema, partitionSpec, sortOrder) {
 	const results = [];
 	await icebergCreateTable({
@@ -65,4 +83,4 @@ async function connectDatasetCatalog(config, opts) {
 async function resolveDatasetDataFiles(conn, opts) {
 	return resolveIcebergDataFiles(conn, opts);
 }
-export { appendDatasetBatches, appendDatasetRows, connectDatasetCatalog, createDatasetTable, resolveDatasetDataFiles };
+export { appendDatasetBatches, appendDatasetRows, connectDatasetCatalog, createIcebergAppendFileResolver as createDatasetAppendFileResolver, createDatasetTable, resolveIcebergAppendFiles as resolveDatasetAppendFiles, resolveDatasetDataFiles, verifyDatasetTable };

package/dist/dataset.d.mts

@@ -105,6 +105,13 @@ export interface IcebergDataset {
   icebergPartitionSpec: () => IcebergPartitionSpec;
   icebergSortOrder: () => IcebergSortOrder | undefined;
   createTable: (conn: IcebergConnection) => Promise<IcebergTableOpResult[]>;
+  /** Inspect current metadata without caches before admitting a provisioned table. */
+  verifyTable: (conn: IcebergConnection) => Promise<{
+    _tag: 'Ok';
+  } | {
+    _tag: 'Err';
+    reason: 'schema-mismatch' | 'partition-mismatch';
+  }>;
   /**
    * One-shot append over an EXISTING connection: identity INT32 guard, dedupe
    * by identity+dims+naturalKey (last-wins), cluster pre-sort (if
@@ -120,6 +127,26 @@ export interface IcebergDataset {
    * across source batches.
    */
   appendBatches: (conn: IcebergConnection, source: AppendBatchSource, opts: AppendBatchesOptions) => Promise<AppendBatchesResult>;
+  /** Fresh, exact snapshot membership. Missing evidence never falls back to current files. */
+  resolveAppendFiles: (conn: IcebergConnection, appendId: string, identity: string | number, dims?: Record<string, string>) => Promise<{
+    _tag: 'Ok';
+    snapshotId: string;
+    files: IcebergListedDataFile[];
+  } | {
+    _tag: 'Err';
+    reason: 'append-unavailable';
+  }>;
+  /** Create once per request. The next request must create a fresh resolver. */
+  createAppendFileResolver: (conn: IcebergConnection) => Promise<((appendId: string, identity: string | number, dims?: Record<string, string>) => ReturnType<IcebergDataset['resolveAppendFiles']>) & {
+    currentFiles: (identity: string | number, dims?: Record<string, string>) => {
+      _tag: 'Ok';
+      files: IcebergListedDataFile[];
+    } | {
+      _tag: 'Err';
+      reason: 'append-unavailable';
+    };
+    confirmCurrent: () => Promise<boolean>;
+  }>;
   /**
    * PURE row processing — the identity INT32 guard, dedupe (identity+dims+
    * naturalKey, last-wins) and cluster pre-sort `appendRows`/`appendSink`

package/dist/dataset.mjs

@@ -180,6 +180,10 @@ function defineIcebergDataset(def) {
 		const { createDatasetTable } = await import("./dataset-runtime.mjs");
 		return createDatasetTable(conn, def.table, schema, partitionSpecIcebird, sortOrder);
 	}
+	async function verifyTable(conn) {
+		const { verifyDatasetTable } = await import("./dataset-runtime.mjs");
+		return verifyDatasetTable(conn, def.table, schema, partitionSpecIcebird);
+	}
 	function process(rows) {
 		const guarded = [];
 		let skipped = 0;
@@ -280,6 +284,20 @@ function defineIcebergDataset(def) {
 	function partitionBoundFilter(identity, months, dims) {
 		return buildManifestPartitionFilter(def.partition, readerPredicate(identity, dims), months);
 	}
+	async function resolveAppendFiles(conn, appendId, identity, dims) {
+		const { resolveDatasetAppendFiles } = await import("./dataset-runtime.mjs");
+		return resolveDatasetAppendFiles(conn, def.table, appendId, readerPredicate(identity, dims));
+	}
+	async function createAppendFileResolver(conn) {
+		const { createDatasetAppendFileResolver } = await import("./dataset-runtime.mjs");
+		const dates = def.columns.filter((column) => column.type === "DATE");
+		const dateColumn = def.partition.find((field) => field.transform === "month")?.sourceColumn ?? (dates.length === 1 ? dates[0]?.name : void 0);
+		const resolve = await createDatasetAppendFileResolver(conn, def.table, dateColumn);
+		return Object.assign((appendId, identity, dims) => resolve(appendId, readerPredicate(identity, dims)), {
+			currentFiles: (identity, dims) => resolve.currentFiles(readerPredicate(identity, dims)),
+			confirmCurrent: resolve.confirmCurrent
+		});
+	}
 	async function resolveDataFiles(conn, identity, range, dims, opts = {}) {
 		const { resolveDatasetDataFiles } = await import("./dataset-runtime.mjs");
 		return resolveDatasetDataFiles(conn, {
@@ -300,8 +318,11 @@ function defineIcebergDataset(def) {
 		icebergPartitionSpec: () => partitionSpecIcebird,
 		icebergSortOrder: () => sortOrder,
 		createTable,
+		verifyTable,
 		appendRows,
 		appendBatches,
+		resolveAppendFiles,
+		createAppendFileResolver,
 		prepareRows: process,
 		appendSink,
 		readerPredicate,

package/dist/index.d.mts

@@ -2,7 +2,7 @@ import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringi
 import { CatalogCache, CatalogCacheGetOutcome, cacheGet, cachePut } from "./catalog-cache.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./schema.mjs";
 import { IcebergFieldSummary, ManifestPartitionFilter, PartitionValueMatch, buildManifestPartitionFilter } from "./partition-prune.mjs";
-import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergDataFileDateBounds, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
+import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergDataFileDateBounds, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, createIcebergNamespace, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
 import { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
 import { toIcebergDayCount } from "./date.mjs";
-export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CatalogCacheGetOutcome, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataFileDateBounds, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };
+export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CatalogCacheGetOutcome, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataFileDateBounds, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, createIcebergNamespace, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/index.mjs

@@ -2,7 +2,7 @@ import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringi
 import { cacheGet, cachePut } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
 import { buildManifestPartitionFilter } from "./partition-prune.mjs";
-import { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
+import { catalogCacheScope, connectIcebergCatalog, createIcebergNamespace, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP } from "./schema.mjs";
 import { defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
-export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, createIcebergNamespace, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "5.2.0",
+  "version": "5.6.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",