@gscdump/lakehouse 1.3.0 → 1.3.2

package/dist/_chunks/catalog.mjs

@@ -1,5 +1,5 @@
 import { stringifyBigintSafe } from "../bigint.mjs";
-import { cachingResolver, icebergAppend, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./libs/icebird.mjs";
+import { cachingResolver, icebergAppend, icebergAppendBatches, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./libs/icebird.mjs";
 function reportCatalogCacheError(cache, operation, key, error) {
 	try {
 		if (cache.onError) cache.onError(operation, key, error);
@@ -240,6 +240,36 @@ async function icebergAppendRetrying(args, options = {}) {
 		await sleep(Math.floor(random() * ceiling));
 	}
 }
+async function icebergAppendBatchesRetrying(args, options) {
+	const maxAttempts = options.maxAttempts ?? 6;
+	const baseDelayMs = options.baseDelayMs ?? 1e3;
+	const maxDelayMs = options.maxDelayMs ?? 2e4;
+	const sleep = options.sleep ?? defaultCommitSleep;
+	const random = options.random ?? Math.random;
+	const appendId = options.appendId;
+	const { batchFactory, ...appendArgs } = args;
+	const stampedArgs = {
+		...appendArgs,
+		snapshotProperties: {
+			...appendArgs.snapshotProperties,
+			[APPEND_ID_SUMMARY_KEY]: appendId
+		}
+	};
+	if (await appendAlreadyLanded(args, appendId)) return false;
+	for (let attempt = 0; attempt < maxAttempts; attempt++) {
+		const err = await icebergAppendBatches({
+			...stampedArgs,
+			batches: batchFactory()
+		}).then(() => void 0, (e) => e);
+		if (err === void 0) return true;
+		if (!isCommitRateLimited(err)) throw err;
+		if (await appendAlreadyLanded(args, appendId)) return true;
+		if (attempt === maxAttempts - 1) throw err;
+		const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
+		await sleep(Math.floor(random() * ceiling));
+	}
+	return false;
+}
 async function deriveAppendId(args) {
 	const records = args.records ?? [];
 	if (records.length === 0) return globalThis.crypto.randomUUID();
@@ -399,4 +429,4 @@ async function resolveIcebergDataFiles(conn, opts) {
 	}
 	return out;
 }
-export { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };

package/dist/_chunks/libs/icebird.mjs

@@ -3145,7 +3145,7 @@ function compareKeys(ka, kb, resultType, desc, nullsFirst) {
 	const c = compare(ka, kb, resultType);
 	return desc ? -c : c;
 }
-async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderId }) {
+async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderId, snapshotId }) {
 	if (!tableUrl) throw new Error("tableUrl is required");
 	if (!resolver?.writer) throw new Error("resolver.writer is required");
 	const writerFn = resolver.writer;
@@ -3156,7 +3156,7 @@ async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderI
 	const schema = metadata.schemas.find((s) => s["schema-id"] === metadata["current-schema-id"]);
 	if (!schema) throw new Error("current schema not found in metadata");
 	validateSchemaForVersion(schema, formatVersion);
-	const snapshotId = newSnapshotId(metadata);
+	const appendSnapshotId = snapshotId ?? newSnapshotId(metadata);
 	const manifestUuid = uuid4();
 	checkWriteFormat(metadata.properties?.["write.format.default"]);
 	const codec = resolveParquetCodec(metadata.properties?.["write.parquet.compression-codec"]);
@@ -3206,7 +3206,7 @@ async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderI
 		writer: manifestWriter,
 		schema,
 		partitionSpec,
-		snapshotId,
+		snapshotId: appendSnapshotId,
 		dataFiles: writtenDataFiles.map((f) => f.dataFile),
 		formatVersion
 	});
@@ -3215,7 +3215,7 @@ async function prepareAppend({ tableUrl, metadata, records, resolver, sortOrderI
 	const addedFilesSize = writtenDataFiles.reduce((sum, f) => sum + f.dataFile.file_size_in_bytes, 0n);
 	const partitions = buildPartitionSummaries(writtenDataFiles.map((f) => f.dataFile.partition), schema, partitionSpec);
 	return {
-		snapshotId,
+		snapshotId: appendSnapshotId,
 		manifestUuid,
 		formatVersion,
 		manifestPath,
@@ -3284,6 +3284,70 @@ async function stageSnapshotForAppend({ tableUrl, metadata, prepared, resolver,
 		writtenFiles: []
 	});
 }
+async function stageSnapshotForAppendBatches({ tableUrl, metadata, prepared, resolver, snapshotProperties }) {
+	if (!tableUrl) throw new Error("tableUrl is required");
+	if (!resolver?.writer) throw new Error("resolver.writer is required");
+	if (prepared.length === 0) throw new Error("prepared batches must not be empty");
+	const first = prepared[0];
+	for (const batch of prepared) {
+		if (batch.snapshotId !== first.snapshotId) throw new Error("prepared batches must share one snapshot id");
+		if (batch.formatVersion !== first.formatVersion) throw new Error("prepared batches must share one format version");
+	}
+	const sequenceNumber = BigInt(metadata["last-sequence-number"] ?? 0) + 1n;
+	const timestampMs = Date.now();
+	const newManifests = prepared.map((batch) => ({
+		manifest_path: batch.manifestPath,
+		manifest_length: batch.manifestLength,
+		partition_spec_id: batch.partitionSpecId,
+		content: 0,
+		sequence_number: sequenceNumber,
+		min_sequence_number: sequenceNumber,
+		added_snapshot_id: first.snapshotId,
+		added_files_count: batch.addedDataFilesCount,
+		existing_files_count: 0,
+		deleted_files_count: 0,
+		added_rows_count: batch.addedRowCount,
+		existing_rows_count: 0n,
+		deleted_rows_count: 0n,
+		partitions: batch.partitions
+	}));
+	const addedDataFilesCount = prepared.reduce((sum, batch) => sum + batch.addedDataFilesCount, 0);
+	const recordsCount = prepared.reduce((sum, batch) => sum + batch.recordsCount, 0);
+	const addedFilesSize = prepared.reduce((sum, batch) => sum + batch.addedFilesSize, 0n);
+	const prevSummary = currentSnapshot(metadata)?.summary;
+	const prevTotals = {
+		records: BigInt(prevSummary?.["total-records"] ?? "0"),
+		size: BigInt(prevSummary?.["total-files-size"] ?? "0"),
+		files: BigInt(prevSummary?.["total-data-files"] ?? "0")
+	};
+	const summary = {
+		operation: "append",
+		"added-data-files": String(addedDataFilesCount),
+		"added-records": String(recordsCount),
+		"added-files-size": String(addedFilesSize),
+		"changed-partition-count": String(addedDataFilesCount),
+		"total-records": String(prevTotals.records + BigInt(recordsCount)),
+		"total-files-size": String(prevTotals.size + addedFilesSize),
+		"total-data-files": String(prevTotals.files + BigInt(addedDataFilesCount)),
+		"total-delete-files": "0",
+		"total-position-deletes": "0",
+		"total-equality-deletes": "0",
+		...snapshotProperties ?? {}
+	};
+	return await buildSnapshotUpdate({
+		tableUrl,
+		metadata,
+		resolver,
+		snapshotId: first.snapshotId,
+		sequenceNumber,
+		manifestUuid: first.manifestUuid,
+		timestampMs,
+		formatVersion: first.formatVersion,
+		newManifests,
+		summary,
+		writtenFiles: []
+	});
+}
 function checkWriteFormat(value) {
 	if (value === void 0) return;
 	if (value.toLowerCase() !== "parquet") throw new Error(`unsupported write.format.default: ${value}`);
@@ -3415,6 +3479,47 @@ async function icebergAppend({ catalog, namespace, table, tableUrl, resolver, re
 		})
 	});
 }
+async function icebergAppendBatches({ catalog, namespace, table, tableUrl, resolver, batches, sortOrderId, snapshotProperties }) {
+	const ctx = await loadTable({
+		catalog,
+		namespace,
+		table,
+		tableUrl,
+		resolver
+	});
+	const writer = requireResolver(ctx.resolver, "icebergAppendBatches");
+	const prepared = [];
+	let snapshotId;
+	for await (const records of batches) {
+		if (records.length === 0) continue;
+		const batch = await prepareAppend({
+			tableUrl: ctx.tableUrl,
+			metadata: ctx.metadata,
+			records,
+			resolver: writer,
+			sortOrderId,
+			snapshotId
+		});
+		snapshotId ??= batch.snapshotId;
+		prepared.push(batch);
+	}
+	if (prepared.length === 0) return ctx.metadata;
+	return await commitWithRetry({
+		catalog,
+		target: {
+			namespace,
+			table
+		},
+		ctx,
+		stage: (workingCtx) => stageSnapshotForAppendBatches({
+			tableUrl: workingCtx.tableUrl,
+			metadata: workingCtx.metadata,
+			prepared,
+			resolver: requireResolver(workingCtx.resolver, "icebergAppendBatches"),
+			snapshotProperties
+		})
+	});
+}
 async function icebergCreateTable({ catalog, namespace, table, tableUrl, schema, partitionSpec, sortOrder, properties, formatVersion, stageCreate }) {
 	if (catalog.type === "rest") {
 		if (!namespace || !table) throw new Error("namespace and table are required for rest catalogs");
@@ -3746,4 +3851,4 @@ function s3SignedResolver({ accessKeyId, secretAccessKey, sessionToken, region,
 	return () => new Promise((resolve) => setTimeout(resolve, 0));
 })();
 new TextEncoder();
-export { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };
+export { cachingResolver, icebergAppend, icebergAppendBatches, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/index.d.mts

@@ -83,6 +83,15 @@ interface AppendResult {
   /** Rows dropped by the identity guard (missing/out-of-range required identity value). */
   skipped: number;
 }
+type AppendBatchSource = () => Iterable<readonly Record<string, unknown>[]> | AsyncIterable<readonly Record<string, unknown>[]>;
+interface AppendBatchesOptions extends AppendCommitOptions {
+  /** Stable across job retries; recorded in snapshot metadata for landed checks. */
+  appendId: string;
+}
+interface AppendBatchesResult extends AppendResult {
+  /** False when this append id was already present before the call. */
+  committed: boolean;
+}
 interface AppendSinkCloseResult extends AppendResult {
   flushed: boolean;
   error?: unknown;
@@ -112,6 +121,12 @@ interface IcebergDataset {
    * date column pre-converted via {@link toIcebergDayCount}.
    */
   appendRows: (conn: IcebergConnection, rows: readonly Record<string, unknown>[], opts?: AppendCommitOptions) => Promise<AppendResult>;
+  /**
+   * Lazily process bounded row batches, writing multiple data files into one
+   * append snapshot and one catalog commit. Dedupe keys must be disjoint
+   * across source batches.
+   */
+  appendBatches: (conn: IcebergConnection, source: AppendBatchSource, opts: AppendBatchesOptions) => Promise<AppendBatchesResult>;
   /**
    * PURE row processing — the identity INT32 guard, dedupe (identity+dims+
    * naturalKey, last-wins) and cluster pre-sort `appendRows`/`appendSink`
@@ -143,4 +158,4 @@ interface IcebergDataset {
  * from the def. See {@link IcebergDatasetDef}.
  */
 declare function defineIcebergDataset(def: IcebergDatasetDef): IcebergDataset;
-export { type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };
+export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/index.mjs

@@ -1,6 +1,6 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
 import { icebergCreateTable } from "./_chunks/libs/icebird.mjs";
-import { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
+import { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING } from "./schema.mjs";
 const INT32_MIN = -2147483648;
 const INT32_MAX = 2147483647;
@@ -140,7 +140,10 @@ function buildRowProcessor(def, tableSpec) {
 			}
 		}
 		for (const name of dimNames) out[name] = def.dims[name].toPartitionValue(String(row[name]));
-		for (const col of def.columns) out[col.name] = coerceBigIntToNumber(row[col.name]);
+		for (const col of def.columns) {
+			const value = row[col.name];
+			out[col.name] = col.type === "LONG" ? typeof value === "number" ? BigInt(value) : value : coerceBigIntToNumber(value);
+		}
 		return out;
 	}
 	function dedupe(rows) {
@@ -240,6 +243,35 @@ function defineIcebergDataset(def) {
 			skipped
 		};
 	}
+	async function appendBatches(conn, source, opts) {
+		let accepted = 0;
+		let skipped = 0;
+		const batchFactory = async function* () {
+			accepted = 0;
+			skipped = 0;
+			for await (const rows of source()) {
+				const prepared = process(rows);
+				accepted += prepared.records.length;
+				skipped += prepared.skipped;
+				if (prepared.records.length > 0) yield prepared.records;
+			}
+		};
+		const committed = await icebergAppendBatchesRetrying({
+			catalog: conn.catalog,
+			namespace: conn.namespace,
+			table: def.table,
+			resolver: conn.resolver,
+			batchFactory
+		}, {
+			...opts.commitRetry,
+			appendId: opts.appendId
+		});
+		return {
+			accepted,
+			skipped,
+			committed
+		};
+	}
 	function appendSink(opts) {
 		let buffer = [];
 		let connection;
@@ -320,6 +352,7 @@ function defineIcebergDataset(def) {
 		icebergSortOrder: () => sortOrder,
 		createTable,
 		appendRows,
+		appendBatches,
 		prepareRows: process,
 		appendSink,
 		readerPredicate,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "1.3.0",
+  "version": "1.3.2",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",