@gscdump/lakehouse 0.38.1 → 0.39.0

package/dist/_chunks/catalog.mjs

@@ -45,9 +45,10 @@ function toUint8(bytes) {
 	if (bytes == null) return null;
 	return bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
 }
+const UTF8_DECODER = new TextDecoder();
 function decodeString(bytes) {
 	const u = toUint8(bytes);
-	return u == null ? null : new TextDecoder().decode(u);
+	return u == null ? null : UTF8_DECODER.decode(u);
 }
 function decodeMonthInt(bytes) {
 	const u = toUint8(bytes);
@@ -57,32 +58,40 @@ function decodeMonthInt(bytes) {
 function buildManifestPartitionFilter(partitionSpec, matches, wantedMonths) {
 	const fieldIndex = (name) => partitionSpec.findIndex((f) => f.name === name || f.sourceColumn === name);
 	const monthFieldIndex = partitionSpec.findIndex((f) => f.transform === "month");
+	const stringMatches = matches.flatMap((match) => {
+		if (match.encoding !== "string") return [];
+		const index = fieldIndex(match.field);
+		return index < 0 ? [] : [{
+			index,
+			value: String(match.value)
+		}];
+	});
+	const wantedMonthValues = wantedMonths ? [...wantedMonths].filter(Number.isFinite).sort((a, b) => a - b) : [];
+	const hasWantedMonthInRange = (lo, hi) => {
+		let left = 0;
+		let right = wantedMonthValues.length;
+		while (left < right) {
+			const middle = left + right >>> 1;
+			if (wantedMonthValues[middle] < lo) left = middle + 1;
+			else right = middle;
+		}
+		return left < wantedMonthValues.length && wantedMonthValues[left] <= hi;
+	};
 	return (partitions) => {
 		if (!partitions || partitions.length === 0) return true;
-		for (const match of matches) {
-			if (match.encoding !== "string") continue;
-			const idx = fieldIndex(match.field);
-			if (idx < 0) continue;
-			const summary = partitions[idx];
+		for (const match of stringMatches) {
+			const summary = partitions[match.index];
 			if (!summary || summary.lower_bound == null && summary.upper_bound == null) continue;
 			const lo = decodeString(summary.lower_bound);
 			const hi = decodeString(summary.upper_bound);
-			const wantStr = String(match.value);
-			if (lo != null && hi != null && (wantStr < lo || wantStr > hi)) return false;
+			if (lo != null && hi != null && (match.value < lo || match.value > hi)) return false;
 		}
-		if (wantedMonths && wantedMonths.size > 0 && monthFieldIndex >= 0) {
+		if (wantedMonthValues.length > 0 && monthFieldIndex >= 0) {
 			const monthSummary = partitions[monthFieldIndex];
 			if (monthSummary && (monthSummary.lower_bound != null || monthSummary.upper_bound != null)) {
 				const lo = decodeMonthInt(monthSummary.lower_bound);
 				const hi = decodeMonthInt(monthSummary.upper_bound);
-				if (lo != null && hi != null) {
-					let anyInRange = false;
-					for (const wm of wantedMonths) if (wm >= lo && wm <= hi) {
-						anyInRange = true;
-						break;
-					}
-					if (!anyInRange) return false;
-				}
+				if (lo != null && hi != null && !hasWantedMonthInRange(lo, hi)) return false;
 			}
 		}
 		return true;

package/dist/index.mjs

@@ -122,6 +122,7 @@ function buildRowProcessor(def, tableSpec) {
 	const idNames = identityColumnNames(def.identity);
 	const idEncodings = identityEncodings(def.identity);
 	const dimNames = def.dims ? Object.keys(def.dims) : [];
+	const identityColumns = tableSpec.identityColumns;
 	function guard(row) {
 		const out = {};
 		for (const name of idNames) {
@@ -142,11 +143,14 @@ function buildRowProcessor(def, tableSpec) {
 	}
 	function dedupe(rows) {
 		if (rows.length < 2) return rows;
-		const keyCols = tableSpec.identityColumns;
 		const seen = /* @__PURE__ */ new Map();
 		for (const rec of rows) {
-			const k = keyCols.map((c) => `${rec[c] ?? ""}`).join("\0");
-			seen.set(k, rec);
+			let identity = "";
+			for (let index = 0; index < identityColumns.length; index++) {
+				if (index > 0) identity += "\0";
+				identity += `${rec[identityColumns[index]] ?? ""}`;
+			}
+			seen.set(identity, rec);
 		}
 		return seen.size === rows.length ? rows : [...seen.values()];
 	}

package/dist/unsafe-raw.d.mts

@@ -50,6 +50,8 @@ interface SweepUncommittedOrphansOptions {
   dryRun?: boolean;
   /** Injectable clock (ms epoch). Defaults to `Date.now`. */
   now?: () => number;
+  /** Maximum catalog/object-store reads held in flight. Default 4, max 32. */
+  ioConcurrency?: number;
 }
 interface SweepUncommittedOrphansResult {
   /** Tables whose retained snapshots were successfully walked. */

package/dist/unsafe-raw.mjs

@@ -2,7 +2,30 @@ import { icebergAppendRetrying, isCommitRateLimited } from "./_chunks/catalog.mj
 import { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./_chunks/libs/icebird.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
+const DEFAULT_IO_CONCURRENCY = 4;
+const MAX_IO_CONCURRENCY = 32;
 const HOUR_MS = 3600 * 1e3;
+function createIoLimiter(requested) {
+	const concurrency = typeof requested === "number" && Number.isFinite(requested) ? Math.max(1, Math.min(MAX_IO_CONCURRENCY, Math.floor(requested))) : DEFAULT_IO_CONCURRENCY;
+	let active = 0;
+	const waiters = [];
+	return async (operation) => {
+		if (active >= concurrency) await new Promise((resolve) => waiters.push(resolve));
+		active++;
+		try {
+			return await operation();
+		} finally {
+			active--;
+			waiters.shift()?.();
+		}
+	};
+}
+async function settleOrThrow(promises) {
+	const settled = await Promise.allSettled(promises);
+	const failure = settled.find((result) => result.status === "rejected");
+	if (failure) throw failure.reason;
+	return settled.map((result) => result.value);
+}
 function splitBucketKey(filePath) {
 	if (!filePath.startsWith("s3://")) return null;
 	const rest = filePath.slice(5);
@@ -13,11 +36,11 @@ function splitBucketKey(filePath) {
 		key: rest.slice(slash + 1)
 	};
 }
-async function scanTable(conn, table, bucket) {
-	const { metadata } = await restCatalogLoadTable(conn.catalog, {
+async function scanTable(conn, table, bucket, io) {
+	const { metadata } = await io(() => restCatalogLoadTable(conn.catalog, {
 		namespace: conn.namespace,
 		table
-	});
+	}));
 	const location = splitBucketKey(String(metadata.location ?? ""));
 	if (!location || location.bucket !== bucket) return {
 		dataPrefix: null,
@@ -25,34 +48,31 @@ async function scanTable(conn, table, bucket) {
 	};
 	const dataPrefix = `${location.key.replace(/\/+$/, "")}/data/`;
 	const liveKeys = /* @__PURE__ */ new Set();
-	const snapshots = metadata.snapshots ?? [];
-	for (const snapshot of snapshots) {
-		const manifests = await icebergManifests({
-			metadata,
-			resolver: conn.resolver,
-			snapshotId: snapshot["snapshot-id"]
-		});
-		for (const manifest of manifests) for (const entry of manifest.entries) {
-			if (entry.status === 2) continue;
-			const filePath = entry.data_file?.file_path;
-			if (!filePath) continue;
-			const parsed = splitBucketKey(filePath);
-			if (parsed && parsed.bucket === bucket) liveKeys.add(parsed.key);
-		}
+	const manifestsBySnapshot = await settleOrThrow((metadata.snapshots ?? []).map((snapshot) => io(() => icebergManifests({
+		metadata,
+		resolver: conn.resolver,
+		snapshotId: snapshot["snapshot-id"]
+	}))));
+	for (const manifests of manifestsBySnapshot) for (const manifest of manifests) for (const entry of manifest.entries) {
+		if (entry.status === 2) continue;
+		const filePath = entry.data_file?.file_path;
+		if (!filePath) continue;
+		const parsed = splitBucketKey(filePath);
+		if (parsed && parsed.bucket === bucket) liveKeys.add(parsed.key);
 	}
 	return {
 		dataPrefix,
 		liveKeys
 	};
 }
-async function listAllUnderPrefix(s3, prefix) {
+async function listAllUnderPrefix(s3, prefix, io) {
 	const out = [];
 	let cursor;
 	do {
-		const page = await s3.list({
+		const page = await io(() => s3.list({
 			prefix,
 			cursor
-		});
+		}));
 		out.push(...page.objects);
 		cursor = page.truncated ? page.cursor : void 0;
 	} while (cursor);
@@ -64,7 +84,8 @@ async function sweepUncommittedOrphans(opts) {
 	const maxDeletes = opts.maxDeletes ?? DEFAULT_MAX_DELETES;
 	const dryRun = opts.dryRun ?? false;
 	const graceCutoff = (opts.now ?? Date.now)() - graceHours * HOUR_MS;
-	const tables = opts.tables ?? (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name);
+	const io = createIoLimiter(opts.ioConcurrency);
+	const tables = opts.tables ?? (await io(() => restCatalogListTables(conn.catalog, { namespace: conn.namespace }))).map((t) => t.name);
 	if (tables.length === 0) return {
 		scannedTables: [],
 		skippedTables: [],
@@ -75,26 +96,33 @@ async function sweepUncommittedOrphans(opts) {
 		cappedAtLimit: false,
 		reason: "zero-tables"
 	};
-	const scannedTables = [];
-	const skippedTables = [];
-	let liveFileCount = 0;
-	const candidates = [];
-	for (const table of tables) {
-		const { dataPrefix, liveKeys } = await scanTable(conn, table, bucket);
-		if (!dataPrefix) {
-			skippedTables.push(table);
-			continue;
-		}
-		scannedTables.push(table);
-		liveFileCount += liveKeys.size;
-		const listed = await listAllUnderPrefix(s3, dataPrefix);
+	const tableResults = await settleOrThrow(tables.map(async (table) => {
+		const { dataPrefix, liveKeys } = await scanTable(conn, table, bucket, io);
+		if (!dataPrefix) return {
+			table,
+			skipped: true,
+			liveFileCount: 0,
+			candidates: []
+		};
+		const listed = await listAllUnderPrefix(s3, dataPrefix, io);
+		const tableCandidates = [];
 		for (const obj of listed) {
 			if (!obj.key.startsWith(dataPrefix)) continue;
 			if (liveKeys.has(obj.key)) continue;
 			if (obj.uploaded.getTime() > graceCutoff) continue;
-			candidates.push(obj);
+			tableCandidates.push(obj);
 		}
-	}
+		return {
+			table,
+			skipped: false,
+			liveFileCount: liveKeys.size,
+			candidates: tableCandidates
+		};
+	}));
+	const scannedTables = tableResults.filter((result) => !result.skipped).map((result) => result.table);
+	const skippedTables = tableResults.filter((result) => result.skipped).map((result) => result.table);
+	const liveFileCount = tableResults.reduce((sum, result) => sum + result.liveFileCount, 0);
+	const candidates = tableResults.flatMap((result) => result.candidates);
 	const candidateCount = candidates.length;
 	const toDelete = candidates.slice(0, maxDeletes);
 	const cappedAtLimit = candidateCount > maxDeletes;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "0.38.1",
+  "version": "0.39.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",