@gscdump/lakehouse 3.4.1 → 3.4.3

package/dist/catalog.mjs

@@ -5,7 +5,7 @@ import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.27_p
 import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/s3.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
-import { buildManifestPartitionFilter } from "./partition-prune.mjs";
+import { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket } from "./partition-prune.mjs";
 let icebirdWriteModule;
 function useIcebirdWriteModule() {
 	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/write/write.mjs");
@@ -46,18 +46,20 @@ async function connectIcebergCatalog(config, opts = {}) {
 			await cachePut(opts.cache, catalogConfigKey(config), toCache, CATALOG_CONFIG_TTL_MS, now);
 		}
 	}
-	const resolver = withVerifiedWriterByteLengths(cachingResolver(s3SignedResolver({
+	const cacheScope = catalogCacheScope(config);
+	const s3Resolver = s3SignedResolver({
 		accessKeyId: config.s3.accessKeyId,
 		secretAccessKey: config.s3.secretAccessKey,
 		region: config.s3.region ?? "auto",
 		endpoint: config.s3.endpoint,
 		pathStyle: true
-	})));
+	});
+	const resolver = withVerifiedWriterByteLengths(cachingResolver(s3Resolver));
 	return {
 		catalog,
 		resolver,
 		namespace: config.namespace,
-		cacheScope: catalogCacheScope(config)
+		cacheScope
 	};
 }
 function withVerifiedWriterByteLengths(resolver) {
@@ -230,6 +232,7 @@ const SNAPSHOT_REF_TTL_MS = 18e5;
 const RESOLVED_FILES_TTL_MS = 864e5;
 const METADATA_TTL_MS = 864e5;
 const MAX_CACHED_METADATA_BYTES = 2097152;
+const MONTH_FILES_TTL_MS = 2592e6;
 function snapshotRefKey(scope, namespace, table) {
 	return `lh-snapref\0${scope}\0${namespace}\0${table}`;
 }
@@ -242,8 +245,11 @@ async function invalidateSnapshotRef(cache, namespace, table, cacheScope = "") {
 function metadataRefKey(scope, namespace, table, snapshotId) {
 	return `lh-snapmeta\0${scope}\0${namespace}\0${table}\0${snapshotId}`;
 }
+function matchKeyOf(matches) {
+	return [...matches].map((m) => `${m.field}=${m.value}`).sort().join(",");
+}
 function resolvedFilesKey(scope, namespace, table, snapshotId, matches, range) {
-	return `lh-files2\0${scope}\0${namespace}\0${table}\0${snapshotId}\0${[...matches].map((m) => `${m.field}=${m.value}`).sort().join(",")}\0${range.start}..${range.end}`;
+	return `lh-files2\0${scope}\0${namespace}\0${table}\0${snapshotId}\0${matchKeyOf(matches)}\0${range.start}..${range.end}`;
 }
 function monthsInRange(range) {
 	const [sy, sm] = range.start.split("-").map(Number);
@@ -265,6 +271,18 @@ function monthsSinceEpoch(ym) {
 	const [y, m] = ym.split("-").map(Number);
 	return (y - 1970) * 12 + (m - 1);
 }
+function ymFromMonthsSinceEpoch(value) {
+	const y = 1970 + Math.floor(value / 12);
+	const m = (value % 12 + 12) % 12 + 1;
+	return `${y}-${String(m).padStart(2, "0")}`;
+}
+function monthFilesKey(scope, namespace, table, matchKey, monthYm, manifestSetHash) {
+	return `lh-month\0${scope}\0${namespace}\0${table}\0${matchKey}\0${monthYm}\0${manifestSetHash}`;
+}
+async function sha256Hex(text) {
+	const digest = await globalThis.crypto.subtle.digest("SHA-256", new TextEncoder().encode(text));
+	return Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join("");
+}
 const DATE_BOUND_BYTES = 4;
 function boundBytesForField(map, fieldId) {
 	if (map == null) return null;
@@ -342,11 +360,125 @@ async function loadSnapshotId(conn, namespace, table, cache, now) {
 		metadata
 	};
 }
+function toListedFile(entry, matches, monthFieldName, wantedMonths, dateFieldId) {
+	if (entry.status === 2) return null;
+	const df = entry.data_file;
+	if (df.content !== 0) return null;
+	const part = df.partition;
+	for (const match of matches) if (String(part[match.field]) !== String(match.value)) return null;
+	if (monthFieldName) {
+		const month = part[monthFieldName];
+		if (typeof month !== "number" || !wantedMonths.has(month)) return null;
+	}
+	const dateBounds = dateFieldId == null ? null : readDataFileDateBounds(df, dateFieldId);
+	return {
+		filePath: df.file_path,
+		objectKey: stripBucket(df.file_path),
+		bytes: Number(df.file_size_in_bytes),
+		rowCount: Number(df.record_count),
+		...dateBounds ? { dateBounds } : {}
+	};
+}
+async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, monthFieldName, wantedMonths, dateFieldId) {
+	const manifests = await icebergManifests({
+		metadata,
+		resolver: conn.resolver,
+		partitionFilter
+	});
+	const entries = [];
+	for (const m of manifests) for (const entry of m.entries) {
+		const file = toListedFile(entry, matches, monthFieldName, wantedMonths, dateFieldId);
+		if (file) entries.push(file);
+	}
+	return {
+		entries,
+		manifestsWalked: manifests.length,
+		monthsWanted: 0,
+		monthsHit: 0
+	};
+}
+async function resolveViaMonthCache(conn, cache, metadata, partitionFilter, partitionSpec, matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now) {
+	const matchKey = matchKeyOf(matches);
+	const manifestMonths = /* @__PURE__ */ new Map();
+	await icebergManifests({
+		metadata,
+		resolver: conn.resolver,
+		partitionFilter: (partitions, _specId, manifest) => {
+			try {
+				if (partitionFilter(partitions) === false) return false;
+				manifestMonths.set(manifest.manifest_path, manifestMonthBucket(partitionSpec, partitions));
+			} catch {
+				manifestMonths.set(manifest.manifest_path, MULTI_MONTH_MANIFEST);
+			}
+			return false;
+		}
+	});
+	const pathsByMonth = /* @__PURE__ */ new Map();
+	for (const [path, bucket] of manifestMonths) {
+		if (bucket === "multi" || !wantedMonths.has(bucket)) continue;
+		const list = pathsByMonth.get(bucket);
+		if (list) list.push(path);
+		else pathsByMonth.set(bucket, [path]);
+	}
+	const monthsWanted = pathsByMonth.size;
+	const monthKeys = /* @__PURE__ */ new Map();
+	await Promise.all([...pathsByMonth].map(async ([monthValue, paths]) => {
+		const hash = await sha256Hex([...paths].sort().join("\0"));
+		monthKeys.set(monthValue, monthFilesKey(scope, namespace, table, matchKey, ymFromMonthsSinceEpoch(monthValue), hash));
+	}));
+	const cachedByMonth = /* @__PURE__ */ new Map();
+	await Promise.all([...monthKeys].map(async ([monthValue, key]) => {
+		const cached = await cacheGet(cache, key, now);
+		if (cached !== void 0) cachedByMonth.set(monthValue, cached);
+	}));
+	const monthsHit = cachedByMonth.size;
+	const toWalk = /* @__PURE__ */ new Set();
+	for (const [path, bucket] of manifestMonths) if (bucket === "multi" || !cachedByMonth.has(bucket)) toWalk.add(path);
+	const entries = [];
+	for (const cached of cachedByMonth.values()) entries.push(...cached);
+	let manifestsWalked = 0;
+	if (toWalk.size > 0) {
+		const manifests = await icebergManifests({
+			metadata,
+			resolver: conn.resolver,
+			partitionFilter: (_partitions, _specId, manifest) => toWalk.has(manifest.manifest_path)
+		});
+		manifestsWalked = manifests.length;
+		const freshByMonth = /* @__PURE__ */ new Map();
+		for (const m of manifests) {
+			const bucket = manifestMonths.get(m.url);
+			if (bucket !== void 0 && bucket !== "multi" && !freshByMonth.has(bucket)) freshByMonth.set(bucket, []);
+			for (const entry of m.entries) {
+				const file = toListedFile(entry, matches, monthFieldName, wantedMonths, dateFieldId);
+				if (!file) continue;
+				entries.push(file);
+				if (bucket !== void 0 && bucket !== "multi") {
+					const list = freshByMonth.get(bucket);
+					if (list) list.push(file);
+					else freshByMonth.set(bucket, [file]);
+				}
+			}
+		}
+		const puts = [];
+		for (const [monthValue, files] of freshByMonth) {
+			const key = monthKeys.get(monthValue);
+			if (key) puts.push(cachePut(cache, key, files, MONTH_FILES_TTL_MS, now));
+		}
+		await Promise.all(puts);
+	}
+	return {
+		entries,
+		manifestsWalked,
+		monthsWanted,
+		monthsHit
+	};
+}
 async function resolveIcebergDataFiles(conn, opts) {
 	const { namespace, table } = opts;
 	const profiler = opts.profiler;
 	const now = (opts.clock ?? Date.now)();
 	const wantedMonths = new Set(monthsInRange(opts.range).map(monthsSinceEpoch));
+	if (wantedMonths.size === 0) return [];
 	const endSnapshot = profiler?.start("iceberg.snapshot");
 	let { snapshotId, metadata } = await loadSnapshotId(conn, namespace, table, opts.cache, now);
 	endSnapshot?.({ cached: metadata == null && snapshotId != null });
@@ -371,49 +503,28 @@ async function resolveIcebergDataFiles(conn, opts) {
 	}
 	const endWalk = profiler?.start("iceberg.walk");
 	const partitionFilter = buildManifestPartitionFilter(opts.partitionSpec, opts.matches, wantedMonths);
-	const manifests = await icebergManifests({
-		metadata,
-		resolver: conn.resolver,
-		partitionFilter
-	});
 	const monthField = opts.partitionSpec.find((f) => f.transform === "month");
 	const monthFieldName = monthField?.name;
 	const dateFieldId = dateColumnFieldId(metadata, monthField?.sourceColumn);
 	const dayWindow = rangeDayWindow(opts.range);
+	const walk = opts.cache && monthFieldName ? await resolveViaMonthCache(conn, opts.cache, metadata, partitionFilter, opts.partitionSpec, opts.matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now) : await resolveViaFullWalk(conn, metadata, partitionFilter, opts.matches, monthFieldName, wantedMonths, dateFieldId);
 	let boundsPruned = 0;
 	const out = [];
-	for (const m of manifests) for (const entry of m.entries) {
-		if (entry.status === 2) continue;
-		const df = entry.data_file;
-		if (df.content !== 0) continue;
-		const part = df.partition;
-		let matchesAll = true;
-		for (const match of opts.matches) if (String(part[match.field]) !== String(match.value)) {
-			matchesAll = false;
-			break;
-		}
-		if (!matchesAll) continue;
-		if (monthFieldName) {
-			const month = part[monthFieldName];
-			if (typeof month !== "number" || !wantedMonths.has(month)) continue;
-		}
-		const dateBounds = dateFieldId == null ? null : readDataFileDateBounds(df, dateFieldId);
-		if (dateBounds && dayWindow && (dateBounds.maxDay < dayWindow.startDay || dateBounds.minDay > dayWindow.endDay)) {
+	for (const file of walk.entries) {
+		if (file.dateBounds && dayWindow && (file.dateBounds.maxDay < dayWindow.startDay || file.dateBounds.minDay > dayWindow.endDay)) {
 			boundsPruned++;
 			continue;
 		}
-		out.push({
-			filePath: df.file_path,
-			objectKey: stripBucket(df.file_path),
-			bytes: Number(df.file_size_in_bytes),
-			rowCount: Number(df.record_count),
-			...dateBounds ? { dateBounds } : {}
-		});
+		out.push(file);
 	}
+	out.sort((a, b) => a.filePath < b.filePath ? -1 : a.filePath > b.filePath ? 1 : 0);
 	endWalk?.({
-		manifests: manifests.length,
+		manifests: walk.manifestsWalked,
 		files: out.length,
-		boundsPruned
+		boundsPruned,
+		monthsWanted: walk.monthsWanted,
+		monthsHit: walk.monthsHit,
+		manifestsWalked: walk.manifestsWalked
 	});
 	if (opts.cache) {
 		const freshKey = resolvedFilesKey(scope, namespace, table, snapshotId, opts.matches, opts.range);

package/dist/partition-prune.mjs

@@ -54,4 +54,15 @@ function buildManifestPartitionFilter(partitionSpec, matches, wantedMonths) {
 		return true;
 	};
 }
-export { buildManifestPartitionFilter };
+const MULTI_MONTH_MANIFEST = "multi";
+function manifestMonthBucket(partitionSpec, partitions) {
+	const monthFieldIndex = partitionSpec.findIndex((f) => f.transform === "month");
+	if (monthFieldIndex < 0 || !partitions || partitions.length <= monthFieldIndex) return MULTI_MONTH_MANIFEST;
+	const monthSummary = partitions[monthFieldIndex];
+	if (!monthSummary || monthSummary.lower_bound == null && monthSummary.upper_bound == null) return MULTI_MONTH_MANIFEST;
+	const lo = decodeMonthInt(monthSummary.lower_bound);
+	const hi = decodeMonthInt(monthSummary.upper_bound);
+	if (lo == null || hi == null || lo !== hi) return MULTI_MONTH_MANIFEST;
+	return lo;
+}
+export { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "3.4.1",
+  "version": "3.4.3",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",