@gscdump/lakehouse 3.4.1 → 3.4.2

package/dist/catalog.mjs

@@ -5,6 +5,7 @@ import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.27_p
 import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.27_patch_hash=651b5cc1b513184638c98e21a0852c4559044ae60f8449fbe9d81a894ea5b4d7/node_modules/icebird/src/s3.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
+import { wrapManifestCacheResolver } from "./manifest-cache-resolver.mjs";
 import { buildManifestPartitionFilter } from "./partition-prune.mjs";
 let icebirdWriteModule;
 function useIcebirdWriteModule() {
@@ -46,18 +47,21 @@ async function connectIcebergCatalog(config, opts = {}) {
 			await cachePut(opts.cache, catalogConfigKey(config), toCache, CATALOG_CONFIG_TTL_MS, now);
 		}
 	}
-	const resolver = withVerifiedWriterByteLengths(cachingResolver(s3SignedResolver({
+	const cacheScope = catalogCacheScope(config);
+	const s3Resolver = s3SignedResolver({
 		accessKeyId: config.s3.accessKeyId,
 		secretAccessKey: config.s3.secretAccessKey,
 		region: config.s3.region ?? "auto",
 		endpoint: config.s3.endpoint,
 		pathStyle: true
-	})));
+	});
+	const baseResolver = opts.cache ? wrapManifestCacheResolver(s3Resolver, opts.cache, cacheScope, opts.clock) : s3Resolver;
+	const resolver = withVerifiedWriterByteLengths(cachingResolver(baseResolver));
 	return {
 		catalog,
 		resolver,
 		namespace: config.namespace,
-		cacheScope: catalogCacheScope(config)
+		cacheScope
 	};
 }
 function withVerifiedWriterByteLengths(resolver) {

package/dist/manifest-cache-resolver.mjs

@@ -0,0 +1,50 @@
+import { cacheGet, cachePut } from "./catalog-cache.mjs";
+const MANIFEST_AVRO_PATH = /\/metadata\/[^/]+\.avro$/;
+const MANIFEST_CACHE_TTL_MS = 6048e5;
+const MAX_CACHED_MANIFEST_BYTES = 1048576;
+const BASE64_CHUNK_BYTES = 32768;
+function isManifestObjectPath(path) {
+	return MANIFEST_AVRO_PATH.test(path);
+}
+function manifestCacheKey(scope, path) {
+	return `lh-manifest\0${scope}\0${path}`;
+}
+function bytesToBase64(bytes) {
+	let binary = "";
+	for (let offset = 0; offset < bytes.length; offset += BASE64_CHUNK_BYTES) {
+		const chunk = bytes.subarray(offset, offset + BASE64_CHUNK_BYTES);
+		binary += String.fromCharCode(...chunk);
+	}
+	return btoa(binary);
+}
+function base64ToBytes(b64) {
+	const binary = atob(b64);
+	const bytes = new Uint8Array(binary.length);
+	for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+	return bytes;
+}
+function bufferedAsyncBuffer(bytes) {
+	return {
+		byteLength: bytes.byteLength,
+		slice(start = 0, end = bytes.byteLength) {
+			return bytes.buffer.slice(bytes.byteOffset + start, bytes.byteOffset + end);
+		}
+	};
+}
+function wrapManifestCacheResolver(base, cache, scope, clock = Date.now) {
+	return {
+		...base,
+		async reader(path, byteLength) {
+			if (!isManifestObjectPath(path)) return base.reader(path, byteLength);
+			const now = clock();
+			const key = manifestCacheKey(scope, path);
+			const cached = await cacheGet(cache, key, now);
+			if (cached) return bufferedAsyncBuffer(base64ToBytes(cached.b64));
+			const ab = await base.reader(path, byteLength);
+			const bytes = new Uint8Array(await ab.slice(0, ab.byteLength));
+			if (bytes.byteLength <= MAX_CACHED_MANIFEST_BYTES) await cachePut(cache, key, { b64: bytesToBase64(bytes) }, MANIFEST_CACHE_TTL_MS, now);
+			return bufferedAsyncBuffer(bytes);
+		}
+	};
+}
+export { wrapManifestCacheResolver };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "3.4.1",
+  "version": "3.4.2",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",