@gscdump/engine 0.32.3 → 0.32.5

package/dist/iceberg/index.mjs

@@ -219,11 +219,16 @@ async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "st
 async function listIcebergTables(conn) {
 	return (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name).sort();
 }
-const SNAPSHOT_REF_TTL_MS = 3e4;
+const SNAPSHOT_REF_TTL_MS = 300 * 1e3;
 const RESOLVED_FILES_TTL_MS = 1440 * 60 * 1e3;
+const METADATA_TTL_MS = 1440 * 60 * 1e3;
+const MAX_CACHED_METADATA_BYTES = 2 * 1024 * 1024;
 function snapshotRefKey(namespace, table) {
 	return `gsc-snapref\0${namespace}\0${table}`;
 }
+function metadataRefKey(namespace, table, snapshotId) {
+	return `gsc-snapmeta\0${namespace}\0${table}\0${snapshotId}`;
+}
 function resolvedFilesKey(namespace, table, snapshotId, siteId, searchType, wantedMonths) {
 	return `gsc-files\0${namespace}\0${table}\0${snapshotId}\0${siteId}\0${searchType}\0${[...wantedMonths].sort((a, b) => a - b).join(",")}`;
 }
@@ -267,7 +272,12 @@ async function loadSnapshotId(conn, opts, now) {
 	});
 	const raw = metadata["current-snapshot-id"];
 	const snapshotId = raw == null ? null : String(raw);
-	if (opts.cache) await cachePut(opts.cache, snapshotRefKey(conn.namespace, opts.table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
+	if (opts.cache) {
+		await cachePut(opts.cache, snapshotRefKey(conn.namespace, opts.table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
+		if (snapshotId != null) {
+			if (JSON.stringify(metadata).length <= MAX_CACHED_METADATA_BYTES) await cachePut(opts.cache, metadataRefKey(conn.namespace, opts.table, snapshotId), metadata, METADATA_TTL_MS, now);
+		}
+	}
 	return {
 		snapshotId,
 		metadata
@@ -288,6 +298,10 @@ async function listIcebergDataFiles(conn, opts) {
 		endCache?.({ hit: cached !== void 0 });
 		if (cached !== void 0) return cached;
 	}
+	if (!metadata && opts.cache) {
+		const cachedMeta = await cacheGet(opts.cache, metadataRefKey(conn.namespace, opts.table, snapshotId), now);
+		if (cachedMeta != null) metadata = cachedMeta;
+	}
 	if (!metadata) {
 		const reloaded = await loadSnapshotId(conn, {
 			...opts,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.32.3",
+  "version": "0.32.5",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.32.3",
-    "gscdump": "0.32.3"
+    "gscdump": "0.32.5",
+    "@gscdump/contracts": "0.32.5"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",