@gscdump/lakehouse 0.35.5 → 0.35.7

package/dist/_chunks/catalog.d.mts

@@ -121,6 +121,15 @@ interface IcebergConnection {
   resolver: ReturnType<typeof cachingResolver>;
   /** The namespace the dataset's table lives under. */
   namespace: string;
+  /**
+   * Catalog identity baked into every cross-isolate cache key. Namespaces are
+   * NOT unique across catalogs (every per-team catalog uses the same 'gsc'
+   * namespace) while deployments share one KV, so a key of (namespace, table)
+   * alone lets one team's snapshot pointer/metadata poison every other team's
+   * reads. Set by {@link connectIcebergCatalog}; hand-built connections may
+   * omit it ONLY when their cache is not shared across catalogs.
+   */
+  cacheScope?: string;
 }
 declare const ICEBERG_TYPE_MAP: Record<IcebergColumnType, IcebergPrimitiveType>;
 /** Options for {@link connectIcebergCatalog}. */
@@ -139,6 +148,12 @@ interface ConnectIcebergOptions {
  * `fetch`, no node builtins.
  */
 declare function connectIcebergCatalog(config: IcebergCatalogConfig, opts?: ConnectIcebergOptions): Promise<IcebergConnection>;
+/**
+ * The catalog-identity component of every cross-isolate cache key. Exported so
+ * write paths that only hold the config (not a connection) can invalidate the
+ * exact keys readers populate — see {@link invalidateSnapshotRef}.
+ */
+declare function catalogCacheScope(config: Pick<IcebergCatalogConfig, 'catalogUri' | 'warehouse'>): string;
 /**
  * Ensure the catalog namespace exists. Idempotent — an "already exists"
  * response from the REST catalog is swallowed.
@@ -236,6 +251,17 @@ interface ResolveIcebergDataFilesOptions {
   clock?: () => number;
   profiler?: QueryProfiler;
 }
+/**
+ * Drop the cached snapshot pointer for `(namespace, table)` so the next read
+ * re-loads the table and sees the just-committed snapshot immediately instead
+ * of after {@link SNAPSHOT_REF_TTL_MS} (writers call this post-commit; without
+ * it worst-case reader staleness is TTL + the downstream ref's own TTL). Only
+ * the snapshot REF is dropped: `lh-snapmeta`/`lh-files` entries are keyed by
+ * snapshotId, so stale ones age out harmlessly and fresh ones rebuild on the
+ * next read. Best-effort like every cache path here — a failed delete only
+ * means TTL-bounded staleness, never an error.
+ */
+declare function invalidateSnapshotRef(cache: CatalogCache, namespace: string, table: string, cacheScope?: string): Promise<void>;
 /**
  * List the parquet data files in the current snapshot of `table`, filtered to
  * one partition slice (`matches` + `range`). Generic over the partition spec —
@@ -243,4 +269,4 @@ interface ResolveIcebergDataFilesOptions {
  * the def's own spec + identity/dims values.
  */
 declare function resolveIcebergDataFiles(conn: IcebergConnection, opts: ResolveIcebergDataFilesOptions): Promise<IcebergListedDataFile[]>;
-export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };

package/dist/_chunks/catalog.mjs

@@ -116,9 +116,13 @@ async function connectIcebergCatalog(config, opts = {}) {
 	return {
 		catalog,
 		resolver,
-		namespace: config.namespace
+		namespace: config.namespace,
+		cacheScope: catalogCacheScope(config)
 	};
 }
+function catalogCacheScope(config) {
+	return `${config.catalogUri}\0${config.warehouse}`;
+}
 async function ensureIcebergNamespace(conn) {
 	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
 }
@@ -201,14 +205,17 @@ const SNAPSHOT_REF_TTL_MS = 1800 * 1e3;
 const RESOLVED_FILES_TTL_MS = 1440 * 60 * 1e3;
 const METADATA_TTL_MS = 1440 * 60 * 1e3;
 const MAX_CACHED_METADATA_BYTES = 2 * 1024 * 1024;
-function snapshotRefKey(namespace, table) {
-	return `lh-snapref\0${namespace}\0${table}`;
+function snapshotRefKey(scope, namespace, table) {
+	return `lh-snapref\0${scope}\0${namespace}\0${table}`;
+}
+async function invalidateSnapshotRef(cache, namespace, table, cacheScope = "") {
+	await cache.storage.removeItem(snapshotRefKey(cacheScope, namespace, table)).catch(() => {});
 }
-function metadataRefKey(namespace, table, snapshotId) {
-	return `lh-snapmeta\0${namespace}\0${table}\0${snapshotId}`;
+function metadataRefKey(scope, namespace, table, snapshotId) {
+	return `lh-snapmeta\0${scope}\0${namespace}\0${table}\0${snapshotId}`;
 }
-function resolvedFilesKey(namespace, table, snapshotId, matches, wantedMonths) {
-	return `lh-files\0${namespace}\0${table}\0${snapshotId}\0${[...matches].map((m) => `${m.field}=${m.value}`).sort().join(",")}\0${[...wantedMonths].sort((a, b) => a - b).join(",")}`;
+function resolvedFilesKey(scope, namespace, table, snapshotId, matches, wantedMonths) {
+	return `lh-files\0${scope}\0${namespace}\0${table}\0${snapshotId}\0${[...matches].map((m) => `${m.field}=${m.value}`).sort().join(",")}\0${[...wantedMonths].sort((a, b) => a - b).join(",")}`;
 }
 function monthsInRange(range) {
 	const [sy, sm] = range.start.split("-").map(Number);
@@ -237,8 +244,9 @@ function stripBucket(filePath) {
 	return slash >= 0 ? rest.slice(slash + 1) : rest;
 }
 async function loadSnapshotId(conn, namespace, table, cache, now) {
+	const scope = conn.cacheScope ?? "";
 	if (cache) {
-		const cached = await cacheGet(cache, snapshotRefKey(namespace, table), now);
+		const cached = await cacheGet(cache, snapshotRefKey(scope, namespace, table), now);
 		if (cached !== void 0) return {
 			snapshotId: cached,
 			metadata: null
@@ -251,9 +259,9 @@ async function loadSnapshotId(conn, namespace, table, cache, now) {
 	const raw = metadata["current-snapshot-id"];
 	const snapshotId = raw == null ? null : String(raw);
 	if (cache) {
-		await cachePut(cache, snapshotRefKey(namespace, table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
+		await cachePut(cache, snapshotRefKey(scope, namespace, table), snapshotId, SNAPSHOT_REF_TTL_MS, now);
 		if (snapshotId != null) {
-			if (JSON.stringify(metadata).length <= MAX_CACHED_METADATA_BYTES) await cachePut(cache, metadataRefKey(namespace, table, snapshotId), metadata, METADATA_TTL_MS, now);
+			if (JSON.stringify(metadata).length <= MAX_CACHED_METADATA_BYTES) await cachePut(cache, metadataRefKey(scope, namespace, table, snapshotId), metadata, METADATA_TTL_MS, now);
 		}
 	}
 	return {
@@ -270,7 +278,8 @@ async function resolveIcebergDataFiles(conn, opts) {
 	let { snapshotId, metadata } = await loadSnapshotId(conn, namespace, table, opts.cache, now);
 	endSnapshot?.({ cached: metadata == null && snapshotId != null });
 	if (snapshotId == null) return [];
-	const filesKey = resolvedFilesKey(namespace, table, snapshotId, opts.matches, wantedMonths);
+	const scope = conn.cacheScope ?? "";
+	const filesKey = resolvedFilesKey(scope, namespace, table, snapshotId, opts.matches, wantedMonths);
 	if (opts.cache) {
 		const endCache = profiler?.start("iceberg.cache");
 		const cached = await cacheGet(opts.cache, filesKey, now);
@@ -278,7 +287,7 @@ async function resolveIcebergDataFiles(conn, opts) {
 		if (cached !== void 0) return cached;
 	}
 	if (!metadata && opts.cache) {
-		const cachedMeta = await cacheGet(opts.cache, metadataRefKey(namespace, table, snapshotId), now);
+		const cachedMeta = await cacheGet(opts.cache, metadataRefKey(scope, namespace, table, snapshotId), now);
 		if (cachedMeta != null) metadata = cachedMeta;
 	}
 	if (!metadata) {
@@ -323,9 +332,9 @@ async function resolveIcebergDataFiles(conn, opts) {
 		files: out.length
 	});
 	if (opts.cache) {
-		const freshKey = resolvedFilesKey(namespace, table, snapshotId, opts.matches, wantedMonths);
+		const freshKey = resolvedFilesKey(scope, namespace, table, snapshotId, opts.matches, wantedMonths);
 		await cachePut(opts.cache, freshKey, out, RESOLVED_FILES_TTL_MS, now);
 	}
 	return out;
 }
-export { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };

package/dist/index.d.mts

@@ -1,4 +1,4 @@
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
 /**
  * Closed identity shapes (ADR-0021 amendments 2 + 4). `'site-int'` is the
@@ -156,4 +156,4 @@ interface RunPyIcebergWriterOptions {
 }
 declare function resolvePyIcebergPython(override?: string): string;
 declare function runPyIcebergWriter<T extends PyIcebergWriterResult>(options: RunPyIcebergWriterOptions): Promise<T>;
-export { type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type PyIcebergWriterResult, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, type RunPyIcebergWriterOptions, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, toIcebergDayCount };
+export { type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type PyIcebergWriterResult, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, type RunPyIcebergWriterOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, toIcebergDayCount };

package/dist/index.mjs

@@ -1,5 +1,5 @@
 import { icebergCreateTable } from "./_chunks/libs/icebird.mjs";
-import { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
+import { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
 import process from "node:process";
 const INT32_MIN = -2147483648;
 const INT32_MAX = 2147483647;
@@ -353,4 +353,4 @@ async function runPyIcebergWriter(options) {
 	});
 }
 const DEFAULT_PARTITION_KEY_ENCODING = "int";
-export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, toIcebergDayCount };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, toIcebergDayCount };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "0.35.5",
+  "version": "0.35.7",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",