@gscdump/lakehouse 4.1.0 → 4.2.1

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7 → icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f}/node_modules/icebird/src/manifest.mjs

@@ -2,7 +2,20 @@ import { avroMetadata } from "./avro/avro.metadata.mjs";
 import { avroRead } from "./avro/avro.read.mjs";
 import { fetchAvroRecords, urlResolver } from "./fetch.mjs";
 const MANIFEST_FETCH_CONCURRENCY = 8;
-async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }) {
+function memoRead(cache, key, read, retain = true) {
+	if (!cache) return read();
+	const existing = cache.get(key);
+	if (existing) return existing;
+	const pending = Promise.resolve().then(read);
+	cache.set(key, pending);
+	pending.then(() => {
+		if (!retain && cache.get(key) === pending) cache.delete(key);
+	}, () => {
+		if (cache.get(key) === pending) cache.delete(key);
+	});
+	return pending;
+}
+async function icebergManifests({ metadata, resolver, snapshotId, partitionFilter, manifestCache }) {
 	resolver ??= urlResolver();
 	const rawTarget = snapshotId ?? metadata["current-snapshot-id"];
 	if (rawTarget == null || rawTarget < 0) throw new Error("No current snapshot id found in table metadata");
@@ -12,8 +25,8 @@ async function icebergManifests({ metadata, resolver, snapshotId, partitionFilte
 	let manifests;
 	if (snapshot["manifest-list"]) {
 		const manifestListUrl = snapshot["manifest-list"];
-		manifests = await fetchAvroRecords(manifestListUrl, resolver);
-	} else if (snapshot.manifests) manifests = await resolveInlineManifests(snapshot, resolver);
+		manifests = await memoRead(manifestCache?.lists, manifestListUrl, () => fetchAvroRecords(manifestListUrl, resolver));
+	} else if (snapshot.manifests) manifests = await memoRead(manifestCache?.lists, `inline:${targetId}`, () => resolveInlineManifests(snapshot, resolver));
 	else throw new Error("No manifest information found in snapshot");
 	if (partitionFilter) manifests = manifests.filter((manifest) => {
 		try {
@@ -22,7 +35,7 @@ async function icebergManifests({ metadata, resolver, snapshotId, partitionFilte
 			return true;
 		}
 	});
-	return await fetchManifests(manifests, resolver);
+	return await fetchManifests(manifests, resolver, manifestCache);
 }
 async function resolveInlineManifests(snapshot, resolver) {
 	return await mapWithConcurrency(snapshot.manifests ?? [], async (manifestPath) => {
@@ -85,8 +98,8 @@ async function fetchOneManifest(manifest, resolver) {
 		entries
 	};
 }
-async function fetchManifests(manifests, resolver) {
-	return await mapWithConcurrency(manifests, (manifest) => fetchOneManifest(manifest, resolver));
+async function fetchManifests(manifests, resolver, manifestCache) {
+	return await mapWithConcurrency(manifests, (manifest) => memoRead(manifestCache?.entries, manifest.manifest_path, () => fetchOneManifest(manifest, resolver), false));
 }
 async function mapWithConcurrency(items, fn) {
 	const results = new Array(items.length);

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7 → icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f}/node_modules/icebird/types/manifest.d.mts

@@ -3,6 +3,14 @@ export type ManifestList = {
   url: string;
   entries: ManifestEntry[];
 }[];
+/** Decoded, immutable manifest data for one caller-owned table snapshot. */
+export type ManifestReadCache = {
+  lists: Map<string, Promise<Manifest[]>>;
+  entries: Map<string, Promise<{
+    url: string;
+    entries: ManifestEntry[];
+  }>>;
+};
 /**
  * Predicate applied before fetching a manifest's entries.
  */
@@ -20,9 +28,10 @@ export type ManifestPartitionFilter = (partitions: Manifest['partitions'], parti
  * @param {number | bigint} [options.snapshotId] - Optional snapshot id; defaults to `current-snapshot-id`.
  * @returns {Promise<ManifestList>}
  */
-export declare function icebergManifests({ metadata, resolver, snapshotId, partitionFilter }: {
+export declare function icebergManifests({ metadata, resolver, snapshotId, partitionFilter, manifestCache }: {
   metadata: TableMetadata;
   resolver?: Resolver;
   snapshotId?: number | bigint;
   partitionFilter?: ManifestPartitionFilter;
+  manifestCache?: ManifestReadCache;
 }): Promise<ManifestList>;

package/dist/catalog.d.mts

@@ -1,9 +1,10 @@
-import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/write/write.mjs";
+import { ManifestReadCache } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/manifest.mjs";
+import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/write/write.mjs";
 import { CatalogCache } from "./catalog-cache.mjs";
 import { IcebergPrimitiveType, IcebergS3Config } from "./schema.mjs";
 import { PartitionValueMatch } from "./partition-prune.mjs";
-import { restCatalogConnect } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/fetch.mjs";
+import { restCatalogConnect, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/fetch.mjs";
 /** A field in an icebird table `Schema`. */
 export interface IcebergSchemaField {
   id: number;
@@ -54,6 +55,7 @@ export interface IcebergCatalogConfig {
   /** R2 S3 credentials for the warehouse objects. */
   s3: IcebergS3Config;
 }
+declare const connectionReadBatch: unique symbol;
 /** The connected catalog context + a signed S3 resolver — the icebird call inputs. */
 export interface IcebergConnection {
   /** icebird REST catalog context, passed as `{ catalog }` to icebird write fns. */
@@ -71,6 +73,16 @@ export interface IcebergConnection {
    * omit it ONLY when their cache is not shared across catalogs.
    */
   cacheScope?: string;
+  /** Concurrent reads share decoded metadata only while their batch is active. */
+  [connectionReadBatch]?: ConnectionReadBatch;
+}
+interface ConnectionReadBatch {
+  active: number;
+  snapshots: Map<CatalogCache['storage'] | undefined, Map<string, Promise<{
+    snapshotId: string | null;
+    metadata: LoadedTableMetadata | null;
+  }>>>;
+  manifests: Map<string, ManifestReadCache>;
 }
 /** Options for {@link connectIcebergCatalog}. */
 export interface ConnectIcebergOptions {
@@ -250,6 +262,7 @@ export interface ResolveIcebergDataFilesOptions {
  * means TTL-bounded staleness, never an error.
  */
 export declare function invalidateSnapshotRef(cache: CatalogCache, namespace: string, table: string, cacheScope?: string): Promise<void>;
+type LoadedTableMetadata = Awaited<ReturnType<typeof restCatalogLoadTable>>['metadata'];
 /**
  * List the parquet data files in the current snapshot of `table`, filtered to
  * one partition slice (`matches` + `range`). Generic over the partition spec —

package/dist/catalog.mjs

@@ -1,14 +1,43 @@
 import { stringifyBigintSafe } from "./bigint.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/catalog/rest.mjs";
-import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/fetch.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/fetch.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/s3.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { toIcebergDayCount } from "./date.mjs";
 import { MULTI_MONTH_MANIFEST, buildManifestPartitionFilter, manifestMonthBucket } from "./partition-prune.mjs";
+const connectionReadBatch = Symbol("iceberg-connection-read-batch");
+function beginConnectionRead(conn) {
+	const batch = conn[connectionReadBatch];
+	if (!batch) return () => {};
+	batch.active++;
+	return () => {
+		batch.active--;
+		if (batch.active === 0) {
+			batch.snapshots.clear();
+			batch.manifests.clear();
+		}
+	};
+}
+function manifestReadCacheFor(conn, namespace, table, metadata) {
+	const batch = conn[connectionReadBatch];
+	if (!batch || batch.active === 0) return void 0;
+	const snapshotId = metadata["current-snapshot-id"];
+	if (snapshotId == null) return void 0;
+	const key = `${conn.cacheScope ?? ""}\0${namespace}\0${table}\0${snapshotId}`;
+	let cache = batch.manifests.get(key);
+	if (!cache) {
+		cache = {
+			lists: /* @__PURE__ */ new Map(),
+			entries: /* @__PURE__ */ new Map()
+		};
+		batch.manifests.set(key, cache);
+	}
+	return cache;
+}
 let icebirdWriteModule;
 function useIcebirdWriteModule() {
-	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/write/write.mjs");
+	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs");
 	return icebirdWriteModule;
 }
 const CATALOG_CONFIG_TTL_MS = 36e5;
@@ -59,7 +88,12 @@ async function connectIcebergCatalog(config, opts = {}) {
 		catalog,
 		resolver,
 		namespace: config.namespace,
-		cacheScope
+		cacheScope,
+		[connectionReadBatch]: {
+			active: 0,
+			snapshots: /* @__PURE__ */ new Map(),
+			manifests: /* @__PURE__ */ new Map()
+		}
 	};
 }
 function withVerifiedWriterByteLengths(resolver) {
@@ -346,6 +380,25 @@ function stripBucket(filePath) {
 	return slash >= 0 ? rest.slice(slash + 1) : rest;
 }
 async function loadSnapshotId(conn, namespace, table, cache, now) {
+	const batch = conn[connectionReadBatch];
+	if (!batch || batch.active === 0) return loadSnapshotIdUnshared(conn, namespace, table, cache, now);
+	const storage = cache?.storage;
+	let pendingByTable = batch.snapshots.get(storage);
+	if (!pendingByTable) {
+		pendingByTable = /* @__PURE__ */ new Map();
+		batch.snapshots.set(storage, pendingByTable);
+	}
+	const key = `${conn.cacheScope ?? ""}\0${namespace}\0${table}`;
+	const existing = pendingByTable.get(key);
+	if (existing) return existing;
+	const pending = loadSnapshotIdUnshared(conn, namespace, table, cache, now);
+	pendingByTable.set(key, pending);
+	pending.then(void 0, () => {
+		if (pendingByTable?.get(key) === pending) pendingByTable.delete(key);
+	});
+	return pending;
+}
+async function loadSnapshotIdUnshared(conn, namespace, table, cache, now) {
 	const scope = conn.cacheScope ?? "";
 	if (cache) {
 		const cached = await cacheGet(cache, snapshotRefKey(scope, namespace, table), now);
@@ -394,11 +447,12 @@ function toListedFile(entry, matches, monthFieldName, wantedMonths, dateFieldId)
 		...dateBounds ? { dateBounds } : {}
 	};
 }
-async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, monthFieldName, wantedMonths, dateFieldId) {
+async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, monthFieldName, wantedMonths, dateFieldId, manifestCache) {
 	const manifests = await icebergManifests({
 		metadata,
 		resolver: conn.resolver,
-		partitionFilter
+		partitionFilter,
+		manifestCache
 	});
 	const entries = [];
 	for (const m of manifests) for (const entry of m.entries) {
@@ -412,12 +466,13 @@ async function resolveViaFullWalk(conn, metadata, partitionFilter, matches, mont
 		monthsHit: 0
 	};
 }
-async function resolveViaMonthCache(conn, cache, metadata, partitionFilter, partitionSpec, matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now) {
+async function resolveViaMonthCache(conn, cache, metadata, partitionFilter, partitionSpec, matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now, manifestCache) {
 	const matchKey = matchKeyOf(matches);
 	const manifestMonths = /* @__PURE__ */ new Map();
 	await icebergManifests({
 		metadata,
 		resolver: conn.resolver,
+		manifestCache,
 		partitionFilter: (partitions, _specId, manifest) => {
 			try {
 				if (partitionFilter(partitions) === false) return false;
@@ -456,6 +511,7 @@ async function resolveViaMonthCache(conn, cache, metadata, partitionFilter, part
 		const manifests = await icebergManifests({
 			metadata,
 			resolver: conn.resolver,
+			manifestCache,
 			partitionFilter: (_partitions, _specId, manifest) => toWalk.has(manifest.manifest_path)
 		});
 		manifestsWalked = manifests.length;
@@ -488,7 +544,11 @@ async function resolveViaMonthCache(conn, cache, metadata, partitionFilter, part
 		monthsHit
 	};
 }
-async function resolveIcebergDataFiles(conn, opts) {
+function resolveIcebergDataFiles(conn, opts) {
+	const release = beginConnectionRead(conn);
+	return resolveIcebergDataFilesInBatch(conn, opts).finally(release);
+}
+async function resolveIcebergDataFilesInBatch(conn, opts) {
 	const { namespace, table } = opts;
 	const profiler = opts.profiler;
 	const now = (opts.clock ?? Date.now)();
@@ -522,7 +582,8 @@ async function resolveIcebergDataFiles(conn, opts) {
 	const monthFieldName = monthField?.name;
 	const dateFieldId = dateColumnFieldId(metadata, monthField?.sourceColumn);
 	const dayWindow = rangeDayWindow(opts.range);
-	const walk = opts.cache && monthFieldName ? await resolveViaMonthCache(conn, opts.cache, metadata, partitionFilter, opts.partitionSpec, opts.matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now) : await resolveViaFullWalk(conn, metadata, partitionFilter, opts.matches, monthFieldName, wantedMonths, dateFieldId);
+	const manifestCache = manifestReadCacheFor(conn, namespace, table, metadata);
+	const walk = opts.cache && monthFieldName ? await resolveViaMonthCache(conn, opts.cache, metadata, partitionFilter, opts.partitionSpec, opts.matches, monthFieldName, wantedMonths, dateFieldId, scope, namespace, table, now, manifestCache) : await resolveViaFullWalk(conn, metadata, partitionFilter, opts.matches, monthFieldName, wantedMonths, dateFieldId, manifestCache);
 	let boundsPruned = 0;
 	const out = [];
 	for (const file of walk.entries) {

package/dist/dataset-runtime.mjs

@@ -1,5 +1,5 @@
 import { connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
-import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/write/write.mjs";
+import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs";
 async function createDatasetTable(conn, table, schema, partitionSpec, sortOrder) {
 	const results = [];
 	await icebergCreateTable({

package/dist/orphan-sweep.mjs

@@ -1,6 +1,6 @@
-import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/manifest.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/index.mjs";
+import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/index.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
 const DEFAULT_IO_CONCURRENCY = 4;

package/dist/partition-prune.d.mts

@@ -11,13 +11,9 @@ export type ManifestPartitionFilter = (partitions: IcebergFieldSummary[] | undef
 /**
  * One identity/dims value to prune an `identity`-transform partition field by.
  *
- * `'int32'`-encoded fields are deliberately NOT pruned here (mirrors the
- * original engine behavior): an INT column's bound bytes are a 4-byte int, but
- * per-team catalogs are single-tenant, so identity pruning on them saves ~nothing
- * — the per-file partition check in the resolver remains the authoritative
- * correctness filter. Only `'string'`-encoded identity fields are pruned by
- * lexicographic UTF-8 bound comparison (the truncated-string-stats case R2 SQL
- * needs the workaround for).
+ * Identity summaries use the declared field encoding: UTF-8 for strings and
+ * four-byte little-endian for INT. Unknown bounds keep the manifest. The
+ * per-file partition check remains the authoritative correctness filter.
  */
 export interface PartitionValueMatch {
   /** Partition field name as declared in the partition spec (e.g. `'site_id'`). */

package/dist/partition-prune.mjs

@@ -7,11 +7,12 @@ function decodeString(bytes) {
 	const u = toUint8(bytes);
 	return u == null ? null : UTF8_DECODER.decode(u);
 }
-function decodeMonthInt(bytes) {
+function decodeInt32(bytes) {
 	const u = toUint8(bytes);
-	if (u == null) return null;
+	if (u?.byteLength !== 4) return null;
 	return new DataView(u.buffer, u.byteOffset, u.byteLength).getInt32(0, true);
 }
+const decodeMonthInt = decodeInt32;
 function buildManifestPartitionFilter(partitionSpec, matches, wantedMonths) {
 	const fieldIndex = (name) => partitionSpec.findIndex((f) => f.name === name || f.sourceColumn === name);
 	const monthFieldIndex = partitionSpec.findIndex((f) => f.transform === "month");
@@ -23,6 +24,14 @@ function buildManifestPartitionFilter(partitionSpec, matches, wantedMonths) {
 			value: String(match.value)
 		}];
 	});
+	const intMatches = matches.flatMap((match) => {
+		if (match.encoding !== "int32" || typeof match.value !== "number" || !Number.isInteger(match.value) || match.value < -2147483648 || match.value > 2147483647) return [];
+		const index = fieldIndex(match.field);
+		return index < 0 || partitionSpec[index]?.transform !== "identity" ? [] : [{
+			index,
+			value: match.value
+		}];
+	});
 	const wantedMonthValues = wantedMonths ? [...wantedMonths].filter(Number.isFinite).sort((a, b) => a - b) : [];
 	const hasWantedMonthInRange = (lo, hi) => {
 		let left = 0;
@@ -43,6 +52,13 @@ function buildManifestPartitionFilter(partitionSpec, matches, wantedMonths) {
 			const hi = decodeString(summary.upper_bound);
 			if (lo != null && hi != null && (match.value < lo || match.value > hi)) return false;
 		}
+		for (const match of intMatches) {
+			const summary = partitions[match.index];
+			if (!summary || summary.lower_bound == null || summary.upper_bound == null) continue;
+			const lo = decodeInt32(summary.lower_bound);
+			const hi = decodeInt32(summary.upper_bound);
+			if (lo != null && hi != null && lo <= hi && (match.value < lo || match.value > hi)) return false;
+		}
 		if (wantedMonthValues.length > 0 && monthFieldIndex >= 0) {
 			const monthSummary = partitions[monthFieldIndex];
 			if (monthSummary && (monthSummary.lower_bound != null || monthSummary.upper_bound != null)) {

package/dist/unsafe-raw.d.mts

@@ -1,7 +1,7 @@
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/write/write.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/manifest.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/write/write.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/catalog/rest.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/s3.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/manifest.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/types/index.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/s3.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/unsafe-raw.mjs

@@ -1,7 +1,7 @@
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/s3.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/write/write.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7/node_modules/icebird/src/index.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "4.1.0",
+  "version": "4.2.1",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",
@@ -73,6 +73,7 @@
   "devDependencies": {
     "@types/node": "^26.6.2",
     "hyparquet": "^1.31.1",
+    "hyparquet-writer": "^0.16.10",
     "icebird": "^0.8.30",
     "unstorage": "^1.17.5",
     "vitest": "^5.0.1"

package/dist/_virtual/node_modules/.pnpm/{icebird@0.8.30_patch_hash=baf49f0ba810036770cc2fd19ec2c43cbccd764be0f7aa04af13b186e90227a7 → icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f}/node_modules/icebird/types/index.d.mts

@@ -1,3 +1,4 @@
+import { icebergManifests } from "./manifest.mjs";
 import { icebergAppend, icebergCreateTable, icebergDropTable } from "./write/write.mjs";
 import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./catalog/rest.mjs";
 import { cachingResolver } from "./fetch.mjs";
@@ -7,7 +8,6 @@ import { s3SignedResolver } from "./s3.mjs";
 import "../../../../squirreling@0.16.6/node_modules/squirreling/src/index.mjs";
 import "./read.mjs";
 import "./metadata.mjs";
-import { icebergManifests } from "./manifest.mjs";
 import "./sql/icebergDataSource.mjs";
 import "./sql/icebergQuery.mjs";
 export { icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };