@gscdump/lakehouse 7.6.0 → 7.8.0

package/dist/dataset-runtime.mjs

@@ -1,6 +1,6 @@
-import { restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
+import { restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/catalog/rest.mjs";
 import { connectIcebergCatalog, createIcebergAppendFileResolver, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergAppendFiles, resolveIcebergDataFiles } from "./catalog.mjs";
-import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs";
+import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/write.mjs";
 async function verifyDatasetTable(conn, table, schema, partitionSpec) {
 	const { metadata } = await restCatalogLoadTable(conn.catalog, {
 		namespace: conn.namespace,

package/dist/dataset.d.mts

@@ -1,4 +1,6 @@
+import { ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
 import { CatalogCache } from "./catalog-cache.mjs";
+import "./manifest-read-limits.mjs";
 import { IcebergColumnType, IcebergPartitionField, IcebergTableSpec } from "./schema.mjs";
 import { ManifestPartitionFilter, PartitionValueMatch } from "./partition-prune.mjs";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergSchema, IcebergSortOrder, IcebergTableOpResult, QueryProfiler } from "./catalog.mjs";
@@ -97,6 +99,8 @@ export interface ResolveDataFilesOptions {
   cache?: CatalogCache;
   clock?: () => number;
   profiler?: QueryProfiler;
+  /** Uncached Avro read limits; catalog metadata/snapshot caching is unchanged. */
+  readLimits?: ManifestReadLimits;
 }
 export interface IcebergDataset {
   readonly def: IcebergDatasetDef;

package/dist/dataset.mjs

@@ -1,4 +1,5 @@
 import { coerceBigIntToNumber } from "./bigint.mjs";
+import { captureManifestReadLimits } from "./manifest-read-limits.mjs";
 import { buildManifestPartitionFilter } from "./partition-prune.mjs";
 import { ICEBERG_TYPE_MAP } from "./schema.mjs";
 const INT32_MIN = -2147483648;
@@ -299,6 +300,7 @@ function defineIcebergDataset(def) {
 		});
 	}
 	async function resolveDataFiles(conn, identity, range, dims, opts = {}) {
+		const readLimits = captureManifestReadLimits(opts.readLimits);
 		const { resolveDatasetDataFiles } = await import("./dataset-runtime.mjs");
 		return resolveDatasetDataFiles(conn, {
 			namespace: def.namespace,
@@ -308,7 +310,8 @@ function defineIcebergDataset(def) {
 			range,
 			cache: opts.cache,
 			clock: opts.clock,
-			profiler: opts.profiler
+			profiler: opts.profiler,
+			readLimits
 		});
 	}
 	return {

package/dist/index.d.mts

@@ -1,8 +1,10 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
+import { ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
 import { CatalogCache, CatalogCacheGetOutcome, cacheGet, cachePut } from "./catalog-cache.mjs";
+import "./manifest-read-limits.mjs";
 import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./schema.mjs";
 import { IcebergFieldSummary, ManifestPartitionFilter, PartitionValueMatch, buildManifestPartitionFilter } from "./partition-prune.mjs";
 import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergDataFileDateBounds, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, createIcebergNamespace, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
 import { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
 import { toIcebergDayCount } from "./date.mjs";
-export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CatalogCacheGetOutcome, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataFileDateBounds, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, createIcebergNamespace, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };
+export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CatalogCacheGetOutcome, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataFileDateBounds, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type ManifestReadLimits, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, createIcebergNamespace, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/manifest-merge.mjs

@@ -1,12 +1,12 @@
-import { restCatalogLoadTable, restCatalogUpdateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
+import { restCatalogLoadTable, restCatalogUpdateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/catalog/rest.mjs";
 import { ByteWriter } from "./_virtual/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/bytewriter.mjs";
 import "./_virtual/node_modules/.pnpm/hyparquet-writer@0.16.10_patch_hash=d26d002aede61ad95dd633d6557b56897cdcbc45dfc3a5d93fa0e9075ad5425f/node_modules/hyparquet-writer/src/node.mjs";
-import { uuid4 } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/utils.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
+import { uuid4 } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/utils.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/manifest.mjs";
 import { invalidateSnapshotRef } from "./catalog.mjs";
-import { writeExistingDataManifest } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/manifest.mjs";
-import { buildPartitionSummaries, buildSnapshotUpdate, loadPriorManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/snapshot.mjs";
-import { newSnapshotId } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/stage.mjs";
+import { writeExistingDataManifest } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/manifest.mjs";
+import { buildPartitionSummaries, buildSnapshotUpdate, loadPriorManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/snapshot.mjs";
+import { newSnapshotId } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/stage.mjs";
 const MAX_MANIFEST_BYTES = 4194304;
 const MAX_INPUT_BYTES = 8388608;
 const MAX_ENTRIES = 1e4;

package/dist/manifest-read-limits.d.mts

@@ -0,0 +1,2 @@
+import { ManifestReadLimits } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
+export type { ManifestReadLimits };

package/dist/manifest-read-limits.mjs

@@ -0,0 +1,26 @@
+const fields = [
+	"maxObjectBytes",
+	"maxTotalObjectBytes",
+	"maxManifests",
+	"maxEntries",
+	"maxInflatedBlockBytes",
+	"maxTotalInflatedBytes",
+	"maxValueBytes",
+	"maxDecodedValues",
+	"maxHeaderBytes",
+	"maxSchemaDepth"
+];
+function captureManifestReadLimits(options) {
+	if (options === void 0) return void 0;
+	if (!options || typeof options !== "object" || Array.isArray(options)) throw new TypeError("Invalid manifest read limits");
+	for (const key of Object.keys(options)) if (!fields.includes(key)) throw new TypeError(`Unknown manifest read limit: ${key}`);
+	const limits = {};
+	for (const key of fields) {
+		const value = options[key];
+		if (!Number.isSafeInteger(value) || value <= 0) throw new RangeError(`Invalid manifest read limit: ${key}`);
+		limits[key] = value;
+	}
+	if (limits.maxSchemaDepth > 128) throw new RangeError("Manifest schema depth must not exceed 128");
+	return Object.freeze(limits);
+}
+export { captureManifestReadLimits };

package/dist/orphan-sweep.mjs

@@ -1,6 +1,6 @@
-import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/index.mjs";
+import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/index.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
 const DEFAULT_IO_CONCURRENCY = 4;

package/dist/resolver-byte-cache.mjs

@@ -0,0 +1,85 @@
+import { cachedAsyncBuffer } from "./_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/utils.mjs";
+import "./_virtual/node_modules/.pnpm/hyparquet@1.31.1/node_modules/hyparquet/src/node.mjs";
+const CONNECTION_CACHE_MAX_BYTES = 16777216;
+function byteCappedCachingResolver(base, maxBytes = CONNECTION_CACHE_MAX_BYTES) {
+	const entries = /* @__PURE__ */ new Map();
+	let totalBytes = 0;
+	function touch(path, entry) {
+		entries.delete(path);
+		entries.set(path, entry);
+	}
+	function evict(path, account) {
+		if (entries.get(path)?.account !== account) return;
+		entries.delete(path);
+		totalBytes -= account.bytes;
+	}
+	function retain(path, account, bytes) {
+		const entry = entries.get(path);
+		if (entry?.account !== account) return;
+		account.bytes += bytes;
+		totalBytes += bytes;
+		touch(path, entry);
+		if (account.bytes > maxBytes) {
+			evict(path, account);
+			return;
+		}
+		for (const [oldest, old] of entries) {
+			if (totalBytes <= maxBytes) break;
+			if (old.account.bytes > 0) evict(oldest, old.account);
+		}
+	}
+	function counted(path, account, file) {
+		return {
+			byteLength: file.byteLength,
+			slice(start, end) {
+				return Promise.resolve(file.slice(start, end)).then((bytes) => {
+					retain(path, account, bytes.byteLength);
+					return bytes;
+				}, (error) => {
+					evict(path, account);
+					throw error;
+				});
+			}
+		};
+	}
+	function invalidate(path) {
+		const entry = entries.get(path);
+		if (entry) evict(path, entry.account);
+	}
+	const out = { reader(path, byteLength) {
+		const hit = entries.get(path);
+		if (hit) {
+			touch(path, hit);
+			return hit.buffer;
+		}
+		const account = { bytes: 0 };
+		const buffer = (async () => cachedAsyncBuffer(counted(path, account, await base.reader(path, byteLength))))();
+		entries.set(path, {
+			buffer,
+			account
+		});
+		buffer.catch(() => evict(path, account));
+		return buffer;
+	} };
+	if (base.writer) {
+		const baseWriter = base.writer;
+		out.writer = (path, options) => {
+			const writer = baseWriter(path, options);
+			const finish = writer.finish.bind(writer);
+			writer.finish = async function() {
+				await finish();
+				invalidate(path);
+			};
+			return writer;
+		};
+	}
+	if (base.deleter) {
+		const baseDeleter = base.deleter;
+		out.deleter = async (path) => {
+			await baseDeleter(path);
+			invalidate(path);
+		};
+	}
+	return out;
+}
+export { CONNECTION_CACHE_MAX_BYTES, byteCappedCachingResolver };

package/dist/unsafe-raw.d.mts

@@ -1,7 +1,7 @@
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/manifest.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/write/write.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/catalog/rest.mjs";
+import { ManifestReadLimits, icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/manifest.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/write/write.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/catalog/rest.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/s3.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/index.mjs";
-export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/s3.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/types/index.mjs";
+export { type ManifestReadLimits, icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/unsafe-raw.mjs

@@ -1,7 +1,7 @@
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/manifest.mjs";
-import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/s3.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/s3.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/write/write.mjs";
-import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/index.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/write/write.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=1359a91da68522a736babe444880ae391f19455fbe47c346f76cec92c5c2e471/node_modules/icebird/src/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "7.6.0",
+  "version": "7.8.0",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",

package/dist/_virtual/node_modules/.pnpm/hyparquet-compressors@1.1.2/node_modules/hyparquet-compressors/src/gzip.mjs

@@ -1,173 +0,0 @@
-import { fixedDistance, fixedDistanceExtraBits, fixedDistanceMap, fixedLength, fixedLengthExtraBits, fixedLengthMap, huffMap } from "./gzip.huffman.mjs";
-const codeLengthIndexMap = new Uint8Array([
-	16,
-	17,
-	18,
-	0,
-	8,
-	7,
-	9,
-	6,
-	10,
-	5,
-	11,
-	4,
-	12,
-	3,
-	13,
-	2,
-	14,
-	1,
-	15
-]);
-function bits(input, pos, mask) {
-	const o = pos / 8 | 0;
-	return (input[o] | input[o + 1] << 8) >> (pos & 7) & mask;
-}
-function bits16(d, p) {
-	const o = p / 8 | 0;
-	return (d[o] | d[o + 1] << 8 | d[o + 2] << 16) >> (p & 7);
-}
-function shft(p) {
-	return (p + 7) / 8 | 0;
-}
-function gzipStart(input, i) {
-	if (input[i++] !== 31 || input[i++] !== 139 || input[i++] !== 8) return 0;
-	const flag = input[i++];
-	i += 6;
-	if (flag & 4) i += (input[i + 10] | input[i + 11] << 8) + 2;
-	for (let zs = (flag >> 3 & 1) + (flag >> 4 & 1); zs > 0; zs -= Number(!input[i++]));
-	return i + (flag & 2);
-}
-function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
-	let out = output ?? /* @__PURE__ */ new Uint8Array(1024);
-	if (!(input.length - inputIndex)) return out;
-	const payloadStart = gzipStart(input, inputIndex);
-	if (payloadStart === input.length - 8) return out;
-	if (payloadStart > input.length - 8) throw new Error("unexpected EOF");
-	let pos = payloadStart * 8;
-	let final = 0;
-	let lengthBits = 0;
-	let distBits = 0;
-	let lengthMap;
-	let distMap;
-	const totalBits = input.length * 8;
-	function ensureSize(length) {
-		if (!output && length > out.length) {
-			const old = out;
-			out = new Uint8Array(Math.max(old.length * 2, length));
-			out.set(old);
-		}
-	}
-	do {
-		if (!lengthMap) {
-			final = bits(input, pos, 1);
-			const type = bits(input, pos + 1, 3);
-			pos += 3;
-			if (!type) {
-				const s = shft(pos) + 4;
-				const l = input[s - 4] | input[s - 3] << 8;
-				const t = s + l;
-				if (t > input.length) throw new Error("unexpected EOF");
-				ensureSize(outputIndex + l);
-				out.set(input.subarray(s, t), outputIndex);
-				outputIndex += l;
-				pos = t * 8;
-				continue;
-			} else if (type === 1) {
-				lengthMap = fixedLengthMap;
-				distMap = fixedDistanceMap;
-				lengthBits = 9;
-				distBits = 5;
-			} else if (type === 2) {
-				const hLiteral = bits(input, pos, 31) + 257;
-				const hcLengths = bits(input, pos + 10, 15) + 4;
-				const tl = hLiteral + bits(input, pos + 5, 31) + 1;
-				pos += 14;
-				const lengthDistanceTree = new Uint8Array(tl);
-				const codeLengthTree = /* @__PURE__ */ new Uint8Array(19);
-				for (let i = 0; i < hcLengths; ++i) codeLengthTree[codeLengthIndexMap[i]] = bits(input, pos + i * 3, 7);
-				pos += hcLengths * 3;
-				const codeLengthBits = Math.max(...codeLengthTree);
-				const clbMask = (1 << codeLengthBits) - 1;
-				const codeLengthMap = huffMap(codeLengthTree, codeLengthBits, 1);
-				for (let i = 0; i < tl;) {
-					const r = codeLengthMap[bits(input, pos, clbMask)];
-					pos += r & 15;
-					const symbol = r >> 4;
-					if (symbol < 16) lengthDistanceTree[i++] = symbol;
-					else {
-						let copy = 0;
-						let n = 0;
-						if (symbol === 16) {
-							n = 3 + bits(input, pos, 3);
-							pos += 2;
-							copy = lengthDistanceTree[i - 1];
-						} else if (symbol === 17) {
-							n = 3 + bits(input, pos, 7);
-							pos += 3;
-						} else if (symbol === 18) {
-							n = 11 + bits(input, pos, 127);
-							pos += 7;
-						}
-						while (n--) lengthDistanceTree[i++] = copy;
-					}
-				}
-				const lengthTree = lengthDistanceTree.subarray(0, hLiteral);
-				const distanceTree = lengthDistanceTree.subarray(hLiteral);
-				lengthBits = Math.max(...lengthTree);
-				distBits = Math.max(...distanceTree);
-				lengthMap = huffMap(lengthTree, lengthBits, 1);
-				distMap = huffMap(distanceTree, distBits, 1);
-			} else throw new Error("invalid block type");
-			if (pos > totalBits) throw new Error("unexpected EOF");
-		}
-		ensureSize(outputIndex + 131072);
-		const lms = (1 << lengthBits) - 1;
-		const dms = (1 << distBits) - 1;
-		for (;;) {
-			const code = lengthMap[bits16(input, pos) & lms];
-			const sym = code >> 4;
-			pos += code & 15;
-			if (pos > totalBits) throw new Error("unexpected EOF");
-			if (!code) throw new Error("invalid length/literal");
-			if (sym < 256) out[outputIndex++] = sym;
-			else if (sym === 256) {
-				lengthMap = void 0;
-				break;
-			} else {
-				let add = sym - 254;
-				if (sym > 264) {
-					const index = sym - 257;
-					const b = fixedLengthExtraBits[index];
-					add = bits(input, pos, (1 << b) - 1) + fixedLength[index];
-					pos += b;
-				}
-				if (!distMap) throw new Error("invalid distance map");
-				const d = distMap[bits16(input, pos) & dms];
-				const dsym = d >> 4;
-				if (!d) throw new Error("invalid distance");
-				pos += d & 15;
-				let dt = fixedDistance[dsym];
-				if (dsym > 3) {
-					const b = fixedDistanceExtraBits[dsym];
-					dt += bits16(input, pos) & (1 << b) - 1;
-					pos += b;
-				}
-				if (pos > totalBits) throw new Error("unexpected EOF");
-				const end = outputIndex + add;
-				if (outputIndex < dt) throw new Error("unexpected dictionary case");
-				ensureSize(end);
-				for (; outputIndex < end; outputIndex++) out[outputIndex] = out[outputIndex - dt];
-			}
-		}
-		if (lengthMap) final = 1;
-	} while (!final);
-	if (outputIndex < out.length) {
-		const nextBlock = Math.ceil(pos / 8) + 8;
-		gunzip(input, out, nextBlock, outputIndex);
-	}
-	if (!output) return out.subarray(0, outputIndex);
-	return out;
-}
-export { gunzip };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/src/avro/avro.metadata.mjs

@@ -1,65 +0,0 @@
-const smallLongs = Array.from({ length: 128 }, (_, i) => BigInt(i >>> 1 ^ -(i & 1)));
-function readZigZag(reader) {
-	let result = 0;
-	let shift = 0;
-	while (true) {
-		const byte = reader.view.getUint8(reader.offset++);
-		result |= (byte & 127) << shift;
-		if (!(byte & 128)) return result >>> 1 ^ -(result & 1);
-		shift += 7;
-	}
-}
-function readZigZagBigInt(reader) {
-	const first = reader.view.getUint8(reader.offset++);
-	if (first < 128) return smallLongs[first];
-	let prefix = first & 127;
-	for (let shift = 7; shift < 28; shift += 7) {
-		const byte = reader.view.getUint8(reader.offset++);
-		prefix |= (byte & 127) << shift;
-		if (!(byte & 128)) return BigInt(prefix >>> 1 ^ -(prefix & 1));
-	}
-	let high = 0;
-	let factor = 1;
-	while (true) {
-		const byte = reader.view.getUint8(reader.offset++);
-		high += (byte & 127) * factor;
-		if (!(byte & 128)) {
-			const result = BigInt(high) << 27n | BigInt(prefix >>> 1);
-			return prefix & 1 ? ~result : result;
-		}
-		factor *= 128;
-	}
-}
-function readAvroString(reader) {
-	const length = readZigZag(reader);
-	const bytes = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, length);
-	reader.offset += length;
-	return new TextDecoder().decode(bytes);
-}
-function avroMetadata(reader) {
-	if (reader.view.getUint32(reader.offset) !== 1331849729) throw new Error("avro invalid magic bytes");
-	reader.offset += 4;
-	const metadata = {};
-	let mapCount = readZigZag(reader);
-	while (mapCount !== 0) {
-		if (mapCount < 0) {
-			mapCount = -mapCount;
-			readZigZag(reader);
-		}
-		for (let i = 0; i < mapCount; i++) {
-			const key = readAvroString(reader);
-			metadata[key] = readAvroString(reader);
-		}
-		mapCount = readZigZag(reader);
-	}
-	metadata["avro.schema"] = JSON.parse(metadata["avro.schema"]);
-	if (metadata["schema"]) metadata["schema"] = JSON.parse(metadata["schema"]);
-	if (metadata["iceberg.schema"]) metadata["iceberg.schema"] = JSON.parse(metadata["iceberg.schema"]);
-	const syncMarker = new Uint8Array(reader.view.buffer, reader.view.byteOffset + reader.offset, 16);
-	reader.offset += 16;
-	return {
-		metadata,
-		syncMarker
-	};
-}
-export { avroMetadata, readZigZag, readZigZagBigInt };

package/dist/_virtual/node_modules/.pnpm/icebird@0.8.30_patch_hash=a400e343a199b1ca18601f96348011745ac6518fc07ffa71d5adfdd2ccc7d14f/node_modules/icebird/types/fetch.d.mts

@@ -1,36 +0,0 @@
-import "../../../../hyparquet@1.31.1/node_modules/hyparquet/types/node.mjs";
-import { Resolver } from "../src/types.mjs";
-/**
- * Wrap a `Resolver` so reads of the same path share one HTTP fetch and
- * subsequent range reads share one in-memory buffer. Writes and deletes
- * through the same resolver invalidate the cache entry for their path on
- * success, so a commit pipeline (`icebergAppend` → reload metadata) sees the
- * new bytes without manual invalidation.
- *
- * - `reader(path)` is memoized by path. The returned buffer is passed through
- *   `cachedAsyncBuffer` so range reads within a single file are also deduped.
- *   `byteLength` is a fetch hint and not part of the cache key — the bytes
- *   are the same either way.
- * - `writer(path).finish()` invalidates the cache entry only when the
- *   underlying finish resolves; a rejected finish (e.g. an `If-None-Match: *`
- *   collision returning 412/409) leaves the cached bytes intact, since the
- *   server-side object hasn't changed.
- * - `deleter(path)` invalidates the cache entry on success.
- * - `writer` and `deleter` are omitted when the base resolver omits them, so
- *   wrapping a read-only resolver still yields a read-only resolver.
- *
- * Iceberg writes are almost entirely create-new-path (data parquets, manifest
- * avros, snapshot avros, vN.metadata.json all live at fresh paths per
- * commit). Only `version-hint.text` and equivalent catalog-pointer files are
- * truly mutated in place, so path-level invalidation is sufficient to keep
- * single-process write-then-read pipelines consistent.
- *
- * Cross-process freshness is out of scope: a reader in process B does not
- * see writes committed by process A through a different cache. Use a
- * short-lived resolver, an external coordination layer, or wrap with TTL /
- * conditional-GET revalidation if you need that.
- *
- * @param {Resolver} base
- * @returns {Resolver}
- */
-export declare function cachingResolver(base: Resolver): Resolver;