@gscdump/lakehouse 1.4.1 → 1.4.2

package/dist/catalog.d.mts

@@ -1,12 +1,9 @@
 import { CatalogCache } from "./catalog-cache.mjs";
-import { IcebergColumnType, IcebergS3Config } from "./schema.mjs";
+import { IcebergPrimitiveType, IcebergS3Config } from "./schema.mjs";
 import { PartitionValueMatch } from "./partition-prune.mjs";
-import { icebergAppend } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/write/write.mjs";
-import { restCatalogConnect } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/catalog/rest.mjs";
-import { cachingResolver } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/fetch.mjs";
-import "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/index.mjs";
-/** icebird's lowercase Iceberg primitive types (subset we use). */
-type IcebergPrimitiveType = 'string' | 'int' | 'long' | 'double' | 'date' | 'boolean';
+import { icebergAppend } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/write/write.mjs";
+import { restCatalogConnect } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/fetch.mjs";
 /** A field in an icebird table `Schema`. */
 interface IcebergSchemaField {
   id: number;
@@ -75,7 +72,6 @@ interface IcebergConnection {
    */
   cacheScope?: string;
 }
-declare const ICEBERG_TYPE_MAP: Record<IcebergColumnType, IcebergPrimitiveType>;
 /** Options for {@link connectIcebergCatalog}. */
 interface ConnectIcebergOptions {
   /**
@@ -142,6 +138,7 @@ interface CommitRetryOptions {
    */
   appendId?: string;
 }
+type IcebergAppendArgs = Parameters<typeof icebergAppend>[0];
 /**
  * True when `err` is an R2 Data Catalog commit rate-limit response
  * (`429 too many commits to this table`).
@@ -153,7 +150,7 @@ declare function isCommitRateLimited(err: unknown): boolean;
  * idempotency guard so a 429 whose commit actually landed is never re-applied
  * (see `deriveAppendId`/`appendAlreadyLanded`).
  */
-declare function icebergAppendRetrying(args: Parameters<typeof icebergAppend>[0], options?: CommitRetryOptions): Promise<void>;
+declare function icebergAppendRetrying(args: IcebergAppendArgs, options?: CommitRetryOptions): Promise<void>;
 /** A data file in the current snapshot's manifest, scoped to one partition. */
 interface IcebergListedDataFile {
   filePath: string;
@@ -213,4 +210,4 @@ declare function invalidateSnapshotRef(cache: CatalogCache, namespace: string, t
  * the def's own spec + identity/dims values.
  */
 declare function resolveIcebergDataFiles(conn: IcebergConnection, opts: ResolveIcebergDataFilesOptions): Promise<IcebergListedDataFile[]>;
-export { CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { CommitRetryOptions, ConnectIcebergOptions, IcebergAppendArgs, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };

package/dist/catalog.mjs

@@ -1,20 +1,15 @@
 import { stringifyBigintSafe } from "./bigint.mjs";
-import { cachingResolver } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/fetch.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
-import { icebergAppend, icebergAppendBatches, icebergDropTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
-import { s3SignedResolver } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/s3.mjs";
-import "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/fetch.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/s3.mjs";
 import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
 import { buildManifestPartitionFilter } from "./partition-prune.mjs";
-const ICEBERG_TYPE_MAP = {
-	STRING: "string",
-	INT: "int",
-	LONG: "long",
-	DOUBLE: "double",
-	DATE: "date",
-	BOOLEAN: "boolean"
-};
+let icebirdWriteModule;
+function useIcebirdWriteModule() {
+	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs");
+	return icebirdWriteModule;
+}
 const CATALOG_CONFIG_TTL_MS = 3600 * 1e3;
 function catalogConfigKey(config) {
 	return `lakehouse-catalog-cfg\0${config.catalogUri}\0${config.warehouse}`;
@@ -111,6 +106,7 @@ async function listIcebergTables(conn) {
 async function dropIcebergTables(conn, tables) {
 	const targets = tables ?? (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name);
 	const results = [];
+	const { icebergDropTable } = await useIcebirdWriteModule();
 	for (const table of targets) await icebergDropTable({
 		catalog: conn.catalog,
 		namespace: conn.namespace,
@@ -137,6 +133,7 @@ function defaultCommitSleep(ms) {
 	return new Promise((resolve) => setTimeout(resolve, ms));
 }
 async function icebergAppendRetrying(args, options = {}) {
+	const { icebergAppend } = await useIcebirdWriteModule();
 	const maxAttempts = options.maxAttempts ?? 6;
 	const baseDelayMs = options.baseDelayMs ?? 1e3;
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
@@ -162,6 +159,7 @@ async function icebergAppendRetrying(args, options = {}) {
 	}
 }
 async function icebergAppendBatchesRetrying(args, options) {
+	const { icebergAppendBatches } = await useIcebirdWriteModule();
 	const maxAttempts = options.maxAttempts ?? 6;
 	const baseDelayMs = options.baseDelayMs ?? 1e3;
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
@@ -350,4 +348,4 @@ async function resolveIcebergDataFiles(conn, opts) {
 	}
 	return out;
 }
-export { ICEBERG_TYPE_MAP, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };

package/dist/dataset-runtime.mjs

@@ -0,0 +1,68 @@
+import { connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
+import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
+async function createDatasetTable(conn, table, schema, partitionSpec, sortOrder) {
+	const results = [];
+	await icebergCreateTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		schema,
+		partitionSpec,
+		...sortOrder ? { sortOrder } : {}
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (error) => results.push({
+		table,
+		ok: false,
+		error: error instanceof Error ? error.message : String(error)
+	}));
+	return results;
+}
+async function appendDatasetRows(conn, table, records, commitRetry) {
+	await icebergAppendRetrying({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		resolver: conn.resolver,
+		records
+	}, commitRetry);
+}
+async function appendDatasetBatches(conn, table, source, prepare, opts) {
+	let accepted = 0;
+	let skipped = 0;
+	const batchFactory = async function* () {
+		accepted = 0;
+		skipped = 0;
+		for await (const rows of source()) {
+			const prepared = prepare(rows);
+			accepted += prepared.records.length;
+			skipped += prepared.skipped;
+			if (prepared.records.length > 0) yield prepared.records;
+		}
+	};
+	const committed = await icebergAppendBatchesRetrying({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		resolver: conn.resolver,
+		batchFactory
+	}, {
+		...opts.commitRetry,
+		appendId: opts.appendId
+	});
+	return {
+		accepted,
+		skipped,
+		committed
+	};
+}
+async function connectDatasetCatalog(config, opts) {
+	const conn = await connectIcebergCatalog(config, opts);
+	await ensureIcebergNamespace(conn);
+	return conn;
+}
+async function resolveDatasetDataFiles(conn, opts) {
+	return resolveIcebergDataFiles(conn, opts);
+}
+export { appendDatasetBatches, appendDatasetRows, connectDatasetCatalog, createDatasetTable, resolveDatasetDataFiles };

package/dist/dataset.d.mts

@@ -58,14 +58,6 @@ interface IcebergDatasetDef {
  * to the frozen inline constants it replaces.
  */
 declare function deriveTableSpec(def: IcebergDatasetDef): IcebergTableSpec;
-/**
- * Convert a `YYYY-MM-DD` string / `Date` / already-numeric day-count to the
- * integer "days since the Unix epoch" the Iceberg `date` type stores.
- * hyparquet-writer mis-encodes Date-valued dictionary columns, so callers
- * should feed this INTO their row before `appendRows`/`appendSink.emit` for
- * any `month`-partitioned date column.
- */
-declare function toIcebergDayCount(value: string | Date | number): number;
 /** Options shared by `appendRows` / `appendSink`. */
 interface AppendCommitOptions {
   commitRetry?: CommitRetryOptions;
@@ -159,4 +151,4 @@ interface IcebergDataset {
  * from the def. See {@link IcebergDatasetDef}.
  */
 declare function defineIcebergDataset(def: IcebergDatasetDef): IcebergDataset;
-export { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec, toIcebergDayCount };
+export { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec };

package/dist/dataset.mjs

@@ -1,11 +1,8 @@
 import { coerceBigIntToNumber } from "./bigint.mjs";
-import { icebergCreateTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
-import "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
 import { buildManifestPartitionFilter } from "./partition-prune.mjs";
-import { ICEBERG_TYPE_MAP, connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
+import { ICEBERG_TYPE_MAP } from "./schema.mjs";
 const INT32_MIN = -2147483648;
 const INT32_MAX = 2147483647;
-const DAY_MILLIS = 864e5;
 function identityColumnNames(identity) {
 	return identity.kind === "site-int" ? ["site_id"] : identity.columns.map((c) => c.name);
 }
@@ -104,17 +101,6 @@ function icebergSortOrderFromSpec(spec) {
 		}))
 	};
 }
-function toIcebergDayCount(value) {
-	if (typeof value === "number") return value;
-	if (value instanceof Date) {
-		const ms = value.getTime();
-		if (Number.isNaN(ms)) throw new TypeError("toIcebergDayCount: invalid Date (NaN)");
-		return Math.floor(ms / DAY_MILLIS);
-	}
-	const ms = Date.parse(`${value}T00:00:00Z`);
-	if (Number.isNaN(ms)) throw new TypeError(`toIcebergDayCount: invalid date string '${value}'`);
-	return Math.floor(ms / DAY_MILLIS);
-}
 function asInt32(value) {
 	if (typeof value !== "string" && typeof value !== "number" && typeof value !== "bigint") return null;
 	const n = Number(value);
@@ -191,23 +177,8 @@ function defineIcebergDataset(def) {
 	const sortOrder = icebergSortOrderFromSpec(tableSpec);
 	const { guard, dedupe, sort } = buildRowProcessor(def, tableSpec);
 	async function createTable(conn) {
-		const results = [];
-		await icebergCreateTable({
-			catalog: conn.catalog,
-			namespace: conn.namespace,
-			table: def.table,
-			schema,
-			partitionSpec: partitionSpecIcebird,
-			...sortOrder ? { sortOrder } : {}
-		}).then(() => results.push({
-			table: def.table,
-			ok: true
-		}), (e) => results.push({
-			table: def.table,
-			ok: false,
-			error: e instanceof Error ? e.message : String(e)
-		}));
-		return results;
+		const { createDatasetTable } = await import("./dataset-runtime.mjs");
+		return createDatasetTable(conn, def.table, schema, partitionSpecIcebird, sortOrder);
 	}
 	function process(rows) {
 		const guarded = [];
@@ -232,57 +203,20 @@ function defineIcebergDataset(def) {
 			accepted: 0,
 			skipped
 		};
-		await icebergAppendRetrying({
-			catalog: conn.catalog,
-			namespace: conn.namespace,
-			table: def.table,
-			resolver: conn.resolver,
-			records
-		}, opts.commitRetry);
+		const { appendDatasetRows } = await import("./dataset-runtime.mjs");
+		await appendDatasetRows(conn, def.table, records, opts.commitRetry);
 		return {
 			accepted: records.length,
 			skipped
 		};
 	}
 	async function appendBatches(conn, source, opts) {
-		let accepted = 0;
-		let skipped = 0;
-		const batchFactory = async function* () {
-			accepted = 0;
-			skipped = 0;
-			for await (const rows of source()) {
-				const prepared = process(rows);
-				accepted += prepared.records.length;
-				skipped += prepared.skipped;
-				if (prepared.records.length > 0) yield prepared.records;
-			}
-		};
-		const committed = await icebergAppendBatchesRetrying({
-			catalog: conn.catalog,
-			namespace: conn.namespace,
-			table: def.table,
-			resolver: conn.resolver,
-			batchFactory
-		}, {
-			...opts.commitRetry,
-			appendId: opts.appendId
-		});
-		return {
-			accepted,
-			skipped,
-			committed
-		};
+		const { appendDatasetBatches } = await import("./dataset-runtime.mjs");
+		return appendDatasetBatches(conn, def.table, source, process, opts);
 	}
 	function appendSink(opts) {
 		let buffer = [];
 		let connection;
-		function connect() {
-			connection ??= connectIcebergCatalog(opts.catalog, opts.connect).then(async (conn) => {
-				await ensureIcebergNamespace(conn);
-				return conn;
-			});
-			return connection;
-		}
 		return {
 			emit(rows) {
 				for (const r of rows) buffer.push(r);
@@ -296,7 +230,20 @@ function defineIcebergDataset(def) {
 				const rows = buffer;
 				buffer = [];
 				try {
-					const result = await appendRows(await connect(), rows, { commitRetry: opts.commitRetry });
+					const runtime = await import("./dataset-runtime.mjs");
+					connection ??= runtime.connectDatasetCatalog(opts.catalog, opts.connect);
+					const conn = await connection;
+					const { records, skipped } = process(rows);
+					if (records.length === 0) return {
+						flushed: false,
+						accepted: 0,
+						skipped
+					};
+					await runtime.appendDatasetRows(conn, def.table, records, opts.commitRetry);
+					const result = {
+						accepted: records.length,
+						skipped
+					};
 					if (result.accepted > 0 && opts.ledger) await opts.ledger.record();
 					return {
 						flushed: result.accepted > 0,
@@ -334,7 +281,8 @@ function defineIcebergDataset(def) {
 		return buildManifestPartitionFilter(def.partition, readerPredicate(identity, dims), months);
 	}
 	async function resolveDataFiles(conn, identity, range, dims, opts = {}) {
-		return resolveIcebergDataFiles(conn, {
+		const { resolveDatasetDataFiles } = await import("./dataset-runtime.mjs");
+		return resolveDatasetDataFiles(conn, {
 			namespace: def.namespace,
 			table: def.table,
 			partitionSpec: def.partition,
@@ -361,4 +309,4 @@ function defineIcebergDataset(def) {
 		resolveDataFiles
 	};
 }
-export { defineIcebergDataset, deriveTableSpec, toIcebergDayCount };
+export { defineIcebergDataset, deriveTableSpec };

package/dist/date.d.mts

@@ -0,0 +1,8 @@
+/**
+ * Convert a `YYYY-MM-DD` string / `Date` / already-numeric day-count to the
+ * integer "days since the Unix epoch" the Iceberg `date` type stores.
+ * hyparquet-writer mis-encodes Date-valued dictionary columns, so callers
+ * should feed this into their row before writing a month-partitioned date.
+ */
+declare function toIcebergDayCount(value: string | Date | number): number;
+export { toIcebergDayCount };

package/dist/date.mjs

@@ -0,0 +1,13 @@
+const DAY_MILLIS = 864e5;
+function toIcebergDayCount(value) {
+	if (typeof value === "number") return value;
+	if (value instanceof Date) {
+		const ms = value.getTime();
+		if (Number.isNaN(ms)) throw new TypeError("toIcebergDayCount: invalid Date (NaN)");
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	const ms = Date.parse(`${value}T00:00:00Z`);
+	if (Number.isNaN(ms)) throw new TypeError(`toIcebergDayCount: invalid date string '${value}'`);
+	return Math.floor(ms / DAY_MILLIS);
+}
+export { toIcebergDayCount };

package/dist/index.d.mts

@@ -1,7 +1,8 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
 import { CatalogCache, cacheGet, cachePut } from "./catalog-cache.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./schema.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./schema.mjs";
 import { IcebergFieldSummary, ManifestPartitionFilter, PartitionValueMatch, buildManifestPartitionFilter } from "./partition-prune.mjs";
-import { CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
-import { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec, toIcebergDayCount } from "./dataset.mjs";
+import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
+import { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
+import { toIcebergDayCount } from "./date.mjs";
 export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/index.mjs

@@ -1,7 +1,8 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
 import { cacheGet, cachePut } from "./catalog-cache.mjs";
 import { buildManifestPartitionFilter } from "./partition-prune.mjs";
-import { ICEBERG_TYPE_MAP, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
-import { defineIcebergDataset, deriveTableSpec, toIcebergDayCount } from "./dataset.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING } from "./schema.mjs";
+import { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP } from "./schema.mjs";
+import { defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
+import { toIcebergDayCount } from "./date.mjs";
 export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/orphan-sweep.mjs

@@ -1,6 +1,6 @@
-import { restCatalogListTables, restCatalogLoadTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
-import "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
+import { restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
 const DEFAULT_GRACE_HOURS = 48;
 const DEFAULT_MAX_DELETES = 500;
 const DEFAULT_IO_CONCURRENCY = 4;

package/dist/schema.d.mts

@@ -18,6 +18,9 @@ interface IcebergS3Config {
 }
 /** Iceberg-native column type. */
 type IcebergColumnType = 'STRING' | 'INT' | 'LONG' | 'DOUBLE' | 'DATE' | 'BOOLEAN';
+/** icebird's lowercase Iceberg primitive types (subset we use). */
+type IcebergPrimitiveType = 'string' | 'int' | 'long' | 'double' | 'date' | 'boolean';
+declare const ICEBERG_TYPE_MAP: Record<IcebergColumnType, IcebergPrimitiveType>;
 interface IcebergColumn {
   /** Column name as written into the Iceberg table (snake_case). */
   name: string;
@@ -71,4 +74,4 @@ interface IcebergTableSpec {
   /** Physical write-order columns (dimension-first) — undefined skips sort-order. */
   clusterKey?: readonly string[];
 }
-export { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding };

package/dist/schema.mjs

@@ -1,2 +1,10 @@
+const ICEBERG_TYPE_MAP = {
+	STRING: "string",
+	INT: "int",
+	LONG: "long",
+	DOUBLE: "double",
+	DATE: "date",
+	BOOLEAN: "boolean"
+};
 const DEFAULT_PARTITION_KEY_ENCODING = "int";
-export { DEFAULT_PARTITION_KEY_ENCODING };
+export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP };

package/dist/unsafe-raw.d.mts

@@ -1,7 +1,7 @@
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/write/write.mjs";
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/catalog/rest.mjs";
-import { s3SignedResolver } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/s3.mjs";
-import { icebergManifests } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/manifest.mjs";
-import "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/index.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/write/write.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/catalog/rest.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/s3.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/manifest.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/types/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/unsafe-raw.mjs

@@ -1,7 +1,7 @@
-import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
-import { icebergManifests } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
-import { icebergAppend, icebergCreateTable, icebergDropTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
-import { s3SignedResolver } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/s3.mjs";
-import "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/s3.mjs";
 import { icebergAppendRetrying } from "./catalog.mjs";
+import { icebergAppend, icebergCreateTable, icebergDropTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
+import "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
 export { icebergAppend, icebergAppendRetrying, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver };

package/dist/vendor/hysnappy-purejs.mjs

@@ -1,4 +1,4 @@
-import { snappyUncompress } from "../node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/snappy.mjs";
+import { snappyUncompress } from "../_virtual/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/snappy.mjs";
 function decode(input, outputLength) {
 	const output = new Uint8Array(outputLength);
 	snappyUncompress(input, output);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/lakehouse",
   "type": "module",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "Dataset-agnostic Iceberg lakehouse layer + dataset registry for R2 Data Catalog producers (ADR-0021).",
   "author": {
     "name": "Harlan Wilton",
@@ -31,6 +31,16 @@
       "import": "./dist/bigint.mjs",
       "default": "./dist/bigint.mjs"
     },
+    "./dataset": {
+      "types": "./dist/dataset.d.mts",
+      "import": "./dist/dataset.mjs",
+      "default": "./dist/dataset.mjs"
+    },
+    "./date": {
+      "types": "./dist/date.d.mts",
+      "import": "./dist/date.mjs",
+      "default": "./dist/date.mjs"
+    },
     "./schema": {
       "types": "./dist/schema.d.mts",
       "import": "./dist/schema.mjs",

package/dist/node_modules/.pnpm/hyparquet-compressors@1.1.1/node_modules/hyparquet-compressors/src/compressors.mjs

@@ -1,7 +0,0 @@
-import "../../../../fzstd@0.1.1/node_modules/fzstd/esm/index.mjs";
-import { snappyUncompressor } from "../../../../../../vendor/hysnappy-purejs.mjs";
-import "./gzip.mjs";
-import "./brotli.mjs";
-import "./lz4.mjs";
-snappyUncompressor();
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/assemble.mjs

@@ -1,3 +0,0 @@
-import "./schema.mjs";
-import "./variant.mjs";
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/bloom.mjs

@@ -1,14 +0,0 @@
-import "./thrift.mjs";
-import "./xxhash.mjs";
-new TextEncoder();
-new Uint32Array([
-	1203114875,
-	1150766481,
-	2284105051,
-	2729912477,
-	1884591559,
-	770785867,
-	2667333959,
-	1550580529
-]);
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/column.mjs

@@ -1,8 +0,0 @@
-import "./constants.mjs";
-import "./convert.mjs";
-import "./schema.mjs";
-import "./thrift.mjs";
-import "./assemble.mjs";
-import "./plain.mjs";
-import "./datapage.mjs";
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/constants.mjs

@@ -1 +0,0 @@
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/convert.mjs

@@ -1,11 +0,0 @@
-import "./wkb.mjs";
-new TextDecoder();
-function parseDecimal(bytes) {
-	if (!bytes.length) return 0;
-	let value = 0n;
-	for (const byte of bytes) value = value * 256n + BigInt(byte);
-	const bits = bytes.length * 8;
-	if (value >= 2n ** BigInt(bits - 1)) value -= 2n ** BigInt(bits);
-	return Number(value);
-}
-export { parseDecimal };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/datapage.mjs

@@ -1,6 +0,0 @@
-import "./schema.mjs";
-import "./delta.mjs";
-import "./encoding.mjs";
-import "./plain.mjs";
-import "./snappy.mjs";
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/delta.mjs

@@ -1,2 +0,0 @@
-import "./thrift.mjs";
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/encoding.mjs

@@ -1,2 +0,0 @@
-import "./thrift.mjs";
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/filter.mjs

@@ -1,3 +0,0 @@
-import "./bloom.mjs";
-import "./utils.mjs";
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/geoparquet.mjs

@@ -1 +0,0 @@
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/index.mjs

@@ -1,7 +0,0 @@
-import "./metadata.mjs";
-import "./indexes.mjs";
-import { asyncBufferFromUrl, cachedAsyncBuffer } from "./utils.mjs";
-import "./snappy.mjs";
-import "./read.mjs";
-import "./query.mjs";
-export { asyncBufferFromUrl, cachedAsyncBuffer };

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/indexes.mjs

@@ -1,5 +0,0 @@
-import "./constants.mjs";
-import "./convert.mjs";
-import "./thrift.mjs";
-import "./metadata.mjs";
-export {};

package/dist/node_modules/.pnpm/hyparquet@1.26.2/node_modules/hyparquet/src/metadata.mjs

@@ -1,8 +0,0 @@
-import "./constants.mjs";
-import "./convert.mjs";
-import "./schema.mjs";
-import "./thrift.mjs";
-import "./geoparquet.mjs";
-const defaultInitialFetchSize = 1 << 19;
-new TextDecoder();
-export { defaultInitialFetchSize };