@gscdump/lakehouse 1.4.0 → 1.4.2

package/dist/date.mjs

@@ -0,0 +1,13 @@
+const DAY_MILLIS = 864e5;
+function toIcebergDayCount(value) {
+	if (typeof value === "number") return value;
+	if (value instanceof Date) {
+		const ms = value.getTime();
+		if (Number.isNaN(ms)) throw new TypeError("toIcebergDayCount: invalid Date (NaN)");
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	const ms = Date.parse(`${value}T00:00:00Z`);
+	if (Number.isNaN(ms)) throw new TypeError(`toIcebergDayCount: invalid date string '${value}'`);
+	return Math.floor(ms / DAY_MILLIS);
+}
+export { toIcebergDayCount };

package/dist/index.d.mts

@@ -1,161 +1,8 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
-/**
- * Closed identity shapes (ADR-0021 amendments 2 + 4). `'site-int'` is the
- * team-scoped numeric Catalog Site Id (nuxtseo ADR-0091) stored under the
- * fixed column name `site_id`. `'columns'` is the general N-column identity
- * with per-column encoding, replacing the original opaque `'custom'` shape.
- */
-type DatasetIdentity = {
-  kind: 'site-int';
-  encoding?: 'int' | 'string';
-} | {
-  kind: 'columns';
-  columns: readonly {
-    name: string;
-    encoding: 'int32' | 'string';
-  }[];
-};
-/** A per-slice dimension (GSC's `searchType`). Absent for plain snapshot datasets. */
-interface DatasetDim {
-  toPartitionValue: (v: string) => string | number;
-  /** Required (amendment 5) — how the reader decodes this column's manifest bounds. */
-  boundEncoding: 'int32' | 'string';
-}
-interface IcebergDatasetColumnDef {
-  name: string;
-  type: IcebergColumnType;
-  required: boolean;
-}
-interface IcebergDatasetLedger {
-  key: readonly string[];
-}
-interface IcebergDatasetDef {
-  namespace: string;
-  table: string;
-  identity: DatasetIdentity;
-  /** Extra per-slice dimensions; absent for snapshot datasets. */
-  dims?: Record<string, DatasetDim>;
-  /** Data columns (non-identity, non-dims) — includes any partition source column (e.g. `date`). */
-  columns: readonly IcebergDatasetColumnDef[];
-  /** Partition fields; identity columns must appear here. */
-  partition: readonly IcebergPartitionField[];
-  /** Natural-key columns. Full dedupe key = identity + dims + naturalKey (amendment 3). */
-  naturalKey: readonly string[];
-  clusterKey?: readonly string[];
-  /** Consumer-owned exactly-once ledger shape — folded into `appendSink().close()` (amendment 6). */
-  ledger?: IcebergDatasetLedger;
-}
-/**
- * Derive the full {@link IcebergTableSpec} from a dataset def. Field ids are
- * assigned sequentially in ONE fixed order — identity columns, then dims
- * columns, then declared data columns — starting at 1. This mirrors the
- * engine's original `icebergTableSpec` contract (identity columns first, data
- * columns from a fixed base) so a def-derived spec for an existing table
- * (`crawl.pages`, `lighthouse.scans`, `dataforseo.keywords`) is BYTE-IDENTICAL
- * to the frozen inline constants it replaces.
- */
-declare function deriveTableSpec(def: IcebergDatasetDef): IcebergTableSpec;
-/**
- * Convert a `YYYY-MM-DD` string / `Date` / already-numeric day-count to the
- * integer "days since the Unix epoch" the Iceberg `date` type stores.
- * hyparquet-writer mis-encodes Date-valued dictionary columns, so callers
- * should feed this INTO their row before `appendRows`/`appendSink.emit` for
- * any `month`-partitioned date column.
- */
-declare function toIcebergDayCount(value: string | Date | number): number;
-/** Options shared by `appendRows` / `appendSink`. */
-interface AppendCommitOptions {
-  commitRetry?: CommitRetryOptions;
-}
-interface AppendSinkOptions extends AppendCommitOptions {
-  catalog: IcebergCatalogConfig;
-  connect?: ConnectIcebergOptions;
-  /** Invoked ONLY after a successful flush (amendment 6) — never on a failed/empty close. */
-  ledger?: {
-    record: () => Promise<void> | void;
-  };
-}
-interface AppendResult {
-  /** Rows accepted after the identity guard + dedupe (what was actually appended). */
-  accepted: number;
-  /** Rows dropped by the identity guard (missing/out-of-range required identity value). */
-  skipped: number;
-}
-type AppendBatchSource = () => Iterable<readonly Record<string, unknown>[]> | AsyncIterable<readonly Record<string, unknown>[]>;
-interface AppendBatchesOptions extends AppendCommitOptions {
-  /** Stable across job retries; recorded in snapshot metadata for landed checks. */
-  appendId: string;
-}
-interface AppendBatchesResult extends AppendResult {
-  /** False when this append id was already present before the call. */
-  committed: boolean;
-}
-interface AppendSinkCloseResult extends AppendResult {
-  flushed: boolean;
-  error?: unknown;
-}
-interface AppendSink {
-  emit: (rows: readonly Record<string, unknown>[]) => void;
-  close: () => Promise<AppendSinkCloseResult>;
-}
-interface ResolveDataFilesOptions {
-  cache?: CatalogCache;
-  clock?: () => number;
-  profiler?: QueryProfiler;
-}
-interface IcebergDataset {
-  readonly def: IcebergDatasetDef;
-  readonly tableSpec: IcebergTableSpec;
-  icebergSchema: () => IcebergSchema;
-  icebergPartitionSpec: () => IcebergPartitionSpec;
-  icebergSortOrder: () => IcebergSortOrder | undefined;
-  createTable: (conn: IcebergConnection) => Promise<IcebergTableOpResult[]>;
-  /**
-   * One-shot append over an EXISTING connection: identity INT32 guard, dedupe
-   * by identity+dims+naturalKey (last-wins), cluster pre-sort (if
-   * `clusterKey` is set), then a single `icebergAppendRetrying` commit. `rows`
-   * must already be in final storage-column shape — identity/dims/data
-   * columns keyed by their STORED column names, with any `month`-partitioned
-   * date column pre-converted via {@link toIcebergDayCount}.
-   */
-  appendRows: (conn: IcebergConnection, rows: readonly Record<string, unknown>[], opts?: AppendCommitOptions) => Promise<AppendResult>;
-  /**
-   * Lazily process bounded row batches, writing multiple data files into one
-   * append snapshot and one catalog commit. Dedupe keys must be disjoint
-   * across source batches.
-   */
-  appendBatches: (conn: IcebergConnection, source: AppendBatchSource, opts: AppendBatchesOptions) => Promise<AppendBatchesResult>;
-  /**
-   * PURE row processing — the identity INT32 guard, dedupe (identity+dims+
-   * naturalKey, last-wins) and cluster pre-sort `appendRows`/`appendSink`
-   * apply before committing, exposed standalone with NO network/icebird call.
-   * Lets a consumer that owns its own commit call-site (e.g. one still
-   * calling a frozen `icebergAppendRetrying` import directly, for test-mock
-   * compatibility) still route its dedupe/guard/sort logic through the
-   * dataset definition.
-   */
-  prepareRows: (rows: readonly Record<string, unknown>[]) => {
-    records: Record<string, unknown>[];
-    skipped: number;
-  };
-  /** Buffered multi-emit sink — owns its own connect/close lifecycle (ADR-0021's primary contract). */
-  appendSink: (opts: AppendSinkOptions) => AppendSink;
-  /** `PartitionValueMatch[]` for one identity value (+ optional dims), for manual manifest filtering. */
-  readerPredicate: (identity: string | number, dims?: Record<string, string>) => PartitionValueMatch[];
-  /** A ready-to-use manifest partition filter for one identity value + wanted months. */
-  partitionBoundFilter: (identity: string | number, months: ReadonlySet<number>, dims?: Record<string, string>) => ManifestPartitionFilter;
-  /** Resolve the data files for one identity value across a date range. */
-  resolveDataFiles: (conn: IcebergConnection, identity: string | number, range: {
-    start: string;
-    end: string;
-  }, dims?: Record<string, string>, opts?: ResolveDataFilesOptions) => Promise<IcebergListedDataFile[]>;
-}
-/**
- * Declare an Iceberg dataset — one `namespace.table` — deriving its schema,
- * partition spec, dedupe key, INT32 guard, append path, and reader helpers
- * from the def. See {@link IcebergDatasetDef}.
- */
-declare function defineIcebergDataset(def: IcebergDatasetDef): IcebergDataset;
+import { CatalogCache, cacheGet, cachePut } from "./catalog-cache.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./schema.mjs";
+import { IcebergFieldSummary, ManifestPartitionFilter, PartitionValueMatch, buildManifestPartitionFilter } from "./partition-prune.mjs";
+import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
+import { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
+import { toIcebergDayCount } from "./date.mjs";
 export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/index.mjs

@@ -1,363 +1,8 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
-import { icebergCreateTable } from "./_chunks/libs/icebird.mjs";
-import { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING } from "./schema.mjs";
-const INT32_MIN = -2147483648;
-const INT32_MAX = 2147483647;
-const DAY_MILLIS = 864e5;
-function identityColumnNames(identity) {
-	return identity.kind === "site-int" ? ["site_id"] : identity.columns.map((c) => c.name);
-}
-function identityEncodings(identity) {
-	const map = /* @__PURE__ */ new Map();
-	if (identity.kind === "site-int") map.set("site_id", (identity.encoding ?? "int") === "int" ? "int32" : "string");
-	else for (const c of identity.columns) map.set(c.name, c.encoding);
-	return map;
-}
-function identityColumnType(encoding) {
-	return encoding === "int32" ? "INT" : "STRING";
-}
-function deriveTableSpec(def) {
-	const encodings = identityEncodings(def.identity);
-	const idNames = identityColumnNames(def.identity);
-	const dimNames = def.dims ? Object.keys(def.dims) : [];
-	let fieldId = 1;
-	const columns = [];
-	for (const name of idNames) columns.push({
-		name,
-		type: identityColumnType(encodings.get(name)),
-		required: true,
-		fieldId: fieldId++
-	});
-	for (const name of dimNames) {
-		const enc = def.dims[name].boundEncoding;
-		columns.push({
-			name,
-			type: identityColumnType(enc),
-			required: true,
-			fieldId: fieldId++
-		});
-	}
-	for (const col of def.columns) columns.push({
-		name: col.name,
-		type: col.type,
-		required: col.required,
-		fieldId: fieldId++
-	});
-	return {
-		namespace: def.namespace,
-		table: def.table,
-		columns,
-		partitionSpec: def.partition,
-		naturalKey: def.naturalKey,
-		identityColumns: [
-			...idNames,
-			...dimNames,
-			...def.naturalKey
-		],
-		clusterKey: def.clusterKey
-	};
-}
-function icebergSchemaFromSpec(spec) {
-	return {
-		"type": "struct",
-		"schema-id": 0,
-		"fields": spec.columns.map((col) => ({
-			id: col.fieldId,
-			name: col.name,
-			required: col.required,
-			type: ICEBERG_TYPE_MAP[col.type]
-		}))
-	};
-}
-function icebergPartitionSpecFromSpec(spec) {
-	const fieldId = (name) => {
-		const col = spec.columns.find((c) => c.name === name);
-		if (!col) throw new Error(`lakehouse: table '${spec.table}' has no '${name}' column`);
-		return col.fieldId;
-	};
-	return {
-		"spec-id": 0,
-		"fields": spec.partitionSpec.map((p, i) => ({
-			"source-id": fieldId(p.sourceColumn),
-			"field-id": 1e3 + i,
-			"name": p.name,
-			"transform": p.transform
-		}))
-	};
-}
-function icebergSortOrderFromSpec(spec) {
-	if (!spec.clusterKey || spec.clusterKey.length === 0) return void 0;
-	const fieldId = (name) => {
-		const col = spec.columns.find((c) => c.name === name);
-		if (!col) throw new Error(`lakehouse: table '${spec.table}' has no '${name}' column`);
-		return col.fieldId;
-	};
-	return {
-		"order-id": 1,
-		"fields": spec.clusterKey.map((col) => ({
-			"source-id": fieldId(col),
-			"transform": "identity",
-			"direction": "asc",
-			"null-order": "nulls-last"
-		}))
-	};
-}
-function toIcebergDayCount(value) {
-	if (typeof value === "number") return value;
-	if (value instanceof Date) {
-		const ms = value.getTime();
-		if (Number.isNaN(ms)) throw new TypeError("toIcebergDayCount: invalid Date (NaN)");
-		return Math.floor(ms / DAY_MILLIS);
-	}
-	const ms = Date.parse(`${value}T00:00:00Z`);
-	if (Number.isNaN(ms)) throw new TypeError(`toIcebergDayCount: invalid date string '${value}'`);
-	return Math.floor(ms / DAY_MILLIS);
-}
-function asInt32(value) {
-	if (typeof value !== "string" && typeof value !== "number" && typeof value !== "bigint") return null;
-	const n = Number(value);
-	if (!Number.isSafeInteger(n) || n < INT32_MIN || n > INT32_MAX) return null;
-	return n;
-}
-function buildRowProcessor(def, tableSpec) {
-	const idNames = identityColumnNames(def.identity);
-	const idEncodings = identityEncodings(def.identity);
-	const dimNames = def.dims ? Object.keys(def.dims) : [];
-	const identityColumns = tableSpec.identityColumns;
-	function guard(row) {
-		const out = {};
-		for (const name of idNames) {
-			const enc = idEncodings.get(name);
-			const raw = row[name];
-			if (enc === "int32") {
-				const n = asInt32(raw);
-				if (n === null) return null;
-				out[name] = n;
-			} else {
-				if (raw == null || raw === "") return null;
-				out[name] = String(raw);
-			}
-		}
-		for (const name of dimNames) out[name] = def.dims[name].toPartitionValue(String(row[name]));
-		for (const col of def.columns) {
-			const value = row[col.name];
-			out[col.name] = col.type === "LONG" ? typeof value === "number" ? BigInt(value) : value : coerceBigIntToNumber(value);
-		}
-		return out;
-	}
-	function dedupe(rows) {
-		if (rows.length < 2) return rows;
-		const seen = /* @__PURE__ */ new Map();
-		for (const rec of rows) {
-			let identity = "";
-			for (let index = 0; index < identityColumns.length; index++) {
-				if (index > 0) identity += "\0";
-				identity += `${rec[identityColumns[index]] ?? ""}`;
-			}
-			seen.set(identity, rec);
-		}
-		return seen.size === rows.length ? rows : [...seen.values()];
-	}
-	function sort(rows) {
-		const cols = def.clusterKey;
-		if (!cols || cols.length === 0 || rows.length < 2) return rows;
-		return rows.slice().sort((a, b) => {
-			for (const col of cols) {
-				const av = a[col];
-				const bv = b[col];
-				if (av === bv) continue;
-				if (av == null) return -1;
-				if (bv == null) return 1;
-				if (typeof av === "number" && typeof bv === "number") return av - bv;
-				const as = String(av);
-				const bs = String(bv);
-				if (as !== bs) return as < bs ? -1 : 1;
-			}
-			return 0;
-		});
-	}
-	return {
-		guard,
-		dedupe,
-		sort
-	};
-}
-function defineIcebergDataset(def) {
-	const tableSpec = deriveTableSpec(def);
-	const schema = icebergSchemaFromSpec(tableSpec);
-	const partitionSpecIcebird = icebergPartitionSpecFromSpec(tableSpec);
-	const sortOrder = icebergSortOrderFromSpec(tableSpec);
-	const { guard, dedupe, sort } = buildRowProcessor(def, tableSpec);
-	async function createTable(conn) {
-		const results = [];
-		await icebergCreateTable({
-			catalog: conn.catalog,
-			namespace: conn.namespace,
-			table: def.table,
-			schema,
-			partitionSpec: partitionSpecIcebird,
-			...sortOrder ? { sortOrder } : {}
-		}).then(() => results.push({
-			table: def.table,
-			ok: true
-		}), (e) => results.push({
-			table: def.table,
-			ok: false,
-			error: e instanceof Error ? e.message : String(e)
-		}));
-		return results;
-	}
-	function process(rows) {
-		const guarded = [];
-		let skipped = 0;
-		for (const row of rows) {
-			const rec = guard(row);
-			if (rec === null) skipped++;
-			else guarded.push(rec);
-		}
-		return {
-			records: sort(dedupe(guarded)),
-			skipped
-		};
-	}
-	async function appendRows(conn, rows, opts = {}) {
-		if (rows.length === 0) return {
-			accepted: 0,
-			skipped: 0
-		};
-		const { records, skipped } = process(rows);
-		if (records.length === 0) return {
-			accepted: 0,
-			skipped
-		};
-		await icebergAppendRetrying({
-			catalog: conn.catalog,
-			namespace: conn.namespace,
-			table: def.table,
-			resolver: conn.resolver,
-			records
-		}, opts.commitRetry);
-		return {
-			accepted: records.length,
-			skipped
-		};
-	}
-	async function appendBatches(conn, source, opts) {
-		let accepted = 0;
-		let skipped = 0;
-		const batchFactory = async function* () {
-			accepted = 0;
-			skipped = 0;
-			for await (const rows of source()) {
-				const prepared = process(rows);
-				accepted += prepared.records.length;
-				skipped += prepared.skipped;
-				if (prepared.records.length > 0) yield prepared.records;
-			}
-		};
-		const committed = await icebergAppendBatchesRetrying({
-			catalog: conn.catalog,
-			namespace: conn.namespace,
-			table: def.table,
-			resolver: conn.resolver,
-			batchFactory
-		}, {
-			...opts.commitRetry,
-			appendId: opts.appendId
-		});
-		return {
-			accepted,
-			skipped,
-			committed
-		};
-	}
-	function appendSink(opts) {
-		let buffer = [];
-		let connection;
-		function connect() {
-			connection ??= connectIcebergCatalog(opts.catalog, opts.connect).then(async (conn) => {
-				await ensureIcebergNamespace(conn);
-				return conn;
-			});
-			return connection;
-		}
-		return {
-			emit(rows) {
-				for (const r of rows) buffer.push(r);
-			},
-			async close() {
-				if (buffer.length === 0) return {
-					flushed: false,
-					accepted: 0,
-					skipped: 0
-				};
-				const rows = buffer;
-				buffer = [];
-				try {
-					const result = await appendRows(await connect(), rows, { commitRetry: opts.commitRetry });
-					if (result.accepted > 0 && opts.ledger) await opts.ledger.record();
-					return {
-						flushed: result.accepted > 0,
-						...result
-					};
-				} catch (error) {
-					return {
-						flushed: false,
-						accepted: 0,
-						skipped: 0,
-						error
-					};
-				}
-			}
-		};
-	}
-	function readerPredicate(identity, dims) {
-		const idNames = identityColumnNames(def.identity);
-		const idEncodings = identityEncodings(def.identity);
-		const matches = idNames.map((name) => ({
-			field: name,
-			value: idEncodings.get(name) === "int32" ? Number(identity) : identity,
-			encoding: idEncodings.get(name) === "int32" ? "int32" : "string"
-		}));
-		if (def.dims && dims) {
-			for (const [name, dim] of Object.entries(def.dims)) if (name in dims) matches.push({
-				field: name,
-				value: dim.toPartitionValue(dims[name]),
-				encoding: dim.boundEncoding
-			});
-		}
-		return matches;
-	}
-	function partitionBoundFilter(identity, months, dims) {
-		return buildManifestPartitionFilter(def.partition, readerPredicate(identity, dims), months);
-	}
-	async function resolveDataFiles(conn, identity, range, dims, opts = {}) {
-		return resolveIcebergDataFiles(conn, {
-			namespace: def.namespace,
-			table: def.table,
-			partitionSpec: def.partition,
-			matches: readerPredicate(identity, dims),
-			range,
-			cache: opts.cache,
-			clock: opts.clock,
-			profiler: opts.profiler
-		});
-	}
-	return {
-		def,
-		tableSpec,
-		icebergSchema: () => schema,
-		icebergPartitionSpec: () => partitionSpecIcebird,
-		icebergSortOrder: () => sortOrder,
-		createTable,
-		appendRows,
-		appendBatches,
-		prepareRows: process,
-		appendSink,
-		readerPredicate,
-		partitionBoundFilter,
-		resolveDataFiles
-	};
-}
+import { cacheGet, cachePut } from "./catalog-cache.mjs";
+import { buildManifestPartitionFilter } from "./partition-prune.mjs";
+import { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP } from "./schema.mjs";
+import { defineIcebergDataset, deriveTableSpec } from "./dataset.mjs";
+import { toIcebergDayCount } from "./date.mjs";
 export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_TYPE_MAP, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };

package/dist/maintenance.d.mts

@@ -1,94 +1,3 @@
-import { IcebergConnection, isCommitRateLimited } from "./_chunks/catalog.mjs";
-/** One listed object under a data prefix. */
-interface SweepListedObject {
-  key: string;
-  uploaded: Date;
-}
-/** One page of a prefix listing. */
-interface SweepListPage {
-  objects: readonly SweepListedObject[];
-  truncated: boolean;
-  cursor?: string;
-}
-/**
- * Minimal storage client `sweepUncommittedOrphans` needs: list-by-prefix +
- * batch delete against the warehouse bucket. Deliberately NOT an S3 SDK
- * client shape — this interface is structurally satisfied by Cloudflare's
- * native `R2Bucket` Workers binding (`env.R2_ANALYTICS.list({ prefix })` /
- * `env.R2_ANALYTICS.delete(keys)`), so a caller running inside a Worker can
- * pass the binding straight through with no adapter. Any other
- * S3-compatible client can be wrapped to this shape.
- */
-interface SweepStorageClient {
-  list: (options: {
-    prefix: string;
-    cursor?: string;
-  }) => Promise<SweepListPage>;
-  delete: (keys: string[]) => Promise<void>;
-}
-interface SweepUncommittedOrphansOptions {
-  conn: IcebergConnection;
-  s3: SweepStorageClient;
-  /**
-   * Bare R2 bucket name backing `conn`'s warehouse. Every scanned table's
-   * `location` must resolve into this exact bucket; a table whose location
-   * points elsewhere is SKIPPED — never guessed at, never touched.
-   */
-  bucket: string;
-  /**
-   * Restrict the sweep to these tables; defaults to every table currently
-   * in `conn.namespace` (`conn` already carries the namespace scope, so
-   * there's no separate `namespace` field here).
-   */
-  tables?: readonly string[];
-  /** Only delete objects older than this many hours. Default 48. */
-  graceHours?: number;
-  /** Hard cap on deletions in one run. Default 500. */
-  maxDeletes?: number;
-  /** Compute the delete set but never call `s3.delete`. Default false. */
-  dryRun?: boolean;
-  /** Injectable clock (ms epoch). Defaults to `Date.now`. */
-  now?: () => number;
-  /** Maximum catalog/object-store reads held in flight. Default 4, max 32. */
-  ioConcurrency?: number;
-}
-interface SweepUncommittedOrphansResult {
-  /** Tables whose retained snapshots were successfully walked. */
-  scannedTables: string[];
-  /** Tables skipped because their `location` didn't resolve into `bucket` — never scanned, never touched. */
-  skippedTables: string[];
-  /** Count of live (referenced-by-some-retained-snapshot) files found across `scannedTables`. */
-  liveFileCount: number;
-  /** Objects under a scanned table's data prefix with no snapshot reference anywhere, past `graceHours`. */
-  candidateCount: number;
-  /** Keys deleted (or, under `dryRun`, that WOULD be deleted). */
-  deleted: string[];
-  dryRun: boolean;
-  /** True when `candidateCount` exceeded `maxDeletes` — remaining orphans are left for the next run. */
-  cappedAtLimit: boolean;
-  /** Set when the sweep did nothing because the table scope listed zero tables (fail-closed — see class docs). */
-  reason?: 'zero-tables';
-}
-/**
- * Sweep data files written under a table's `data/` prefix that were NEVER
- * referenced by any snapshot the catalog still retains — a crashed/partial
- * writer's leftovers. This is the one orphan class Cloudflare's own R2 Data
- * Catalog snapshot-expiration GC does not cover.
- *
- * HARD SAFETY:
- * - A listing failure (table load / manifest walk throwing) PROPAGATES.
- *   It is never caught and treated as "no live files" — that would make
- *   every real data file look orphaned. Callers must let this reject and
- *   must not proceed to delete on catch.
- * - Zero tables in scope is treated as "couldn't determine what's live",
- *   not "nothing to protect" — the sweep no-ops with `reason: 'zero-tables'`
- *   rather than listing/deleting anything.
- * - Deletions are capped at `maxDeletes` per run (default 500).
- * - `dryRun` computes the exact candidate set without ever calling `s3.delete`.
- * - Only ever lists/deletes keys under a scanned table's own
- *   `<table.location>/data/` prefix, and every candidate is re-checked to
- *   actually start with that prefix before being considered for deletion —
- *   never a bare bucket-root listing, never a key outside a data prefix.
- */
-declare function sweepUncommittedOrphans(opts: SweepUncommittedOrphansOptions): Promise<SweepUncommittedOrphansResult>;
+import { isCommitRateLimited } from "./catalog.mjs";
+import { SweepListPage, SweepListedObject, SweepStorageClient, SweepUncommittedOrphansOptions, SweepUncommittedOrphansResult, sweepUncommittedOrphans } from "./orphan-sweep.mjs";
 export { type SweepListPage, type SweepListedObject, type SweepStorageClient, type SweepUncommittedOrphansOptions, type SweepUncommittedOrphansResult, isCommitRateLimited, sweepUncommittedOrphans };