@gscdump/lakehouse 1.4.0 → 1.4.2

package/dist/{_chunks/catalog.mjs → catalog.mjs}

@@ -1,99 +1,15 @@
-import { stringifyBigintSafe } from "../bigint.mjs";
-import { cachingResolver, icebergAppend, icebergAppendBatches, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "./libs/icebird.mjs";
-function reportCatalogCacheError(cache, operation, key, error) {
-	try {
-		if (cache.onError) cache.onError(operation, key, error);
-		else console.warn(`[gscdump/lakehouse] cache ${operation} failed for ${key}`, error);
-	} catch (reportError) {
-		console.warn(`[gscdump/lakehouse] cache error reporter failed during ${operation} for ${key}`, reportError);
-	}
-}
-async function cacheGet(cache, key, now) {
-	const boxed = await cache.storage.getItem(key).catch((error) => {
-		reportCatalogCacheError(cache, "get", key, error);
-		return null;
-	});
-	if (!boxed || typeof boxed.exp !== "number" || boxed.exp <= now) return void 0;
-	return boxed.v;
-}
-function cachePut(cache, key, value, ttlMs, now) {
-	const boxed = {
-		v: value,
-		exp: now + ttlMs
-	};
-	const write = cache.storage.setItem(key, boxed, { ttl: Math.ceil(ttlMs / 1e3) }).catch((error) => {
-		reportCatalogCacheError(cache, "set", key, error);
-	});
-	if (cache.defer) {
-		cache.defer(write);
-		return Promise.resolve();
-	}
-	return write;
-}
-function toUint8(bytes) {
-	if (bytes == null) return null;
-	return bytes instanceof Uint8Array ? bytes : new Uint8Array(bytes);
-}
-const UTF8_DECODER = new TextDecoder();
-function decodeString(bytes) {
-	const u = toUint8(bytes);
-	return u == null ? null : UTF8_DECODER.decode(u);
-}
-function decodeMonthInt(bytes) {
-	const u = toUint8(bytes);
-	if (u == null) return null;
-	return new DataView(u.buffer, u.byteOffset, u.byteLength).getInt32(0, true);
-}
-function buildManifestPartitionFilter(partitionSpec, matches, wantedMonths) {
-	const fieldIndex = (name) => partitionSpec.findIndex((f) => f.name === name || f.sourceColumn === name);
-	const monthFieldIndex = partitionSpec.findIndex((f) => f.transform === "month");
-	const stringMatches = matches.flatMap((match) => {
-		if (match.encoding !== "string") return [];
-		const index = fieldIndex(match.field);
-		return index < 0 ? [] : [{
-			index,
-			value: String(match.value)
-		}];
-	});
-	const wantedMonthValues = wantedMonths ? [...wantedMonths].filter(Number.isFinite).sort((a, b) => a - b) : [];
-	const hasWantedMonthInRange = (lo, hi) => {
-		let left = 0;
-		let right = wantedMonthValues.length;
-		while (left < right) {
-			const middle = left + right >>> 1;
-			if (wantedMonthValues[middle] < lo) left = middle + 1;
-			else right = middle;
-		}
-		return left < wantedMonthValues.length && wantedMonthValues[left] <= hi;
-	};
-	return (partitions) => {
-		if (!partitions || partitions.length === 0) return true;
-		for (const match of stringMatches) {
-			const summary = partitions[match.index];
-			if (!summary || summary.lower_bound == null && summary.upper_bound == null) continue;
-			const lo = decodeString(summary.lower_bound);
-			const hi = decodeString(summary.upper_bound);
-			if (lo != null && hi != null && (match.value < lo || match.value > hi)) return false;
-		}
-		if (wantedMonthValues.length > 0 && monthFieldIndex >= 0) {
-			const monthSummary = partitions[monthFieldIndex];
-			if (monthSummary && (monthSummary.lower_bound != null || monthSummary.upper_bound != null)) {
-				const lo = decodeMonthInt(monthSummary.lower_bound);
-				const hi = decodeMonthInt(monthSummary.upper_bound);
-				if (lo != null && hi != null && !hasWantedMonthInRange(lo, hi)) return false;
-			}
-		}
-		return true;
-	};
+import { stringifyBigintSafe } from "./bigint.mjs";
+import { restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/catalog/rest.mjs";
+import { cachingResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/fetch.mjs";
+import { icebergManifests } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/manifest.mjs";
+import { s3SignedResolver } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/s3.mjs";
+import { cacheGet, cachePut, reportCatalogCacheError } from "./catalog-cache.mjs";
+import { buildManifestPartitionFilter } from "./partition-prune.mjs";
+let icebirdWriteModule;
+function useIcebirdWriteModule() {
+	icebirdWriteModule ??= import("./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs");
+	return icebirdWriteModule;
 }
-const ICEBERG_TYPE_MAP = {
-	STRING: "string",
-	INT: "int",
-	LONG: "long",
-	DOUBLE: "double",
-	DATE: "date",
-	BOOLEAN: "boolean"
-};
 const CATALOG_CONFIG_TTL_MS = 3600 * 1e3;
 function catalogConfigKey(config) {
 	return `lakehouse-catalog-cfg\0${config.catalogUri}\0${config.warehouse}`;
@@ -190,6 +106,7 @@ async function listIcebergTables(conn) {
 async function dropIcebergTables(conn, tables) {
 	const targets = tables ?? (await restCatalogListTables(conn.catalog, { namespace: conn.namespace })).map((t) => t.name);
 	const results = [];
+	const { icebergDropTable } = await useIcebirdWriteModule();
 	for (const table of targets) await icebergDropTable({
 		catalog: conn.catalog,
 		namespace: conn.namespace,
@@ -216,6 +133,7 @@ function defaultCommitSleep(ms) {
 	return new Promise((resolve) => setTimeout(resolve, ms));
 }
 async function icebergAppendRetrying(args, options = {}) {
+	const { icebergAppend } = await useIcebirdWriteModule();
 	const maxAttempts = options.maxAttempts ?? 6;
 	const baseDelayMs = options.baseDelayMs ?? 1e3;
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
@@ -241,6 +159,7 @@ async function icebergAppendRetrying(args, options = {}) {
 	}
 }
 async function icebergAppendBatchesRetrying(args, options) {
+	const { icebergAppendBatches } = await useIcebirdWriteModule();
 	const maxAttempts = options.maxAttempts ?? 6;
 	const baseDelayMs = options.baseDelayMs ?? 1e3;
 	const maxDelayMs = options.maxDelayMs ?? 2e4;
@@ -429,4 +348,4 @@ async function resolveIcebergDataFiles(conn, opts) {
 	}
 	return out;
 }
-export { ICEBERG_TYPE_MAP, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };
+export { catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, invalidateSnapshotRef, isCommitRateLimited, listIcebergTables, resolveIcebergDataFiles };

package/dist/dataset-runtime.mjs

@@ -0,0 +1,68 @@
+import { connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
+import { icebergCreateTable } from "./_virtual/node_modules/.pnpm/icebird@0.8.15_patch_hash=d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
+async function createDatasetTable(conn, table, schema, partitionSpec, sortOrder) {
+	const results = [];
+	await icebergCreateTable({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		schema,
+		partitionSpec,
+		...sortOrder ? { sortOrder } : {}
+	}).then(() => results.push({
+		table,
+		ok: true
+	}), (error) => results.push({
+		table,
+		ok: false,
+		error: error instanceof Error ? error.message : String(error)
+	}));
+	return results;
+}
+async function appendDatasetRows(conn, table, records, commitRetry) {
+	await icebergAppendRetrying({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		resolver: conn.resolver,
+		records
+	}, commitRetry);
+}
+async function appendDatasetBatches(conn, table, source, prepare, opts) {
+	let accepted = 0;
+	let skipped = 0;
+	const batchFactory = async function* () {
+		accepted = 0;
+		skipped = 0;
+		for await (const rows of source()) {
+			const prepared = prepare(rows);
+			accepted += prepared.records.length;
+			skipped += prepared.skipped;
+			if (prepared.records.length > 0) yield prepared.records;
+		}
+	};
+	const committed = await icebergAppendBatchesRetrying({
+		catalog: conn.catalog,
+		namespace: conn.namespace,
+		table,
+		resolver: conn.resolver,
+		batchFactory
+	}, {
+		...opts.commitRetry,
+		appendId: opts.appendId
+	});
+	return {
+		accepted,
+		skipped,
+		committed
+	};
+}
+async function connectDatasetCatalog(config, opts) {
+	const conn = await connectIcebergCatalog(config, opts);
+	await ensureIcebergNamespace(conn);
+	return conn;
+}
+async function resolveDatasetDataFiles(conn, opts) {
+	return resolveIcebergDataFiles(conn, opts);
+}
+export { appendDatasetBatches, appendDatasetRows, connectDatasetCatalog, createDatasetTable, resolveDatasetDataFiles };

package/dist/dataset.d.mts

@@ -0,0 +1,154 @@
+import { CatalogCache } from "./catalog-cache.mjs";
+import { IcebergColumnType, IcebergPartitionField, IcebergTableSpec } from "./schema.mjs";
+import { ManifestPartitionFilter, PartitionValueMatch } from "./partition-prune.mjs";
+import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergSchema, IcebergSortOrder, IcebergTableOpResult, QueryProfiler } from "./catalog.mjs";
+/**
+ * Closed identity shapes (ADR-0021 amendments 2 + 4). `'site-int'` is the
+ * team-scoped numeric Catalog Site Id (nuxtseo ADR-0091) stored under the
+ * fixed column name `site_id`. `'columns'` is the general N-column identity
+ * with per-column encoding, replacing the original opaque `'custom'` shape.
+ */
+type DatasetIdentity = {
+  kind: 'site-int';
+  encoding?: 'int' | 'string';
+} | {
+  kind: 'columns';
+  columns: readonly {
+    name: string;
+    encoding: 'int32' | 'string';
+  }[];
+};
+/** A per-slice dimension (GSC's `searchType`). Absent for plain snapshot datasets. */
+interface DatasetDim {
+  toPartitionValue: (v: string) => string | number;
+  /** Required (amendment 5) — how the reader decodes this column's manifest bounds. */
+  boundEncoding: 'int32' | 'string';
+}
+interface IcebergDatasetColumnDef {
+  name: string;
+  type: IcebergColumnType;
+  required: boolean;
+}
+interface IcebergDatasetLedger {
+  key: readonly string[];
+}
+interface IcebergDatasetDef {
+  namespace: string;
+  table: string;
+  identity: DatasetIdentity;
+  /** Extra per-slice dimensions; absent for snapshot datasets. */
+  dims?: Record<string, DatasetDim>;
+  /** Data columns (non-identity, non-dims) — includes any partition source column (e.g. `date`). */
+  columns: readonly IcebergDatasetColumnDef[];
+  /** Partition fields; identity columns must appear here. */
+  partition: readonly IcebergPartitionField[];
+  /** Natural-key columns. Full dedupe key = identity + dims + naturalKey (amendment 3). */
+  naturalKey: readonly string[];
+  clusterKey?: readonly string[];
+  /** Consumer-owned exactly-once ledger shape — folded into `appendSink().close()` (amendment 6). */
+  ledger?: IcebergDatasetLedger;
+}
+/**
+ * Derive the full {@link IcebergTableSpec} from a dataset def. Field ids are
+ * assigned sequentially in ONE fixed order — identity columns, then dims
+ * columns, then declared data columns — starting at 1. This mirrors the
+ * engine's original `icebergTableSpec` contract (identity columns first, data
+ * columns from a fixed base) so a def-derived spec for an existing table
+ * (`crawl.pages`, `lighthouse.scans`, `dataforseo.keywords`) is BYTE-IDENTICAL
+ * to the frozen inline constants it replaces.
+ */
+declare function deriveTableSpec(def: IcebergDatasetDef): IcebergTableSpec;
+/** Options shared by `appendRows` / `appendSink`. */
+interface AppendCommitOptions {
+  commitRetry?: CommitRetryOptions;
+}
+interface AppendSinkOptions extends AppendCommitOptions {
+  catalog: IcebergCatalogConfig;
+  connect?: ConnectIcebergOptions;
+  /** Invoked ONLY after a successful flush (amendment 6) — never on a failed/empty close. */
+  ledger?: {
+    record: () => Promise<void> | void;
+  };
+}
+interface AppendResult {
+  /** Rows accepted after the identity guard + dedupe (what was actually appended). */
+  accepted: number;
+  /** Rows dropped by the identity guard (missing/out-of-range required identity value). */
+  skipped: number;
+}
+type AppendBatchSource = () => Iterable<readonly Record<string, unknown>[]> | AsyncIterable<readonly Record<string, unknown>[]>;
+interface AppendBatchesOptions extends AppendCommitOptions {
+  /** Stable across job retries; recorded in snapshot metadata for landed checks. */
+  appendId: string;
+}
+interface AppendBatchesResult extends AppendResult {
+  /** False when this append id was already present before the call. */
+  committed: boolean;
+}
+interface AppendSinkCloseResult extends AppendResult {
+  flushed: boolean;
+  error?: unknown;
+}
+interface AppendSink {
+  emit: (rows: readonly Record<string, unknown>[]) => void;
+  close: () => Promise<AppendSinkCloseResult>;
+}
+interface ResolveDataFilesOptions {
+  cache?: CatalogCache;
+  clock?: () => number;
+  profiler?: QueryProfiler;
+}
+interface IcebergDataset {
+  readonly def: IcebergDatasetDef;
+  readonly tableSpec: IcebergTableSpec;
+  icebergSchema: () => IcebergSchema;
+  icebergPartitionSpec: () => IcebergPartitionSpec;
+  icebergSortOrder: () => IcebergSortOrder | undefined;
+  createTable: (conn: IcebergConnection) => Promise<IcebergTableOpResult[]>;
+  /**
+   * One-shot append over an EXISTING connection: identity INT32 guard, dedupe
+   * by identity+dims+naturalKey (last-wins), cluster pre-sort (if
+   * `clusterKey` is set), then a single `icebergAppendRetrying` commit. `rows`
+   * must already be in final storage-column shape — identity/dims/data
+   * columns keyed by their STORED column names, with any `month`-partitioned
+   * date column pre-converted via {@link toIcebergDayCount}.
+   */
+  appendRows: (conn: IcebergConnection, rows: readonly Record<string, unknown>[], opts?: AppendCommitOptions) => Promise<AppendResult>;
+  /**
+   * Lazily process bounded row batches, writing multiple data files into one
+   * append snapshot and one catalog commit. Dedupe keys must be disjoint
+   * across source batches.
+   */
+  appendBatches: (conn: IcebergConnection, source: AppendBatchSource, opts: AppendBatchesOptions) => Promise<AppendBatchesResult>;
+  /**
+   * PURE row processing — the identity INT32 guard, dedupe (identity+dims+
+   * naturalKey, last-wins) and cluster pre-sort `appendRows`/`appendSink`
+   * apply before committing, exposed standalone with NO network/icebird call.
+   * Lets a consumer that owns its own commit call-site (e.g. one still
+   * calling a frozen `icebergAppendRetrying` import directly, for test-mock
+   * compatibility) still route its dedupe/guard/sort logic through the
+   * dataset definition.
+   */
+  prepareRows: (rows: readonly Record<string, unknown>[]) => {
+    records: Record<string, unknown>[];
+    skipped: number;
+  };
+  /** Buffered multi-emit sink — owns its own connect/close lifecycle (ADR-0021's primary contract). */
+  appendSink: (opts: AppendSinkOptions) => AppendSink;
+  /** `PartitionValueMatch[]` for one identity value (+ optional dims), for manual manifest filtering. */
+  readerPredicate: (identity: string | number, dims?: Record<string, string>) => PartitionValueMatch[];
+  /** A ready-to-use manifest partition filter for one identity value + wanted months. */
+  partitionBoundFilter: (identity: string | number, months: ReadonlySet<number>, dims?: Record<string, string>) => ManifestPartitionFilter;
+  /** Resolve the data files for one identity value across a date range. */
+  resolveDataFiles: (conn: IcebergConnection, identity: string | number, range: {
+    start: string;
+    end: string;
+  }, dims?: Record<string, string>, opts?: ResolveDataFilesOptions) => Promise<IcebergListedDataFile[]>;
+}
+/**
+ * Declare an Iceberg dataset — one `namespace.table` — deriving its schema,
+ * partition spec, dedupe key, INT32 guard, append path, and reader helpers
+ * from the def. See {@link IcebergDatasetDef}.
+ */
+declare function defineIcebergDataset(def: IcebergDatasetDef): IcebergDataset;
+export { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec };

package/dist/dataset.mjs

@@ -0,0 +1,312 @@
+import { coerceBigIntToNumber } from "./bigint.mjs";
+import { buildManifestPartitionFilter } from "./partition-prune.mjs";
+import { ICEBERG_TYPE_MAP } from "./schema.mjs";
+const INT32_MIN = -2147483648;
+const INT32_MAX = 2147483647;
+function identityColumnNames(identity) {
+	return identity.kind === "site-int" ? ["site_id"] : identity.columns.map((c) => c.name);
+}
+function identityEncodings(identity) {
+	const map = /* @__PURE__ */ new Map();
+	if (identity.kind === "site-int") map.set("site_id", (identity.encoding ?? "int") === "int" ? "int32" : "string");
+	else for (const c of identity.columns) map.set(c.name, c.encoding);
+	return map;
+}
+function identityColumnType(encoding) {
+	return encoding === "int32" ? "INT" : "STRING";
+}
+function deriveTableSpec(def) {
+	const encodings = identityEncodings(def.identity);
+	const idNames = identityColumnNames(def.identity);
+	const dimNames = def.dims ? Object.keys(def.dims) : [];
+	let fieldId = 1;
+	const columns = [];
+	for (const name of idNames) columns.push({
+		name,
+		type: identityColumnType(encodings.get(name)),
+		required: true,
+		fieldId: fieldId++
+	});
+	for (const name of dimNames) {
+		const enc = def.dims[name].boundEncoding;
+		columns.push({
+			name,
+			type: identityColumnType(enc),
+			required: true,
+			fieldId: fieldId++
+		});
+	}
+	for (const col of def.columns) columns.push({
+		name: col.name,
+		type: col.type,
+		required: col.required,
+		fieldId: fieldId++
+	});
+	return {
+		namespace: def.namespace,
+		table: def.table,
+		columns,
+		partitionSpec: def.partition,
+		naturalKey: def.naturalKey,
+		identityColumns: [
+			...idNames,
+			...dimNames,
+			...def.naturalKey
+		],
+		clusterKey: def.clusterKey
+	};
+}
+function icebergSchemaFromSpec(spec) {
+	return {
+		"type": "struct",
+		"schema-id": 0,
+		"fields": spec.columns.map((col) => ({
+			id: col.fieldId,
+			name: col.name,
+			required: col.required,
+			type: ICEBERG_TYPE_MAP[col.type]
+		}))
+	};
+}
+function icebergPartitionSpecFromSpec(spec) {
+	const fieldId = (name) => {
+		const col = spec.columns.find((c) => c.name === name);
+		if (!col) throw new Error(`lakehouse: table '${spec.table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"spec-id": 0,
+		"fields": spec.partitionSpec.map((p, i) => ({
+			"source-id": fieldId(p.sourceColumn),
+			"field-id": 1e3 + i,
+			"name": p.name,
+			"transform": p.transform
+		}))
+	};
+}
+function icebergSortOrderFromSpec(spec) {
+	if (!spec.clusterKey || spec.clusterKey.length === 0) return void 0;
+	const fieldId = (name) => {
+		const col = spec.columns.find((c) => c.name === name);
+		if (!col) throw new Error(`lakehouse: table '${spec.table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"order-id": 1,
+		"fields": spec.clusterKey.map((col) => ({
+			"source-id": fieldId(col),
+			"transform": "identity",
+			"direction": "asc",
+			"null-order": "nulls-last"
+		}))
+	};
+}
+function asInt32(value) {
+	if (typeof value !== "string" && typeof value !== "number" && typeof value !== "bigint") return null;
+	const n = Number(value);
+	if (!Number.isSafeInteger(n) || n < INT32_MIN || n > INT32_MAX) return null;
+	return n;
+}
+function buildRowProcessor(def, tableSpec) {
+	const idNames = identityColumnNames(def.identity);
+	const idEncodings = identityEncodings(def.identity);
+	const dimNames = def.dims ? Object.keys(def.dims) : [];
+	const identityColumns = tableSpec.identityColumns;
+	function guard(row) {
+		const out = {};
+		for (const name of idNames) {
+			const enc = idEncodings.get(name);
+			const raw = row[name];
+			if (enc === "int32") {
+				const n = asInt32(raw);
+				if (n === null) return null;
+				out[name] = n;
+			} else {
+				if (raw == null || raw === "") return null;
+				out[name] = String(raw);
+			}
+		}
+		for (const name of dimNames) out[name] = def.dims[name].toPartitionValue(String(row[name]));
+		for (const col of def.columns) {
+			const value = row[col.name];
+			out[col.name] = col.type === "LONG" ? typeof value === "number" ? BigInt(value) : value : coerceBigIntToNumber(value);
+		}
+		return out;
+	}
+	function dedupe(rows) {
+		if (rows.length < 2) return rows;
+		const seen = /* @__PURE__ */ new Map();
+		for (const rec of rows) {
+			let identity = "";
+			for (let index = 0; index < identityColumns.length; index++) {
+				if (index > 0) identity += "\0";
+				identity += `${rec[identityColumns[index]] ?? ""}`;
+			}
+			seen.set(identity, rec);
+		}
+		return seen.size === rows.length ? rows : [...seen.values()];
+	}
+	function sort(rows) {
+		const cols = def.clusterKey;
+		if (!cols || cols.length === 0 || rows.length < 2) return rows;
+		return rows.slice().sort((a, b) => {
+			for (const col of cols) {
+				const av = a[col];
+				const bv = b[col];
+				if (av === bv) continue;
+				if (av == null) return -1;
+				if (bv == null) return 1;
+				if (typeof av === "number" && typeof bv === "number") return av - bv;
+				const as = String(av);
+				const bs = String(bv);
+				if (as !== bs) return as < bs ? -1 : 1;
+			}
+			return 0;
+		});
+	}
+	return {
+		guard,
+		dedupe,
+		sort
+	};
+}
+function defineIcebergDataset(def) {
+	const tableSpec = deriveTableSpec(def);
+	const schema = icebergSchemaFromSpec(tableSpec);
+	const partitionSpecIcebird = icebergPartitionSpecFromSpec(tableSpec);
+	const sortOrder = icebergSortOrderFromSpec(tableSpec);
+	const { guard, dedupe, sort } = buildRowProcessor(def, tableSpec);
+	async function createTable(conn) {
+		const { createDatasetTable } = await import("./dataset-runtime.mjs");
+		return createDatasetTable(conn, def.table, schema, partitionSpecIcebird, sortOrder);
+	}
+	function process(rows) {
+		const guarded = [];
+		let skipped = 0;
+		for (const row of rows) {
+			const rec = guard(row);
+			if (rec === null) skipped++;
+			else guarded.push(rec);
+		}
+		return {
+			records: sort(dedupe(guarded)),
+			skipped
+		};
+	}
+	async function appendRows(conn, rows, opts = {}) {
+		if (rows.length === 0) return {
+			accepted: 0,
+			skipped: 0
+		};
+		const { records, skipped } = process(rows);
+		if (records.length === 0) return {
+			accepted: 0,
+			skipped
+		};
+		const { appendDatasetRows } = await import("./dataset-runtime.mjs");
+		await appendDatasetRows(conn, def.table, records, opts.commitRetry);
+		return {
+			accepted: records.length,
+			skipped
+		};
+	}
+	async function appendBatches(conn, source, opts) {
+		const { appendDatasetBatches } = await import("./dataset-runtime.mjs");
+		return appendDatasetBatches(conn, def.table, source, process, opts);
+	}
+	function appendSink(opts) {
+		let buffer = [];
+		let connection;
+		return {
+			emit(rows) {
+				for (const r of rows) buffer.push(r);
+			},
+			async close() {
+				if (buffer.length === 0) return {
+					flushed: false,
+					accepted: 0,
+					skipped: 0
+				};
+				const rows = buffer;
+				buffer = [];
+				try {
+					const runtime = await import("./dataset-runtime.mjs");
+					connection ??= runtime.connectDatasetCatalog(opts.catalog, opts.connect);
+					const conn = await connection;
+					const { records, skipped } = process(rows);
+					if (records.length === 0) return {
+						flushed: false,
+						accepted: 0,
+						skipped
+					};
+					await runtime.appendDatasetRows(conn, def.table, records, opts.commitRetry);
+					const result = {
+						accepted: records.length,
+						skipped
+					};
+					if (result.accepted > 0 && opts.ledger) await opts.ledger.record();
+					return {
+						flushed: result.accepted > 0,
+						...result
+					};
+				} catch (error) {
+					return {
+						flushed: false,
+						accepted: 0,
+						skipped: 0,
+						error
+					};
+				}
+			}
+		};
+	}
+	function readerPredicate(identity, dims) {
+		const idNames = identityColumnNames(def.identity);
+		const idEncodings = identityEncodings(def.identity);
+		const matches = idNames.map((name) => ({
+			field: name,
+			value: idEncodings.get(name) === "int32" ? Number(identity) : identity,
+			encoding: idEncodings.get(name) === "int32" ? "int32" : "string"
+		}));
+		if (def.dims && dims) {
+			for (const [name, dim] of Object.entries(def.dims)) if (name in dims) matches.push({
+				field: name,
+				value: dim.toPartitionValue(dims[name]),
+				encoding: dim.boundEncoding
+			});
+		}
+		return matches;
+	}
+	function partitionBoundFilter(identity, months, dims) {
+		return buildManifestPartitionFilter(def.partition, readerPredicate(identity, dims), months);
+	}
+	async function resolveDataFiles(conn, identity, range, dims, opts = {}) {
+		const { resolveDatasetDataFiles } = await import("./dataset-runtime.mjs");
+		return resolveDatasetDataFiles(conn, {
+			namespace: def.namespace,
+			table: def.table,
+			partitionSpec: def.partition,
+			matches: readerPredicate(identity, dims),
+			range,
+			cache: opts.cache,
+			clock: opts.clock,
+			profiler: opts.profiler
+		});
+	}
+	return {
+		def,
+		tableSpec,
+		icebergSchema: () => schema,
+		icebergPartitionSpec: () => partitionSpecIcebird,
+		icebergSortOrder: () => sortOrder,
+		createTable,
+		appendRows,
+		appendBatches,
+		prepareRows: process,
+		appendSink,
+		readerPredicate,
+		partitionBoundFilter,
+		resolveDataFiles
+	};
+}
+export { defineIcebergDataset, deriveTableSpec };

package/dist/date.d.mts

@@ -0,0 +1,8 @@
+/**
+ * Convert a `YYYY-MM-DD` string / `Date` / already-numeric day-count to the
+ * integer "days since the Unix epoch" the Iceberg `date` type stores.
+ * hyparquet-writer mis-encodes Date-valued dictionary columns, so callers
+ * should feed this into their row before writing a month-partitioned date.
+ */
+declare function toIcebergDayCount(value: string | Date | number): number;
+export { toIcebergDayCount };