@gscdump/engine 1.4.0 → 1.4.1

package/dist/iceberg/index.mjs

@@ -1,245 +1,4 @@
-import { engineErrors } from "../errors.mjs";
-import { TABLE_METADATA } from "../_chunks/schema.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
-import { icebergCreateTable } from "../_chunks/libs/icebird.mjs";
-import { err, ok } from "gscdump/result";
-import { ICEBERG_TYPE_MAP, connectIcebergCatalog, dropIcebergTables as dropIcebergTables$1, resolveIcebergDataFiles } from "@gscdump/lakehouse";
-import { icebergAppendRetrying } from "@gscdump/lakehouse/unsafe-raw";
-function icebergSchemaFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
-	return {
-		"type": "struct",
-		"schema-id": 0,
-		"fields": icebergSchemasFor(encoding)[table].columns.map((col) => ({
-			id: col.fieldId,
-			name: col.name,
-			required: col.required,
-			type: ICEBERG_TYPE_MAP[col.type]
-		}))
-	};
-}
-function icebergPartitionSpecFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
-	const fields = icebergSchemasFor(encoding)[table].columns;
-	const fieldId = (name) => {
-		const col = fields.find((c) => c.name === name);
-		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
-		return col.fieldId;
-	};
-	return {
-		"spec-id": 0,
-		"fields": ICEBERG_PARTITION_SPEC.map((p, i) => ({
-			"source-id": fieldId(p.sourceColumn),
-			"field-id": 1e3 + i,
-			"name": p.name,
-			"transform": p.transform
-		}))
-	};
-}
-function icebergSortOrderFor(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
-	const fields = icebergSchemasFor(encoding)[table].columns;
-	const fieldId = (name) => {
-		const col = fields.find((c) => c.name === name);
-		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
-		return col.fieldId;
-	};
-	return {
-		"order-id": 1,
-		"fields": TABLE_METADATA[table].clusterKey.map((col) => ({
-			"source-id": fieldId(col),
-			"transform": "identity",
-			"direction": "asc",
-			"null-order": "nulls-last"
-		}))
-	};
-}
-async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
-	const results = [];
-	for (const table of tables) await icebergCreateTable({
-		catalog: conn.catalog,
-		namespace: conn.namespace,
-		table,
-		schema: icebergSchemaFor(table, encoding),
-		partitionSpec: icebergPartitionSpecFor(table, encoding),
-		sortOrder: icebergSortOrderFor(table, encoding)
-	}).then(() => results.push({
-		table,
-		outcome: ok(void 0)
-	}), (e) => results.push({
-		table,
-		outcome: err(engineErrors.icebergTableOpFailed("create", table, e))
-	}));
-	return results;
-}
-async function dropIcebergTables(conn, tables) {
-	return (await dropIcebergTables$1(conn, tables)).map((r) => ({
-		table: r.table,
-		outcome: r.ok ? ok(void 0) : err(engineErrors.icebergTableOpFailed("drop", r.table, r.error))
-	}));
-}
-async function listIcebergDataFiles(conn, opts) {
-	const matches = (opts.encoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "string" ? [{
-		field: "site_id",
-		value: opts.siteId,
-		encoding: "string"
-	}, {
-		field: "search_type",
-		value: opts.searchType,
-		encoding: "string"
-	}] : [{
-		field: "site_id",
-		value: opts.siteId,
-		encoding: "int32"
-	}, {
-		field: "search_type",
-		value: opts.searchType,
-		encoding: "int32"
-	}];
-	return resolveIcebergDataFiles(conn, {
-		namespace: conn.namespace,
-		table: opts.table,
-		partitionSpec: ICEBERG_PARTITION_SPEC,
-		matches,
-		range: opts.range,
-		cache: opts.cache,
-		clock: opts.clock,
-		profiler: opts.profiler
-	});
-}
-const DAY_MILLIS = 864e5;
-const INT32_MIN = -2147483648;
-const INT32_MAX = 2147483647;
-function toIcebergDate(value) {
-	if (typeof value === "string") {
-		const ms = Date.parse(`${value}T00:00:00Z`);
-		if (Number.isNaN(ms)) throw new TypeError(`toIcebergDate: invalid date string '${value}'`);
-		return Math.floor(ms / DAY_MILLIS);
-	}
-	if (value instanceof Date) {
-		const ms = value.getTime();
-		if (Number.isNaN(ms)) throw new TypeError("toIcebergDate: invalid Date (NaN)");
-		return Math.floor(ms / DAY_MILLIS);
-	}
-	return value;
-}
-function coerceJsonSafe(value) {
-	if (typeof value === "bigint") return Number(value);
-	return value;
-}
-function toIntPartitionSiteId(value) {
-	if (value == null || typeof value === "string" && value.trim() === "") throw new TypeError("toRecords: slice.ctx.siteId is required for int partition encoding");
-	if (typeof value !== "string" && typeof value !== "number" && typeof value !== "bigint") throw new TypeError(`toRecords: int partition site_id must be a safe integer, got '${String(value)}'`);
-	const siteId = Number(value);
-	if (!Number.isSafeInteger(siteId)) throw new TypeError(`toRecords: int partition site_id must be a safe integer, got '${String(value)}'`);
-	if (siteId < INT32_MIN || siteId > INT32_MAX) throw new TypeError(`toRecords: int partition site_id must fit Iceberg INT, got '${String(value)}'`);
-	return siteId;
-}
-function dedupeByIdentity(table, records) {
-	if (records.length < 2) return records;
-	const key = gscDataset(table, "int").tableSpec.identityColumns;
-	const seen = /* @__PURE__ */ new Map();
-	for (const rec of records) {
-		let identity = "";
-		for (let index = 0; index < key.length; index++) {
-			if (index > 0) identity += "\0";
-			identity += `${rec[key[index]] ?? ""}`;
-		}
-		seen.set(identity, rec);
-	}
-	return seen.size === records.length ? records : [...seen.values()];
-}
-function sortByClusterKey(table, records) {
-	const cols = gscDataset(table, "int").tableSpec.clusterKey ?? [];
-	if (cols.length === 0 || records.length < 2) return records;
-	return records.slice().sort((a, b) => {
-		for (const col of cols) {
-			const av = a[col];
-			const bv = b[col];
-			if (av === bv) continue;
-			if (av == null) return -1;
-			if (bv == null) return 1;
-			if (typeof av === "number" && typeof bv === "number") return av - bv;
-			const as = String(av);
-			const bs = String(bv);
-			if (as !== bs) return as < bs ? -1 : 1;
-		}
-		return 0;
-	});
-}
-function toRecords(slice, rows, encoding) {
-	const siteVal = encoding === "int" ? toIntPartitionSiteId(slice.ctx.siteId) : slice.ctx.siteId ?? "";
-	const searchVal = gscDataset(slice.table, encoding).def.dims.search_type.toPartitionValue(slice.searchType);
-	return rows.map((row) => {
-		const out = {};
-		for (const k in row) out[k] = coerceJsonSafe(row[k]);
-		out.date = toIcebergDate(out.date);
-		out.site_id = siteVal;
-		out.search_type = searchVal;
-		return out;
-	});
-}
-function createIcebergAppendSink(options) {
-	let connection;
-	const encoding = options.encoding ?? DEFAULT_PARTITION_KEY_ENCODING;
-	const buffers = /* @__PURE__ */ new Map();
-	function connect() {
-		connection ??= connectIcebergCatalog(options.catalog, options.connect);
-		return connection;
-	}
-	return {
-		capabilities: { appendOnly: true },
-		async emit(slice, rows) {
-			if (rows.length === 0) return { rowCount: 0 };
-			const records = toRecords(slice, rows, encoding);
-			const buffer = buffers.get(slice.table);
-			if (buffer) for (let i = 0; i < records.length; i++) buffer.push(records[i]);
-			else buffers.set(slice.table, records);
-			return { rowCount: records.length };
-		},
-		async close() {
-			const flushed = [];
-			const failed = [];
-			if (buffers.size === 0) return {
-				flushed,
-				failed
-			};
-			const conn = await connect().then((c) => c, (cause) => {
-				connection = void 0;
-				return { connectError: cause };
-			});
-			if ("connectError" in conn) {
-				for (const [table, records] of buffers) if (records.length > 0) failed.push({
-					table,
-					error: engineErrors.sinkTableFlushFailed(table, conn.connectError)
-				});
-				buffers.clear();
-				return {
-					flushed,
-					failed
-				};
-			}
-			for (const [table, records] of buffers) {
-				if (records.length === 0) continue;
-				const deduped = sortByClusterKey(table, dedupeByIdentity(table, records));
-				await icebergAppendRetrying({
-					catalog: conn.catalog,
-					namespace: conn.namespace,
-					table,
-					resolver: conn.resolver,
-					records: deduped
-				}, options.commitRetry).then(() => {
-					flushed.push(table);
-				}, (cause) => {
-					failed.push({
-						table,
-						error: engineErrors.sinkTableFlushFailed(table, cause)
-					});
-				});
-			}
-			buffers.clear();
-			return {
-				flushed,
-				failed
-			};
-		}
-	};
-}
+import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "./schema.mjs";
+import { createIcebergTables, dropIcebergTables, icebergPartitionSpecFor, icebergSchemaFor, icebergSortOrderFor, listIcebergDataFiles } from "./catalog.mjs";
+import { createIcebergAppendSink } from "./append-sink.mjs";
 export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, createIcebergAppendSink, createIcebergTables, dropIcebergTables, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isIcebergTable, listIcebergDataFiles };

package/dist/iceberg/overwrite-writer.d.mts

@@ -0,0 +1,161 @@
+import { Row } from "../storage.mjs";
+import { IcebergS3Config } from "./schema.mjs";
+import { Sink, SinkCloseResult, SliceOverwriteWriter } from "../sink.mjs";
+/** Connection details for the Iceberg REST catalog the writer targets. */
+interface OverwriteWriterCatalogConfig {
+  /**
+   * Iceberg REST catalog endpoint, e.g. `http://localhost:8181` (POC) or the
+   *  R2 Data Catalog endpoint (prod).
+   */
+  catalogUri: string;
+  /** Catalog namespace the 5 fact tables live under, e.g. `gsc`. */
+  namespace: string;
+  /** Warehouse identifier the catalog resolves table locations under. */
+  warehouse: string;
+  /** S3-compatible object store backing the catalog. */
+  s3: IcebergS3Config;
+  /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
+  catalogToken?: string;
+}
+/**
+ * The JSON job wire-format shared with `scripts/iceberg-writer.py`.
+ * `op` is always `overwrite` here.
+ */
+interface OverwriteJob {
+  op: 'overwrite';
+  catalogUri: string;
+  namespace: string;
+  warehouse: string;
+  s3: IcebergS3Config;
+  table: string;
+  spec: unknown;
+  siteId: string;
+  searchType: string;
+  date: string;
+  /** Data columns only — the script injects `site_id` / `search_type`. */
+  rows: readonly Row[];
+}
+/**
+ * Per-site delete job — removes EVERY row for one `siteId` across a whole
+ * table (all dates / search types). Recovery op for clearing a corrupt site
+ * (e.g. a cross-run append double) from a shared per-team shard before a clean
+ * re-backfill, without touching sibling sites. Carries no rows. `siteId` is a
+ * `number` for an INT-encoded shard, a `string` for STRING encoding.
+ */
+interface DeleteJob {
+  op: 'delete';
+  catalogUri: string;
+  namespace: string;
+  warehouse: string;
+  s3: IcebergS3Config;
+  /** Bearer token for a token-authed REST catalog (R2 Data Catalog). */
+  catalogToken?: string;
+  table: string;
+  siteId: string | number;
+}
+/** Any job the shared PyIceberg backend can execute. */
+type PyIcebergJob = OverwriteJob | DeleteJob;
+/** Result wire-format from the PyIceberg backend. */
+interface OverwriteJobResult {
+  /** Rows written (emit/overwrite) or removed (delete). */
+  rowCount?: number;
+  error?: string;
+}
+/**
+ * A transport that executes one PyIceberg job and returns its result.
+ * `subprocessBackend` and `httpBackend` are the two shipped implementations;
+ * tests inject a fake. The param is the `OverwriteJob | DeleteJob` union, so a
+ * backend value is still assignable where the narrower `OverwriteBackend` is
+ * expected (a wider-param function accepts the narrower call).
+ */
+type OverwriteBackend = (job: PyIcebergJob) => Promise<OverwriteJobResult>;
+interface IcebergOverwriteWriterOptions {
+  catalog: OverwriteWriterCatalogConfig;
+  /** The transport that runs the PyIceberg job. */
+  backend: OverwriteBackend;
+}
+/**
+ * The trailing-window Iceberg overwrite writer. Conforms to the frozen
+ * `SliceOverwriteWriter` contract; `overwriteSlice` builds one PyIceberg
+ * overwrite job per call and runs it through the configured backend.
+ */
+interface IcebergOverwriteWriter extends SliceOverwriteWriter {
+  /** Release any backend resources. Idempotent. */
+  close: () => Promise<SinkCloseResult>;
+}
+/**
+ * Build a trailing-window Iceberg overwrite writer over the given backend.
+ * Pure — all I/O is the backend's; the writer only shapes the job.
+ */
+declare function createIcebergOverwriteWriter(opts: IcebergOverwriteWriterOptions): IcebergOverwriteWriter;
+/** Options for the PyIceberg subprocess backend. */
+interface SubprocessBackendOptions {
+  /** Python interpreter. Defaults to `$GSCDUMP_ICEBERG_PYTHON` then `python3`. */
+  python?: string;
+  /**
+   * Path to the PyIceberg writer script. Defaults to
+   *  `<engine>/scripts/iceberg-writer.py`.
+   */
+  writerScript?: string;
+}
+/**
+ * Backend that runs the PyIceberg overwrite by spawning the shared writer
+ * script (`scripts/iceberg-writer.py`). Node-only — used by local integration
+ * tests and a Node-side scheduled job box. The script reads the job JSON on
+ * stdin and writes `{rowCount}` / `{error}` on stdout.
+ */
+declare function subprocessBackend(opts?: SubprocessBackendOptions): OverwriteBackend;
+/** Options for the PyIceberg HTTP backend (the Cloudflare Container in prod). */
+interface HttpBackendOptions {
+  /**
+   * URL of the PyIceberg overwrite service — a POST endpoint accepting the
+   *  `OverwriteJob` JSON and returning `OverwriteJobResult`.
+   */
+  endpoint: string;
+  /** Optional bearer token for the service. */
+  token?: string;
+  /** Injectable fetch — defaults to the global. */
+  fetch?: typeof fetch;
+}
+/**
+ * Backend that POSTs the overwrite job to a PyIceberg HTTP service — the
+ * Cloudflare Container running `iceberg-writer.py` behind an HTTP shim. This
+ * is the prod transport: the request-path Worker holds no Python and never
+ * does an Iceberg metadata commit itself.
+ */
+declare function httpBackend(opts: HttpBackendOptions): OverwriteBackend;
+/**
+ * Adapt an `IcebergOverwriteWriter` into a full `Sink` whose `emit` is also a
+ * partition-overwrite — convenient for tests so a single Iceberg writer backs
+ * both append and revision paths against the local stack. Prod keeps `emit`
+ * on the Pipeline; only `overwriteSlice` delegates here.
+ */
+declare function overwriteWriterAsSink(writer: IcebergOverwriteWriter): Sink & SliceOverwriteWriter;
+/** Per-table outcome of a {@link deleteSiteFromShard} run. */
+interface DeleteSiteResult {
+  table: string;
+  /** Rows removed, when the backend reported a count. */
+  rowCount?: number;
+  /** Present when this table's delete failed; the sweep continues regardless. */
+  error?: string;
+}
+/**
+ * Remove every row for one `siteId` from a per-team Iceberg shard, across the
+ * given tables. Recovery op: clear a corrupt site (e.g. a cross-run append
+ * double) before a clean re-backfill, leaving sibling sites in the shared shard
+ * untouched. `site_id` is an identity partition column, so PyIceberg prunes
+ * whole data files (no row-level delete files).
+ *
+ * Runs SEQUENTIALLY (one commit per table at a time) so the per-table catalog
+ * commit-rate ceiling is never tripped. A table failure is captured in its
+ * result row, not thrown — the sweep always attempts every table so a partial
+ * failure is visible and re-runnable (delete is idempotent: re-deleting an
+ * already-empty site is a no-op).
+ */
+declare function deleteSiteFromShard(args: {
+  catalog: OverwriteWriterCatalogConfig;
+  backend: OverwriteBackend;
+  siteId: string | number;
+  tables: readonly string[];
+}): Promise<DeleteSiteResult[]>;
+export { DeleteJob, DeleteSiteResult, HttpBackendOptions, IcebergOverwriteWriter, IcebergOverwriteWriterOptions, OverwriteBackend, OverwriteJob, OverwriteJobResult, OverwriteWriterCatalogConfig, PyIcebergJob, SubprocessBackendOptions, createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/dist/iceberg/overwrite-writer.mjs

@@ -0,0 +1,102 @@
+import { resolvePyIcebergPython, runPyIcebergWriter } from "./pyiceberg-runtime.mjs";
+import { ICEBERG_SCHEMAS, assertIcebergTable } from "./schema.mjs";
+function createIcebergOverwriteWriter(opts) {
+	const { catalog, backend } = opts;
+	async function overwriteSlice(slice, rows) {
+		const siteId = slice.ctx.siteId;
+		if (!siteId) throw new Error("overwriteSlice: slice.ctx.siteId is required for the Iceberg partition key");
+		const table = assertIcebergTable(slice.table);
+		const job = {
+			op: "overwrite",
+			catalogUri: catalog.catalogUri,
+			namespace: catalog.namespace,
+			warehouse: catalog.warehouse,
+			s3: catalog.s3,
+			table,
+			spec: ICEBERG_SCHEMAS[table],
+			siteId,
+			searchType: slice.searchType,
+			date: slice.date,
+			rows
+		};
+		const res = await backend(job);
+		if (res.error) throw new Error(`overwriteSlice: PyIceberg backend failed: ${res.error}`);
+		return { rowCount: res.rowCount ?? rows.length };
+	}
+	return {
+		overwriteSlice,
+		async close() {
+			return {
+				flushed: [],
+				failed: []
+			};
+		}
+	};
+}
+function subprocessBackend(opts = {}) {
+	return async (job) => {
+		const { dirname, join } = await import("node:path");
+		const { fileURLToPath } = await import("node:url");
+		return runPyIcebergWriter({
+			python: resolvePyIcebergPython(opts.python),
+			script: opts.writerScript ?? join(dirname(fileURLToPath(import.meta.url)), "..", "scripts", "iceberg-writer.py"),
+			job,
+			label: "iceberg overwrite subprocess",
+			processErrorAsParseFailure: true
+		});
+	};
+}
+function httpBackend(opts) {
+	const doFetch = opts.fetch ?? fetch;
+	return async (job) => {
+		const res = await doFetch(opts.endpoint, {
+			method: "POST",
+			headers: {
+				"content-type": "application/json",
+				...opts.token ? { authorization: `Bearer ${opts.token}` } : {}
+			},
+			body: JSON.stringify(job)
+		});
+		if (!res.ok) {
+			const body = await res.text().catch(() => "");
+			return { error: `HTTP ${res.status}${body ? `: ${body}` : ""}` };
+		}
+		return await res.json();
+	};
+}
+function overwriteWriterAsSink(writer) {
+	return {
+		capabilities: {
+			canOverwrite: true,
+			appendOnly: false
+		},
+		emit: (slice, rows) => writer.overwriteSlice(slice, rows),
+		overwriteSlice: (slice, rows) => writer.overwriteSlice(slice, rows),
+		close: () => writer.close()
+	};
+}
+async function deleteSiteFromShard(args) {
+	const { catalog, backend, siteId, tables } = args;
+	const results = [];
+	for (const table of tables) {
+		const res = await backend({
+			op: "delete",
+			catalogUri: catalog.catalogUri,
+			namespace: catalog.namespace,
+			warehouse: catalog.warehouse,
+			s3: catalog.s3,
+			...catalog.catalogToken ? { catalogToken: catalog.catalogToken } : {},
+			table,
+			siteId
+		}).catch((err) => ({ error: err instanceof Error ? err.message : String(err) }));
+		results.push(res.error ? {
+			table,
+			error: res.error
+		} : {
+			table,
+			rowCount: res.rowCount
+		});
+	}
+	return results;
+}
+export { createIcebergOverwriteWriter, deleteSiteFromShard, httpBackend, overwriteWriterAsSink, subprocessBackend };

package/dist/iceberg/pyiceberg-runtime.mjs

@@ -0,0 +1,34 @@
+import process from "node:process";
+const PYICEBERG_PYTHON_ENV = "GSCDUMP_ICEBERG_PYTHON";
+const DEFAULT_PYICEBERG_PYTHON = "python3";
+function resolvePyIcebergPython(override) {
+	return override ?? process.env[PYICEBERG_PYTHON_ENV] ?? DEFAULT_PYICEBERG_PYTHON;
+}
+async function runPyIcebergWriter(options) {
+	const { execFile } = await import("node:child_process");
+	return new Promise((resolve, reject) => {
+		execFile(options.python, [options.script], { maxBuffer: 64 * 1024 * 1024 }, (err, stdout, stderr) => {
+			let parsed;
+			let parseError;
+			if (stdout.trim()) try {
+				parsed = JSON.parse(stdout);
+			} catch (error) {
+				parseError = error;
+			}
+			if (parsed && !(err && options.rejectOnProcessError)) {
+				resolve(parsed);
+				return;
+			}
+			if (err) {
+				if (options.processErrorAsParseFailure) {
+					reject(/* @__PURE__ */ new Error(`${options.label} produced no parseable output (${err.message})${stderr ? `: ${stderr}` : ""}`));
+					return;
+				}
+				reject(/* @__PURE__ */ new Error(`${options.label} process failed (${err.message})${stderr ? `: ${stderr}` : ""}`));
+				return;
+			}
+			reject(new Error(`${options.label} produced no parseable output: ${stdout || stderr}`, { cause: parseError }));
+		}).stdin?.end(JSON.stringify(options.job));
+	});
+}
+export { resolvePyIcebergPython, runPyIcebergWriter };

package/dist/iceberg/schema.d.mts

@@ -0,0 +1,69 @@
+import { SearchType } from "../storage.mjs";
+import { IcebergColumn, IcebergPartitionField, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1 } from "@gscdump/lakehouse/schema";
+import { TableName } from "@gscdump/contracts";
+/** The 9 fact tables that exist as global Iceberg tables. */
+type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;
+/** The 9 Iceberg table names, in canonical order. */
+declare const ICEBERG_TABLES: readonly IcebergTableName[];
+/**
+ * Stable `search_type` enum → int map for `'int'`-encoded catalogs. Engine-owned
+ * and FROZEN: never renumber or reuse an id (it's the on-disk partition value).
+ */
+declare const SEARCH_TYPE_INT: Record<SearchType, number>;
+/** Reverse of {@link SEARCH_TYPE_INT} — int → `search_type`, for read-result mapping. */
+declare const INT_SEARCH_TYPE: Record<number, SearchType>;
+/**
+ * The two partition-identity columns Iceberg rows carry that the legacy
+ * parquet path encoded in the object-key prefix instead. Field ids 1–2 are
+ * the first two columns; per-table metric/dimension columns follow
+ * contiguously from id 3 (see `ICEBERG_FIELD_ID_BASE`).
+ */
+declare const ICEBERG_PARTITION_COLUMNS: readonly IcebergColumn[];
+/**
+ * The two partition-identity columns for a given {@link PartitionKeyEncoding}.
+ * `'string'` returns {@link ICEBERG_PARTITION_COLUMNS} verbatim; `'int'` swaps
+ * BOTH to INT — `site_id` (the app's small `user_sites.int_id`; ≪ 2.1B sites, so
+ * INT is ample) and `search_type` (its fixed enum code). Integer identity columns
+ * avoid R2 SQL's truncated-string-stats equality undercount and restore pruning.
+ * Field ids are unchanged (1, 2) — only the column types differ.
+ */
+declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): readonly IcebergColumn[];
+/**
+ * First field id used for per-table (non-partition) columns — immediately
+ * after the two partition-identity columns (`site_id`=1, `search_type`=2).
+ * No longer read by `defineDataset` below (field ids are now assigned by
+ * `defineIcebergDataset` itself, identity-then-dims-then-columns) — kept as a
+ * documented, byte-identity-tested constant for existing external readers.
+ */
+declare const ICEBERG_FIELD_ID_BASE = 3;
+/** Shared partition spec — identical across every table. */
+declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
+/**
+ * Derive the full Iceberg table spec for a table + encoding. Thin wrapper
+ * over {@link gscDataset}'s `tableSpec` (ADR-0021 R2-FIXES C5) — kept as a
+ * named GSC-specific export. Generic schema authoring types are owned by
+ * `@gscdump/lakehouse`.
+ *
+ * CONTRACT NOTE: implementation agents must treat the RETURNED VALUE as the
+ * source of truth — do not hand-list columns elsewhere.
+ */
+declare function icebergTableSpec(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergTableSpec;
+/** All Iceberg table specs in legacy `'string'` encoding, keyed by table name. */
+declare const ICEBERG_SCHEMAS_STRING: Record<IcebergTableName, IcebergTableSpec>;
+/** All Iceberg table specs in `'int'` encoding (INT site_id + INT search_type). */
+declare const ICEBERG_SCHEMAS_INT: Record<IcebergTableName, IcebergTableSpec>;
+/** All Iceberg table specs for the default new-catalog encoding. */
+declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
+/** Table specs for the given encoding (`'int'` default). */
+declare function icebergSchemasFor(encoding?: PartitionKeyEncoding): Record<IcebergTableName, IcebergTableSpec>;
+/** True when `table` is one of the canonical {@link ICEBERG_TABLES}. */
+declare function isIcebergTable(table: string): table is IcebergTableName;
+/**
+ * Narrow an arbitrary table name to a canonical {@link IcebergTableName},
+ * throwing a clear error otherwise. Guards write paths that index
+ * `ICEBERG_SCHEMAS` (a `Record<IcebergTableName, …>`) — a non-canonical name
+ * silently yields `undefined` there, propagating a corrupt/empty spec into the
+ * Iceberg job instead of failing loudly.
+ */
+declare function assertIcebergTable(table: string): IcebergTableName;
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, type IcebergS3Config, IcebergTableName, type PartitionKeyEncoding$1 as PartitionKeyEncoding, SEARCH_TYPE_INT, type SearchType, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };

package/dist/{_chunks/schema2.mjs → iceberg/schema.mjs}

@@ -1,4 +1,5 @@
-import { SCHEMAS, TABLE_METADATA } from "./schema.mjs";
+import { TABLE_METADATA } from "../drizzle-schema.mjs";
+import { SCHEMAS } from "../schema.mjs";
 import { defineIcebergDataset } from "@gscdump/lakehouse";
 import { DEFAULT_PARTITION_KEY_ENCODING, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1 } from "@gscdump/lakehouse/schema";
 function mapColumnType(t) {
@@ -124,4 +125,4 @@ function assertIcebergTable(table) {
 	if (!isIcebergTable(table)) throw new Error(`Unknown Iceberg table '${table}'. Expected one of: ${ICEBERG_TABLES.join(", ")}`);
 	return table;
 }
-export { DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };
+export { DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, GSC_DATASETS_INT, GSC_DATASETS_STRING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };