@gscdump/lakehouse 1.4.0 → 1.4.1

package/dist/dataset.mjs

@@ -0,0 +1,364 @@
+import { coerceBigIntToNumber } from "./bigint.mjs";
+import { icebergCreateTable } from "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/write/write.mjs";
+import "./node_modules/.pnpm/icebird@0.8.15_patch_hash_d1902372376af18885e7c7dce807f1ecab86c2f86a9d1ff65d51e3b63b64b057/node_modules/icebird/src/index.mjs";
+import { buildManifestPartitionFilter } from "./partition-prune.mjs";
+import { ICEBERG_TYPE_MAP, connectIcebergCatalog, ensureIcebergNamespace, icebergAppendBatchesRetrying, icebergAppendRetrying, resolveIcebergDataFiles } from "./catalog.mjs";
+const INT32_MIN = -2147483648;
+const INT32_MAX = 2147483647;
+const DAY_MILLIS = 864e5;
+function identityColumnNames(identity) {
+	return identity.kind === "site-int" ? ["site_id"] : identity.columns.map((c) => c.name);
+}
+function identityEncodings(identity) {
+	const map = /* @__PURE__ */ new Map();
+	if (identity.kind === "site-int") map.set("site_id", (identity.encoding ?? "int") === "int" ? "int32" : "string");
+	else for (const c of identity.columns) map.set(c.name, c.encoding);
+	return map;
+}
+function identityColumnType(encoding) {
+	return encoding === "int32" ? "INT" : "STRING";
+}
+function deriveTableSpec(def) {
+	const encodings = identityEncodings(def.identity);
+	const idNames = identityColumnNames(def.identity);
+	const dimNames = def.dims ? Object.keys(def.dims) : [];
+	let fieldId = 1;
+	const columns = [];
+	for (const name of idNames) columns.push({
+		name,
+		type: identityColumnType(encodings.get(name)),
+		required: true,
+		fieldId: fieldId++
+	});
+	for (const name of dimNames) {
+		const enc = def.dims[name].boundEncoding;
+		columns.push({
+			name,
+			type: identityColumnType(enc),
+			required: true,
+			fieldId: fieldId++
+		});
+	}
+	for (const col of def.columns) columns.push({
+		name: col.name,
+		type: col.type,
+		required: col.required,
+		fieldId: fieldId++
+	});
+	return {
+		namespace: def.namespace,
+		table: def.table,
+		columns,
+		partitionSpec: def.partition,
+		naturalKey: def.naturalKey,
+		identityColumns: [
+			...idNames,
+			...dimNames,
+			...def.naturalKey
+		],
+		clusterKey: def.clusterKey
+	};
+}
+function icebergSchemaFromSpec(spec) {
+	return {
+		"type": "struct",
+		"schema-id": 0,
+		"fields": spec.columns.map((col) => ({
+			id: col.fieldId,
+			name: col.name,
+			required: col.required,
+			type: ICEBERG_TYPE_MAP[col.type]
+		}))
+	};
+}
+function icebergPartitionSpecFromSpec(spec) {
+	const fieldId = (name) => {
+		const col = spec.columns.find((c) => c.name === name);
+		if (!col) throw new Error(`lakehouse: table '${spec.table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"spec-id": 0,
+		"fields": spec.partitionSpec.map((p, i) => ({
+			"source-id": fieldId(p.sourceColumn),
+			"field-id": 1e3 + i,
+			"name": p.name,
+			"transform": p.transform
+		}))
+	};
+}
+function icebergSortOrderFromSpec(spec) {
+	if (!spec.clusterKey || spec.clusterKey.length === 0) return void 0;
+	const fieldId = (name) => {
+		const col = spec.columns.find((c) => c.name === name);
+		if (!col) throw new Error(`lakehouse: table '${spec.table}' has no '${name}' column`);
+		return col.fieldId;
+	};
+	return {
+		"order-id": 1,
+		"fields": spec.clusterKey.map((col) => ({
+			"source-id": fieldId(col),
+			"transform": "identity",
+			"direction": "asc",
+			"null-order": "nulls-last"
+		}))
+	};
+}
+function toIcebergDayCount(value) {
+	if (typeof value === "number") return value;
+	if (value instanceof Date) {
+		const ms = value.getTime();
+		if (Number.isNaN(ms)) throw new TypeError("toIcebergDayCount: invalid Date (NaN)");
+		return Math.floor(ms / DAY_MILLIS);
+	}
+	const ms = Date.parse(`${value}T00:00:00Z`);
+	if (Number.isNaN(ms)) throw new TypeError(`toIcebergDayCount: invalid date string '${value}'`);
+	return Math.floor(ms / DAY_MILLIS);
+}
+function asInt32(value) {
+	if (typeof value !== "string" && typeof value !== "number" && typeof value !== "bigint") return null;
+	const n = Number(value);
+	if (!Number.isSafeInteger(n) || n < INT32_MIN || n > INT32_MAX) return null;
+	return n;
+}
+function buildRowProcessor(def, tableSpec) {
+	const idNames = identityColumnNames(def.identity);
+	const idEncodings = identityEncodings(def.identity);
+	const dimNames = def.dims ? Object.keys(def.dims) : [];
+	const identityColumns = tableSpec.identityColumns;
+	function guard(row) {
+		const out = {};
+		for (const name of idNames) {
+			const enc = idEncodings.get(name);
+			const raw = row[name];
+			if (enc === "int32") {
+				const n = asInt32(raw);
+				if (n === null) return null;
+				out[name] = n;
+			} else {
+				if (raw == null || raw === "") return null;
+				out[name] = String(raw);
+			}
+		}
+		for (const name of dimNames) out[name] = def.dims[name].toPartitionValue(String(row[name]));
+		for (const col of def.columns) {
+			const value = row[col.name];
+			out[col.name] = col.type === "LONG" ? typeof value === "number" ? BigInt(value) : value : coerceBigIntToNumber(value);
+		}
+		return out;
+	}
+	function dedupe(rows) {
+		if (rows.length < 2) return rows;
+		const seen = /* @__PURE__ */ new Map();
+		for (const rec of rows) {
+			let identity = "";
+			for (let index = 0; index < identityColumns.length; index++) {
+				if (index > 0) identity += "\0";
+				identity += `${rec[identityColumns[index]] ?? ""}`;
+			}
+			seen.set(identity, rec);
+		}
+		return seen.size === rows.length ? rows : [...seen.values()];
+	}
+	function sort(rows) {
+		const cols = def.clusterKey;
+		if (!cols || cols.length === 0 || rows.length < 2) return rows;
+		return rows.slice().sort((a, b) => {
+			for (const col of cols) {
+				const av = a[col];
+				const bv = b[col];
+				if (av === bv) continue;
+				if (av == null) return -1;
+				if (bv == null) return 1;
+				if (typeof av === "number" && typeof bv === "number") return av - bv;
+				const as = String(av);
+				const bs = String(bv);
+				if (as !== bs) return as < bs ? -1 : 1;
+			}
+			return 0;
+		});
+	}
+	return {
+		guard,
+		dedupe,
+		sort
+	};
+}
+function defineIcebergDataset(def) {
+	const tableSpec = deriveTableSpec(def);
+	const schema = icebergSchemaFromSpec(tableSpec);
+	const partitionSpecIcebird = icebergPartitionSpecFromSpec(tableSpec);
+	const sortOrder = icebergSortOrderFromSpec(tableSpec);
+	const { guard, dedupe, sort } = buildRowProcessor(def, tableSpec);
+	async function createTable(conn) {
+		const results = [];
+		await icebergCreateTable({
+			catalog: conn.catalog,
+			namespace: conn.namespace,
+			table: def.table,
+			schema,
+			partitionSpec: partitionSpecIcebird,
+			...sortOrder ? { sortOrder } : {}
+		}).then(() => results.push({
+			table: def.table,
+			ok: true
+		}), (e) => results.push({
+			table: def.table,
+			ok: false,
+			error: e instanceof Error ? e.message : String(e)
+		}));
+		return results;
+	}
+	function process(rows) {
+		const guarded = [];
+		let skipped = 0;
+		for (const row of rows) {
+			const rec = guard(row);
+			if (rec === null) skipped++;
+			else guarded.push(rec);
+		}
+		return {
+			records: sort(dedupe(guarded)),
+			skipped
+		};
+	}
+	async function appendRows(conn, rows, opts = {}) {
+		if (rows.length === 0) return {
+			accepted: 0,
+			skipped: 0
+		};
+		const { records, skipped } = process(rows);
+		if (records.length === 0) return {
+			accepted: 0,
+			skipped
+		};
+		await icebergAppendRetrying({
+			catalog: conn.catalog,
+			namespace: conn.namespace,
+			table: def.table,
+			resolver: conn.resolver,
+			records
+		}, opts.commitRetry);
+		return {
+			accepted: records.length,
+			skipped
+		};
+	}
+	async function appendBatches(conn, source, opts) {
+		let accepted = 0;
+		let skipped = 0;
+		const batchFactory = async function* () {
+			accepted = 0;
+			skipped = 0;
+			for await (const rows of source()) {
+				const prepared = process(rows);
+				accepted += prepared.records.length;
+				skipped += prepared.skipped;
+				if (prepared.records.length > 0) yield prepared.records;
+			}
+		};
+		const committed = await icebergAppendBatchesRetrying({
+			catalog: conn.catalog,
+			namespace: conn.namespace,
+			table: def.table,
+			resolver: conn.resolver,
+			batchFactory
+		}, {
+			...opts.commitRetry,
+			appendId: opts.appendId
+		});
+		return {
+			accepted,
+			skipped,
+			committed
+		};
+	}
+	function appendSink(opts) {
+		let buffer = [];
+		let connection;
+		function connect() {
+			connection ??= connectIcebergCatalog(opts.catalog, opts.connect).then(async (conn) => {
+				await ensureIcebergNamespace(conn);
+				return conn;
+			});
+			return connection;
+		}
+		return {
+			emit(rows) {
+				for (const r of rows) buffer.push(r);
+			},
+			async close() {
+				if (buffer.length === 0) return {
+					flushed: false,
+					accepted: 0,
+					skipped: 0
+				};
+				const rows = buffer;
+				buffer = [];
+				try {
+					const result = await appendRows(await connect(), rows, { commitRetry: opts.commitRetry });
+					if (result.accepted > 0 && opts.ledger) await opts.ledger.record();
+					return {
+						flushed: result.accepted > 0,
+						...result
+					};
+				} catch (error) {
+					return {
+						flushed: false,
+						accepted: 0,
+						skipped: 0,
+						error
+					};
+				}
+			}
+		};
+	}
+	function readerPredicate(identity, dims) {
+		const idNames = identityColumnNames(def.identity);
+		const idEncodings = identityEncodings(def.identity);
+		const matches = idNames.map((name) => ({
+			field: name,
+			value: idEncodings.get(name) === "int32" ? Number(identity) : identity,
+			encoding: idEncodings.get(name) === "int32" ? "int32" : "string"
+		}));
+		if (def.dims && dims) {
+			for (const [name, dim] of Object.entries(def.dims)) if (name in dims) matches.push({
+				field: name,
+				value: dim.toPartitionValue(dims[name]),
+				encoding: dim.boundEncoding
+			});
+		}
+		return matches;
+	}
+	function partitionBoundFilter(identity, months, dims) {
+		return buildManifestPartitionFilter(def.partition, readerPredicate(identity, dims), months);
+	}
+	async function resolveDataFiles(conn, identity, range, dims, opts = {}) {
+		return resolveIcebergDataFiles(conn, {
+			namespace: def.namespace,
+			table: def.table,
+			partitionSpec: def.partition,
+			matches: readerPredicate(identity, dims),
+			range,
+			cache: opts.cache,
+			clock: opts.clock,
+			profiler: opts.profiler
+		});
+	}
+	return {
+		def,
+		tableSpec,
+		icebergSchema: () => schema,
+		icebergPartitionSpec: () => partitionSpecIcebird,
+		icebergSortOrder: () => sortOrder,
+		createTable,
+		appendRows,
+		appendBatches,
+		prepareRows: process,
+		appendSink,
+		readerPredicate,
+		partitionBoundFilter,
+		resolveDataFiles
+	};
+}
+export { defineIcebergDataset, deriveTableSpec, toIcebergDayCount };

package/dist/index.d.mts

@@ -1,161 +1,7 @@
 import { bigintJsonReplacer, coerceBigIntToNumber, encodeJsonBigintSafe, stringifyBigintSafe } from "./bigint.mjs";
-import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
-import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
-/**
- * Closed identity shapes (ADR-0021 amendments 2 + 4). `'site-int'` is the
- * team-scoped numeric Catalog Site Id (nuxtseo ADR-0091) stored under the
- * fixed column name `site_id`. `'columns'` is the general N-column identity
- * with per-column encoding, replacing the original opaque `'custom'` shape.
- */
-type DatasetIdentity = {
-  kind: 'site-int';
-  encoding?: 'int' | 'string';
-} | {
-  kind: 'columns';
-  columns: readonly {
-    name: string;
-    encoding: 'int32' | 'string';
-  }[];
-};
-/** A per-slice dimension (GSC's `searchType`). Absent for plain snapshot datasets. */
-interface DatasetDim {
-  toPartitionValue: (v: string) => string | number;
-  /** Required (amendment 5) — how the reader decodes this column's manifest bounds. */
-  boundEncoding: 'int32' | 'string';
-}
-interface IcebergDatasetColumnDef {
-  name: string;
-  type: IcebergColumnType;
-  required: boolean;
-}
-interface IcebergDatasetLedger {
-  key: readonly string[];
-}
-interface IcebergDatasetDef {
-  namespace: string;
-  table: string;
-  identity: DatasetIdentity;
-  /** Extra per-slice dimensions; absent for snapshot datasets. */
-  dims?: Record<string, DatasetDim>;
-  /** Data columns (non-identity, non-dims) — includes any partition source column (e.g. `date`). */
-  columns: readonly IcebergDatasetColumnDef[];
-  /** Partition fields; identity columns must appear here. */
-  partition: readonly IcebergPartitionField[];
-  /** Natural-key columns. Full dedupe key = identity + dims + naturalKey (amendment 3). */
-  naturalKey: readonly string[];
-  clusterKey?: readonly string[];
-  /** Consumer-owned exactly-once ledger shape — folded into `appendSink().close()` (amendment 6). */
-  ledger?: IcebergDatasetLedger;
-}
-/**
- * Derive the full {@link IcebergTableSpec} from a dataset def. Field ids are
- * assigned sequentially in ONE fixed order — identity columns, then dims
- * columns, then declared data columns — starting at 1. This mirrors the
- * engine's original `icebergTableSpec` contract (identity columns first, data
- * columns from a fixed base) so a def-derived spec for an existing table
- * (`crawl.pages`, `lighthouse.scans`, `dataforseo.keywords`) is BYTE-IDENTICAL
- * to the frozen inline constants it replaces.
- */
-declare function deriveTableSpec(def: IcebergDatasetDef): IcebergTableSpec;
-/**
- * Convert a `YYYY-MM-DD` string / `Date` / already-numeric day-count to the
- * integer "days since the Unix epoch" the Iceberg `date` type stores.
- * hyparquet-writer mis-encodes Date-valued dictionary columns, so callers
- * should feed this INTO their row before `appendRows`/`appendSink.emit` for
- * any `month`-partitioned date column.
- */
-declare function toIcebergDayCount(value: string | Date | number): number;
-/** Options shared by `appendRows` / `appendSink`. */
-interface AppendCommitOptions {
-  commitRetry?: CommitRetryOptions;
-}
-interface AppendSinkOptions extends AppendCommitOptions {
-  catalog: IcebergCatalogConfig;
-  connect?: ConnectIcebergOptions;
-  /** Invoked ONLY after a successful flush (amendment 6) — never on a failed/empty close. */
-  ledger?: {
-    record: () => Promise<void> | void;
-  };
-}
-interface AppendResult {
-  /** Rows accepted after the identity guard + dedupe (what was actually appended). */
-  accepted: number;
-  /** Rows dropped by the identity guard (missing/out-of-range required identity value). */
-  skipped: number;
-}
-type AppendBatchSource = () => Iterable<readonly Record<string, unknown>[]> | AsyncIterable<readonly Record<string, unknown>[]>;
-interface AppendBatchesOptions extends AppendCommitOptions {
-  /** Stable across job retries; recorded in snapshot metadata for landed checks. */
-  appendId: string;
-}
-interface AppendBatchesResult extends AppendResult {
-  /** False when this append id was already present before the call. */
-  committed: boolean;
-}
-interface AppendSinkCloseResult extends AppendResult {
-  flushed: boolean;
-  error?: unknown;
-}
-interface AppendSink {
-  emit: (rows: readonly Record<string, unknown>[]) => void;
-  close: () => Promise<AppendSinkCloseResult>;
-}
-interface ResolveDataFilesOptions {
-  cache?: CatalogCache;
-  clock?: () => number;
-  profiler?: QueryProfiler;
-}
-interface IcebergDataset {
-  readonly def: IcebergDatasetDef;
-  readonly tableSpec: IcebergTableSpec;
-  icebergSchema: () => IcebergSchema;
-  icebergPartitionSpec: () => IcebergPartitionSpec;
-  icebergSortOrder: () => IcebergSortOrder | undefined;
-  createTable: (conn: IcebergConnection) => Promise<IcebergTableOpResult[]>;
-  /**
-   * One-shot append over an EXISTING connection: identity INT32 guard, dedupe
-   * by identity+dims+naturalKey (last-wins), cluster pre-sort (if
-   * `clusterKey` is set), then a single `icebergAppendRetrying` commit. `rows`
-   * must already be in final storage-column shape — identity/dims/data
-   * columns keyed by their STORED column names, with any `month`-partitioned
-   * date column pre-converted via {@link toIcebergDayCount}.
-   */
-  appendRows: (conn: IcebergConnection, rows: readonly Record<string, unknown>[], opts?: AppendCommitOptions) => Promise<AppendResult>;
-  /**
-   * Lazily process bounded row batches, writing multiple data files into one
-   * append snapshot and one catalog commit. Dedupe keys must be disjoint
-   * across source batches.
-   */
-  appendBatches: (conn: IcebergConnection, source: AppendBatchSource, opts: AppendBatchesOptions) => Promise<AppendBatchesResult>;
-  /**
-   * PURE row processing — the identity INT32 guard, dedupe (identity+dims+
-   * naturalKey, last-wins) and cluster pre-sort `appendRows`/`appendSink`
-   * apply before committing, exposed standalone with NO network/icebird call.
-   * Lets a consumer that owns its own commit call-site (e.g. one still
-   * calling a frozen `icebergAppendRetrying` import directly, for test-mock
-   * compatibility) still route its dedupe/guard/sort logic through the
-   * dataset definition.
-   */
-  prepareRows: (rows: readonly Record<string, unknown>[]) => {
-    records: Record<string, unknown>[];
-    skipped: number;
-  };
-  /** Buffered multi-emit sink — owns its own connect/close lifecycle (ADR-0021's primary contract). */
-  appendSink: (opts: AppendSinkOptions) => AppendSink;
-  /** `PartitionValueMatch[]` for one identity value (+ optional dims), for manual manifest filtering. */
-  readerPredicate: (identity: string | number, dims?: Record<string, string>) => PartitionValueMatch[];
-  /** A ready-to-use manifest partition filter for one identity value + wanted months. */
-  partitionBoundFilter: (identity: string | number, months: ReadonlySet<number>, dims?: Record<string, string>) => ManifestPartitionFilter;
-  /** Resolve the data files for one identity value across a date range. */
-  resolveDataFiles: (conn: IcebergConnection, identity: string | number, range: {
-    start: string;
-    end: string;
-  }, dims?: Record<string, string>, opts?: ResolveDataFilesOptions) => Promise<IcebergListedDataFile[]>;
-}
-/**
- * Declare an Iceberg dataset — one `namespace.table` — deriving its schema,
- * partition spec, dedupe key, INT32 guard, append path, and reader helpers
- * from the def. See {@link IcebergDatasetDef}.
- */
-declare function defineIcebergDataset(def: IcebergDatasetDef): IcebergDataset;
+import { CatalogCache, cacheGet, cachePut } from "./catalog-cache.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./schema.mjs";
+import { IcebergFieldSummary, ManifestPartitionFilter, PartitionValueMatch, buildManifestPartitionFilter } from "./partition-prune.mjs";
+import { CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, QueryProfiler, ResolveIcebergDataFilesOptions, catalogCacheScope, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles } from "./catalog.mjs";
+import { AppendBatchSource, AppendBatchesOptions, AppendBatchesResult, AppendCommitOptions, AppendResult, AppendSink, AppendSinkCloseResult, AppendSinkOptions, DatasetDim, DatasetIdentity, IcebergDataset, IcebergDatasetColumnDef, IcebergDatasetDef, IcebergDatasetLedger, ResolveDataFilesOptions, defineIcebergDataset, deriveTableSpec, toIcebergDayCount } from "./dataset.mjs";
 export { type AppendBatchSource, type AppendBatchesOptions, type AppendBatchesResult, type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, bigintJsonReplacer, buildManifestPartitionFilter, cacheGet, cachePut, catalogCacheScope, coerceBigIntToNumber, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, encodeJsonBigintSafe, ensureIcebergNamespace, invalidateSnapshotRef, listIcebergTables, resolveIcebergDataFiles, stringifyBigintSafe, toIcebergDayCount };