@gscdump/engine 0.33.10 → 0.35.2

package/dist/_chunks/resolver.mjs

@@ -2,7 +2,7 @@ import { SCHEMAS, drizzleSchema } from "./schema.mjs";
 import { enumeratePartitions } from "./compaction.mjs";
 import { escapeLike } from "../sql-fragments.mjs";
 import "../planner.mjs";
-import "./schema2.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING } from "./schema2.mjs";
 import { UnresolvableDatasetError, buildLogicalComparisonPlan, buildLogicalPlan, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
 import { PgDialect, pgTable, varchar } from "drizzle-orm/pg-core";
 import { normalizeUrl } from "gscdump/normalize";
@@ -492,7 +492,7 @@ function createR2SqlResolverAdapter(options = {}) {
 		tableRef: (tk) => sql.raw(`"${tk}"`),
 		queryDimTableRef: () => sql.raw("\"query_dim\"")
 	});
-	if ((options.partitionKeyEncoding ?? "int") === "int") return adapter;
+	if ((options.partitionKeyEncoding ?? DEFAULT_PARTITION_KEY_ENCODING) === "int") return adapter;
 	return {
 		...adapter,
 		siteIdColRef: (tk) => sql`CONCAT(${adapter.siteIdColRef(tk)}, '')`,

package/dist/_chunks/schema2.mjs

@@ -1,4 +1,14 @@
-import { SCHEMAS } from "./schema.mjs";
+import { SCHEMAS, TABLE_METADATA } from "./schema.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, defineIcebergDataset } from "@gscdump/lakehouse";
+function mapColumnType(t) {
+	switch (t) {
+		case "VARCHAR": return "STRING";
+		case "INTEGER": return "INT";
+		case "BIGINT": return "LONG";
+		case "DOUBLE": return "DOUBLE";
+		case "DATE": return "DATE";
+	}
+}
 const ICEBERG_TABLES = [
 	"pages",
 	"queries",
@@ -10,7 +20,6 @@ const ICEBERG_TABLES = [
 	"search_appearance_queries",
 	"search_appearance_page_queries"
 ];
-const DEFAULT_PARTITION_KEY_ENCODING = "int";
 const SEARCH_TYPE_INT = {
 	web: 1,
 	image: 2,
@@ -31,7 +40,7 @@ const ICEBERG_PARTITION_COLUMNS = [{
 	required: true,
 	fieldId: 2
 }];
-function icebergPartitionColumns(encoding = "int") {
+function icebergPartitionColumns(encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	if (encoding === "string") return ICEBERG_PARTITION_COLUMNS;
 	return [{
 		name: "site_id",
@@ -63,38 +72,47 @@ const ICEBERG_PARTITION_SPEC = [
 		name: "date_month"
 	}
 ];
-function mapColumnType(t) {
-	switch (t) {
-		case "VARCHAR": return "STRING";
-		case "INTEGER": return "INT";
-		case "BIGINT": return "LONG";
-		case "DOUBLE": return "DOUBLE";
-		case "DATE": return "DATE";
-	}
-}
-function icebergTableSpec(table, encoding = "int") {
-	const base = SCHEMAS[table];
-	const dataColumns = base.columns.map((col, i) => ({
+function dataColumnsFor(table) {
+	return SCHEMAS[table].columns.map((col) => ({
 		name: col.name,
 		type: mapColumnType(col.type),
-		required: !col.nullable,
-		fieldId: 3 + i
+		required: !col.nullable
 	}));
+}
+function searchTypeDim(encoding) {
+	return {
+		toPartitionValue: (v) => encoding === "int" ? SEARCH_TYPE_INT[v] : v,
+		boundEncoding: encoding === "int" ? "int32" : "string"
+	};
+}
+function gscDatasetDef(table, encoding) {
+	const meta = TABLE_METADATA[table];
 	return {
+		namespace: "gsc",
 		table,
-		columns: [...icebergPartitionColumns(encoding), ...dataColumns],
-		partitionSpec: ICEBERG_PARTITION_SPEC,
-		identityColumns: [
-			"site_id",
-			"search_type",
-			...base.sortKey
-		]
+		identity: {
+			kind: "site-int",
+			encoding
+		},
+		dims: { search_type: searchTypeDim(encoding) },
+		columns: dataColumnsFor(table),
+		partition: ICEBERG_PARTITION_SPEC,
+		naturalKey: meta.sortKey,
+		clusterKey: meta.clusterKey
 	};
 }
+const GSC_DATASETS_STRING = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, defineIcebergDataset(gscDatasetDef(t, "string"))]));
+const GSC_DATASETS_INT = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, defineIcebergDataset(gscDatasetDef(t, "int"))]));
+function gscDataset(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
+	return (encoding === "int" ? GSC_DATASETS_INT : GSC_DATASETS_STRING)[table];
+}
+function icebergTableSpec(table, encoding = DEFAULT_PARTITION_KEY_ENCODING) {
+	return gscDataset(table, encoding).tableSpec;
+}
 const ICEBERG_SCHEMAS_STRING = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "string")]));
 const ICEBERG_SCHEMAS_INT = Object.fromEntries(ICEBERG_TABLES.map((t) => [t, icebergTableSpec(t, "int")]));
 const ICEBERG_SCHEMAS = ICEBERG_SCHEMAS_INT;
-function icebergSchemasFor(encoding = "int") {
+function icebergSchemasFor(encoding = DEFAULT_PARTITION_KEY_ENCODING) {
 	return encoding === "int" ? ICEBERG_SCHEMAS_INT : ICEBERG_SCHEMAS_STRING;
 }
 const ICEBERG_TABLE_SET = new Set(ICEBERG_TABLES);
@@ -105,4 +123,4 @@ function assertIcebergTable(table) {
 	if (!isIcebergTable(table)) throw new Error(`Unknown Iceberg table '${table}'. Expected one of: ${ICEBERG_TABLES.join(", ")}`);
 	return table;
 }
-export { DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };
+export { DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, gscDataset, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable };

package/dist/_chunks/sink.d.mts

@@ -1,77 +1,13 @@
-import { QueryProfiler, Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
+import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
 import { EngineError } from "./errors.mjs";
-import { Storage } from "./libs/unstorage.mjs";
-import { cachingResolver, icebergAppend, restCatalogConnect } from "./libs/icebird.mjs";
 import { Result } from "gscdump/result";
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING as DEFAULT_PARTITION_KEY_ENCODING$1, IcebergCatalogConfig, IcebergColumn, IcebergColumn as IcebergColumn$1, IcebergColumnType, IcebergConnection, IcebergConnection as IcebergConnection$1, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionField as IcebergPartitionField$1, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableSpec, IcebergTableSpec as IcebergTableSpec$1, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1, QueryProfiler, connectIcebergCatalog, ensureIcebergNamespace, listIcebergTables } from "@gscdump/lakehouse";
+import { icebergAppendRetrying, isCommitRateLimited } from "@gscdump/lakehouse/unsafe-raw";
 import { TableName } from "@gscdump/contracts";
-/** Injected catalog cache: an unstorage `Storage` plus an optional defer hook. */
-interface CatalogCache {
-  /** unstorage storage instance — the driver is the caller's choice. */
-  storage: Storage;
-  /**
-   * Optional hook to run a cache WRITE off the response critical path, e.g.
-   * Cloudflare's `ctx.waitUntil`. When omitted the writer awaits the put
-   * inline so it is never cut off when the response returns.
-   */
-  defer?: (write: Promise<unknown>) => void;
-}
-/**
- * S3-compatible credentials for the Iceberg warehouse object store (R2 in prod,
- * MinIO in the POC). The single definition shared by every catalog/writer/sink
- * that signs warehouse object access — keep this contract in one place so the
- * credential shape cannot drift between the icebird and PyIceberg paths.
- */
-interface IcebergS3Config {
-  /** S3 endpoint host (POC MinIO: `localhost:9100`; prod: the R2 S3 endpoint). */
-  endpoint: string;
-  accessKeyId: string;
-  secretAccessKey: string;
-  /** Defaults to `'auto'` (R2's region). */
-  region?: string;
-}
 /** The 9 fact tables that exist as global Iceberg tables. */
 type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;
 /** The 9 Iceberg table names, in canonical order. */
 declare const ICEBERG_TABLES: readonly IcebergTableName[];
-/**
- * Iceberg-native column type. Superset-mapped from the engine `ColumnType`;
- * `LONG` is Iceberg's name for 64-bit integers, `STRING` for varchar.
- */
-type IcebergColumnType = 'STRING' | 'INT' | 'LONG' | 'DOUBLE' | 'DATE';
-interface IcebergColumn {
-  /** Column name as written into the Iceberg table (snake_case). */
-  name: string;
-  type: IcebergColumnType;
-  /** Iceberg field nullability. Partition identity columns are never null. */
-  required: boolean;
-  /**
-   * Stable Iceberg field id. Field ids — not names — are the schema-evolution
-   * identity in Iceberg; never reuse or renumber an id once a table is live.
-   */
-  fieldId: number;
-}
-/**
- * Partition-key encoding for the two identity columns (`site_id`, `search_type`).
- *
- * - `'string'` (legacy): both columns are STRING. Correct, but R2 SQL's
- *   string min/max statistics are truncated in predicate pushdown, so a bare
- *   `WHERE site_id='<uuid>'` UNDERCOUNTS — callers must CONCAT(col,'') to stay
- *   correct, which defeats partition pruning.
- * - `'int'`: BOTH `site_id` and `search_type` are INT. Integer statistics are
- *   fixed-width and never truncated, so `WHERE site_id=<n>` is both correct AND
- *   prunes (empirically confirmed 2026-06-19, gscdump.com probe-int64-partition;
- *   INT equality proven via the search_type column in the engine e2e canary). A
- *   small INT site_id is ample (≪ 2.1B sites) — no LONG/BigInt needed. The caller
- *   maps the UUID `site_id` ↔ int (app-owned, per-tenant serial) and uses
- *   {@link SEARCH_TYPE_INT} for `search_type` (engine-owned, fixed enum).
- *
- * New per-team catalogs are provisioned `'int'`; existing catalogs stay
- * `'string'`. Callers that read or write legacy catalogs must pass
- * `encoding: 'string'` explicitly.
- */
-type PartitionKeyEncoding = 'string' | 'int';
-/** Default for new Iceberg/R2 Data Catalog tables. */
-declare const DEFAULT_PARTITION_KEY_ENCODING: PartitionKeyEncoding;
 /**
  * Stable `search_type` enum → int map for `'int'`-encoded catalogs. Engine-owned
  * and FROZEN: never renumber or reuse an id (it's the on-disk partition value).
@@ -79,31 +15,6 @@ declare const DEFAULT_PARTITION_KEY_ENCODING: PartitionKeyEncoding;
 declare const SEARCH_TYPE_INT: Record<SearchType, number>;
 /** Reverse of {@link SEARCH_TYPE_INT} — int → `search_type`, for read-result mapping. */
 declare const INT_SEARCH_TYPE: Record<number, SearchType>;
-/** Iceberg partition transform applied to a source column. */
-type IcebergPartitionTransform = 'identity' | 'month';
-interface IcebergPartitionField {
-  /** Source column the transform reads. */
-  sourceColumn: 'site_id' | 'search_type' | 'date';
-  transform: IcebergPartitionTransform;
-  /** Partition field name as it appears in Iceberg metadata. */
-  name: string;
-}
-interface IcebergTableSpec {
-  table: IcebergTableName;
-  columns: readonly IcebergColumn[];
-  /**
-   * Partition spec — shared by every table: identity(site_id),
-   * identity(search_type), month(date).
-   */
-  partitionSpec: readonly IcebergPartitionField[];
-  /**
-   * Natural-key columns: a row is uniquely identified by this tuple within
-   * its partition. Drives partition-overwrite revision correctness and
-   * dedup. Mirrors `TABLE_METADATA[table].sortKey` plus `site_id` +
-   * `search_type`.
-   */
-  identityColumns: readonly string[];
-}
 /**
  * The two partition-identity columns Iceberg rows carry that the legacy
  * parquet path encoded in the object-key prefix instead. Field ids 1–2 are
@@ -123,22 +34,18 @@ declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): reado
 /**
  * First field id used for per-table (non-partition) columns — immediately
  * after the two partition-identity columns (`site_id`=1, `search_type`=2).
- *
- * ADVISORY ONLY. The icebird spike (2026-05-22) established that R2 Data
- * Catalog's `createTable` endpoint re-assigns field ids sequentially and does
- * NOT preserve caller-supplied ids. The contiguous numbering here matches what
- * the catalog produces (`site_id`=1, `search_type`=2, data columns 3, 4, …) so
- * the contract describes reality, but ids are authoritatively assigned by the
- * catalog. Iceberg still guarantees ids are stable once a table exists.
+ * No longer read by `defineDataset` below (field ids are now assigned by
+ * `defineIcebergDataset` itself, identity-then-dims-then-columns) — kept as a
+ * documented, byte-identity-tested constant for existing external readers.
  */
 declare const ICEBERG_FIELD_ID_BASE = 3;
 /** Shared partition spec — identical across every table. */
 declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
 /**
- * Derive the full Iceberg table spec for a table from the engine `SCHEMAS`
- * (drizzle-derived column set) plus the shared partition-identity columns.
- * Field ids are assigned deterministically from `ICEBERG_FIELD_ID_BASE` in
- * column declaration order so the same schema always yields the same ids.
+ * Derive the full Iceberg table spec for a table + encoding. Thin wrapper
+ * over {@link gscDataset}'s `tableSpec` (ADR-0021 R2-FIXES C5) — kept as a
+ * named export so existing `@gscdump/engine/iceberg` consumers see no change
+ * to their import surface.
  *
  * CONTRACT NOTE: implementation agents must treat the RETURNED VALUE as the
  * source of truth — do not hand-list columns elsewhere.
@@ -162,286 +69,91 @@ declare function isIcebergTable(table: string): table is IcebergTableName;
  * Iceberg job instead of failing loudly.
  */
 declare function assertIcebergTable(table: string): IcebergTableName;
-/** icebird's lowercase Iceberg primitive types (subset we use). */
-type IcebergPrimitiveType = 'string' | 'int' | 'long' | 'double' | 'date';
-/** A field in an icebird table `Schema`. */
-interface IcebergSchemaField {
-  id: number;
-  name: string;
-  required: boolean;
-  type: IcebergPrimitiveType;
-}
-/** An icebird table `Schema` (Iceberg `struct`). */
-interface IcebergSchema {
-  'type': 'struct';
-  'schema-id': number;
-  'fields': IcebergSchemaField[];
-}
-/** A field in an icebird `PartitionSpec`. */
-interface IcebergPartitionSpecField {
-  'source-id': number;
-  'field-id': number;
-  'name': string;
-  'transform': 'identity' | 'month';
-}
-/** An icebird `PartitionSpec`. */
-interface IcebergPartitionSpec {
-  'spec-id': number;
-  'fields': IcebergPartitionSpecField[];
-}
-/** A field in an icebird `SortOrder`. */
-interface IcebergSortOrderField {
-  'source-id': number;
-  'transform': 'identity';
-  'direction': 'asc' | 'desc';
-  'null-order': 'nulls-first' | 'nulls-last';
-}
-/** An icebird `SortOrder` (Iceberg write-order). */
-interface IcebergSortOrder {
-  'order-id': number;
-  'fields': IcebergSortOrderField[];
-}
-/** Everything needed to talk to the R2 Data Catalog. */
-interface IcebergCatalogConfig {
-  /** REST catalog URI, e.g. `https://catalog.cloudflarestorage.com/<acct>/<warehouse>`. */
-  catalogUri: string;
-  /** Warehouse identifier, e.g. `<acct>_gscdump-analytics`. */
-  warehouse: string;
-  /** Catalog namespace the 5 fact tables live under (e.g. `gsc`). */
-  namespace: string;
-  /** Bearer token for the REST catalog (`R2_CATALOG_TOKEN`). */
-  catalogToken: string;
-  /** R2 S3 credentials for the warehouse objects. */
-  s3: IcebergS3Config;
-}
-/** The connected catalog context + a signed S3 resolver — the icebird call inputs. */
-interface IcebergConnection {
-  /** icebird REST catalog context, passed as `{ catalog }` to icebird write fns. */
-  catalog: Awaited<ReturnType<typeof restCatalogConnect>>;
-  /** icebird S3 resolver (caching-wrapped), passed as `{ resolver }` to icebird fns. */
-  resolver: ReturnType<typeof cachingResolver>;
-  /** The namespace the fact tables live under. */
-  namespace: string;
-}
 /**
- * Build the icebird `Schema` for one of the 5 fact tables from the frozen
+ * Build the icebird `Schema` for one of the 9 fact tables from the frozen
  * `ICEBERG_SCHEMAS` contract. Field ids are advisory — R2 Data Catalog
- * re-assigns them on `createTable` (see `ICEBERG_FIELD_ID_BASE`).
+ * re-assigns them on `createTable`.
  */
-declare function icebergSchemaFor(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergSchema;
+declare function icebergSchemaFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): {
+  type: "struct";
+  'schema-id': number;
+  fields: {
+    id: number;
+    name: string;
+    required: boolean;
+    type: import("@gscdump/lakehouse").IcebergPrimitiveType;
+  }[];
+};
 /**
- * Build the icebird `PartitionSpec` for one of the 5 fact tables: the locked
- * spec `identity(site_id) + identity(search_type) + month(date)`. Each
- * partition field's `source-id` is resolved to the real column field id from
- * {@link icebergSchemaFor}.
+ * Build the icebird `PartitionSpec` for one of the 9 fact tables: the locked
+ * spec `identity(site_id) + identity(search_type) + month(date)`.
  */
-declare function icebergPartitionSpecFor(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergPartitionSpec;
+declare function icebergPartitionSpecFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): {
+  'spec-id': number;
+  fields: {
+    'source-id': number;
+    'field-id': number;
+    name: string;
+    transform: import("@gscdump/lakehouse").IcebergPartitionTransform;
+  }[];
+};
 /**
  * Build the icebird `SortOrder` for a fact table from its `clusterKey`
- * (dimension-first, then `date`) — e.g. `pages` → sort by `url`, then `date`.
- *
- * Declared so any sort-aware compaction (a self-run `icebergRewrite`, or R2
- * managed compaction if/when it honors sort order) re-clusters merged files the
- * same way the append path already orders them ({@link sortByClusterKey} in
- * `append-sink.ts`). R2's managed compaction currently only bin-packs small
- * files without re-sorting, so this is forward-looking: it costs nothing today
- * (the table simply carries the metadata) and means a future sort-aware pass
- * produces globally clustered files for free, maximizing row-group skipping on
- * the DuckDB-over-R2 read path. clusterKey columns are all non-null, so the
- * null ordering is moot; `identity`/`asc` mirrors the physical write order.
+ * (dimension-first, then `date`).
  */
-declare function icebergSortOrderFor(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergSortOrder;
-/** Options for {@link connectIcebergCatalog}. */
-interface ConnectIcebergOptions {
-  /**
-   * Optional cross-isolate cache (any unstorage driver). When supplied, the
-   * `/v1/config` REST probe is served from cache on a warm catalog, removing
-   * one serial network hop from cold-isolate connects. The bearer token is
-   * NEVER cached — only the warehouse-static routing config (`url`, `prefix`,
-   * `defaults`, `overrides`) is; `requestInit` is rebuilt from `config`.
-   */
-  cache?: CatalogCache;
-  /** Injectable clock for the cache TTL. Defaults to `Date.now`. */
-  clock?: () => number;
-}
-/**
- * Connect to the R2 Data Catalog: a REST catalog context + a signed S3
- * resolver. Runs in Node and in `workerd` — SigV4 is Web Crypto, I/O is
- * `fetch`, no node builtins.
- *
- * With a `cache`, the `/v1/config` probe is skipped on a warm catalog and the
- * context is rebuilt from the cached routing config plus the freshly-derived
- * bearer `requestInit`. icebird reads only `url`/`prefix`/`requestInit` from
- * the context downstream, so this is a faithful, secret-free reconstruction.
- */
-declare function connectIcebergCatalog(config: IcebergCatalogConfig, opts?: ConnectIcebergOptions): Promise<IcebergConnection>;
-/** Tunable retry policy for {@link icebergAppendRetrying}. */
-interface CommitRetryOptions {
-  /** Total attempts, including the first. Default 6. */
-  maxAttempts?: number;
-  /** Base (ms) for the exponential back-off ceiling. Default 1000. */
-  baseDelayMs?: number;
-  /** Hard cap (ms) on the back-off ceiling. Default 20_000. */
-  maxDelayMs?: number;
-  /** Injectable sleep — tests pass a synchronous no-op. */
-  sleep?: (ms: number) => Promise<void>;
-  /** Injectable RNG for the jitter — tests pass a deterministic value. */
-  random?: () => number;
-  /**
-   * Idempotency token stamped into the appended snapshot's summary
-   * (`gscdump.append-id`) and matched by the landed-check. When omitted it is
-   * DERIVED from the records' content (a SHA-256 over each row's full
-   * key/value, order-independent — see {@link deriveAppendId}), making the
-   * token STABLE across PROCESSES, not just within one call's retry loop.
-   *
-   * Why content-derived (superseding the old "random per call" choice): the
-   * double we actually hit is a CROSS-RUN re-append — a job commits the
-   * Iceberg snapshot, then dies/evicts before its D1 ledger write, so a queue
-   * RETRY (or operator re-resync) re-runs from scratch, re-emits the identical
-   * buffer, and appends a second copy. A random per-call id can't catch that
-   * (the retry is a new process → new id). A content hash CAN: the retry
-   * re-derives the same id, the pre-append landed-check finds it, and skips.
-   *
-   * The old "content hash risks a false skip / data loss" fear does NOT apply
-   * to this fact model: a date is appended exactly once when finalized and
-   * never revised (revisions go through the OVERWRITE path), and pagination
-   * pages carry DIFFERENT rows → a different content hash → never falsely
-   * skipped. Two appends collide only if their full row-sets are byte-identical
-   * — which is precisely a duplicate that SHOULD be skipped. Injectable for
-   * deterministic tests.
-   */
-  appendId?: string;
-}
-/**
- * True when `err` is an R2 Data Catalog commit rate-limit response
- * (`429 too many commits to this table`). Matches a numeric `status` of 429
- * or the message text, so it holds whether icebird surfaces the raw HTTP
- * error or a wrapped `Error`.
- */
-declare function isCommitRateLimited(err: unknown): boolean;
-/**
- * `icebergAppend` wrapped with retry on R2 Data Catalog 429 commit
- * rate-limits, using full-jitter exponential back-off. icebird already
- * retries 412/409 internally; 429 is the gap this closes. Non-429 errors
- * (and 429s that survive every attempt) propagate unchanged.
- *
- * IDEMPOTENT RE-RUN. A 429 that escapes icebird's internal loop is retried HERE
- * by re-running the whole `icebergAppend`, which re-prepares + re-uploads the
- * data files. That is safe ONLY if the prior attempt's commit did NOT land — but
- * R2 Data Catalog can apply a commit and STILL return 429 (rate-limit on the
- * response path), and a lost ack looks identical. A blind re-run then appends a
- * SECOND copy of the same rows → silent double-count (observed: a re-resynced
- * site read 2× its true totals). To close this, every attempt stamps a per-call
- * `gscdump.append-id` into the snapshot summary, and BEFORE retrying or giving
- * up we reload the table and check whether that id already landed
- * ({@link appendAlreadyLanded}); if so the append succeeded and we return without
- * re-appending. A non-landed 429 still re-runs (the previous attempt's parquet is
- * an orphan, reclaimed by R2 cleanup) — same as before, no double.
- */
-declare function icebergAppendRetrying(args: Parameters<typeof icebergAppend>[0], options?: CommitRetryOptions): Promise<void>;
+declare function icebergSortOrderFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): {
+  'order-id': number;
+  fields: {
+    'source-id': number;
+    transform: "identity";
+    direction: "asc";
+    'null-order': "nulls-last";
+  }[];
+};
 /**
  * Outcome of a single table create/drop: the table name plus a `Result` —
  * `Ok(void)` on success, `Err(iceberg-table-op-failed)` carrying the failure
- * message (and original `cause`) when the catalog rejects the op (e.g. "table
- * already exists", a 5xx). Per-table so a partial provisioning run is fully
- * observable; the human-readable string lives on `error.message`.
+ * message when the catalog rejects the op. Per-table so a partial
+ * provisioning run is fully observable.
  */
 interface IcebergTableOpResult {
   table: string;
   outcome: Result<void, EngineError>;
 }
 /**
- * Ensure the catalog namespace exists. Idempotent — an "already exists"
- * response from the REST catalog is swallowed.
+ * Create the global Iceberg fact tables with the locked partition spec and
+ * the schema derived from {@link ICEBERG_SCHEMAS}. Per-table errors are
+ * captured rather than thrown so a partial run is observable.
  */
-declare function ensureIcebergNamespace(conn: IcebergConnection): Promise<void>;
+declare function createIcebergTables(conn: IcebergConnection, tables?: readonly IcebergTableName[], encoding?: PartitionKeyEncoding$1): Promise<IcebergTableOpResult[]>;
 /**
- * Create the global Iceberg fact tables with the locked partition spec
- * (`identity(site_id) + identity(search_type) + month(date)`) and the schema
- * derived from {@link ICEBERG_SCHEMAS}. Per-table errors are captured rather
- * than thrown so a partial run is observable; "table already exists" surfaces
- * as a failed result. Used by the app's one-off provisioning script.
- */
-declare function createIcebergTables(conn: IcebergConnection, tables?: readonly IcebergTableName[], encoding?: PartitionKeyEncoding): Promise<IcebergTableOpResult[]>;
-/**
- * List the table names currently in the catalog namespace.
- *
- * A genuinely-empty namespace resolves to `[]`. A LIST *failure* (catalog
- * unreachable, 401/403, 5xx) propagates rather than being masked as an empty
- * list — callers must be able to tell "no tables" from "couldn't ask".
+ * Drop tables from the catalog namespace, purging their data objects.
+ * Defaults to every table currently in the namespace. Delegates the walk to
+ * `@gscdump/lakehouse`'s generic `dropIcebergTables`, mapping its simpler
+ * `{table, ok, error?}` result back onto the engine's original `Result`-typed
+ * {@link IcebergTableOpResult} contract.
  */
-declare function listIcebergTables(conn: IcebergConnection): Promise<string[]>;
-/** A data file in the current snapshot's manifest, scoped to one partition. */
-interface IcebergListedDataFile {
-  /** Raw Iceberg `data_file.file_path` (e.g. `s3://gscdump-analytics/.../x.parquet`). */
-  filePath: string;
-  /** Object key relative to the warehouse bucket (the part after `s3://<bucket>/`). */
-  objectKey: string;
-  bytes: number;
-  rowCount: number;
-}
+declare function dropIcebergTables$1(conn: IcebergConnection, tables?: readonly string[]): Promise<IcebergTableOpResult[]>;
 interface ListIcebergDataFilesOptions {
   table: IcebergTableName;
   /** Partition identity column. `number` for `'int'`-encoded catalogs. */
   siteId: string | number;
   /** Partition identity column. `number` (int code) for `'int'`-encoded catalogs. */
   searchType: string | number;
-  /**
-   * Partition-key encoding of the catalog. `'int'` changes how manifest-summary
-   * bounds are decoded (int bytes vs UTF-8) and how the per-file partition value
-   * is compared. Defaults to `'int'` for new catalogs; pass `'string'` for
-   * legacy catalogs.
-   */
-  encoding?: PartitionKeyEncoding;
-  /**
-   * Inclusive date range. Every month touched by `[start, end]` is scanned;
-   * `month(date)` is the third partition transform.
-   */
+  encoding?: PartitionKeyEncoding$1;
   range: {
     start: string;
     end: string;
   };
-  /**
-   * Optional cross-isolate cache (any unstorage driver). When supplied, the
-   * snapshot pointer is cached short (so a warm catalog skips `loadTable`) and
-   * the resolved file list is cached long, content-addressed by snapshot id
-   * (so it skips the manifest walk). Omit it to read straight from the catalog.
-   */
-  cache?: CatalogCache;
-  /** Injectable clock for the cache TTLs. Defaults to `Date.now`. */
+  cache?: import('@gscdump/lakehouse').CatalogCache;
   clock?: () => number;
-  /**
-   * Optional read-path profiler. Emits `iceberg.snapshot` (snapshot-pointer
-   * load), `iceberg.cache` (resolved-files lookup + hit/miss), and
-   * `iceberg.walk` (manifest fetch + entry scan, with manifest/file counts) —
-   * the catalog cold-start breakdown a hosted reader wants in `Server-Timing`.
-   */
   profiler?: QueryProfiler;
 }
 /**
- * List the parquet data files in the current snapshot of `table`, filtered to a
- * single partition slice `(siteId, searchType, month(date) ∈ range)`.
- *
- * The shared `gsc.<table>` tables are multi-tenant, so a naive walk is O(all
- * tenants). This prunes the manifest LIST by partition summaries before
- * fetching any manifest's entries (see {@link buildPartitionFilter}), making
- * the fetch count independent of tenant count, and — when an unstorage `cache`
- * is supplied — skips the `loadTable` round-trip on a warm snapshot pointer and
- * the manifest walk entirely on a resolved-files hit. The final entry-level
- * partition filter is the authoritative correctness check; pruning only avoids
- * reading manifests that cannot match.
- *
- * Skips deleted entries (status=2) and non-data file types (delete files).
- * Returns object keys + bytes + rowCount so the caller can build presigned
- * URLs without re-walking the catalog.
+ * List the parquet data files in the current snapshot of `table`, filtered to
+ * a single partition slice `(siteId, searchType, month(date) ∈ range)`.
  */
-declare function listIcebergDataFiles(conn: IcebergConnection, opts: ListIcebergDataFilesOptions): Promise<IcebergListedDataFile[]>;
-/**
- * Drop tables from the catalog namespace, purging their data objects.
- * Defaults to every table currently in the namespace — used to clear the
- * wrong-spec Pipelines-provisioned `gsc.*` tables before re-creating them.
- */
-declare function dropIcebergTables(conn: IcebergConnection, tables?: readonly string[]): Promise<IcebergTableOpResult[]>;
+declare function listIcebergDataFiles(conn: IcebergConnection, opts: ListIcebergDataFilesOptions): Promise<import("@gscdump/lakehouse").IcebergListedDataFile[]>;
 /**
  * Identifies one fact slice — the atomic unit a sink emits.
  * `(table, site, searchType, date)`. `userId` rides along on `ctx` for
@@ -554,15 +266,6 @@ interface IcebergAppendSinkOptions extends SinkOptions {
    * INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
    * {@link import('./iceberg/schema').PartitionKeyEncoding}.
    */
-  encoding?: PartitionKeyEncoding;
-}
-/** `LocalIcebergSink` options — points at the local Iceberg REST catalog. */
-interface LocalIcebergSinkOptions extends SinkOptions {
-  /** Iceberg REST catalog URI (POC: `apache/iceberg-rest-fixture`). */
-  catalogUri: string;
-  /** Catalog namespace the 5 tables live under. */
-  namespace: string;
-  /** S3-compatible warehouse location (POC: MinIO). */
-  warehouse: string;
+  encoding?: PartitionKeyEncoding$1;
 }
-export { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, IcebergCatalogConfig, IcebergColumn, IcebergColumnType, IcebergConnection, IcebergListedDataFile, IcebergPartitionField, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPartitionTransform, IcebergPrimitiveType, IcebergS3Config, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, IcebergTableSpec, ListIcebergDataFilesOptions, LocalIcebergSinkOptions, PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };
+export { type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING$1 as DEFAULT_PARTITION_KEY_ENCODING, ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, type IcebergCatalogConfig, type IcebergColumn$1 as IcebergColumn, type IcebergColumnType, type IcebergConnection$1 as IcebergConnection, type IcebergListedDataFile, type IcebergPartitionField$1 as IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, IcebergTableName, IcebergTableOpResult, type IcebergTableSpec$1 as IcebergTableSpec, ListIcebergDataFilesOptions, type PartitionKeyEncoding$1 as PartitionKeyEncoding, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, connectIcebergCatalog, createIcebergTables, dropIcebergTables$1 as dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables };

package/dist/adapters/duckdb-node.mjs

@@ -1,10 +1,10 @@
 import { arrowToRows } from "../arrow-utils.mjs";
 import { createRequire } from "node:module";
-import { join } from "node:path";
-import { fileURLToPath } from "node:url";
-import process from "node:process";
 import { unlinkSync } from "node:fs";
 import { tmpdir } from "node:os";
+import { join } from "node:path";
+import process from "node:process";
+import { fileURLToPath } from "node:url";
 import { ConsoleLogger, NODE_RUNTIME, VoidLogger, createDuckDB } from "@duckdb/duckdb-wasm/dist/duckdb-node-blocking.cjs";
 const require_ = createRequire(typeof __filename !== "undefined" ? __filename : typeof import.meta !== "undefined" ? fileURLToPath(import.meta.url) : process.cwd());
 let singleton = null;