@gscdump/engine 1.4.0 → 1.4.2

package/dist/_chunks/sink.d.mts

@@ -1,257 +0,0 @@
-import { Row as Row$1, SearchType, TenantCtx as TenantCtx$1 } from "./storage.mjs";
-import { EngineError } from "./errors.mjs";
-import { icebergCreateTable } from "./libs/icebird.mjs";
-import { Result } from "gscdump/result";
-import { CommitRetryOptions, ConnectIcebergOptions, IcebergCatalogConfig, IcebergConnection, IcebergListedDataFile, QueryProfiler } from "@gscdump/lakehouse";
-import { IcebergColumn, IcebergPartitionField, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding, PartitionKeyEncoding as PartitionKeyEncoding$1 } from "@gscdump/lakehouse/schema";
-import { TableName } from "@gscdump/contracts";
-/** The 9 fact tables that exist as global Iceberg tables. */
-type IcebergTableName = Extract<TableName, 'pages' | 'queries' | 'countries' | 'page_queries' | 'dates' | 'search_appearance' | 'search_appearance_pages' | 'search_appearance_queries' | 'search_appearance_page_queries'>;
-/** The 9 Iceberg table names, in canonical order. */
-declare const ICEBERG_TABLES: readonly IcebergTableName[];
-/**
- * Stable `search_type` enum → int map for `'int'`-encoded catalogs. Engine-owned
- * and FROZEN: never renumber or reuse an id (it's the on-disk partition value).
- */
-declare const SEARCH_TYPE_INT: Record<SearchType, number>;
-/** Reverse of {@link SEARCH_TYPE_INT} — int → `search_type`, for read-result mapping. */
-declare const INT_SEARCH_TYPE: Record<number, SearchType>;
-/**
- * The two partition-identity columns Iceberg rows carry that the legacy
- * parquet path encoded in the object-key prefix instead. Field ids 1–2 are
- * the first two columns; per-table metric/dimension columns follow
- * contiguously from id 3 (see `ICEBERG_FIELD_ID_BASE`).
- */
-declare const ICEBERG_PARTITION_COLUMNS: readonly IcebergColumn[];
-/**
- * The two partition-identity columns for a given {@link PartitionKeyEncoding}.
- * `'string'` returns {@link ICEBERG_PARTITION_COLUMNS} verbatim; `'int'` swaps
- * BOTH to INT — `site_id` (the app's small `user_sites.int_id`; ≪ 2.1B sites, so
- * INT is ample) and `search_type` (its fixed enum code). Integer identity columns
- * avoid R2 SQL's truncated-string-stats equality undercount and restore pruning.
- * Field ids are unchanged (1, 2) — only the column types differ.
- */
-declare function icebergPartitionColumns(encoding?: PartitionKeyEncoding): readonly IcebergColumn[];
-/**
- * First field id used for per-table (non-partition) columns — immediately
- * after the two partition-identity columns (`site_id`=1, `search_type`=2).
- * No longer read by `defineDataset` below (field ids are now assigned by
- * `defineIcebergDataset` itself, identity-then-dims-then-columns) — kept as a
- * documented, byte-identity-tested constant for existing external readers.
- */
-declare const ICEBERG_FIELD_ID_BASE = 3;
-/** Shared partition spec — identical across every table. */
-declare const ICEBERG_PARTITION_SPEC: readonly IcebergPartitionField[];
-/**
- * Derive the full Iceberg table spec for a table + encoding. Thin wrapper
- * over {@link gscDataset}'s `tableSpec` (ADR-0021 R2-FIXES C5) — kept as a
- * named GSC-specific export. Generic schema authoring types are owned by
- * `@gscdump/lakehouse`.
- *
- * CONTRACT NOTE: implementation agents must treat the RETURNED VALUE as the
- * source of truth — do not hand-list columns elsewhere.
- */
-declare function icebergTableSpec(table: IcebergTableName, encoding?: PartitionKeyEncoding): IcebergTableSpec;
-/** All Iceberg table specs in legacy `'string'` encoding, keyed by table name. */
-declare const ICEBERG_SCHEMAS_STRING: Record<IcebergTableName, IcebergTableSpec>;
-/** All Iceberg table specs in `'int'` encoding (INT site_id + INT search_type). */
-declare const ICEBERG_SCHEMAS_INT: Record<IcebergTableName, IcebergTableSpec>;
-/** All Iceberg table specs for the default new-catalog encoding. */
-declare const ICEBERG_SCHEMAS: Record<IcebergTableName, IcebergTableSpec>;
-/** Table specs for the given encoding (`'int'` default). */
-declare function icebergSchemasFor(encoding?: PartitionKeyEncoding): Record<IcebergTableName, IcebergTableSpec>;
-/** True when `table` is one of the canonical {@link ICEBERG_TABLES}. */
-declare function isIcebergTable(table: string): table is IcebergTableName;
-/**
- * Narrow an arbitrary table name to a canonical {@link IcebergTableName},
- * throwing a clear error otherwise. Guards write paths that index
- * `ICEBERG_SCHEMAS` (a `Record<IcebergTableName, …>`) — a non-canonical name
- * silently yields `undefined` there, propagating a corrupt/empty spec into the
- * Iceberg job instead of failing loudly.
- */
-declare function assertIcebergTable(table: string): IcebergTableName;
-type IcebirdCreateTableOptions = Parameters<typeof icebergCreateTable>[0];
-type IcebirdSchema = NonNullable<IcebirdCreateTableOptions['schema']>;
-type IcebirdPartitionSpec = NonNullable<IcebirdCreateTableOptions['partitionSpec']>;
-type IcebirdSortOrder = NonNullable<IcebirdCreateTableOptions['sortOrder']>;
-/**
- * Build the icebird `Schema` for one of the 9 fact tables from the frozen
- * `ICEBERG_SCHEMAS` contract. Field ids are advisory — R2 Data Catalog
- * re-assigns them on `createTable`.
- */
-declare function icebergSchemaFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): IcebirdSchema;
-/**
- * Build the icebird `PartitionSpec` for one of the 9 fact tables: the locked
- * spec `identity(site_id) + identity(search_type) + month(date)`.
- */
-declare function icebergPartitionSpecFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): IcebirdPartitionSpec;
-/**
- * Build the icebird `SortOrder` for a fact table from its `clusterKey`
- * (dimension-first, then `date`).
- */
-declare function icebergSortOrderFor(table: IcebergTableName, encoding?: PartitionKeyEncoding$1): IcebirdSortOrder;
-/**
- * Outcome of a single table create/drop: the table name plus a `Result` —
- * `Ok(void)` on success, `Err(iceberg-table-op-failed)` carrying the failure
- * message when the catalog rejects the op. Per-table so a partial
- * provisioning run is fully observable.
- */
-interface IcebergTableOpResult {
-  table: string;
-  outcome: Result<void, EngineError>;
-}
-/**
- * Create the global Iceberg fact tables with the locked partition spec and
- * the schema derived from {@link ICEBERG_SCHEMAS}. Per-table errors are
- * captured rather than thrown so a partial run is observable.
- */
-declare function createIcebergTables(conn: IcebergConnection, tables?: readonly IcebergTableName[], encoding?: PartitionKeyEncoding$1): Promise<IcebergTableOpResult[]>;
-/**
- * Drop tables from the catalog namespace, purging their data objects.
- * Defaults to every table currently in the namespace. Delegates the walk to
- * `@gscdump/lakehouse`'s generic `dropIcebergTables`, mapping its simpler
- * `{table, ok, error?}` result back onto the engine's original `Result`-typed
- * {@link IcebergTableOpResult} contract.
- */
-declare function dropIcebergTables$1(conn: IcebergConnection, tables?: readonly string[]): Promise<IcebergTableOpResult[]>;
-interface ListIcebergDataFilesOptions {
-  table: IcebergTableName;
-  /** Partition identity column. `number` for `'int'`-encoded catalogs. */
-  siteId: string | number;
-  /** Partition identity column. `number` (int code) for `'int'`-encoded catalogs. */
-  searchType: string | number;
-  encoding?: PartitionKeyEncoding$1;
-  range: {
-    start: string;
-    end: string;
-  };
-  cache?: import('@gscdump/lakehouse').CatalogCache;
-  clock?: () => number;
-  profiler?: QueryProfiler;
-}
-/**
- * List the parquet data files in the current snapshot of `table`, filtered to
- * a single partition slice `(siteId, searchType, month(date) ∈ range)`.
- */
-declare function listIcebergDataFiles(conn: IcebergConnection, opts: ListIcebergDataFilesOptions): Promise<IcebergListedDataFile[]>;
-/**
- * Identifies one fact slice — the atomic unit a sink emits.
- * `(table, site, searchType, date)`. `userId` rides along on `ctx` for
- * tenant-scoped sinks (local/in-memory); the prod Iceberg table is global
- * and keys only on `siteId`.
- */
-interface SinkSlice {
-  ctx: TenantCtx$1;
-  table: IcebergTableName;
-  /** GSC search-type partition. */
-  searchType: SearchType;
-  /** Calendar day (PT), `YYYY-MM-DD`. The slice's `month(date)` partition. */
-  date: string;
-}
-/**
- * Outcome of a sink write. `rowCount` is the number of rows accepted;
- * `bytes` is best-effort — `IcebergAppendSink` does not report it (undefined there).
- */
-interface SinkWriteResult {
-  rowCount: number;
-  bytes?: number;
-}
-/**
- * Static description of a sink. Production ingestion is append-only under the
- * v5 stability-cutoff model. Test and revision-path adapters may expose an
- * overwrite capability explicitly.
- */
-interface SinkCapabilities {
-  /** When true, re-emitting a slice accumulates duplicate rows. */
-  appendOnly: boolean;
-  /** Whether the sink exposes partition overwrite semantics. */
-  canOverwrite?: boolean;
-}
-/** Partition-overwrite writer for revision paths. */
-interface SliceOverwriteWriter {
-  overwriteSlice: (slice: SinkSlice, rows: readonly Row$1[]) => Promise<SinkWriteResult>;
-}
-/**
- * Outcome of `Sink.close()` — which tables' buffered rows reached durable
- * storage and which failed.
- *
- * `IcebergAppendSink.emit` only BUFFERS; the durable Iceberg commit happens
- * in `close()`, one `icebergAppend()` per table. The ingest ledger
- * (`sinkAsIngestEngine`) records a `(site, table, searchType, date)` slice
- * ONLY after the table holding it appears in `flushed` — a table in `failed`
- * leaves its slices un-recorded so the next sync re-emits them. This is what
- * keeps the D1 ledger from ever running ahead of Iceberg.
- */
-interface SinkCloseResult {
-  /** Tables whose buffered rows committed durably. */
-  flushed: IcebergTableName[];
-  /**
-   * Tables whose flush failed — their slices must NOT be ledger-recorded. Each
-   * carries a typed `sink-table-flush-failed` `EngineError`; the human-readable
-   * cause is on `error.message`.
-   */
-  failed: {
-    table: IcebergTableName;
-    error: EngineError;
-  }[];
-}
-interface Sink {
-  readonly capabilities: SinkCapabilities;
-  /**
-   * Emit the fact rows for one slice. Append semantics — for `IcebergAppendSink`
-   * this commits one Iceberg snapshot. Re-emitting the same slice produces
-   * DUPLICATE rows; exactly-once is enforced upstream by the D1 ingested-days
-   * ledger, which only calls `emit` for a slice once.
-   *
-   * `rows` carry the table's data columns; the sink injects the partition
-   * identity columns (`site_id`, `search_type`) from `slice` — callers MUST
-   * NOT pre-populate them.
-   */
-  emit: (slice: SinkSlice, rows: readonly Row$1[]) => Promise<SinkWriteResult>;
-  /**
-   * Flush any buffered rows and release resources, returning which tables
-   * reached durable storage. Idempotent — a second `close()` after a flush
-   * reports empty `flushed`/`failed` (nothing left buffered).
-   *
-   * `IcebergAppendSink` buffers in `emit` and commits one `icebergAppend()`
-   * per table HERE; `flushed`/`failed` reflect those per-table commits.
-   * In-memory / local sinks write durably in `emit`, so they report every
-   * table they received rows for as `flushed`.
-   */
-  close: () => Promise<SinkCloseResult>;
-}
-/** Construction options shared by all sink implementations. */
-interface SinkOptions {
-  now?: () => number;
-}
-/**
- * `IcebergAppendSink`-specific options — the R2 Data Catalog coordinates the
- * sink appends to via `icebird`. The catalog config shape (`IcebergCatalogConfig`)
- * is type-only here so this contract file stays implementation-free.
- */
-interface IcebergAppendSinkOptions extends SinkOptions {
-  /** R2 Data Catalog connection config (catalog URI, warehouse, namespace, token, S3 creds). */
-  catalog: IcebergCatalogConfig;
-  /**
-   * Options forwarded to the lazy catalog connection. Supply a shared
-   * `CatalogCache` here to serve the REST `/v1/config` probe from durable cache
-   * across short-lived Worker isolates.
-   */
-  connect?: ConnectIcebergOptions;
-  /**
-   * Retry policy for the per-table `icebergAppend()` commit, applied on R2
-   * Data Catalog 429 ("too many commits") rate-limits. Optional — production
-   * uses the defaults; tests inject a synchronous `sleep`.
-   */
-  commitRetry?: CommitRetryOptions;
-  /**
-   * Partition-key encoding (default `'int'` for new catalogs). `'int'` writes
-   * BOTH `site_id` and `search_type` as INT — the caller MUST pass the numeric
-   * `site_id` (a numeric string is fine; it's `Number()`-coerced) in
-   * `slice.ctx.siteId`. Pass `'string'` explicitly for legacy catalogs. A small
-   * INT is ample (≪ 2.1B sites), so no LONG/BigInt is involved. See
-   * {@link import('./iceberg/schema').PartitionKeyEncoding}.
-   */
-  encoding?: PartitionKeyEncoding$1;
-}
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_SCHEMAS_STRING, ICEBERG_TABLES, INT_SEARCH_TYPE, IcebergAppendSinkOptions, type IcebergS3Config, IcebergTableName, IcebergTableOpResult, ListIcebergDataFilesOptions, SEARCH_TYPE_INT, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult, SliceOverwriteWriter, assertIcebergTable, createIcebergTables, dropIcebergTables$1 as dropIcebergTables, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergSortOrderFor, icebergTableSpec, isIcebergTable, listIcebergDataFiles };