@gscdump/lakehouse 0.1.0

package/dist/_chunks/libs/ioredis.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/_chunks/libs/lru-cache.d.mts

@@ -0,0 +1 @@
+export { };

package/dist/_chunks/libs/unstorage.d.mts

@@ -0,0 +1,120 @@
+type StorageValue = null | string | number | boolean | object;
+type WatchEvent = "update" | "remove";
+type WatchCallback = (event: WatchEvent, key: string) => any;
+type MaybePromise<T> = T | Promise<T>;
+type MaybeDefined<T> = T extends any ? T : any;
+type Unwatch = () => MaybePromise<void>;
+interface StorageMeta {
+  atime?: Date;
+  mtime?: Date;
+  ttl?: number;
+  [key: string]: StorageValue | Date | undefined;
+}
+type TransactionOptions = Record<string, any>;
+type GetKeysOptions = TransactionOptions & {
+  maxDepth?: number;
+};
+interface DriverFlags {
+  maxDepth?: boolean;
+  ttl?: boolean;
+}
+interface Driver<OptionsT = any, InstanceT = any> {
+  name?: string;
+  flags?: DriverFlags;
+  options?: OptionsT;
+  getInstance?: () => InstanceT;
+  hasItem: (key: string, opts: TransactionOptions) => MaybePromise<boolean>;
+  getItem: (key: string, opts?: TransactionOptions) => MaybePromise<StorageValue>;
+  /** @experimental */
+  getItems?: (items: {
+    key: string;
+    options?: TransactionOptions;
+  }[], commonOptions?: TransactionOptions) => MaybePromise<{
+    key: string;
+    value: StorageValue;
+  }[]>;
+  /** @experimental */
+  getItemRaw?: (key: string, opts: TransactionOptions) => MaybePromise<unknown>;
+  setItem?: (key: string, value: string, opts: TransactionOptions) => MaybePromise<void>;
+  /** @experimental */
+  setItems?: (items: {
+    key: string;
+    value: string;
+    options?: TransactionOptions;
+  }[], commonOptions?: TransactionOptions) => MaybePromise<void>;
+  /** @experimental */
+  setItemRaw?: (key: string, value: any, opts: TransactionOptions) => MaybePromise<void>;
+  removeItem?: (key: string, opts: TransactionOptions) => MaybePromise<void>;
+  getMeta?: (key: string, opts: TransactionOptions) => MaybePromise<StorageMeta | null>;
+  getKeys: (base: string, opts: GetKeysOptions) => MaybePromise<string[]>;
+  clear?: (base: string, opts: TransactionOptions) => MaybePromise<void>;
+  dispose?: () => MaybePromise<void>;
+  watch?: (callback: WatchCallback) => MaybePromise<Unwatch>;
+}
+type StorageDefinition = {
+  items: unknown;
+  [key: string]: unknown;
+};
+type StorageItemMap<T> = T extends StorageDefinition ? T["items"] : T;
+type StorageItemType<T, K> = K extends keyof StorageItemMap<T> ? StorageItemMap<T>[K] : T extends StorageDefinition ? StorageValue : T;
+interface Storage$1<T extends StorageValue = StorageValue> {
+  hasItem<U extends Extract<T, StorageDefinition>, K extends keyof StorageItemMap<U>>(key: K, opts?: TransactionOptions): Promise<boolean>;
+  hasItem(key: string, opts?: TransactionOptions): Promise<boolean>;
+  getItem<U extends Extract<T, StorageDefinition>, K extends string & keyof StorageItemMap<U>>(key: K, ops?: TransactionOptions): Promise<StorageItemType<T, K> | null>;
+  getItem<R = StorageItemType<T, string>>(key: string, opts?: TransactionOptions): Promise<R | null>;
+  /** @experimental */
+  getItems: <U extends T>(items: (string | {
+    key: string;
+    options?: TransactionOptions;
+  })[], commonOptions?: TransactionOptions) => Promise<{
+    key: string;
+    value: U;
+  }[]>;
+  /** @experimental See https://github.com/unjs/unstorage/issues/142 */
+  getItemRaw: <T = any>(key: string, opts?: TransactionOptions) => Promise<MaybeDefined<T> | null>;
+  setItem<U extends Extract<T, StorageDefinition>, K extends keyof StorageItemMap<U>>(key: K, value: StorageItemType<T, K>, opts?: TransactionOptions): Promise<void>;
+  setItem<U extends T>(key: string, value: U, opts?: TransactionOptions): Promise<void>;
+  /** @experimental */
+  setItems: <U extends T>(items: {
+    key: string;
+    value: U;
+    options?: TransactionOptions;
+  }[], commonOptions?: TransactionOptions) => Promise<void>;
+  /** @experimental See https://github.com/unjs/unstorage/issues/142 */
+  setItemRaw: <T = any>(key: string, value: MaybeDefined<T>, opts?: TransactionOptions) => Promise<void>;
+  removeItem<U extends Extract<T, StorageDefinition>, K extends keyof StorageItemMap<U>>(key: K, opts?: (TransactionOptions & {
+    removeMeta?: boolean;
+  }) | boolean): Promise<void>;
+  removeItem(key: string, opts?: (TransactionOptions & {
+    removeMeta?: boolean;
+  }) | boolean): Promise<void>;
+  getMeta: (key: string, opts?: (TransactionOptions & {
+    nativeOnly?: boolean;
+  }) | boolean) => MaybePromise<StorageMeta>;
+  setMeta: (key: string, value: StorageMeta, opts?: TransactionOptions) => Promise<void>;
+  removeMeta: (key: string, opts?: TransactionOptions) => Promise<void>;
+  getKeys: (base?: string, opts?: GetKeysOptions) => Promise<string[]>;
+  clear: (base?: string, opts?: TransactionOptions) => Promise<void>;
+  dispose: () => Promise<void>;
+  watch: (callback: WatchCallback) => Promise<Unwatch>;
+  unwatch: () => Promise<void>;
+  mount: (base: string, driver: Driver) => Storage$1;
+  unmount: (base: string, dispose?: boolean) => Promise<void>;
+  getMount: (key?: string) => {
+    base: string;
+    driver: Driver;
+  };
+  getMounts: (base?: string, options?: {
+    parents?: boolean;
+  }) => {
+    base: string;
+    driver: Driver;
+  }[];
+  keys: Storage$1["getKeys"];
+  get: Storage$1<T>["getItem"];
+  set: Storage$1<T>["setItem"];
+  has: Storage$1<T>["hasItem"];
+  del: Storage$1<T>["removeItem"];
+  remove: Storage$1<T>["removeItem"];
+}
+export { Storage$1 as Storage };

package/dist/_chunks/schema.d.mts

@@ -0,0 +1,74 @@
+/**
+ * Generic Iceberg schema/partition/identity type shapes — the dataset-agnostic
+ * half of `@gscdump/engine/iceberg/schema.ts` (ADR-0021, split per amendment 8).
+ *
+ * TYPES + PURE HELPERS ONLY. The GSC-specific frozen constants
+ * (`IcebergTableName`, `ICEBERG_TABLES`, `ICEBERG_SCHEMAS`, `SEARCH_TYPE_INT`,
+ * `ICEBERG_PARTITION_SPEC`) stay in the engine's own registry instance — they
+ * describe ONE dataset (`gsc.*`), not the mechanism.
+ */
+/** S3-compatible credentials for the Iceberg warehouse object store (R2 in prod). */
+interface IcebergS3Config {
+  /** S3 endpoint host (prod: the R2 S3 endpoint). */
+  endpoint: string;
+  accessKeyId: string;
+  secretAccessKey: string;
+  /** Defaults to `'auto'` (R2's region). */
+  region?: string;
+}
+/** Iceberg-native column type. */
+type IcebergColumnType = 'STRING' | 'INT' | 'LONG' | 'DOUBLE' | 'DATE' | 'BOOLEAN';
+interface IcebergColumn {
+  /** Column name as written into the Iceberg table (snake_case). */
+  name: string;
+  type: IcebergColumnType;
+  /** Iceberg field nullability. Partition identity columns are never null. */
+  required: boolean;
+  /**
+   * Stable Iceberg field id. Field ids — not names — are the schema-evolution
+   * identity in Iceberg; never reuse or renumber an id once a table is live.
+   */
+  fieldId: number;
+}
+/**
+ * Partition-key encoding for identity columns.
+ *
+ * - `'string'` (legacy): identity columns are STRING. Correct, but R2 SQL's
+ *   string min/max statistics are truncated in predicate pushdown, so a bare
+ *   `WHERE site_id='<uuid>'` UNDERCOUNTS.
+ * - `'int'`: identity columns are INT. Integer statistics are fixed-width and
+ *   never truncated, so `WHERE site_id=<n>` is both correct AND prunes.
+ *
+ * New catalogs are provisioned `'int'`; existing legacy catalogs stay `'string'`.
+ */
+type PartitionKeyEncoding = 'string' | 'int';
+/** Default for new Iceberg/R2 Data Catalog tables. */
+declare const DEFAULT_PARTITION_KEY_ENCODING: PartitionKeyEncoding;
+/** Iceberg partition transform applied to a source column. */
+type IcebergPartitionTransform = 'identity' | 'month';
+interface IcebergPartitionField {
+  /** Source column the transform reads. */
+  sourceColumn: string;
+  transform: IcebergPartitionTransform;
+  /** Partition field name as it appears in Iceberg metadata. */
+  name: string;
+}
+interface IcebergTableSpec {
+  namespace: string;
+  table: string;
+  columns: readonly IcebergColumn[];
+  /** Partition spec — identity columns must appear here. */
+  partitionSpec: readonly IcebergPartitionField[];
+  /**
+   * Natural-key columns: a row is uniquely identified by this tuple within
+   * its partition. Drives dedupe. Per amendment 3, the FULL dedupe key is
+   * identity columns + dims columns + `naturalKey` (not `naturalKey` alone —
+   * identity alone would collapse rows across sites/dims).
+   */
+  naturalKey: readonly string[];
+  /** Full dedupe identity: identity columns + dims columns + naturalKey. */
+  identityColumns: readonly string[];
+  /** Physical write-order columns (dimension-first) — undefined skips sort-order. */
+  clusterKey?: readonly string[];
+}
+export { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding };

package/dist/index.d.mts

@@ -0,0 +1,159 @@
+import { CatalogCache, CommitRetryOptions, ConnectIcebergOptions, ICEBERG_TYPE_MAP, IcebergCatalogConfig, IcebergConnection, IcebergFieldSummary, IcebergListedDataFile, IcebergPartitionSpec, IcebergPartitionSpecField, IcebergPrimitiveType, IcebergSchema, IcebergSchemaField, IcebergSortOrder, IcebergSortOrderField, IcebergTableOpResult, ManifestPartitionFilter, PartitionValueMatch, QueryProfiler, ResolveIcebergDataFilesOptions, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, dropIcebergTables, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles } from "./_chunks/catalog.mjs";
+import { DEFAULT_PARTITION_KEY_ENCODING, IcebergColumn, IcebergColumnType, IcebergPartitionField, IcebergPartitionTransform, IcebergS3Config, IcebergTableSpec, PartitionKeyEncoding } from "./_chunks/schema.mjs";
+/**
+ * Closed identity shapes (ADR-0021 amendments 2 + 4). `'site-int'` is the
+ * team-scoped numeric Catalog Site Id (nuxtseo ADR-0091) stored under the
+ * fixed column name `site_id`. `'columns'` is the general N-column identity
+ * with per-column encoding, replacing the original opaque `'custom'` shape.
+ */
+type DatasetIdentity = {
+  kind: 'site-int';
+  encoding?: 'int' | 'string';
+} | {
+  kind: 'columns';
+  columns: readonly {
+    name: string;
+    encoding: 'int32' | 'string';
+  }[];
+};
+/** A per-slice dimension (GSC's `searchType`). Absent for plain snapshot datasets. */
+interface DatasetDim {
+  toPartitionValue: (v: string) => string | number;
+  /** Required (amendment 5) — how the reader decodes this column's manifest bounds. */
+  boundEncoding: 'int32' | 'string';
+}
+interface IcebergDatasetColumnDef {
+  name: string;
+  type: IcebergColumnType;
+  required: boolean;
+}
+interface IcebergDatasetLedger {
+  key: readonly string[];
+}
+interface IcebergDatasetDef {
+  namespace: string;
+  table: string;
+  identity: DatasetIdentity;
+  /** Extra per-slice dimensions; absent for snapshot datasets. */
+  dims?: Record<string, DatasetDim>;
+  /** Data columns (non-identity, non-dims) — includes any partition source column (e.g. `date`). */
+  columns: readonly IcebergDatasetColumnDef[];
+  /** Partition fields; identity columns must appear here. */
+  partition: readonly IcebergPartitionField[];
+  /** Natural-key columns. Full dedupe key = identity + dims + naturalKey (amendment 3). */
+  naturalKey: readonly string[];
+  clusterKey?: readonly string[];
+  /** Consumer-owned exactly-once ledger shape — folded into `appendSink().close()` (amendment 6). */
+  ledger?: IcebergDatasetLedger;
+}
+/**
+ * Derive the full {@link IcebergTableSpec} from a dataset def. Field ids are
+ * assigned sequentially in ONE fixed order — identity columns, then dims
+ * columns, then declared data columns — starting at 1. This mirrors the
+ * engine's original `icebergTableSpec` contract (identity columns first, data
+ * columns from a fixed base) so a def-derived spec for an existing table
+ * (`crawl.pages`, `lighthouse.scans`, `dataforseo.keywords`) is BYTE-IDENTICAL
+ * to the frozen inline constants it replaces.
+ */
+declare function deriveTableSpec(def: IcebergDatasetDef): IcebergTableSpec;
+/**
+ * Convert a `YYYY-MM-DD` string / `Date` / already-numeric day-count to the
+ * integer "days since the Unix epoch" the Iceberg `date` type stores.
+ * hyparquet-writer mis-encodes Date-valued dictionary columns, so callers
+ * should feed this INTO their row before `appendRows`/`appendSink.emit` for
+ * any `month`-partitioned date column.
+ */
+declare function toIcebergDayCount(value: string | Date | number): number;
+/** Options shared by `appendRows` / `appendSink`. */
+interface AppendCommitOptions {
+  commitRetry?: CommitRetryOptions;
+}
+interface AppendSinkOptions extends AppendCommitOptions {
+  catalog: IcebergCatalogConfig;
+  connect?: ConnectIcebergOptions;
+  /** Invoked ONLY after a successful flush (amendment 6) — never on a failed/empty close. */
+  ledger?: {
+    record: () => Promise<void> | void;
+  };
+}
+interface AppendResult {
+  /** Rows accepted after the identity guard + dedupe (what was actually appended). */
+  accepted: number;
+  /** Rows dropped by the identity guard (missing/out-of-range required identity value). */
+  skipped: number;
+}
+interface AppendSinkCloseResult extends AppendResult {
+  flushed: boolean;
+  error?: unknown;
+}
+interface AppendSink {
+  emit: (rows: readonly Record<string, unknown>[]) => void;
+  close: () => Promise<AppendSinkCloseResult>;
+}
+interface ResolveDataFilesOptions {
+  cache?: CatalogCache;
+  clock?: () => number;
+  profiler?: QueryProfiler;
+}
+interface IcebergDataset {
+  readonly def: IcebergDatasetDef;
+  readonly tableSpec: IcebergTableSpec;
+  icebergSchema: () => IcebergSchema;
+  icebergPartitionSpec: () => IcebergPartitionSpec;
+  icebergSortOrder: () => IcebergSortOrder | undefined;
+  createTable: (conn: IcebergConnection) => Promise<IcebergTableOpResult[]>;
+  /**
+   * One-shot append over an EXISTING connection: identity INT32 guard, dedupe
+   * by identity+dims+naturalKey (last-wins), cluster pre-sort (if
+   * `clusterKey` is set), then a single `icebergAppendRetrying` commit. `rows`
+   * must already be in final storage-column shape — identity/dims/data
+   * columns keyed by their STORED column names, with any `month`-partitioned
+   * date column pre-converted via {@link toIcebergDayCount}.
+   */
+  appendRows: (conn: IcebergConnection, rows: readonly Record<string, unknown>[], opts?: AppendCommitOptions) => Promise<AppendResult>;
+  /**
+   * PURE row processing — the identity INT32 guard, dedupe (identity+dims+
+   * naturalKey, last-wins) and cluster pre-sort `appendRows`/`appendSink`
+   * apply before committing, exposed standalone with NO network/icebird call.
+   * Lets a consumer that owns its own commit call-site (e.g. one still
+   * calling a frozen `icebergAppendRetrying` import directly, for test-mock
+   * compatibility) still route its dedupe/guard/sort logic through the
+   * dataset definition.
+   */
+  prepareRows: (rows: readonly Record<string, unknown>[]) => {
+    records: Record<string, unknown>[];
+    skipped: number;
+  };
+  /** Buffered multi-emit sink — owns its own connect/close lifecycle (ADR-0021's primary contract). */
+  appendSink: (opts: AppendSinkOptions) => AppendSink;
+  /** `PartitionValueMatch[]` for one identity value (+ optional dims), for manual manifest filtering. */
+  readerPredicate: (identity: string | number, dims?: Record<string, string>) => PartitionValueMatch[];
+  /** A ready-to-use manifest partition filter for one identity value + wanted months. */
+  partitionBoundFilter: (identity: string | number, months: ReadonlySet<number>, dims?: Record<string, string>) => ManifestPartitionFilter;
+  /** Resolve the data files for one identity value across a date range. */
+  resolveDataFiles: (conn: IcebergConnection, identity: string | number, range: {
+    start: string;
+    end: string;
+  }, dims?: Record<string, string>, opts?: ResolveDataFilesOptions) => Promise<IcebergListedDataFile[]>;
+}
+/**
+ * Declare an Iceberg dataset — one `namespace.table` — deriving its schema,
+ * partition spec, dedupe key, INT32 guard, append path, and reader helpers
+ * from the def. See {@link IcebergDatasetDef}.
+ */
+declare function defineIcebergDataset(def: IcebergDatasetDef): IcebergDataset;
+interface PyIcebergWriterResult {
+  rowCount?: number;
+  error?: string;
+}
+interface RunPyIcebergWriterOptions {
+  python: string;
+  script: string;
+  job: unknown;
+  label: string;
+  processErrorAsParseFailure?: boolean;
+  rejectOnProcessError?: boolean;
+}
+declare function resolvePyIcebergPython(override?: string): string;
+declare function runPyIcebergWriter<T extends PyIcebergWriterResult>(options: RunPyIcebergWriterOptions): Promise<T>;
+export { type AppendCommitOptions, type AppendResult, type AppendSink, type AppendSinkCloseResult, type AppendSinkOptions, type CatalogCache, type CommitRetryOptions, type ConnectIcebergOptions, DEFAULT_PARTITION_KEY_ENCODING, type DatasetDim, type DatasetIdentity, ICEBERG_TYPE_MAP, type IcebergCatalogConfig, type IcebergColumn, type IcebergColumnType, type IcebergConnection, type IcebergDataset, type IcebergDatasetColumnDef, type IcebergDatasetDef, type IcebergDatasetLedger, type IcebergFieldSummary, type IcebergListedDataFile, type IcebergPartitionField, type IcebergPartitionSpec, type IcebergPartitionSpecField, type IcebergPartitionTransform, type IcebergPrimitiveType, type IcebergS3Config, type IcebergSchema, type IcebergSchemaField, type IcebergSortOrder, type IcebergSortOrderField, type IcebergTableOpResult, type IcebergTableSpec, type ManifestPartitionFilter, type PartitionKeyEncoding, type PartitionValueMatch, type PyIcebergWriterResult, type QueryProfiler, type ResolveDataFilesOptions, type ResolveIcebergDataFilesOptions, type RunPyIcebergWriterOptions, buildManifestPartitionFilter, cacheGet, cachePut, connectIcebergCatalog, defineIcebergDataset, deriveTableSpec, dropIcebergTables, ensureIcebergNamespace, listIcebergTables, resolveIcebergDataFiles, resolvePyIcebergPython, runPyIcebergWriter, toIcebergDayCount };