@gscdump/engine 0.4.0

package/dist/rollups.d.mts

@@ -0,0 +1,207 @@
+import { ColumnDef, Row, TableName, TenantCtx } from "gscdump/contracts";
+interface DataSource {
+  read: (key: string, range?: {
+    offset: number;
+    length: number;
+  }, signal?: AbortSignal) => Promise<Uint8Array>;
+  write: (key: string, bytes: Uint8Array) => Promise<void>;
+  delete: (keys: string[]) => Promise<void>;
+  /**
+   * One-shot listing under a prefix. Implementations may cap the number of
+   * returned keys (typically 10k) — callers iterating full tenant space
+   * should prefer `streamList` when available or narrow the prefix.
+   */
+  list: (prefix: string) => Promise<string[]>;
+  /**
+   * Per-key URI probe. Returns a URI string DuckDB's `httpfs` (or an
+   * equivalent engine that fetches its own I/O) can read directly, or
+   * `undefined` if the key isn't URI-resolvable on this backend and the
+   * caller must fall back to `read(key)` for the bytes.
+   *
+   * Contracts:
+   * - When defined, the returned URI MUST yield byte-identical content to
+   *   `read(key)`. Callers rely on this for correctness.
+   * - Backends with a native URI for every key (filesystem: absolute path,
+   *   R2 via `httpfs`: signed URL) may always return a string.
+   * - Backends without a native URI shape (in-memory) omit the method or
+   *   return `undefined` per call.
+   * - Mixed-per-query is allowed: some keys in one query may return a URI,
+   *   others may not; the executor branches per key.
+   */
+  uri?: (key: string) => string | undefined;
+  /**
+   * Optional — probe the byte size of a key without reading it. Used by
+   * the engine to fill in `WriteResult.bytes` when a codec reports 0 or
+   * unknown but the file is non-trivial.
+   */
+  head?: (key: string) => Promise<{
+    bytes: number;
+  } | undefined>;
+  /**
+   * Optional streaming variant of `list`. Implementations that page
+   * backing-store results (R2, S3) should implement this and yield keys
+   * lazily. `list` may return up to an adapter-defined cap (typically
+   * 10k keys); callers iterating full tenant space must prefer
+   * `streamList` when available, or chunk by narrower prefixes.
+   */
+  streamList?: (prefix: string) => AsyncIterable<string>;
+}
+interface RollupCtx extends TenantCtx {
+  /** When the rollup was built. Stamped into payload + filename. */
+  builtAt: number;
+}
+/**
+ * Tenant-scoped engine surface a rollup builder needs. Subset of
+ * `StorageEngine.runSQL` so rollups stay testable without a full engine.
+ */
+interface RollupEngine {
+  runSQL: (opts: {
+    ctx: TenantCtx;
+    fileSets: Record<string, {
+      table: TableName;
+      partitions?: string[];
+    }>;
+    table?: TableName;
+    sql: string;
+    params?: unknown[];
+  }) => Promise<{
+    rows: Row[];
+  }>;
+}
+/**
+ * One rollup definition. Build runs SQL over the tenant's facts and/or reads
+ * from entity stores via `dataSource`, returning a JSON-serializable payload
+ * that the runner timestamps + writes.
+ */
+interface RollupDef {
+  id: string;
+  /**
+   * Window in days the rollup covers. `null` means full history. Used by
+   * the runner to populate `windowDays` in the payload metadata so readers
+   * can validate freshness.
+   */
+  windowDays: number | null;
+  /**
+   * Storage format. `'json'` (default) wraps the build payload in a
+   * `RollupEnvelope` and writes as a JSON blob. `'parquet'` expects `build`
+   * to return rows matching `parquetColumns` and writes a parquet file plus
+   * a tiny JSON sidecar envelope that points at it, so metadata
+   * (`builtAt` / `windowDays`) stays readable without decoding parquet.
+   */
+  format?: 'json' | 'parquet';
+  /**
+   * Column schema for parquet output. Required when `format === 'parquet'`.
+   * Types map the same way as the fact-table encoder: VARCHAR / DATE go
+   * through BYTE_ARRAY/UTF8; BIGINT → INT64; INTEGER → INT32; DOUBLE → DOUBLE.
+   */
+  parquetColumns?: readonly ColumnDef[];
+  /** Sort-key column names for parquet row-group stats. Optional. */
+  parquetSortKey?: readonly string[];
+  build: (deps: {
+    engine: RollupEngine;
+    ctx: TenantCtx;
+    /**
+     * Tenant-scoped object store. Rollups that aggregate over entity
+     * snapshots (e.g. indexing metadata) read JSON docs through this.
+     * Pure-SQL rollups can ignore it.
+     */
+    dataSource: DataSource;
+    /**
+     * Wall-clock millis when the runner started this rollup. Use for
+     * derived window cutoffs (e.g. trailing-28d boundary) so the SQL can
+     * inline a date literal and stay portable across DuckDB builds that
+     * don't bundle the ICU extension (Workers DuckDB, for one — CURRENT_DATE
+     * lives in ICU).
+     */
+    builtAt: number;
+  }) => Promise<unknown>;
+}
+/**
+ * Wire shape persisted to R2/disk. Readers can rely on the `version` + `builtAt`.
+ * Parquet rollups write this envelope as a sidecar whose `payload` points at
+ * the co-located `.parquet` object via `{ parquetKey, rowCount }`.
+ */
+interface RollupEnvelope<T = unknown> {
+  version: 1;
+  id: string;
+  builtAt: number;
+  windowDays: number | null;
+  payload: T;
+}
+interface ParquetRollupPointer {
+  parquetKey: string;
+  rowCount: number;
+}
+declare function rollupKey(ctx: TenantCtx, id: string, builtAt: number): string;
+declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number): string;
+interface RebuildRollupsOptions {
+  engine: RollupEngine;
+  dataSource: DataSource;
+  ctx: TenantCtx;
+  defs: readonly RollupDef[];
+  now?: () => number;
+}
+interface RebuildRollupResult {
+  id: string;
+  /** JSON envelope key. For parquet rollups this is the sidecar pointer. */
+  objectKey: string;
+  /** Parquet payload key. Present only when `format === 'parquet'`. */
+  parquetKey?: string;
+  /** Envelope byte size; for parquet rollups does NOT include parquet bytes. */
+  bytes: number;
+  /** Parquet payload byte size when `format === 'parquet'`. */
+  parquetBytes?: number;
+  builtAt: number;
+}
+declare function rebuildRollups(opts: RebuildRollupsOptions): Promise<RebuildRollupResult[]>;
+/**
+ * Daily totals across the full history. One row per (date, table) with
+ * clicks + impressions + position. Powers sparklines and headline totals.
+ *
+ * Includes `anonymizedImpressionsPct` per day computed as
+ *   1 - sum(query_grained_impressions) / sum(page_grained_impressions)
+ * — surfaces GSC's anonymous-query gap so the dashboard can warn users not
+ * to trust query-grained breakdowns as comprehensive.
+ */
+declare const dailyTotalsRollup: RollupDef;
+/** Weekly totals, ISO week aligned. Cheap and stable for trend widgets. */
+declare const weeklyTotalsRollup: RollupDef;
+/**
+ * Top 1000 pages by clicks over the trailing 28-day window. JSON for v1;
+ * promote to parquet (`top_pages_28d.parquet`) when the dashboard needs
+ * server-side WHERE filtering on this rollup.
+ */
+declare const topPages28dRollup: RollupDef;
+/**
+ * Top 250 countries by clicks over the trailing 28-day window. Countries
+ * cardinality is bounded (~250 ISO codes), so the list fits in a tiny JSON
+ * payload regardless of traffic shape. Powers a geo-overview widget without
+ * spinning up DuckDB-WASM.
+ */
+declare const topCountries28dRollup: RollupDef;
+/** Top 1000 keywords by clicks over the trailing 28-day window. */
+declare const topKeywords28dRollup: RollupDef;
+/**
+ * Parquet-format companion to `topKeywords28dRollup`. Same shape, but persists
+ * as a parquet object plus JSON sidecar pointer so widgets that need
+ * server-side WHERE (filter by prefix, by clicks threshold, paginate) can scan
+ * it directly with DuckDB-WASM instead of loading all 1000 rows into JS.
+ *
+ * Opt-in: include in the caller's rollup def list alongside (or instead of)
+ * the JSON variant; the runner treats the two as independent ids so they can
+ * coexist during a migration.
+ */
+declare const topKeywords28dParquetRollup: RollupDef;
+/**
+ * Aggregates the per-URL Indexing API metadata entity store (populated by
+ * `gscdump entities indexing snapshot`) into daily counts of `URL_UPDATED`
+ * and `URL_REMOVED` notifications. Covers the third entity-snapshot shape
+ * without needing its own parquet family — publish events are sparse and
+ * aggregate cleanly into a small JSON rollup.
+ *
+ * Safe no-op when the entity store is empty: returns `{ totals: {...}, days: [] }`
+ * so downstream readers don't have to special-case first-run sites.
+ */
+declare const indexingMetadataRollup: RollupDef;
+declare const DEFAULT_ROLLUPS: readonly RollupDef[];
+export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildRollupResult, RebuildRollupsOptions, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, dailyTotalsRollup, indexingMetadataRollup, rebuildRollups, rollupKey, rollupParquetKey, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };