@gscdump/engine 0.9.2 → 0.11.0

package/dist/resolver/index.mjs

@@ -1,3 +1,3 @@
-import { _ as resolveToSQL, a as createResolverAdapter, c as LOGICAL_DATASETS, d as inferLogicalDataset, f as supportsDimensionOnSurface, g as resolveComparisonSQL, h as mergeExtras, i as compileSqlite, l as assertDimensionsSupported, m as buildTotalsSql, n as pgResolverAdapter, o as createSqlFragments, p as buildExtrasQueries, r as compilePg, s as DIMENSION_SURFACES, t as createParquetResolverAdapter, u as dimensionColumn, v as resolveToSQLOptimized } from "../_chunks/pg-adapter.mjs";
-import { a as getFilterDimensions, c as matchesMetricFilter, d as createSqlQuerySource, i as getDimensionFilters, l as matchesTopLevelPage, n as assertSchemaInSync, o as getInternalFilters, r as dimensionValue, s as matchesDimensionFilter, t as isSqlQuerySource, u as metricValue } from "../_chunks/resolver.mjs";
-export { DIMENSION_SURFACES, LOGICAL_DATASETS, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, compilePg, compileSqlite, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, createSqlQuerySource, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isSqlQuerySource, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, supportsDimensionOnSurface };
+import { a as DIMENSION_SURFACES, c as dimensionColumn, d as buildExtrasQueries, f as buildTotalsSql, g as resolveToSQLOptimized, h as resolveToSQL, i as createSqlFragments, l as inferLogicalDataset, m as resolveComparisonSQL, n as pgResolverAdapter, o as LOGICAL_DATASETS, p as mergeExtras, r as createResolverAdapter, s as assertDimensionsSupported, t as createParquetResolverAdapter, u as supportsDimensionOnSurface } from "../_chunks/pg-adapter.mjs";
+import { a as getInternalFilters, c as matchesTopLevelPage, i as getFilterDimensions, l as metricValue, n as dimensionValue, o as matchesDimensionFilter, r as getDimensionFilters, s as matchesMetricFilter, t as assertSchemaInSync } from "../_chunks/resolver.mjs";
+export { DIMENSION_SURFACES, LOGICAL_DATASETS, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, supportsDimensionOnSurface };

package/dist/rollups.d.mts

@@ -0,0 +1,196 @@
+import { a as DataSource } from "./_chunks/storage.mjs";
+import { t as ColumnDef } from "./_chunks/schema.mjs";
+import { TenantCtx } from "gscdump/contracts";
+import * as _$_gscdump_engine_contracts0 from "@gscdump/engine/contracts";
+interface RollupCtx extends TenantCtx {
+  /** When the rollup was built. Stamped into payload + filename. */
+  builtAt: number;
+}
+/**
+ * Tenant-scoped engine surface a rollup builder needs. Subset of
+ * `StorageEngine.runSQL` so rollups stay testable without a full engine.
+ */
+interface RollupEngine {
+  runSQL: (opts: {
+    ctx: TenantCtx;
+    fileSets: Record<string, {
+      table: _$_gscdump_engine_contracts0.TableName;
+      partitions?: string[];
+    }>;
+    table?: _$_gscdump_engine_contracts0.TableName;
+    sql: string;
+    params?: unknown[];
+  }) => Promise<{
+    rows: _$_gscdump_engine_contracts0.Row[];
+  }>;
+}
+/**
+ * One rollup definition. Build runs SQL over the tenant's facts and/or reads
+ * from entity stores via `dataSource`, returning a JSON-serializable payload
+ * that the runner timestamps + writes.
+ */
+interface RollupDef {
+  id: string;
+  /**
+   * Window in days the rollup covers. `null` means full history. Used by
+   * the runner to populate `windowDays` in the payload metadata so readers
+   * can validate freshness.
+   */
+  windowDays: number | null;
+  /**
+   * Storage format. `'json'` (default) wraps the build payload in a
+   * `RollupEnvelope` and writes as a JSON blob. `'parquet'` expects `build`
+   * to return rows matching `parquetColumns` and writes a parquet file plus
+   * a tiny JSON sidecar envelope that points at it, so metadata
+   * (`builtAt` / `windowDays`) stays readable without decoding parquet.
+   */
+  format?: 'json' | 'parquet';
+  /**
+   * Column schema for parquet output. Required when `format === 'parquet'`.
+   * Types map the same way as the fact-table encoder: VARCHAR / DATE go
+   * through BYTE_ARRAY/UTF8; BIGINT → INT64; INTEGER → INT32; DOUBLE → DOUBLE.
+   */
+  parquetColumns?: readonly ColumnDef[];
+  /** Sort-key column names for parquet row-group stats. Optional. */
+  parquetSortKey?: readonly string[];
+  build: (deps: {
+    engine: RollupEngine;
+    ctx: TenantCtx;
+    /**
+     * Tenant-scoped object store. Rollups that aggregate over entity
+     * snapshots (e.g. indexing metadata) read JSON docs through this.
+     * Pure-SQL rollups can ignore it.
+     */
+    dataSource: DataSource;
+    /**
+     * Wall-clock millis when the runner started this rollup. Use for
+     * derived window cutoffs (e.g. trailing-28d boundary) so the SQL can
+     * inline a date literal and stay portable across DuckDB builds that
+     * don't bundle the ICU extension (Workers DuckDB, for one — CURRENT_DATE
+     * lives in ICU).
+     */
+    builtAt: number;
+  }) => Promise<unknown>;
+}
+/**
+ * Wire shape persisted to R2/disk. Readers can rely on the `version` + `builtAt`.
+ * Parquet rollups write this envelope as a sidecar whose `payload` points at
+ * the co-located `.parquet` object via `{ parquetKey, rowCount }`.
+ */
+interface RollupEnvelope<T = unknown> {
+  version: 1;
+  id: string;
+  builtAt: number;
+  windowDays: number | null;
+  payload: T;
+}
+interface ParquetRollupPointer {
+  parquetKey: string;
+  rowCount: number;
+}
+declare function rollupKey(ctx: TenantCtx, id: string, builtAt: number): string;
+declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number): string;
+interface RebuildRollupsOptions {
+  engine: RollupEngine;
+  dataSource: DataSource;
+  ctx: TenantCtx;
+  defs: readonly RollupDef[];
+  now?: () => number;
+}
+interface RebuildRollupResult {
+  id: string;
+  /** JSON envelope key. For parquet rollups this is the sidecar pointer. */
+  objectKey: string;
+  /** Parquet payload key. Present only when `format === 'parquet'`. */
+  parquetKey?: string;
+  /** Envelope byte size; for parquet rollups does NOT include parquet bytes. */
+  bytes: number;
+  /** Parquet payload byte size when `format === 'parquet'`. */
+  parquetBytes?: number;
+  builtAt: number;
+}
+declare function rebuildRollups(opts: RebuildRollupsOptions): Promise<RebuildRollupResult[]>;
+/**
+ * Daily totals across the full history. One row per (date, table) with
+ * clicks + impressions + position. Powers sparklines and headline totals.
+ *
+ * Includes `anonymizedImpressionsPct` per day computed as
+ *   1 - sum(query_grained_impressions) / sum(page_grained_impressions)
+ * — surfaces GSC's anonymous-query gap so the dashboard can warn users not
+ * to trust query-grained breakdowns as comprehensive.
+ */
+declare const dailyTotalsRollup: RollupDef;
+/** Weekly totals, ISO week aligned. Cheap and stable for trend widgets. */
+declare const weeklyTotalsRollup: RollupDef;
+/**
+ * Top 1000 pages by clicks over the trailing 28-day window. JSON for v1;
+ * promote to parquet (`top_pages_28d.parquet`) when the dashboard needs
+ * server-side WHERE filtering on this rollup.
+ */
+declare const topPages28dRollup: RollupDef;
+/**
+ * Top 250 countries by clicks over the trailing 28-day window. Countries
+ * cardinality is bounded (~250 ISO codes), so the list fits in a tiny JSON
+ * payload regardless of traffic shape. Powers a geo-overview widget without
+ * spinning up DuckDB-WASM.
+ */
+declare const topCountries28dRollup: RollupDef;
+/** Top 1000 keywords by clicks over the trailing 28-day window. */
+declare const topKeywords28dRollup: RollupDef;
+/**
+ * Parquet-format companion to `topKeywords28dRollup`. Same shape, but persists
+ * as a parquet object plus JSON sidecar pointer so widgets that need
+ * server-side WHERE (filter by prefix, by clicks threshold, paginate) can scan
+ * it directly with DuckDB-WASM instead of loading all 1000 rows into JS.
+ *
+ * Opt-in: include in the caller's rollup def list alongside (or instead of)
+ * the JSON variant; the runner treats the two as independent ids so they can
+ * coexist during a migration.
+ */
+declare const topKeywords28dParquetRollup: RollupDef;
+/**
+ * Aggregates the per-URL Indexing API metadata entity store (populated by
+ * `gscdump entities indexing snapshot`) into daily counts of `URL_UPDATED`
+ * and `URL_REMOVED` notifications. Covers the third entity-snapshot shape
+ * without needing its own parquet family — publish events are sparse and
+ * aggregate cleanly into a small JSON rollup.
+ *
+ * Safe no-op when the entity store is empty: returns `{ totals: {...}, days: [] }`
+ * so downstream readers don't have to special-case first-run sites.
+ */
+declare const indexingMetadataRollup: RollupDef;
+/**
+ * Indexing-API health by day: per `inspectedAt` date, counts of indexed,
+ * soft-404, redirect, not-found, mobile passes, rich-results passes, and
+ * canonical mismatches. Sourced from the inspections parquet sidecar
+ * (`InspectionStore.parquetUri`), which holds the latest record per URL.
+ *
+ * Empty-payload no-op when the sidecar URI is unavailable (in-memory
+ * `DataSource`, or before `materialize` has run).
+ */
+declare const indexingHealthRollup: RollupDef;
+/**
+ * Per-day index-percent: ratio of (sitemap URLs that received GSC clicks on
+ * that date) / (total live sitemap URLs). Uses a DuckDB JOIN between the
+ * sitemap urls parquet (`SitemapStore.urlsParquetUri`) and the `pages` fact
+ * parquet. Total denominator is the count of live URLs in the urls index;
+ * numerator is per-day distinct loc count where pages.clicks > 0.
+ */
+declare const indexPercentRollup: RollupDef;
+/**
+ * Sitemap-health per-day series materialized from the sitemap-store JSON
+ * index. Each `SitemapRecord` carries `urlCount`, `errors`, `warnings`,
+ * `contentHash`, and `lastDownloaded`. We bucket records by the day of their
+ * `capturedAt` (or `lastDownloaded` fallback) and emit per-day aggregates plus
+ * a snapshot of per-feed stats at the most recent capture.
+ */
+declare const sitemapHealthRollup: RollupDef;
+/**
+ * Trailing-28-day sitemap URL changes: per-day per-feedpath {added, removed}
+ * counts plus rolling top-200 added and removed URLs. Streams from
+ * `SitemapStore.loadDeltas()` so it scales independently of how many feeds
+ * exist on the site.
+ */
+declare const sitemapChanges28dRollup: RollupDef;
+declare const DEFAULT_ROLLUPS: readonly RollupDef[];
+export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildRollupResult, RebuildRollupsOptions, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };