@gscdump/engine 0.28.3 → 0.30.0

package/dist/iceberg/index.mjs

@@ -1,5 +1,5 @@
 import { engineErrors } from "../errors.mjs";
-import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
+import { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, icebergPartitionColumns, icebergSchemasFor, icebergTableSpec, isIcebergTable } from "../_chunks/schema2.mjs";
 import { cachingResolver, icebergAppend, icebergCreateTable, icebergDropTable, icebergManifests, restCatalogConnect, restCatalogCreateNamespace, restCatalogListTables, restCatalogLoadTable, s3SignedResolver } from "../_chunks/libs/icebird.mjs";
 import { err, ok } from "gscdump/result";
 async function cacheGet(cache, key, now) {
@@ -35,21 +35,25 @@ function decodeInt(bytes) {
 	if (u == null) return null;
 	return new DataView(u.buffer, u.byteOffset, u.byteLength).getInt32(0, true);
 }
-function buildPartitionFilter(siteId, searchType, wantedMonths) {
+function buildPartitionFilter(siteId, searchType, wantedMonths, encoding = "string") {
 	return (partitions) => {
 		const parts = partitions;
 		if (!parts || parts.length === 0) return true;
-		const siteSummary = parts[SITE_ID_FIELD_INDEX];
-		if (siteSummary && (siteSummary.lower_bound != null || siteSummary.upper_bound != null)) {
-			const lo = decodeString(siteSummary.lower_bound);
-			const hi = decodeString(siteSummary.upper_bound);
-			if (lo != null && hi != null && (siteId < lo || siteId > hi)) return false;
-		}
-		const searchTypeSummary = parts[SEARCH_TYPE_FIELD_INDEX];
-		if (searchTypeSummary && (searchTypeSummary.lower_bound != null || searchTypeSummary.upper_bound != null)) {
-			const lo = decodeString(searchTypeSummary.lower_bound);
-			const hi = decodeString(searchTypeSummary.upper_bound);
-			if (lo != null && hi != null && (searchType < lo || searchType > hi)) return false;
+		if (encoding === "string") {
+			const siteStr = String(siteId);
+			const searchStr = String(searchType);
+			const siteSummary = parts[SITE_ID_FIELD_INDEX];
+			if (siteSummary && (siteSummary.lower_bound != null || siteSummary.upper_bound != null)) {
+				const lo = decodeString(siteSummary.lower_bound);
+				const hi = decodeString(siteSummary.upper_bound);
+				if (lo != null && hi != null && (siteStr < lo || siteStr > hi)) return false;
+			}
+			const searchTypeSummary = parts[SEARCH_TYPE_FIELD_INDEX];
+			if (searchTypeSummary && (searchTypeSummary.lower_bound != null || searchTypeSummary.upper_bound != null)) {
+				const lo = decodeString(searchTypeSummary.lower_bound);
+				const hi = decodeString(searchTypeSummary.upper_bound);
+				if (lo != null && hi != null && (searchStr < lo || searchStr > hi)) return false;
+			}
 		}
 		const monthSummary = parts[DATE_MONTH_FIELD_INDEX];
 		if (monthSummary && (monthSummary.lower_bound != null || monthSummary.upper_bound != null)) {
@@ -74,11 +78,11 @@ const ICEBERG_TYPE_MAP = {
 	DOUBLE: "double",
 	DATE: "date"
 };
-function icebergSchemaFor(table) {
+function icebergSchemaFor(table, encoding = "string") {
 	return {
 		"type": "struct",
 		"schema-id": 0,
-		"fields": ICEBERG_SCHEMAS[table].columns.map((col) => ({
+		"fields": icebergSchemasFor(encoding)[table].columns.map((col) => ({
 			id: col.fieldId,
 			name: col.name,
 			required: col.required,
@@ -86,8 +90,8 @@ function icebergSchemaFor(table) {
 		}))
 	};
 }
-function icebergPartitionSpecFor(table) {
-	const fields = ICEBERG_SCHEMAS[table].columns;
+function icebergPartitionSpecFor(table, encoding = "string") {
+	const fields = icebergSchemasFor(encoding)[table].columns;
 	const fieldId = (name) => {
 		const col = fields.find((c) => c.name === name);
 		if (!col) throw new Error(`iceberg-catalog: table '${table}' has no '${name}' column`);
@@ -176,14 +180,14 @@ async function icebergAppendRetrying(args, options = {}) {
 async function ensureIcebergNamespace(conn) {
 	await restCatalogCreateNamespace(conn.catalog, { namespace: conn.namespace }).catch(() => {});
 }
-async function createIcebergTables(conn, tables = ICEBERG_TABLES) {
+async function createIcebergTables(conn, tables = ICEBERG_TABLES, encoding = "string") {
 	const results = [];
 	for (const table of tables) await icebergCreateTable({
 		catalog: conn.catalog,
 		namespace: conn.namespace,
 		table,
-		schema: icebergSchemaFor(table),
-		partitionSpec: icebergPartitionSpecFor(table)
+		schema: icebergSchemaFor(table, encoding),
+		partitionSpec: icebergPartitionSpecFor(table, encoding)
 	}).then(() => results.push({
 		table,
 		outcome: ok(void 0)
@@ -275,20 +279,22 @@ async function listIcebergDataFiles(conn, opts) {
 		if (snapshotId == null || !metadata) return [];
 	}
 	const endWalk = profiler?.start("iceberg.walk");
-	const partitionFilter = buildPartitionFilter(opts.siteId, opts.searchType, wantedMonths);
+	const partitionFilter = buildPartitionFilter(opts.siteId, opts.searchType, wantedMonths, opts.encoding ?? "string");
 	const manifests = await icebergManifests({
 		metadata,
 		resolver: conn.resolver,
 		partitionFilter
 	});
+	const wantSite = String(opts.siteId);
+	const wantSearch = String(opts.searchType);
 	const out = [];
 	for (const m of manifests) for (const entry of m.entries) {
 		if (entry.status === 2) continue;
 		const df = entry.data_file;
 		if (df.content !== 0) continue;
 		const part = df.partition;
-		if (part.site_id !== opts.siteId) continue;
-		if (part.search_type !== opts.searchType) continue;
+		if (String(part.site_id) !== wantSite) continue;
+		if (String(part.search_type) !== wantSearch) continue;
 		const month = part.date_month;
 		if (typeof month !== "number" || !wantedMonths.has(month)) continue;
 		out.push({
@@ -326,6 +332,8 @@ async function dropIcebergTables(conn, tables) {
 	return results;
 }
 const DAY_MILLIS = 864e5;
+const INT32_MIN = -2147483648;
+const INT32_MAX = 2147483647;
 function toIcebergDate(value) {
 	if (typeof value === "string") {
 		const ms = Date.parse(`${value}T00:00:00Z`);
@@ -343,6 +351,14 @@ function coerceJsonSafe(value) {
 	if (typeof value === "bigint") return Number(value);
 	return value;
 }
+function toIntPartitionSiteId(value) {
+	if (value == null || typeof value === "string" && value.trim() === "") throw new TypeError("toRecords: slice.ctx.siteId is required for int partition encoding");
+	if (typeof value !== "string" && typeof value !== "number" && typeof value !== "bigint") throw new TypeError(`toRecords: int partition site_id must be a safe integer, got '${String(value)}'`);
+	const siteId = Number(value);
+	if (!Number.isSafeInteger(siteId)) throw new TypeError(`toRecords: int partition site_id must be a safe integer, got '${String(value)}'`);
+	if (siteId < INT32_MIN || siteId > INT32_MAX) throw new TypeError(`toRecords: int partition site_id must fit Iceberg INT, got '${String(value)}'`);
+	return siteId;
+}
 function dedupeByIdentity(table, records) {
 	if (records.length < 2) return records;
 	const key = ICEBERG_SCHEMAS[table].identityColumns;
@@ -353,19 +369,21 @@ function dedupeByIdentity(table, records) {
 	}
 	return seen.size === records.length ? records : [...seen.values()];
 }
-function toRecords(slice, rows) {
-	const siteId = slice.ctx.siteId ?? "";
+function toRecords(slice, rows, encoding) {
+	const siteVal = encoding === "int" ? toIntPartitionSiteId(slice.ctx.siteId) : slice.ctx.siteId ?? "";
+	const searchVal = encoding === "int" ? SEARCH_TYPE_INT[slice.searchType] : slice.searchType;
 	return rows.map((row) => {
 		const out = {};
 		for (const k in row) out[k] = coerceJsonSafe(row[k]);
 		out.date = toIcebergDate(out.date);
-		out.site_id = siteId;
-		out.search_type = slice.searchType;
+		out.site_id = siteVal;
+		out.search_type = searchVal;
 		return out;
 	});
 }
 function createIcebergAppendSink(options) {
 	let connection;
+	const encoding = options.encoding ?? "string";
 	const buffers = /* @__PURE__ */ new Map();
 	function connect() {
 		connection ??= connectIcebergCatalog(options.catalog);
@@ -375,7 +393,7 @@ function createIcebergAppendSink(options) {
 		capabilities: { appendOnly: true },
 		async emit(slice, rows) {
 			if (rows.length === 0) return { rowCount: 0 };
-			const records = toRecords(slice, rows);
+			const records = toRecords(slice, rows, encoding);
 			const buffer = buffers.get(slice.table);
 			if (buffer) for (let i = 0; i < records.length; i++) buffer.push(records[i]);
 			else buffers.set(slice.table, records);
@@ -429,4 +447,4 @@ function createIcebergAppendSink(options) {
 		}
 	};
 }
-export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_TABLES, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionSpecFor, icebergSchemaFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };
+export { ICEBERG_FIELD_ID_BASE, ICEBERG_PARTITION_COLUMNS, ICEBERG_PARTITION_SPEC, ICEBERG_SCHEMAS, ICEBERG_SCHEMAS_INT, ICEBERG_TABLES, INT_SEARCH_TYPE, SEARCH_TYPE_INT, assertIcebergTable, connectIcebergCatalog, createIcebergAppendSink, createIcebergTables, dropIcebergTables, ensureIcebergNamespace, icebergAppendRetrying, icebergCreateTable, icebergManifests, icebergPartitionColumns, icebergPartitionSpecFor, icebergSchemaFor, icebergSchemasFor, icebergTableSpec, isCommitRateLimited, isIcebergTable, listIcebergDataFiles, listIcebergTables, restCatalogLoadTable };

package/dist/ingest.mjs

@@ -26,7 +26,7 @@ function toPath(gscUrl) {
 	}
 }
 function toSumPosition(apiPosition, impressions) {
-	return (apiPosition - 1) * Math.max(impressions, 1);
+	return ((apiPosition >= 1 ? apiPosition : 1) - 1) * Math.max(impressions, 1);
 }
 function transformGscRow(table, apiRow, options = {}) {
 	const keys = apiRow.keys;
@@ -76,8 +76,10 @@ function transformGscRow(table, apiRow, options = {}) {
 		};
 	}
 	if (table === "hourly_pages") {
-		const hour = String(keys[0] ?? "");
-		const date = hour.slice(0, 10);
+		const hourStamp = String(keys[0] ?? "");
+		const date = hourStamp.slice(0, 10);
+		const hour = Number.parseInt(hourStamp.slice(11, 13), 10);
+		if (!Number.isInteger(hour) || hour < 0 || hour > 23) throw new Error(`hourly_pages: cannot derive hour-of-day from '${hourStamp}'`);
 		return {
 			date,
 			row: {

package/dist/resolver/index.d.mts

@@ -1,7 +1,7 @@
 import { SearchType as SearchType$1, TableName as TableName$1 } from "../_chunks/storage.mjs";
 import { ComparisonFilter, ExtraQuery, ResolvedComparisonSQL, ResolvedSQL, ResolvedSQLOptimized, ResolverAdapter, ResolverOptions } from "../_chunks/types.mjs";
-import { PgTableKey, createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "../_chunks/pg-adapter.mjs";
-import { LogicalDataset, LogicalDataset as LogicalDataset$1, PlannerCapabilities, UnresolvableDatasetError, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
+import { PgTableKey, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "../_chunks/pg-adapter.mjs";
+import { LogicalDataset, LogicalDataset as LogicalDataset$1, LogicalQueryPlan, PlannerCapabilities, UnresolvableDatasetError, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
 import { SQL } from "drizzle-orm";
 import { BuilderState, Dimension, FilterInput, InternalFilter, Metric } from "gscdump/query";
 import { Grain, TableName } from "@gscdump/contracts";
@@ -35,6 +35,19 @@ interface SqlFragmentsConfig<TableKey extends string> {
    * against the alias.
    */
   tableRef?: (tableKey: TableKey) => SQL;
+  /**
+   * Opt-in correctness for canonical-primary lookups. When true, the
+   * `queryCanonical` dimension expression falls back to the raw `query` when
+   * the stored `query_canonical` is NULL (no normalizer ran at ingest) or `''`
+   * (a fully-stripped query like "free online"), i.e.
+   * `COALESCE(NULLIF(query_canonical, ''), query)`. This makes canonical a
+   * TOTAL key, valid for GROUP BY / comparison joins.
+   *
+   * Default (false) preserves legacy behaviour: the raw nullable column, so a
+   * NULL/'' bucket pollutes top results and — because `NULL = NULL` is UNKNOWN
+   * — double-counts in the gaining/losing FULL OUTER JOIN. See ADR-0018.
+   */
+  canonicalFallback?: boolean;
 }
 interface SqlFragments<TableKey extends string> {
   METRIC_NAMES: Metric[];
@@ -65,6 +78,16 @@ interface CreateResolverAdapterConfig<TableKey extends string> extends SqlFragme
   capabilities: PlannerCapabilities;
 }
 declare function createResolverAdapter<TableKey extends string>(config: CreateResolverAdapterConfig<TableKey>): ResolverAdapter<TableKey>;
+/**
+ * True when `plan` can be served from the canonical-grained rollup instead of
+ * the raw `queries` fact partitions. Conservative: anything that would read a
+ * dropped column or the raw row grain disqualifies the query, so a false
+ * negative just falls back to live aggregation (correct, slower) — never wrong
+ * data.
+ */
+declare function planCoveredByCanonicalRollup(plan: LogicalQueryPlan): boolean;
+/** State-level convenience: build the plan then gate. */
+declare function canonicalRollupCovers(state: BuilderState, capabilities: PlannerCapabilities): boolean;
 declare function resolveToSQLOptimized<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): ResolvedSQLOptimized;
 declare function resolveToSQL<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): ResolvedSQL;
 declare function buildTotalsSql<TK extends string>(state: BuilderState, options: ResolverOptions<TK>): {
@@ -77,14 +100,6 @@ declare function mergeExtras(rows: Record<string, unknown>[], extrasResults: {
   key: string;
   results: Record<string, unknown>[];
 }[]): Record<string, unknown>[];
-declare function getInternalFilters(filter: FilterInput | undefined): InternalFilter[];
-declare function getDimensionFilters(filter: FilterInput | undefined, isMetricDimension: (dim: string) => dim is Metric): InternalFilter[];
-declare function getFilterDimensions(filter: FilterInput | undefined, isMetricDimension: (dim: string) => dim is Metric): Dimension[];
-declare function metricValue(row: Record<string, unknown>, metric: string): number;
-declare function dimensionValue(row: Record<string, unknown>, dimension: string): string;
-declare function matchesDimensionFilter(row: Record<string, unknown>, filter: InternalFilter): boolean;
-declare function matchesMetricFilter(row: Record<string, unknown>, filter: InternalFilter): boolean;
-declare function matchesTopLevelPage(row: Record<string, unknown>): boolean;
 interface RunQueryCtx {
   userId: string;
   siteId: string;
@@ -118,6 +133,60 @@ interface RunSQLFn {
     rows: Array<Record<string, unknown>>;
   }>;
 }
+/**
+ * Optional overlay that serves a resolver extra (e.g. canonical-variant
+ * grouping, keyed `'canonicalExtras'`) from a precomputed source — typically a
+ * materialised rollup — instead of the live window-function SQL. Return the
+ * rows in the exact shape the live extra produces (`mergeExtras` consumes
+ * either source unchanged), or `null` to decline so the caller falls back to
+ * the live query. Pure seam: storage/tenant routing lives in the host's
+ * implementation, not here. See ADR-0017.
+ */
+interface ResolveExtraFn {
+  (opts: {
+    key: string;
+    state: BuilderState;
+    ctx: RunQueryCtx;
+    dateRange: {
+      startDate: string;
+      endDate: string;
+    };
+  }): Promise<Array<Record<string, unknown>> | null>;
+}
+interface RunOptimizedQueryOptions {
+  /** Overlay tried per extra before the live SQL; absent → today's live path. */
+  resolveExtra?: ResolveExtraFn;
+  /**
+   * Opt-in canonical-primary correctness: group/compare `queryCanonical` as a
+   * total key (NULL/'' folds to the raw `query`). Default false = legacy raw
+   * nullable column. See ADR-0018.
+   */
+  canonicalFallback?: boolean;
+  /**
+   * Opt-in canonical-primary performance (ADR-0018 Gap 2): object keys of the
+   * `query_canonical_daily` rollup parquet(s). When supplied AND the query is
+   * coverable (`canonicalRollupCovers`) AND `canonicalFallback` is on AND the
+   * window is within the rollup's coverage, the MAIN query reads these
+   * pre-summed `(query_canonical × date)` rows instead of re-aggregating raw
+   * partitions; variant extras still read raw. Ignored (live path) on any miss,
+   * so a mis-wired host degrades to correct-but-slow, never wrong.
+   *
+   * `canonicalFallback` is REQUIRED: the rollup is built with
+   * `COALESCE(NULLIF(query_canonical, ''), query)` (fallback semantics), so
+   * serving it to a legacy (`canonicalFallback: false`) caller would change
+   * NULL/'' rows from legacy buckets to raw-query keys. The rollup is already
+   * null-free, so the rollup READ itself runs without fallback.
+   *
+   * `coversThrough` (ISO `YYYY-MM-DD`, the rollup's newest covered date) gates
+   * staleness: the source is used only when `dateRange.endDate <= coversThrough`,
+   * else the live path serves the window so the recent tail is never silently
+   * undercounted. Omit to assert full coverage (use with care).
+   */
+  canonicalSource?: {
+    keys: string[];
+    coversThrough?: string;
+  };
+}
 interface OptimizedQueryResult {
   rows: Array<Record<string, unknown>>;
   totalCount: number;
@@ -140,7 +209,7 @@ interface ComparisonQueryResult {
 declare function runOptimizedQuery(runSQL: RunSQLFn, ctx: RunQueryCtx, state: BuilderState, dateRange: {
   startDate: string;
   endDate: string;
-}): Promise<OptimizedQueryResult>;
+}, options?: RunOptimizedQueryOptions): Promise<OptimizedQueryResult>;
 declare function runComparisonQuery(runSQL: RunSQLFn, ctx: RunQueryCtx, current: BuilderState, previous: BuilderState, windows: {
   current: {
     startDate: string;
@@ -150,7 +219,55 @@ declare function runComparisonQuery(runSQL: RunSQLFn, ctx: RunQueryCtx, current:
     startDate: string;
     endDate: string;
   };
-}, filter?: ComparisonFilter): Promise<ComparisonQueryResult>;
+}, filter?: ComparisonFilter, options?: {
+  canonicalFallback?: boolean;
+  canonicalSource?: {
+    keys: string[];
+    coversThrough?: string;
+  };
+}): Promise<ComparisonQueryResult>;
+/**
+ * Host-supplied reader: return the materialised rollup's rows for an
+ * `(id, tenant, slice)`, in the exact shape the live extra produces, or `null`
+ * when no rollup exists (first sync, never built, stale) so the overlay
+ * declines and the resolver falls back to the live query. Typically wired with
+ * `readLatestRollup` + a `read_parquet` of the pointer.
+ *
+ * `dateRange` is the request window. `query_canonical_variants` is full-history
+ * (its grouping/variant metrics span all dates), but `buildExtrasQueries`
+ * windows the live `canonicalExtras` to the requested range — so for a narrow
+ * window the reader MUST decline (return `null`) rather than attach
+ * out-of-window variantCount/canonicalName/variants. A common rule: serve only
+ * when the request window covers full history.
+ */
+interface RollupRowsReader {
+  (opts: {
+    id: string;
+    ctx: {
+      userId: string;
+      siteId: string;
+    };
+    searchType?: SearchType$1;
+    dateRange: {
+      startDate: string;
+      endDate: string;
+    };
+  }): Promise<Array<Record<string, unknown>> | null>;
+}
+/**
+ * Build a {@link ResolveExtraFn} that serves resolver extras from materialised
+ * rollups when one is mapped for the extra's key, else returns `null` to fall
+ * back to the live SQL. Pure wiring around the host's `readRollupRows`.
+ */
+declare function createRollupExtrasOverlay(readRollupRows: RollupRowsReader): ResolveExtraFn;
+declare function getInternalFilters(filter: FilterInput | undefined): InternalFilter[];
+declare function getDimensionFilters(filter: FilterInput | undefined, isMetricDimension: (dim: string) => dim is Metric): InternalFilter[];
+declare function getFilterDimensions(filter: FilterInput | undefined, isMetricDimension: (dim: string) => dim is Metric): Dimension[];
+declare function metricValue(row: Record<string, unknown>, metric: string): number;
+declare function dimensionValue(row: Record<string, unknown>, dimension: string): string;
+declare function matchesDimensionFilter(row: Record<string, unknown>, filter: InternalFilter): boolean;
+declare function matchesMetricFilter(row: Record<string, unknown>, filter: InternalFilter): boolean;
+declare function matchesTopLevelPage(row: Record<string, unknown>): boolean;
 interface AssertSchemaInSyncOptions {
   /** Label used in the thrown error (e.g. 'browser', 'sqlite'). */
   label: string;
@@ -164,4 +281,4 @@ interface AssertSchemaInSyncOptions {
   mode: 'exact' | 'superset';
 }
 declare function assertSchemaInSync(options: AssertSchemaInSyncOptions): void;
-export { type AssertSchemaInSyncOptions, type ComparisonFilter, type ComparisonQueryResult, type CreateResolverAdapterConfig, DIMENSION_SURFACES, type DimensionBinding, type DimensionSurface, type ExtraQuery, LOGICAL_DATASETS, type LogicalDataset, type LogicalDatasetDefinition, type OptimizedQueryResult, type PgTableKey, type ResolvedComparisonSQL, type ResolvedSQL, type ResolvedSQLOptimized, type ResolverAdapter, type ResolverOptions, type RunQueryCtx, type RunSQLFn, type SqlFragments, type SqlFragmentsConfig, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
+export { type AssertSchemaInSyncOptions, type ComparisonFilter, type ComparisonQueryResult, type CreateResolverAdapterConfig, DIMENSION_SURFACES, type DimensionBinding, type DimensionSurface, type ExtraQuery, LOGICAL_DATASETS, type LogicalDataset, type LogicalDatasetDefinition, type OptimizedQueryResult, type PgTableKey, type ResolveExtraFn, type ResolvedComparisonSQL, type ResolvedSQL, type ResolvedSQLOptimized, type ResolverAdapter, type ResolverAdapterOptions, type ResolverOptions, type RollupRowsReader, type RunOptimizedQueryOptions, type RunQueryCtx, type RunSQLFn, type SqlFragments, type SqlFragmentsConfig, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };

package/dist/resolver/index.mjs

@@ -1,2 +1,2 @@
-import { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface } from "../_chunks/resolver.mjs";
-export { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
+import { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface } from "../_chunks/resolver.mjs";
+export { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };

package/dist/rollups.d.mts

@@ -283,6 +283,47 @@ declare const topKeywords28dRollup: RollupDef;
  * coexist during a migration.
  */
 declare const topKeywords28dParquetRollup: RollupDef;
+/**
+ * Materialises canonical-query variant grouping so the read path
+ * (`buildExtrasQueries` in `resolver/compile.ts`) becomes a passthrough scan
+ * instead of two window passes (`ROW_NUMBER`/`COUNT` over `PARTITION BY
+ * query_canonical`) plus a `GROUP_CONCAT` over the whole `queries` table on
+ * every request — work that is single-threaded under DuckDB-WASM/Workers and
+ * scales with table size. See ADR-0017.
+ *
+ * One row per `query_canonical` group, columns named 1:1 with the live query's
+ * output (`joinKey`, `variantCount`, `canonicalName`, `variants`) so
+ * `mergeExtras` consumes either source unchanged. `variants` packs the top-10
+ * variants as `query:::clicks:::impressions:::position` joined by `||`,
+ * identical to the live composer.
+ *
+ * Full history (`windowDays: null`), not a trailing window: grouping metadata
+ * is global (which variant is canonical, how many variants exist) and stays
+ * stable across requests rather than shifting with each query's date range.
+ * Reflects the last sync/compaction, not the live tail — readers that need the
+ * tail can layer a recent-overlay later (the envelope carries `builtAt`).
+ */
+declare const queryCanonicalVariantsRollup: RollupDef;
+/**
+ * Canonical-grained fact aggregate (ADR-0018 Gap 2): pre-sums the raw
+ * `(query × date)` query rows to `(query_canonical × date)`, so canonical-
+ * primary top/gaining/losing reads a small pre-aggregated table instead of
+ * re-collapsing variants on every request. Metrics are additive, so summing
+ * these per-date sums over a window is exact — identical to aggregating the raw
+ * rows.
+ *
+ * Null-free by construction: groups by `COALESCE(NULLIF(query_canonical, ''),
+ * query)`, the same total-key expression the opt-in read path uses (ADR-0018
+ * Gap 1), so the rollup never carries a NULL/'' canonical bucket and the read
+ * path needs no fallback when pointed at it.
+ *
+ * Date-grained full history (`windowDays: null`): one rollup serves every date
+ * range (reads filter by `date`) and both windows of a comparison. Opt-in (not
+ * in `DEFAULT_ROLLUPS`); the host points the main query's file set at it for
+ * queries the rollup covers (see `canonicalRollupCovers` /
+ * `RunOptimizedQueryOptions.canonicalSource`).
+ */
+declare const queryCanonicalDailyRollup: RollupDef;
 /**
  * Aggregates the per-URL Indexing API metadata entity store (populated by
  * `gscdump entities indexing snapshot`) into daily counts of `URL_UPDATED`
@@ -360,4 +401,12 @@ declare function rebuildDailyFromHourly(opts: RebuildDailyFromHourlyOptions): Pr
   rowsWritten: number;
 }>;
 declare const DEFAULT_ROLLUPS: readonly RollupDef[];
-export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+/**
+ * Canonical-primary rollups (ADR-0017 / ADR-0018). Opt-in — kept out of
+ * `DEFAULT_ROLLUPS` because they only pay off once the consumer queries by
+ * `queryCanonical` and wires the read seams (`resolveExtra` /
+ * `canonicalSource`). Hosts opt in by concatenating these onto their def list
+ * (CLI: `gscdump rollups --with-canonical`).
+ */
+declare const CANONICAL_ROLLUPS: readonly RollupDef[];
+export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -534,6 +534,136 @@ const topKeywords28dParquetRollup = {
 		}));
 	}
 };
+const queryCanonicalVariantsRollup = {
+	id: "query_canonical_variants",
+	windowDays: null,
+	format: "parquet",
+	parquetColumns: [
+		{
+			name: "joinKey",
+			type: "VARCHAR",
+			nullable: false
+		},
+		{
+			name: "variantCount",
+			type: "BIGINT",
+			nullable: false
+		},
+		{
+			name: "canonicalName",
+			type: "VARCHAR",
+			nullable: true
+		},
+		{
+			name: "variants",
+			type: "VARCHAR",
+			nullable: true
+		}
+	],
+	parquetSortKey: ["joinKey"],
+	async build({ engine, ctx, searchType }) {
+		const parts = await engine.listPartitions({
+			ctx,
+			table: "queries",
+			...searchType !== void 0 ? { searchType } : {}
+		});
+		if (parts.length === 0) return [];
+		const partitions = parts.map((p) => p.partition);
+		return (await engine.runSQL({
+			ctx,
+			table: "queries",
+			fileSets: { FILES: {
+				table: "queries",
+				partitions
+			} },
+			...searchType !== void 0 ? { searchType } : {},
+			sql: `
+        WITH per_variant AS (
+          SELECT
+            COALESCE(NULLIF(query_canonical, ''), query) AS joinKey,
+            query AS query,
+            SUM(clicks) AS clicks,
+            SUM(impressions) AS impressions,
+            SUM(sum_position) AS sum_pos,
+            ROW_NUMBER() OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query) ORDER BY SUM(clicks) DESC) AS rn,
+            COUNT(*) OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query)) AS variantCount
+          FROM read_parquet({{FILES}}, union_by_name = true)
+          GROUP BY COALESCE(NULLIF(query_canonical, ''), query), query
+        )
+        SELECT
+          joinKey,
+          MAX(variantCount)::BIGINT AS variantCount,
+          MAX(CASE WHEN rn = 1 THEN query END) AS canonicalName,
+          GROUP_CONCAT(CASE WHEN rn <= 10 THEN query || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') AS variants
+        FROM per_variant
+        GROUP BY joinKey
+      `
+		})).rows.map((r) => ({
+			joinKey: String(r.joinKey),
+			variantCount: BigInt(r.variantCount),
+			canonicalName: r.canonicalName == null ? null : String(r.canonicalName),
+			variants: r.variants == null ? null : String(r.variants)
+		}));
+	}
+};
+const queryCanonicalDailyRollup = {
+	id: "query_canonical_daily",
+	windowDays: null,
+	format: "parquet",
+	parquetColumns: [
+		{
+			name: "query_canonical",
+			type: "VARCHAR",
+			nullable: false
+		},
+		{
+			name: "date",
+			type: "DATE",
+			nullable: false
+		},
+		{
+			name: "clicks",
+			type: "BIGINT",
+			nullable: false
+		},
+		{
+			name: "impressions",
+			type: "BIGINT",
+			nullable: false
+		},
+		{
+			name: "sum_position",
+			type: "DOUBLE",
+			nullable: false
+		}
+	],
+	parquetSortKey: ["date", "query_canonical"],
+	async build({ engine, ctx, searchType }) {
+		return (await runWindowed({
+			engine,
+			ctx,
+			table: "queries",
+			...searchType !== void 0 ? { searchType } : {},
+			sqlFor: (w) => `
+        SELECT
+          COALESCE(NULLIF(query_canonical, ''), query) AS query_canonical,
+          CAST(date AS VARCHAR) AS date,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
+        GROUP BY COALESCE(NULLIF(query_canonical, ''), query), date
+      `
+		})).map((r) => ({
+			query_canonical: String(r.query_canonical),
+			date: String(r.date),
+			clicks: BigInt(r.clicks),
+			impressions: BigInt(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
 const indexingMetadataRollup = {
 	id: "indexing_metadata",
 	windowDays: null,
@@ -560,7 +690,7 @@ const indexingMetadataRollup = {
 				if (!latestRemove || r.latestRemoveAt > latestRemove) latestRemove = r.latestRemoveAt;
 			}
 		}
-		const days = new Set([...updatesByDay.keys(), ...removesByDay.keys()]);
+		const days = /* @__PURE__ */ new Set([...updatesByDay.keys(), ...removesByDay.keys()]);
 		const perDay = Array.from(days).sort().map((day) => ({
 			day,
 			updates: updatesByDay.get(day) ?? 0,
@@ -845,4 +975,5 @@ const DEFAULT_ROLLUPS = [
 	sitemapHealthRollup,
 	sitemapChanges28dRollup
 ];
-export { DEFAULT_ROLLUPS, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
+export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/schema.d.mts

@@ -1,2 +1,2 @@
-import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
-export { type ColumnDef, type ColumnType, type DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+import { ColumnDef, ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
+export { type ColumnDef, type ColumnType, type DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/schema.mjs

@@ -1,2 +1,2 @@
-import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
-export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+import { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries } from "./_chunks/schema.mjs";
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };