@gscdump/engine 0.32.11 → 0.33.0

package/dist/index.d.mts

@@ -6,7 +6,7 @@ import { InspectionVerdict, SchedulePolicy, ScheduleState, fixedPolicy, inspecti
 import { IcebergTableName, Sink, SinkCapabilities, SinkCloseResult, SinkOptions, SinkSlice, SinkWriteResult } from "./_chunks/sink.mjs";
 import { GscApiRow, IngestOptions, RowAccumulator, RowAccumulatorOptions, assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import { FILES_PLACEHOLDER, ResolvedQuery, resolveParquetSQL, substituteNamedFiles } from "./_chunks/planner.mjs";
-import { createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "./_chunks/pg-adapter.mjs";
+import { createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter } from "./_chunks/pg-adapter.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { ENGINE_QUERY_CAPABILITIES, createSqlQuerySource } from "./_chunks/index.mjs";
 import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
@@ -209,4 +209,4 @@ declare const MIN_SYNC_IMPRESSIONS = 1;
 declare const MIN_COUNTRY_IMPRESSIONS = 10;
 declare const MAX_SITEMAP_URLS_PER_SITE = 50000;
 declare const MAX_TRACKED_URLS_PER_SITE = 200000;
-export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export { type CodecCtx, type ColumnDef, type ColumnType, type CompactionThresholds, type CompactionTier, type CreateIngestAccumulatorOptions, DEFAULT_SEARCH_TYPE, type DataSource, type DateWeight, type DrizzleSchema, type DuckDBFactory, type DuckDBHandle, ENGINE_QUERY_CAPABILITIES, EngineError, EngineErrorKind, type EngineOptions, FILES_PLACEHOLDER, type FileSetRef, type FinalizeOptions, type FinalizeResult, type GcCtx, type Grain, type GscApiRow, type InMemorySink, type IngestAccumulator, type IngestAccumulatorCtx, type IngestAccumulatorEngine, type IngestAccumulatorHooks, type IngestOptions, type InspectionVerdict, type ListLiveFilter, type LockScope, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, type ManifestEntry, type ManifestPurgeResult, type ManifestStore, type ParquetCodec, type PurgeFilter, type PurgeResult, type PurgeUrlsResult, type QueryCtx, type QueryExecuteOptions, type QueryExecuteResult, type QueryExecutor, type QueryProfiler, type QueryResult, type QuerySpan, ROW_LIMIT_R2, type ResolvedQuery, type Row, type RowAccumulator, type RowAccumulatorOptions, type RunSQLOptions, SCHEMAS, type SchedulePolicy, type ScheduleState, type SearchType, type Sink, type SinkCapabilities, type SinkCloseResult, type SinkOptions, type SinkSlice, type SinkWriteResult, type StorageEngine, type StoredRow, type SyncState, type SyncStateDetail, type SyncStateFilter, type SyncStateKind, type SyncStateScope, type SyncTableName, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, type TableName, type TableSchema, type TableTier, type TenantCtx, type TieredTableName, WEIGHT_PRIORITY, type Watermark, type WatermarkFilter, type WatermarkScope, type WriteCtx, type WriteResult, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };

package/dist/index.mjs

@@ -8,7 +8,7 @@ import { bindLiterals, formatLiteral } from "./sql-bind.mjs";
 import { MAX_DAY_BYTES, canonicalEmptyParquetSchema, createDuckDBCodec, createDuckDBExecutor, createStorageEngine } from "./_chunks/engine.mjs";
 import { assembleDatesRow, createRowAccumulator, toPath, toSumPosition, transformGscRow } from "./ingest.mjs";
 import "./planner.mjs";
-import { createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "./_chunks/resolver.mjs";
+import { createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter } from "./_chunks/resolver.mjs";
 import { rebuildDailyFromHourly } from "./rollups.mjs";
 import { fixedPolicy, inspectionPolicy, sitemapPolicy } from "./schedule.mjs";
 import { err, ok, unwrapResult } from "gscdump/result";
@@ -308,4 +308,4 @@ const MIN_SYNC_IMPRESSIONS = 1;
 const MIN_COUNTRY_IMPRESSIONS = 10;
 const MAX_SITEMAP_URLS_PER_SITE = 5e4;
 const MAX_TRACKED_URLS_PER_SITE = 2e5;
-export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };
+export { DEFAULT_SEARCH_TYPE, ENGINE_QUERY_CAPABILITIES, FILES_PLACEHOLDER, MAX_DAY_BYTES, MAX_GSC_PAGES_R2, MAX_SITEMAP_URLS_PER_SITE, MAX_TRACKED_URLS_PER_SITE, MIN_COUNTRY_IMPRESSIONS, MIN_SYNC_IMPRESSIONS, ROW_LIMIT_R2, SCHEMAS, TABLES_BY_SEARCH_TYPE, TABLE_METADATA, TABLE_TIERS, TIER_PRIORITY, WEIGHT_PRIORITY, allTables, assembleDatesRow, bindLiterals, canonicalEmptyParquetSchema, coerceRow, coerceRows, collectSpans, countries, createDuckDBCodec, createDuckDBExecutor, createIcebergResolverAdapter, createInMemorySink, createIngestAccumulator, createNoopIngestAccumulator, createParquetResolverAdapter, createQueryProfiler, createR2SqlResolverAdapter, createRowAccumulator, createSqlQuerySource, createStorageEngine, currentSchemaVersion, dates, dayPartition, dimensionToColumn, drizzleSchema, engineErrorToException, engineErrors, enumeratePartitions, fixedPolicy, formatEngineError, formatLiteral, getDateWeight, getTableTier, getTablesForTier, hourPartition, hourly_pages, inferLegacyTier, inferSearchType, inferTable, inspectionPolicy, isEngineError, objectKey, page_queries, pages, parseEnabledSearchTypes, pgResolverAdapter, queries, rebuildDailyFromHourly, resolveParquetSQL, sitemapPolicy, substituteNamedFiles, toPath, toSumPosition, transformGscRow, validateEnabledSearchTypes, validateEnabledSearchTypesResult };

package/dist/ingest.d.mts

@@ -16,12 +16,7 @@ interface GscApiRow {
   position: number;
 }
 interface IngestOptions {
-  /**
-   * Canonical form of a query string, stored alongside `query` as
-   * `query_canonical`. Site-specific (e.g. synonym groups, stemming); if
-   * omitted, `query_canonical` is null. Applied to `queries` +
-   * `page_queries` tables only.
-   */
+  /** @deprecated Canonical query data is built through query_dim, not fact rows. */
   normalizeQuery?: (query: string) => string | null | undefined;
   /** Date for one-day `searchAppearance` total queries, whose keys omit date. */
   date?: string;

package/dist/ingest.mjs

@@ -54,7 +54,6 @@ function transformGscRow(table, apiRow, options = {}) {
 			date,
 			row: {
 				query,
-				query_canonical: options.normalizeQuery?.(query) ?? null,
 				date,
 				clicks,
 				impressions,
@@ -122,13 +121,11 @@ function transformGscRow(table, apiRow, options = {}) {
 	if (table === "search_appearance_queries") {
 		const query = String(keys[0] ?? "");
 		const date = String(keys[1] ?? "");
-		const query_canonical = options.normalizeQuery?.(query) ?? null;
 		return {
 			date,
 			row: {
 				searchAppearance: String(options.searchAppearance ?? ""),
 				query,
-				query_canonical,
 				date,
 				clicks,
 				impressions,
@@ -139,14 +136,12 @@ function transformGscRow(table, apiRow, options = {}) {
 	if (table === "search_appearance_page_queries") {
 		const query = String(keys[1] ?? "");
 		const date = String(keys[2] ?? "");
-		const query_canonical = options.normalizeQuery?.(query) ?? null;
 		return {
 			date,
 			row: {
 				searchAppearance: String(options.searchAppearance ?? ""),
 				url: toPath(String(keys[0] ?? "")),
 				query,
-				query_canonical,
 				date,
 				clicks,
 				impressions,
@@ -157,13 +152,11 @@ function transformGscRow(table, apiRow, options = {}) {
 	if (table === "dates") throw new Error("`dates` rows must be built via assembleDatesRow, not transformGscRow");
 	const query = String(keys[1] ?? "");
 	const date = String(keys[2] ?? "");
-	const query_canonical = options.normalizeQuery?.(query) ?? null;
 	return {
 		date,
 		row: {
 			url: toPath(String(keys[0] ?? "")),
 			query,
-			query_canonical,
 			date,
 			clicks,
 			impressions,

package/dist/resolver/index.d.mts

@@ -1,6 +1,6 @@
 import { SearchType as SearchType$1, TableName as TableName$1 } from "../_chunks/storage.mjs";
 import { ComparisonFilter, ExtraQuery, ResolvedComparisonSQL, ResolvedSQL, ResolvedSQLOptimized, ResolverAdapter, ResolverOptions } from "../_chunks/types.mjs";
-import { PgTableKey, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter } from "../_chunks/pg-adapter.mjs";
+import { PgTableKey, R2SqlResolverAdapterOptions, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter } from "../_chunks/pg-adapter.mjs";
 import { LogicalDataset, LogicalDataset as LogicalDataset$1, LogicalQueryPlan, PlannerCapabilities, UnresolvableDatasetError, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
 import { SQL } from "drizzle-orm";
 import { BuilderState, Dimension, FilterInput, InternalFilter, Metric } from "gscdump/query";
@@ -36,17 +36,19 @@ interface SqlFragmentsConfig<TableKey extends string> {
    */
   tableRef?: (tableKey: TableKey) => SQL;
   /**
-   * Opt-in correctness for canonical-primary lookups. When true, the
-   * `queryCanonical` dimension expression falls back to the raw `query` when
-   * the stored `query_canonical` is NULL (no normalizer ran at ingest) or `''`
-   * (a fully-stripped query like "free online"), i.e.
-   * `COALESCE(NULLIF(query_canonical, ''), query)`. This makes canonical a
-   * TOTAL key, valid for GROUP BY / comparison joins.
+   * Where `queryCanonical` comes from when compiling fact-table reads.
    *
-   * Default (false) preserves legacy behaviour: the raw nullable column, so a
-   * NULL/'' bucket pollutes top results and — because `NULL = NULL` is UNKNOWN
-   * — double-counts in the gaining/losing FULL OUTER JOIN. See ADR-0018.
+   * - `queryDim` (default): LEFT JOIN a query dimension relation and derive
+   *   `COALESCE(query_dim.query_canonical, fact.query)`.
+   * - `column`: read a `query_canonical` column from the primary relation. This
+   *   is reserved for derived canonical rollup relations, not fact tables.
    */
+  queryCanonicalSource?: 'queryDim' | 'column';
+  /** Relation used for the query-dimension join when `queryCanonicalSource=queryDim`. */
+  queryDimTableRef?: () => SQL;
+  /** When true, also joins query_dim on site_id (SQLite/D1 dimension table). */
+  queryDimSiteScoped?: boolean;
+  /** @deprecated Canonical reads now derive from query_dim or canonical rollups. */
   canonicalFallback?: boolean;
 }
 interface SqlFragments<TableKey extends string> {
@@ -59,6 +61,9 @@ interface SqlFragments<TableKey extends string> {
   urlToPathExpr: (col: string) => string;
   colRef: (tableKey: TableKey, colName: string) => SQL;
   tableRef: (tableKey: TableKey) => SQL;
+  fromSql: (tableKey: TableKey, options?: {
+    queryCanonical?: boolean;
+  }) => SQL;
   dateColRef: (tableKey: TableKey) => SQL;
   siteIdColRef?: (tableKey: TableKey) => SQL;
   searchTypeColRef?: (tableKey: TableKey) => SQL;
@@ -81,9 +86,8 @@ declare function createResolverAdapter<TableKey extends string>(config: CreateRe
 /**
  * True when `plan` can be served from the canonical-grained rollup instead of
  * the raw `queries` fact partitions. Conservative: anything that would read a
- * dropped column or the raw row grain disqualifies the query, so a false
- * negative just falls back to live aggregation (correct, slower) — never wrong
- * data.
+ * dropped column or the raw row grain disqualifies the query. Callers decide
+ * whether that miss is a typed raw fallback or a hard coverage error.
  */
 declare function planCoveredByCanonicalRollup(plan: LogicalQueryPlan): boolean;
 /** State-level convenience: build the plan then gate. */
@@ -133,6 +137,21 @@ interface RunSQLFn {
     rows: Array<Record<string, unknown>>;
   }>;
 }
+interface CanonicalQueryDimSource {
+  /** Object keys for the versioned query_dim parquet. */
+  keys: string[];
+  normalizerVersion: number;
+  intentVersion?: number;
+  builtAt?: number;
+}
+interface PrimaryColumnarSource {
+  /** Pre-resolved compacted/Iceberg parquet object keys for the requested fact table. */
+  keys: string[];
+  /** Oldest covered date (`YYYY-MM-DD`). Omit to assert the source covers all older dates. */
+  coversFrom?: string;
+  /** Newest covered date (`YYYY-MM-DD`). Omit to assert the source covers the current tail. */
+  coversThrough?: string;
+}
 /**
  * Optional overlay that serves a resolver extra (e.g. canonical-variant
  * grouping, keyed `'canonicalExtras'`) from a precomputed source — typically a
@@ -157,36 +176,75 @@ interface RunOptimizedQueryOptions {
   /** Overlay tried per extra before the live SQL; absent → today's live path. */
   resolveExtra?: ResolveExtraFn;
   /**
-   * Opt-in canonical-primary correctness: group/compare `queryCanonical` as a
-   * total key (NULL/'' folds to the raw `query`). Default false = legacy raw
-   * nullable column. See ADR-0018.
+   * Deprecated compatibility flag. Canonical reads now derive from query_dim,
+   * not a nullable fact-table `query_canonical` column.
    */
   canonicalFallback?: boolean;
+  /**
+   * Versioned query dimension backing canonical reads over fact rows. Required
+   * whenever a query groups or filters by `queryCanonical`; a canonical rollup
+   * source may carry the same metadata/key set via `canonicalSource.queryDim`.
+   */
+  queryDim?: CanonicalQueryDimSource;
+  /**
+   * Primary fact-file source for consumer reads. Callers pass compacted or
+   * Iceberg data-file keys with explicit coverage metadata; raw daily
+   * partitions are only used when `primarySourceFallback: 'raw'` is set.
+   */
+  primarySource?: PrimaryColumnarSource;
+  /**
+   * Explicit compatibility escape hatch for raw daily shards. Default is
+   * strict: missing/empty/stale primary coverage fails with
+   * `QuerySourceCoverageError`.
+   */
+  primarySourceFallback?: 'raw';
   /**
    * Opt-in canonical-primary performance (ADR-0018 Gap 2): object keys of the
    * `query_canonical_daily` rollup parquet(s). When supplied AND the query is
-   * coverable (`canonicalRollupCovers`) AND `canonicalFallback` is on AND the
-   * window is within the rollup's coverage, the MAIN query reads these
+   * coverable (`canonicalRollupCovers`) AND the window is within the rollup's
+   * coverage, the MAIN query reads these
    * pre-summed `(query_canonical × date)` rows instead of re-aggregating raw
-   * partitions; variant extras still read raw. Ignored (live path) on any miss,
-   * so a mis-wired host degrades to correct-but-slow, never wrong.
-   *
-   * `canonicalFallback` is REQUIRED: the rollup is built with
-   * `COALESCE(NULLIF(query_canonical, ''), query)` (fallback semantics), so
-   * serving it to a legacy (`canonicalFallback: false`) caller would change
-   * NULL/'' rows from legacy buckets to raw-query keys. The rollup is already
-   * null-free, so the rollup READ itself runs without fallback.
+   * partitions. Variant extras still need fact-grain rows, so they read
+   * `primarySource` when present and raw partitions only via
+   * `primarySourceFallback: 'raw'`.
    *
    * `coversThrough` (ISO `YYYY-MM-DD`, the rollup's newest covered date) gates
    * staleness: the source is used only when `dateRange.endDate <= coversThrough`,
-   * else the live path serves the window so the recent tail is never silently
-   * undercounted. Omit to assert full coverage (use with care).
+   * else strict mode fails so the recent tail is never silently undercounted.
+   * Omit to assert full coverage (use with care).
    */
   canonicalSource?: {
     keys: string[];
     coversThrough?: string;
+    queryDim?: CanonicalQueryDimSource;
+  };
+  /**
+   * @deprecated Canonical-source misses are recorded on `source.fallbacks`;
+   * raw daily fallback is controlled by `primarySourceFallback: 'raw'`.
+   */
+  canonicalSourceFallback?: 'raw';
+  /** Optional version gates for the query dimension backing `canonicalSource`. */
+  canonicalRequirements?: {
+    normalizerVersion?: number;
+    intentVersion?: number;
   };
 }
+type QuerySourceKind = 'canonical-rollup' | 'primary-columnar' | 'raw-partitions';
+type QuerySourceFallbackKind = 'primary-source-missing' | 'primary-source-empty' | 'primary-source-missing-coverage' | 'primary-source-stale-coverage' | 'canonical-source-missing' | 'canonical-source-empty' | 'canonical-source-not-coverable' | 'canonical-source-stale-coverage' | 'query-dim-missing' | 'query-dim-empty' | 'query-dim-stale-version';
+interface QuerySourceFallback {
+  kind: QuerySourceFallbackKind;
+  message: string;
+}
+interface QuerySourceDecision {
+  kind: QuerySourceKind;
+  fallback?: QuerySourceFallback;
+  fallbacks?: QuerySourceFallback[];
+}
+declare class QuerySourceCoverageError extends Error {
+  readonly fallback: QuerySourceFallback;
+  name: string;
+  constructor(fallback: QuerySourceFallback);
+}
 interface OptimizedQueryResult {
   rows: Array<Record<string, unknown>>;
   totalCount: number;
@@ -200,11 +258,14 @@ interface OptimizedQueryResult {
     key: string;
     rows: Array<Record<string, unknown>>;
   }>;
+  source: QuerySourceDecision;
+  extraSource?: QuerySourceDecision;
 }
 interface ComparisonQueryResult {
   rows: Array<Record<string, unknown>>;
   totalCount: number;
   totals: Record<string, unknown>;
+  source: QuerySourceDecision;
 }
 declare function runOptimizedQuery(runSQL: RunSQLFn, ctx: RunQueryCtx, state: BuilderState, dateRange: {
   startDate: string;
@@ -219,13 +280,7 @@ declare function runComparisonQuery(runSQL: RunSQLFn, ctx: RunQueryCtx, current:
     startDate: string;
     endDate: string;
   };
-}, filter?: ComparisonFilter, options?: {
-  canonicalFallback?: boolean;
-  canonicalSource?: {
-    keys: string[];
-    coversThrough?: string;
-  };
-}): Promise<ComparisonQueryResult>;
+}, filter?: ComparisonFilter, options?: RunOptimizedQueryOptions): Promise<ComparisonQueryResult>;
 /**
  * Host-supplied reader: return the materialised rollup's rows for an
  * `(id, tenant, slice)`, in the exact shape the live extra produces, or `null`
@@ -281,4 +336,4 @@ interface AssertSchemaInSyncOptions {
   mode: 'exact' | 'superset';
 }
 declare function assertSchemaInSync(options: AssertSchemaInSyncOptions): void;
-export { type AssertSchemaInSyncOptions, type ComparisonFilter, type ComparisonQueryResult, type CreateResolverAdapterConfig, DIMENSION_SURFACES, type DimensionBinding, type DimensionSurface, type ExtraQuery, LOGICAL_DATASETS, type LogicalDataset, type LogicalDatasetDefinition, type OptimizedQueryResult, type PgTableKey, type ResolveExtraFn, type ResolvedComparisonSQL, type ResolvedSQL, type ResolvedSQLOptimized, type ResolverAdapter, type ResolverAdapterOptions, type ResolverOptions, type RollupRowsReader, type RunOptimizedQueryOptions, type RunQueryCtx, type RunSQLFn, type SqlFragments, type SqlFragmentsConfig, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
+export { type AssertSchemaInSyncOptions, type CanonicalQueryDimSource, type ComparisonFilter, type ComparisonQueryResult, type CreateResolverAdapterConfig, DIMENSION_SURFACES, type DimensionBinding, type DimensionSurface, type ExtraQuery, LOGICAL_DATASETS, type LogicalDataset, type LogicalDatasetDefinition, type OptimizedQueryResult, type PgTableKey, type PrimaryColumnarSource, QuerySourceCoverageError, type QuerySourceDecision, type QuerySourceFallback, type QuerySourceFallbackKind, type QuerySourceKind, type R2SqlResolverAdapterOptions, type ResolveExtraFn, type ResolvedComparisonSQL, type ResolvedSQL, type ResolvedSQLOptimized, type ResolverAdapter, type ResolverAdapterOptions, type ResolverOptions, type RollupRowsReader, type RunOptimizedQueryOptions, type RunQueryCtx, type RunSQLFn, type SqlFragments, type SqlFragmentsConfig, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };

package/dist/resolver/index.mjs

@@ -1,2 +1,2 @@
-import { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface } from "../_chunks/resolver.mjs";
-export { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
+import { DIMENSION_SURFACES, LOGICAL_DATASETS, QuerySourceCoverageError, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface } from "../_chunks/resolver.mjs";
+export { DIMENSION_SURFACES, LOGICAL_DATASETS, QuerySourceCoverageError, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };

package/dist/rollups.d.mts

@@ -361,10 +361,10 @@ declare const queryCanonicalVariantsRollup: RollupDef;
  * these per-date sums over a window is exact — identical to aggregating the raw
  * rows.
  *
- * Null-free by construction: groups by `COALESCE(NULLIF(query_canonical, ''),
- * query)`, the same total-key expression the opt-in read path uses (ADR-0018
- * Gap 1), so the rollup never carries a NULL/'' canonical bucket and the read
- * path needs no fallback when pointed at it.
+ * Null-free by construction: groups by the versioned query dimension when it
+ * exists, with raw query as the fallback, so the rollup never carries a NULL/''
+ * canonical bucket and the read path can treat the rollup's `query_canonical`
+ * column as already-derived.
  *
  * Date-grained full history (`windowDays: null`): one rollup serves every date
  * range (reads filter by `date`) and both windows of a comparison. Opt-in (not

package/dist/rollups.mjs

@@ -598,7 +598,7 @@ const queryCanonicalVariantsRollup = {
 		}
 	],
 	parquetSortKey: ["joinKey"],
-	async build({ engine, ctx, searchType }) {
+	async build({ engine, ctx, dataSource, searchType }) {
 		const parts = await engine.listPartitions({
 			ctx,
 			table: "queries",
@@ -607,28 +607,37 @@ const queryCanonicalVariantsRollup = {
 		if (parts.length === 0) return [];
 		const partitions = parts.map((p) => p.partition);
 		const byCanonical = /* @__PURE__ */ new Map();
+		const dimStore = createQueryDimStore({ dataSource });
+		const useDim = await dimStore.loadMeta(ctx) !== null;
+		const canonExpr = useDim ? "COALESCE(qd.query_canonical, q.query)" : "q.query";
 		let cursor = null;
 		for (;;) {
-			const after = cursor === null ? "" : `AND query > '${cursor.replace(/'/g, "''")}'`;
+			const after = cursor === null ? "" : `AND q.query > '${cursor.replace(/'/g, "''")}'`;
+			const fileSets = { FILES: {
+				table: "queries",
+				partitions
+			} };
+			if (useDim) fileSets.QUERY_DIM = {
+				table: "queries",
+				keys: [dimStore.parquetKey(ctx)]
+			};
 			const { rows } = await engine.runSQL({
 				ctx,
 				table: "queries",
 				...searchType !== void 0 ? { searchType } : {},
-				fileSets: { FILES: {
-					table: "queries",
-					partitions
-				} },
+				fileSets,
 				sql: `
           SELECT
-            COALESCE(NULLIF(query_canonical, ''), query) AS joinKey,
-            query AS query,
-            SUM(clicks) AS clicks,
-            SUM(impressions) AS impressions,
-            SUM(sum_position) AS sum_pos
-          FROM read_parquet({{FILES}}, union_by_name = true)
-          WHERE query IS NOT NULL ${after}
-          GROUP BY COALESCE(NULLIF(query_canonical, ''), query), query
-          ORDER BY query
+            ${canonExpr} AS joinKey,
+            q.query AS query,
+            SUM(q.clicks) AS clicks,
+            SUM(q.impressions) AS impressions,
+            SUM(q.sum_position) AS sum_pos
+          FROM read_parquet({{FILES}}, union_by_name = true) q
+          ${useDim ? "LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query" : ""}
+          WHERE q.query IS NOT NULL ${after}
+          GROUP BY ${canonExpr}, q.query
+          ORDER BY q.query
           LIMIT ${ROLLUP_PAGE_ROWS_WIDE}
         `
 			});
@@ -663,6 +672,8 @@ const queryCanonicalVariantsRollup = {
 		return out;
 	}
 };
+const CANONICAL_DAILY_ROLLUP_FINAL_ID = "query_canonical_daily";
+const CANONICAL_DAILY_PART_STEM = "query_canonical_daily__part";
 const queryCanonicalDailyRollup = {
 	id: "query_canonical_daily",
 	windowDays: null,
@@ -698,7 +709,7 @@ const queryCanonicalDailyRollup = {
 	async build({ engine, ctx, dataSource, searchType }) {
 		const dimStore = createQueryDimStore({ dataSource });
 		const useDim = await dimStore.loadMeta(ctx) !== null;
-		const canonExpr = useDim ? `COALESCE(qd.query_canonical, NULLIF(q.query_canonical, ''), q.query)` : `COALESCE(NULLIF(query_canonical, ''), query)`;
+		const canonExpr = useDim ? `COALESCE(qd.query_canonical, q.query)` : `query`;
 		return (await runWindowed({
 			engine,
 			ctx,
@@ -764,7 +775,7 @@ async function rebuildCanonicalDailyResumable(opts) {
 	const windowsTotal = windows.length;
 	const dimStore = createQueryDimStore({ dataSource });
 	const useDim = await dimStore.loadMeta(ctx) !== null;
-	const canonExpr = useDim ? `COALESCE(qd.query_canonical, NULLIF(q.query_canonical, ''), q.query)` : `COALESCE(NULLIF(query_canonical, ''), query)`;
+	const canonExpr = useDim ? `COALESCE(qd.query_canonical, q.query)` : `query`;
 	const extraFileSets = useDim ? { QUERY_DIM: {
 		table: "queries",
 		keys: [dimStore.parquetKey(ctx)]
@@ -833,8 +844,6 @@ async function rebuildCanonicalDailyResumable(opts) {
 		rowsWritten: batchRows.length
 	};
 }
-const CANONICAL_DAILY_ROLLUP_FINAL_ID = "query_canonical_daily";
-const CANONICAL_DAILY_PART_STEM = "query_canonical_daily__part";
 const indexingMetadataRollup = {
 	id: "indexing_metadata",
 	windowDays: null,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.32.11",
+  "version": "0.33.0",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.32.11",
-    "gscdump": "0.32.11"
+    "@gscdump/contracts": "0.33.0",
+    "gscdump": "0.33.0"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",