@gscdump/engine 0.32.11 → 0.33.0

package/dist/_chunks/engine.mjs

@@ -6,6 +6,20 @@ import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mj
 import { sqlEscape } from "../sql-bind.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
 import { normalizeUrl } from "gscdump/normalize";
+const BUFFER_READ_BATCH_SIZE = 8;
+async function registerBufferedFiles(db, files, dataSource, registered, signal) {
+	for (let i = 0; i < files.length; i += BUFFER_READ_BATCH_SIZE) {
+		signal?.throwIfAborted();
+		const batch = files.slice(i, i + BUFFER_READ_BATCH_SIZE);
+		const buffers = await Promise.all(batch.map((file) => dataSource.read(file.key, void 0, signal)));
+		for (let j = 0; j < batch.length; j++) {
+			signal?.throwIfAborted();
+			const file = batch[j];
+			await db.registerFileBuffer(file.name, buffers[j]);
+			registered.push(file.name);
+		}
+	}
+}
 async function encodeBytes(db, table, rows) {
 	const inName = db.makeTempPath("json");
 	const outName = db.makeTempPath("parquet");
@@ -72,17 +86,19 @@ function createDuckDBCodec(factory) {
 					await db.dropFiles([outName]);
 				}
 			}
-			const inputs = await Promise.all(inputKeys.map((k) => dataSource.read(k)));
 			const inNames = [];
 			const outName = db.makeTempPath("parquet");
 			const registered = [];
-			for (let i = 0; i < inputs.length; i++) {
+			const bufferedInputs = inputKeys.map((key) => {
 				const name = db.makeTempPath("parquet");
-				await db.registerFileBuffer(name, inputs[i]);
 				inNames.push(name);
-				registered.push(name);
-			}
+				return {
+					key,
+					name
+				};
+			});
 			try {
+				await registerBufferedFiles(db, bufferedInputs, dataSource, registered);
 				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
 				await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
 				registered.push(outName);
@@ -127,23 +143,26 @@ function createDuckDBExecutor(factory) {
 		const registered = [];
 		const totalFiles = Object.values(fileKeys).reduce((n, keys) => n + keys.length, 0);
 		const endRegister = profiler?.start("files.register", { files: totalFiles });
-		await Promise.all(Object.entries(fileKeys).map(async ([name, keys]) => {
-			const uris = keys.map((key) => dataSource.uri?.(key));
-			const buffers = await Promise.all(keys.map((key, i) => uris[i] !== void 0 ? Promise.resolve(void 0) : dataSource.read(key, void 0, signal)));
-			const resolved = [];
-			for (let i = 0; i < keys.length; i++) {
-				const uri = uris[i];
-				if (uri !== void 0) resolved.push(uri);
-				else {
-					await db.registerFileBuffer(keys[i], buffers[i]);
-					registered.push(keys[i]);
-					resolved.push(keys[i]);
-				}
-			}
-			placeholders[name] = resolved;
-		}));
-		endRegister?.({ buffered: registered.length });
 		try {
+			await Promise.all(Object.entries(fileKeys).map(async ([name, keys]) => {
+				const resolved = [];
+				const buffered = [];
+				for (let i = 0; i < keys.length; i++) {
+					const key = keys[i];
+					const uri = dataSource.uri?.(key);
+					if (uri !== void 0) resolved.push(uri);
+					else {
+						buffered.push({
+							key,
+							name: key
+						});
+						resolved.push(key);
+					}
+				}
+				await registerBufferedFiles(db, buffered, dataSource, registered, signal);
+				placeholders[name] = resolved;
+			}));
+			endRegister?.({ buffered: registered.length });
 			signal?.throwIfAborted();
 			const finalSql = substituteNamedFiles(rewriteEmptyFileSets(sql, placeholders, table, placeholderTables), placeholders);
 			const endQuery = profiler?.start("query.run");
@@ -274,7 +293,11 @@ async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
 	}
 	return { deleted: retired.length + sweptOrphans + hourlyDeleted };
 }
-const PUSHABLE_COLUMN = { query: "query" };
+const PUSHABLE_COLUMN = {
+	country: "country",
+	query: "query",
+	searchAppearance: "searchAppearance"
+};
 function txLeaf(leaf, columns) {
 	if (leaf.operator !== "equals") return null;
 	const column = PUSHABLE_COLUMN[leaf.dimension];

package/dist/_chunks/pg-adapter.d.mts

@@ -15,11 +15,25 @@ declare const pgResolverAdapter: ResolverAdapter<PgTableKey>;
  */
 interface ResolverAdapterOptions {
   /**
-   * Opt-in canonical-primary correctness: fold NULL/'' `query_canonical` back
-   * to the raw `query` so canonical is a total GROUP BY / join key. Default
-   * false preserves the legacy raw-column behaviour. See ADR-0018.
+   * Deprecated compatibility flag. Fact tables no longer carry
+   * `query_canonical`; canonical reads derive from `query_dim`, or from a
+   * canonical rollup relation when `queryCanonicalSource: 'column'` is set.
    */
   canonicalFallback?: boolean;
+  /**
+   * `queryDim` reads canonical from a joined query dimension. `column` is only
+   * for derived canonical rollup relations whose primary relation already
+   * carries a null-free `query_canonical` output column.
+   */
+  queryCanonicalSource?: 'queryDim' | 'column';
+}
+interface R2SqlResolverAdapterOptions extends ResolverAdapterOptions {
+  /**
+   * R2 SQL string partition equality can undercount on identity partitions;
+   * string-encoded catalogs use CONCAT(col, '') in partition predicates. Int
+   * catalogs do not need the workaround and keep bare equality for pruning.
+   */
+  partitionKeyEncoding?: 'string' | 'int';
 }
 declare function createParquetResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
 /**
@@ -33,4 +47,16 @@ declare function createParquetResolverAdapter(options?: ResolverAdapterOptions):
  * `${namespace}.pages`) before sending to R2 SQL.
  */
 declare function createIcebergResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
-export { PgTableKey, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter };
+/**
+ * R2 SQL adapter for the Iceberg fact tables.
+ *
+ * It shares the multi-tenant Iceberg schema with `createIcebergResolverAdapter`
+ * but models R2 SQL's narrower execution surface: no window-total plans and no
+ * comparison joins. Int-partition catalogs are the default and emit bare
+ * equality predicates for pruning. Legacy string-partition catalogs must pass
+ * `partitionKeyEncoding: 'string'` to emit `CONCAT(partition_col, '') = ?`,
+ * working around R2 SQL's partition-string equality undercount while preserving
+ * bound params.
+ */
+declare function createR2SqlResolverAdapter(options?: R2SqlResolverAdapterOptions): ResolverAdapter<PgTableKey>;
+export { PgTableKey, R2SqlResolverAdapterOptions, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, pgResolverAdapter };

package/dist/_chunks/resolver.mjs

@@ -2,6 +2,7 @@ import { SCHEMAS, drizzleSchema } from "./schema.mjs";
 import { enumeratePartitions } from "./compaction.mjs";
 import { escapeLike } from "../sql-fragments.mjs";
 import "../planner.mjs";
+import "./schema2.mjs";
 import { UnresolvableDatasetError, buildLogicalComparisonPlan, buildLogicalPlan, inferDataset as inferLogicalDataset, isDatasetResolvable } from "gscdump/query/plan";
 import { PgDialect, pgTable, varchar } from "drizzle-orm/pg-core";
 import { normalizeUrl } from "gscdump/normalize";
@@ -9,7 +10,7 @@ import { sql } from "drizzle-orm";
 const DIMENSION_SURFACES = {
 	page: ["api", "stored"],
 	query: ["api", "stored"],
-	queryCanonical: ["stored", "derived"],
+	queryCanonical: ["derived"],
 	country: ["api", "stored"],
 	device: ["api", "stored"],
 	searchAppearance: ["api", "stored"],
@@ -34,7 +35,7 @@ const LOGICAL_DATASETS = {
 		},
 		queryCanonical: {
 			column: "query_canonical",
-			surfaces: ["stored", "derived"]
+			surfaces: ["derived"]
 		},
 		date: {
 			column: "date",
@@ -52,7 +53,7 @@ const LOGICAL_DATASETS = {
 		},
 		queryCanonical: {
 			column: "query_canonical",
-			surfaces: ["stored", "derived"]
+			surfaces: ["derived"]
 		},
 		date: {
 			column: "date",
@@ -108,7 +109,7 @@ const LOGICAL_DATASETS = {
 		},
 		queryCanonical: {
 			column: "query_canonical",
-			surfaces: ["stored", "derived"]
+			surfaces: ["derived"]
 		},
 		date: {
 			column: "date",
@@ -130,7 +131,7 @@ const LOGICAL_DATASETS = {
 		},
 		queryCanonical: {
 			column: "query_canonical",
-			surfaces: ["stored", "derived"]
+			surfaces: ["derived"]
 		},
 		date: {
 			column: "date",
@@ -169,6 +170,13 @@ const METRIC_NAMES = [
 	"ctr",
 	"position"
 ];
+const QUERY_DIM_ALIAS = "query_dim";
+function quoteIdent(id) {
+	return `"${id.replace(/"/g, "\"\"")}"`;
+}
+function qualifiedRaw(alias, column) {
+	return sql.raw(`${quoteIdent(alias)}.${quoteIdent(column)}`);
+}
 function defaultSqliteUrlToPathExpr(col) {
 	return `CASE WHEN ${col} LIKE 'http%' THEN CASE WHEN INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/') > 0 THEN SUBSTR(${col}, INSTR(${col}, '://') + 2 + INSTR(SUBSTR(${col}, INSTR(${col}, '://') + 3), '/')) ELSE '/' END ELSE ${col} END`;
 }
@@ -180,7 +188,7 @@ function buildDimensionColumnMap(datasetToTableKey) {
 	return Object.fromEntries(entries);
 }
 function createSqlFragments(config) {
-	const { schema, datasetToTableKey, metricCast, regexPredicate, tableLabel, includeSiteId, includeSearchType, urlToPathExpr: urlToPathExprOverride, tableRef: tableRefOverride, canonicalFallback = false } = config;
+	const { schema, datasetToTableKey, metricCast, regexPredicate, tableLabel, includeSiteId, includeSearchType, urlToPathExpr: urlToPathExprOverride, tableRef: tableRefOverride, queryCanonicalSource = "queryDim", queryDimTableRef, queryDimSiteScoped = false } = config;
 	const DIM_COLUMN_MAP = buildDimensionColumnMap(datasetToTableKey);
 	function isMetricDimension(dim) {
 		return METRIC_NAMES.includes(dim);
@@ -205,6 +213,11 @@ function createSqlFragments(config) {
 		if (tableRefOverride) return tableRefOverride(tableKey);
 		return sql`${schema[tableKey]}`;
 	}
+	function fromSql(tableKey, options = {}) {
+		const base = tableRef(tableKey);
+		if (!options.queryCanonical || queryCanonicalSource !== "queryDim") return base;
+		return sql`${base} LEFT JOIN ${queryDimTableRef?.() ?? sql.raw(`${quoteIdent(QUERY_DIM_ALIAS)}`)} ON ${queryDimSiteScoped ? sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")} AND ${colRef(tableKey, "site_id")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "site_id")}` : sql`${colRef(tableKey, "query")} = ${qualifiedRaw(QUERY_DIM_ALIAS, "query")}`}`;
+	}
 	function dateColRef(tableKey) {
 		return colRef(tableKey, "date");
 	}
@@ -217,7 +230,10 @@ function createSqlFragments(config) {
 	function dimExprSql(dim, tableKey) {
 		const colName = dimColumn(dim, tableKey);
 		if (dim === "page") return sql.raw(urlToPathExpr(colName));
-		if (canonicalFallback && dim === "queryCanonical") return sql`COALESCE(NULLIF(${colRef(tableKey, colName)}, ''), ${colRef(tableKey, "query")})`;
+		if (dim === "queryCanonical") {
+			if (queryCanonicalSource === "queryDim") return sql`COALESCE(${qualifiedRaw(QUERY_DIM_ALIAS, "query_canonical")}, ${colRef(tableKey, "query")})`;
+			return qualifiedRaw(String(tableKey), "query_canonical");
+		}
 		return colRef(tableKey, colName);
 	}
 	function metricSql(metric, tableKey) {
@@ -298,7 +314,7 @@ function createSqlFragments(config) {
 			if (f.dimension === "date") continue;
 			if (f.operator === "topLevel") continue;
 			const dim = f.dimension;
-			const cRef = colRef(tableKey, dimColumn(dim, tableKey));
+			const cRef = dim === "queryCanonical" ? void 0 : colRef(tableKey, dimColumn(dim, tableKey));
 			const matchExpr = dim === "page" || dim === "queryCanonical" ? dimExprSql(dim, tableKey) : cRef;
 			const patternExpr = dim === "queryCanonical" ? matchExpr : cRef;
 			switch (f.operator) {
@@ -339,6 +355,7 @@ function createSqlFragments(config) {
 		urlToPathExpr,
 		colRef,
 		tableRef,
+		fromSql,
 		dateColRef,
 		siteIdColRef: includeSiteId ? siteIdColRef : void 0,
 		searchTypeColRef: includeSearchType ? searchTypeColRef : void 0,
@@ -361,6 +378,7 @@ function createResolverAdapter(config) {
 		dimColumn: runtime.dimColumn,
 		isMetricDimension: runtime.isMetricDimension,
 		tableRef: runtime.tableRef,
+		fromSql: runtime.fromSql,
 		dateColRef: runtime.dateColRef,
 		urlToPathExpr: runtime.urlToPathExpr,
 		siteIdColRef: runtime.siteIdColRef,
@@ -439,7 +457,9 @@ function createParquetResolverAdapter(options = {}) {
 		...PG_BASE_CONFIG,
 		tableLabel: "parquet-resolver-adapter",
 		canonicalFallback: options.canonicalFallback ?? false,
-		tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`)
+		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
+		tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`),
+		queryDimTableRef: () => sql.raw("read_parquet({{QUERY_DIM}}, union_by_name = true) AS \"query_dim\"")
 	});
 }
 function createIcebergResolverAdapter(options = {}) {
@@ -450,9 +470,35 @@ function createIcebergResolverAdapter(options = {}) {
 		includeSearchType: true,
 		tableLabel: "iceberg-resolver-adapter",
 		canonicalFallback: options.canonicalFallback ?? false,
-		tableRef: (tk) => sql.raw(`"${tk}"`)
+		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
+		tableRef: (tk) => sql.raw(`"${tk}"`),
+		queryDimTableRef: () => sql.raw("\"query_dim\"")
 	});
 }
+function createR2SqlResolverAdapter(options = {}) {
+	const adapter = createResolverAdapter({
+		...PG_BASE_CONFIG,
+		schema: icebergSchema,
+		includeSiteId: true,
+		includeSearchType: true,
+		tableLabel: "r2-sql-resolver-adapter",
+		canonicalFallback: options.canonicalFallback ?? false,
+		queryCanonicalSource: options.queryCanonicalSource ?? "queryDim",
+		capabilities: {
+			regex: false,
+			comparisonJoin: false,
+			windowTotals: false
+		},
+		tableRef: (tk) => sql.raw(`"${tk}"`),
+		queryDimTableRef: () => sql.raw("\"query_dim\"")
+	});
+	if ((options.partitionKeyEncoding ?? "int") === "int") return adapter;
+	return {
+		...adapter,
+		siteIdColRef: (tk) => sql`CONCAT(${adapter.siteIdColRef(tk)}, '')`,
+		searchTypeColRef: (tk) => sql`CONCAT(${adapter.searchTypeColRef(tk)}, '')`
+	};
+}
 const ALLOWED_FILTER_DIMS = /* @__PURE__ */ new Set(["date", "queryCanonical"]);
 function planCoveredByCanonicalRollup(plan) {
 	if (plan.dataset !== "queries") return false;
@@ -548,6 +594,7 @@ function buildScope(state, options) {
 	const groupByDims = plan.groupByDimensions;
 	const hasDate = plan.hasDate;
 	const metrics = plan.metrics;
+	const queryCanonicalUsed = groupByDims.includes("queryCanonical") || plan.dimensionFilters.some((filter) => filter.dimension === "queryCanonical");
 	const wherePredicates = [];
 	if (adapter.siteIdColRef && siteId != null) wherePredicates.push(sql`${adapter.siteIdColRef(tableKey)} = ${siteId}`);
 	if (adapter.searchTypeColRef && searchType != null) wherePredicates.push(sql`${adapter.searchTypeColRef(tableKey)} = ${searchType}`);
@@ -569,7 +616,8 @@ function buildScope(state, options) {
 		having: adapter.havingPredicates(metricFilters, tableKey),
 		dimFilters,
 		startDate: plan.dateRange.startDate,
-		endDate: plan.dateRange.endDate
+		endDate: plan.dateRange.endDate,
+		queryCanonicalUsed
 	};
 }
 function buildComparisonPlan(current, previous, capabilities) {
@@ -584,8 +632,8 @@ function compileCollapsed(adapter, q) {
 }
 function resolveToSQLOptimized(state, options) {
 	const { adapter } = options;
-	const { tableKey, groupByDims, hasDate, metrics, wherePredicates, having } = buildScope(state, options);
-	const table = adapter.tableRef(tableKey);
+	const { tableKey, groupByDims, hasDate, metrics, wherePredicates, having, queryCanonicalUsed } = buildScope(state, options);
+	const table = adapter.fromSql(tableKey, { queryCanonical: queryCanonicalUsed });
 	const schema = adapter.schema;
 	const cteSelect = [];
 	for (const d of groupByDims) {
@@ -632,8 +680,8 @@ function resolveToSQLOptimized(state, options) {
 }
 function resolveToSQL(state, options) {
 	const { adapter } = options;
-	const { tableKey, groupByDims, hasDate, metrics, wherePredicates, having } = buildScope(state, options);
-	const table = adapter.tableRef(tableKey);
+	const { tableKey, groupByDims, hasDate, metrics, wherePredicates, having, queryCanonicalUsed } = buildScope(state, options);
+	const table = adapter.fromSql(tableKey, { queryCanonical: queryCanonicalUsed });
 	const selectExprs = [];
 	for (const d of groupByDims) {
 		const expr = adapter.dimExprSql(d, tableKey);
@@ -666,8 +714,8 @@ function resolveToSQL(state, options) {
 }
 function buildTotalsSql(state, options) {
 	const { adapter } = options;
-	const { tableKey, metrics, wherePredicates } = buildScope(state, options);
-	const table = adapter.tableRef(tableKey);
+	const { tableKey, metrics, wherePredicates, queryCanonicalUsed } = buildScope(state, options);
+	const table = adapter.fromSql(tableKey, { queryCanonical: queryCanonicalUsed });
 	const selectExprs = metrics.map((m) => sql`${adapter.metricSql(m, tableKey)} as ${aliasRaw(m)}`);
 	return compileCollapsed(adapter, wherePredicates.length > 0 ? sql`SELECT ${joinComma(selectExprs)} FROM ${table} WHERE ${joinAnd(wherePredicates)}` : sql`SELECT ${joinComma(selectExprs)} FROM ${table}`);
 }
@@ -677,7 +725,7 @@ function resolveComparisonSQL(current, previous, options, comparisonFilter) {
 	const currentScope = buildScope(current, options);
 	const previousScope = buildScope(previous, options);
 	const { tableKey, groupByDims, metrics, wherePredicates: currentWhere, having } = currentScope;
-	const table = adapter.tableRef(tableKey);
+	const table = adapter.fromSql(tableKey, { queryCanonical: currentScope.queryCanonicalUsed || previousScope.queryCanonicalUsed });
 	const dimSelectExprs = [];
 	for (const d of groupByDims) {
 		const expr = adapter.dimExprSql(d, tableKey);
@@ -735,7 +783,7 @@ function buildExtrasQueries(state, options) {
 	if (!dims.includes("queryCanonical")) return extras;
 	const queriesKey = adapter.tableKeyForDataset("queries");
 	const t = adapter.schema[queriesKey];
-	const table = adapter.tableRef(queriesKey);
+	const table = adapter.fromSql(queriesKey, { queryCanonical: true });
 	const whereParts = [];
 	if (adapter.siteIdColRef && siteId != null) whereParts.push(sql`${adapter.siteIdColRef(queriesKey)} = ${siteId}`);
 	if (adapter.searchTypeColRef && searchType != null) whereParts.push(sql`${adapter.searchTypeColRef(queriesKey)} = ${searchType}`);
@@ -743,7 +791,7 @@ function buildExtrasQueries(state, options) {
 	whereParts.push(sql`${adapter.dateColRef(queriesKey)} <= ${plan.dateRange.endDate}`);
 	const whereExpr = whereParts.length > 0 ? sql`WHERE ${joinAnd(whereParts)}` : sql``;
 	const outerQueryCol = sql.raw("query");
-	const canonKey = sql`COALESCE(NULLIF(${t.query_canonical}, ''), ${t.query})`;
+	const canonKey = adapter.dimExprSql("queryCanonical", queriesKey);
 	const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${canonKey} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${canonKey} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${canonKey}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${canonKey}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, GROUP_CONCAT(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
 	extras.push({
 		key: "canonicalExtras",
@@ -890,9 +938,80 @@ function matchesMetricFilter(row, filter) {
 function matchesTopLevelPage(row) {
 	return (normalizeUrl(dimensionValue(row, "page")).match(/\//g)?.length ?? 0) <= 1;
 }
+var QuerySourceCoverageError = class extends Error {
+	fallback;
+	name = "QuerySourceCoverageError";
+	constructor(fallback) {
+		super(fallback.message);
+		this.fallback = fallback;
+	}
+};
 function canonicalSourceWithinCoverage(source, windowEnd) {
 	return source.coversThrough === void 0 || windowEnd <= source.coversThrough;
 }
+function querySourceDecision(kind, fallbacks = []) {
+	const first = fallbacks[0];
+	return {
+		kind,
+		...first ? {
+			fallback: first,
+			fallbacks
+		} : {}
+	};
+}
+function canonicalFeatureRequested(state, capabilities) {
+	const plan = buildLogicalPlan(state, capabilities);
+	return plan.groupByDimensions.includes("queryCanonical") || plan.dimensionFilters.some((f) => f.dimension === "queryCanonical");
+}
+function fallback(kind, message) {
+	return {
+		kind,
+		message
+	};
+}
+function staleQueryDim(source, requirements) {
+	if (requirements?.normalizerVersion !== void 0 && source.normalizerVersion !== requirements.normalizerVersion) return fallback("query-dim-stale-version", `canonical query dimension normalizer v${source.normalizerVersion} does not match required v${requirements.normalizerVersion}`);
+	if (requirements?.intentVersion !== void 0 && source.intentVersion !== requirements.intentVersion) return fallback("query-dim-stale-version", `canonical query dimension intent v${source.intentVersion ?? "missing"} does not match required v${requirements.intentVersion}`);
+}
+function canonicalQueryDim(options) {
+	return options.queryDim ?? options.canonicalSource?.queryDim;
+}
+function queryDimMiss(source, requirements) {
+	if (!source) return fallback("query-dim-missing", "canonical query requires queryDim; fact tables do not carry query_canonical");
+	if (source.keys.length === 0) return fallback("query-dim-empty", "queryDim has no parquet keys; refusing to run canonical query from an empty dimension");
+	return staleQueryDim(source, requirements);
+}
+function decideCanonicalSource(state, capabilities, options, windowEnd) {
+	if (!canonicalFeatureRequested(state, capabilities)) return querySourceDecision("raw-partitions");
+	const dimMiss = queryDimMiss(canonicalQueryDim(options), options.canonicalRequirements);
+	if (dimMiss) throw new QuerySourceCoverageError(dimMiss);
+	const source = options.canonicalSource;
+	const miss = (() => {
+		if (!source) return fallback("canonical-source-missing", "canonical rollup source missing; falling back to primary facts joined through query_dim");
+		if (source.keys.length === 0) return fallback("canonical-source-empty", "canonicalSource has no parquet keys; falling back to primary facts joined through query_dim");
+		if (!canonicalSourceWithinCoverage(source, windowEnd)) return fallback("canonical-source-stale-coverage", `canonicalSource covers through ${source.coversThrough}, but query needs ${windowEnd}`);
+		if (!canonicalRollupCovers(state, capabilities)) return fallback("canonical-source-not-coverable", "canonicalSource only covers queryCanonical/date reads without raw-grain filters");
+	})();
+	if (!miss) return querySourceDecision("canonical-rollup");
+	return querySourceDecision("raw-partitions", [miss]);
+}
+function primarySourceMiss(source, dateRange) {
+	if (!source) return fallback("primary-source-missing", "primary columnar source is required; pass primarySourceFallback: \"raw\" to use raw daily partitions explicitly");
+	if (source.keys.length === 0) return fallback("primary-source-empty", "primarySource has no parquet keys; refusing to return partial or empty data implicitly");
+	if (source.coversFrom !== void 0 && dateRange.startDate < source.coversFrom) return fallback("primary-source-missing-coverage", `primarySource covers from ${source.coversFrom}, but query starts ${dateRange.startDate}`);
+	if (source.coversThrough !== void 0 && dateRange.endDate > source.coversThrough) return fallback("primary-source-stale-coverage", `primarySource covers through ${source.coversThrough}, but query needs ${dateRange.endDate}`);
+}
+function decidePrimarySource(options, dateRange, priorFallbacks = []) {
+	const miss = primarySourceMiss(options.primarySource, dateRange);
+	if (!miss) return querySourceDecision("primary-columnar", priorFallbacks);
+	const fallbacks = [...priorFallbacks, miss];
+	if (options.primarySourceFallback === "raw") return querySourceDecision("raw-partitions", fallbacks);
+	throw new QuerySourceCoverageError(miss);
+}
+function sourceFallbacks(source) {
+	if (source.fallbacks) return source.fallbacks;
+	return source.fallback ? [source.fallback] : [];
+}
 function runArgs(ctx, partitions) {
 	return {
 		ctx: {
@@ -907,26 +1026,58 @@ function runArgs(ctx, partitions) {
 		...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {}
 	};
 }
+function primaryRunArgs(ctx, keys) {
+	return {
+		ctx: {
+			userId: ctx.userId,
+			siteId: ctx.siteId
+		},
+		table: ctx.table,
+		fileSets: { FILES: {
+			table: ctx.table,
+			keys
+		} },
+		...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {}
+	};
+}
+function withQueryDimFileSet(args, queryDim, enabled) {
+	if (!enabled || !queryDim) return args;
+	return {
+		...args,
+		fileSets: {
+			...args.fileSets,
+			QUERY_DIM: {
+				table: "queries",
+				keys: queryDim.keys
+			}
+		}
+	};
+}
 async function runOptimizedQuery(runSQL, ctx, state, dateRange, options = {}) {
 	const base = runArgs(ctx, enumeratePartitions(dateRange.startDate, dateRange.endDate));
-	const probe = createParquetResolverAdapter({ canonicalFallback: options.canonicalFallback ?? false });
-	const useCanonicalSource = options.canonicalSource !== void 0 && (options.canonicalFallback ?? false) && canonicalSourceWithinCoverage(options.canonicalSource, dateRange.endDate) && canonicalRollupCovers(state, probe.capabilities);
-	const adapter = useCanonicalSource ? createParquetResolverAdapter({ canonicalFallback: false }) : probe;
+	const probe = createParquetResolverAdapter();
+	const canonicalRequested = canonicalFeatureRequested(state, probe.capabilities);
+	const canonicalDecision = decideCanonicalSource(state, probe.capabilities, options, dateRange.endDate);
+	const useCanonicalSource = canonicalDecision.kind === "canonical-rollup";
+	const source = useCanonicalSource ? canonicalDecision : decidePrimarySource(options, dateRange, sourceFallbacks(canonicalDecision));
 	const optimized = resolveToSQLOptimized(state, {
-		adapter,
+		adapter: useCanonicalSource ? createParquetResolverAdapter({ queryCanonicalSource: "column" }) : probe,
 		siteId: void 0
 	});
 	const extras = buildExtrasQueries(state, {
-		adapter,
+		adapter: probe,
 		siteId: void 0
 	});
-	const mainArgs = useCanonicalSource ? {
+	const extraSource = extras.length > 0 ? useCanonicalSource ? decidePrimarySource(options, dateRange) : source : void 0;
+	const queryDim = canonicalQueryDim(options);
+	const mainArgs = withQueryDimFileSet(useCanonicalSource ? {
 		...base,
 		fileSets: { FILES: {
 			table: ctx.table,
 			keys: options.canonicalSource.keys
 		} }
-	} : base;
+	} : source.kind === "primary-columnar" ? primaryRunArgs(ctx, options.primarySource.keys) : base, queryDim, canonicalRequested && !useCanonicalSource);
+	const extraArgs = withQueryDimFileSet(extraSource?.kind === "primary-columnar" ? primaryRunArgs(ctx, options.primarySource.keys) : base, queryDim, extras.length > 0);
 	const resolveExtra = options.resolveExtra;
 	const [optRes, ...extrasRows] = await Promise.all([runSQL({
 		...mainArgs,
@@ -940,7 +1091,7 @@ async function runOptimizedQuery(runSQL, ctx, state, dateRange, options = {}) {
 			dateRange
 		}) : null;
 		return overlaid !== null ? { rows: overlaid } : runSQL({
-			...base,
+			...extraArgs,
 			sql: e.sql,
 			params: e.params
 		});
@@ -963,13 +1114,25 @@ async function runOptimizedQuery(runSQL, ctx, state, dateRange, options = {}) {
 		extras: extras.map((e, i) => ({
 			key: e.key,
 			rows: extrasRows[i].rows
-		}))
+		})),
+		source,
+		...extraSource ? { extraSource } : {}
 	};
 }
 async function runComparisonQuery(runSQL, ctx, current, previous, windows, filter, options = {}) {
-	const probe = createParquetResolverAdapter({ canonicalFallback: options.canonicalFallback ?? false });
-	const useCanonicalSource = options.canonicalSource !== void 0 && (options.canonicalFallback ?? false) && canonicalSourceWithinCoverage(options.canonicalSource, windows.current.endDate > windows.previous.endDate ? windows.current.endDate : windows.previous.endDate) && canonicalRollupCovers(current, probe.capabilities) && canonicalRollupCovers(previous, probe.capabilities);
-	const adapter = useCanonicalSource ? createParquetResolverAdapter({ canonicalFallback: false }) : probe;
+	const probe = createParquetResolverAdapter();
+	const maxWindowEnd = windows.current.endDate > windows.previous.endDate ? windows.current.endDate : windows.previous.endDate;
+	const currentSource = decideCanonicalSource(current, probe.capabilities, options, maxWindowEnd);
+	const previousSource = decideCanonicalSource(previous, probe.capabilities, options, maxWindowEnd);
+	const startDate = windows.current.startDate < windows.previous.startDate ? windows.current.startDate : windows.previous.startDate;
+	const endDate = windows.current.endDate > windows.previous.endDate ? windows.current.endDate : windows.previous.endDate;
+	const comparisonRange = {
+		startDate,
+		endDate
+	};
+	const source = currentSource.kind === "canonical-rollup" && previousSource.kind === "canonical-rollup" ? querySourceDecision("canonical-rollup") : decidePrimarySource(options, comparisonRange, [...sourceFallbacks(currentSource), ...sourceFallbacks(previousSource)]);
+	const useCanonicalSource = source.kind === "canonical-rollup";
+	const adapter = useCanonicalSource ? createParquetResolverAdapter({ queryCanonicalSource: "column" }) : probe;
 	const comparison = resolveComparisonSQL(current, previous, {
 		adapter,
 		siteId: void 0
@@ -978,14 +1141,14 @@ async function runComparisonQuery(runSQL, ctx, current, previous, windows, filte
 		adapter,
 		siteId: void 0
 	});
-	const partitions = enumeratePartitions(windows.current.startDate < windows.previous.startDate ? windows.current.startDate : windows.previous.startDate, windows.current.endDate > windows.previous.endDate ? windows.current.endDate : windows.previous.endDate);
-	const base = useCanonicalSource ? {
+	const partitions = enumeratePartitions(startDate, endDate);
+	const base = withQueryDimFileSet(useCanonicalSource ? {
 		...runArgs(ctx, partitions),
 		fileSets: { FILES: {
 			table: ctx.table,
 			keys: options.canonicalSource.keys
 		} }
-	} : runArgs(ctx, partitions);
+	} : source.kind === "primary-columnar" ? primaryRunArgs(ctx, options.primarySource.keys) : runArgs(ctx, partitions), canonicalQueryDim(options), !useCanonicalSource && (canonicalFeatureRequested(current, probe.capabilities) || canonicalFeatureRequested(previous, probe.capabilities)));
 	const main = await runSQL({
 		...base,
 		sql: comparison.sql,
@@ -1004,7 +1167,8 @@ async function runComparisonQuery(runSQL, ctx, current, previous, windows, filte
 	return {
 		rows: main.rows,
 		totalCount: Number(count.rows[0]?.total ?? 0),
-		totals: totalsRow.rows[0] ?? {}
+		totals: totalsRow.rows[0] ?? {},
+		source
 	};
 }
 function assertSchemaInSync(options) {
@@ -1017,4 +1181,4 @@ function assertSchemaInSync(options) {
 		if (missing.length > 0 || extra.length > 0) throw new Error(`${label} drizzle schema for '${key}' drifted from SCHEMAS. Missing: [${missing.join(", ")}]. Extra: [${extra.join(", ")}].`);
 	}
 }
-export { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
+export { DIMENSION_SURFACES, LOGICAL_DATASETS, QuerySourceCoverageError, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createR2SqlResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };