@gscdump/engine 0.28.3 → 0.30.0

package/dist/_chunks/duckdb.d.mts

@@ -19,7 +19,7 @@ declare function createDuckDBExecutor(factory: DuckDBFactory): QueryExecutor;
 /**
  * Canonical "empty-file" SELECT clause for a table. Codecs that need to
  * emit a schema-correct empty Parquet can wrap this in:
- *   `COPY (SELECT * FROM <clause> WHERE FALSE) TO '<key>' (FORMAT PARQUET)`
+ *   `COPY (SELECT * FROM <clause> WHERE FALSE) TO '<key>' (FORMAT PARQUET, COMPRESSION ZSTD)`
  * to satisfy the ParquetCodec empty-rows invariant.
  */
 declare function canonicalEmptyParquetSchema(table: TableName): string;

package/dist/_chunks/engine.mjs

@@ -1,6 +1,7 @@
 import { dayPartition, hourPartition, inferSearchType, objectKey, tenantPrefix } from "./layout.mjs";
-import { SCHEMAS, currentSchemaVersion, dedupeByNaturalKey } from "./schema.mjs";
+import { SCHEMAS, currentSchemaVersion, dateColumnsFor, dedupeByNaturalKey } from "./schema.mjs";
 import { compactTieredImpl, dedupeOverlappingTiers, splitOverlappingTiers } from "./compaction.mjs";
+import { dateReplaceClause as dateReplaceClause$1 } from "../sql-fragments.mjs";
 import { compileLogicalQueryPlan, substituteNamedFiles } from "./parquet-plan.mjs";
 import { sqlEscape } from "../sql-bind.mjs";
 import { buildLogicalPlan } from "gscdump/query/plan";
@@ -13,7 +14,7 @@ async function encodeBytes(db, table, rows) {
 	await db.registerFileBuffer(inName, jsonBytes);
 	registered.push(inName);
 	try {
-		const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`;
+		const sql = rows.length === 0 ? `COPY (SELECT * FROM ${emptyTableSchema(table)} WHERE FALSE) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)` : `COPY (SELECT * FROM read_json_auto('${sqlEscape(inName)}', format='array', columns=${columnsJson(table)})) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`;
 		await db.query(sql);
 		registered.push(outName);
 		return await db.copyFileToBuffer(outName);
@@ -58,7 +59,7 @@ function createDuckDBCodec(factory) {
 				const outName = db.makeTempPath("parquet");
 				const fileList = inputUris.map((u) => `'${sqlEscape(u)}'`).join(", ");
 				try {
-					await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
+					await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
 					const bytes = await db.copyFileToBuffer(outName);
 					const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
 					const rowCount = Number(countRows[0]?.n ?? 0);
@@ -83,7 +84,7 @@ function createDuckDBCodec(factory) {
 			}
 			try {
 				const fileList = inNames.map((n) => `'${sqlEscape(n)}'`).join(", ");
-				await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET)`);
+				await db.query(`COPY (${dedupedMergeSql(ctx.table, fileList)}) TO '${sqlEscape(outName)}' (FORMAT PARQUET, COMPRESSION ZSTD)`);
 				registered.push(outName);
 				const bytes = await db.copyFileToBuffer(outName);
 				const countRows = await db.query(`SELECT count(*)::BIGINT AS n FROM read_parquet('${sqlEscape(outName)}')`);
@@ -162,9 +163,7 @@ function canonicalEmptyParquetSchema(table) {
 }
 function dateReplaceClause(table) {
 	if (!table) return "";
-	const dateCols = SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
-	if (dateCols.length === 0) return "";
-	return `REPLACE (${dateCols.map((n) => `strftime(CAST(${n} AS DATE), '%Y-%m-%d') AS ${n}`).join(", ")})`;
+	return dateReplaceClause$1(dateColumnsFor(table), "string");
 }
 function columnList(table) {
 	return SCHEMAS[table].columns.map((c) => c.name).join(", ");
@@ -272,6 +271,52 @@ async function gcOrphansImpl(deps, now, graceMs, opts = {}) {
 	}
 	return { deleted: retired.length + sweptOrphans + hourlyDeleted };
 }
+const PUSHABLE_COLUMN = { query: "query" };
+function txLeaf(leaf, columns) {
+	if (leaf.operator !== "equals") return null;
+	const column = PUSHABLE_COLUMN[leaf.dimension];
+	if (!column || !columns.has(column)) return null;
+	return { [column]: { $eq: leaf.expression } };
+}
+function txExact(node, columns) {
+	const groupType = node._groupType ?? "and";
+	const leafParts = [];
+	for (const leaf of node._filters) {
+		const t = txLeaf(leaf, columns);
+		if (!t) return null;
+		leafParts.push(t);
+	}
+	if (groupType === "or") {
+		if (node._nestedGroups?.length || leafParts.length === 0) return null;
+		return leafParts.length === 1 ? leafParts[0] : { $or: leafParts };
+	}
+	const parts = leafParts;
+	for (const group of node._nestedGroups ?? []) {
+		const t = txExact(group, columns);
+		if (!t) return null;
+		parts.push(t);
+	}
+	if (parts.length === 0) return null;
+	return parts.length === 1 ? parts[0] : { $and: parts };
+}
+function extractParquetPushdown(state, table) {
+	const filter = state?.filter;
+	const schema = SCHEMAS[table];
+	if (!filter || !schema) return void 0;
+	const columns = new Set(schema.columns.map((c) => c.name));
+	if ((filter._groupType ?? "and") === "or") return txExact(filter, columns) ?? void 0;
+	const parts = [];
+	for (const leaf of filter._filters) {
+		const t = txLeaf(leaf, columns);
+		if (t) parts.push(t);
+	}
+	for (const group of filter._nestedGroups ?? []) {
+		const t = txExact(group, columns);
+		if (t) parts.push(t);
+	}
+	if (parts.length === 0) return void 0;
+	return parts.length === 1 ? parts[0] : { $and: parts };
+}
 const URL_PURGE_TABLES = ["pages", "page_queries"];
 const MAX_DAY_BYTES = 100 * 1024 * 1024;
 const URL_COLUMNS = /* @__PURE__ */ new Set();
@@ -463,6 +508,7 @@ function createStorageEngine(opts) {
 			dataSource,
 			table,
 			signal: opts.signal,
+			...opts.pushdownFilters ? { pushdownFilters: opts.pushdownFilters } : {},
 			...profiler ? { profiler } : {}
 		});
 		endExec?.({ rows: result.rows.length });
@@ -476,6 +522,7 @@ function createStorageEngine(opts) {
 		const plan = buildLogicalPlan(state, { regex: true });
 		const table = ctx.table ?? plan.dataset;
 		const resolved = compileLogicalQueryPlan(plan, table);
+		const pushdown = extractParquetPushdown(state, table);
 		return runSQL({
 			ctx: {
 				userId: ctx.userId,
@@ -489,6 +536,7 @@ function createStorageEngine(opts) {
 			sql: resolved.sql,
 			params: resolved.params,
 			signal: ctx.signal,
+			...pushdown ? { pushdownFilters: { FILES: pushdown } } : {},
 			...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {},
 			...ctx.profiler ? { profiler: ctx.profiler } : {}
 		});

package/dist/_chunks/index.d.mts

@@ -60,8 +60,12 @@ interface CreateSqlQuerySourceOptions<TKey extends string> {
   execute: (sql: string, params: unknown[]) => Promise<QueryRow[]>;
   /** Tenant id for multi-tenant dialects; forwarded to `resolveToSQL`. */
   siteId?: string | number;
-  /** Search-type scope for multi-tenant dialects; forwarded to `resolveToSQL`. */
-  searchType?: string;
+  /**
+   * Search-type scope for multi-tenant dialects; forwarded to `resolveToSQL`.
+   * `number` = int-encoded code (`SEARCH_TYPE_INT`) for INT `search_type`
+   * catalogs (bound bare so the int partition prunes); `string` otherwise.
+   */
+  searchType?: string | number;
   /** Additional capability flags merged on top of `adapter.capabilities`. */
   extraCapabilities?: Partial<SourceCapabilities>;
 }

package/dist/_chunks/libs/hyparquet-compressors.mjs

@@ -175,8 +175,8 @@ function nextTableBitSize(count, len, root_bits) {
 }
 function buildHuffmanTable(root_table, table, root_bits, code_lengths, code_lengths_size) {
 	const start_table = table;
-	const count = new Int32Array(16);
-	const offset = new Int32Array(16);
+	const count = /* @__PURE__ */ new Int32Array(16);
+	const offset = /* @__PURE__ */ new Int32Array(16);
 	const sorted = new Int32Array(code_lengths_size);
 	for (let i = 0; i < code_lengths_size; i++) count[code_lengths[i]]++;
 	offset[1] = 0;
@@ -220,7 +220,7 @@ function readHuffmanCode(alphabet_size, tables, table, br) {
 	if (simple_code_or_skip === 1) {
 		let max_bits_counter = alphabet_size - 1;
 		let max_bits = 0;
-		const symbols = new Int32Array(4);
+		const symbols = /* @__PURE__ */ new Int32Array(4);
 		const num_symbols = br.readBits(2) + 1;
 		while (max_bits_counter) {
 			max_bits_counter >>= 1;
@@ -505,7 +505,7 @@ const fixedDistanceExtraBits = new Uint8Array([
 	0
 ]);
 function freb(eb, start) {
-	const base = new Uint16Array(31);
+	const base = /* @__PURE__ */ new Uint16Array(31);
 	for (let i = 0; i < 31; i++) base[i] = start += 1 << eb[i - 1];
 	const rev = new Int32Array(base[30]);
 	for (let i = 1; i < 30; i++) for (let j = base[i]; j < base[i + 1]; ++j) rev[j] = j - base[i] << 5 | i;
@@ -518,7 +518,7 @@ const { base: fixedLength, rev: revfl } = freb(fixedLengthExtraBits, 2);
 fixedLength[28] = 258;
 revfl[258] = 28;
 const { base: fixedDistance } = freb(fixedDistanceExtraBits, 0);
-const rev = new Uint16Array(32768);
+const rev = /* @__PURE__ */ new Uint16Array(32768);
 for (let i = 0; i < 32768; i++) {
 	let x = (i & 43690) >> 1 | (i & 21845) << 1;
 	x = (x & 52428) >> 2 | (x & 13107) << 2;
@@ -546,12 +546,12 @@ function huffMap(cd, maxBits, r) {
 	}
 	return co;
 }
-const fixedLengthTree = new Uint8Array(288);
+const fixedLengthTree = /* @__PURE__ */ new Uint8Array(288);
 for (let i = 0; i < 144; i++) fixedLengthTree[i] = 8;
 for (let i = 144; i < 256; i++) fixedLengthTree[i] = 9;
 for (let i = 256; i < 280; i++) fixedLengthTree[i] = 7;
 for (let i = 280; i < 288; i++) fixedLengthTree[i] = 8;
-const fixedDistanceTree = new Uint8Array(32);
+const fixedDistanceTree = /* @__PURE__ */ new Uint8Array(32);
 for (let i = 0; i < 32; i++) fixedDistanceTree[i] = 5;
 const fixedLengthMap = /*#__PURE__*/ huffMap(fixedLengthTree, 9, 1);
 const fixedDistanceMap = /*#__PURE__*/ huffMap(fixedDistanceTree, 5, 1);
@@ -2420,7 +2420,7 @@ function gzipStart(input, i) {
 	return i + (flag & 2);
 }
 function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
-	let out = output ?? new Uint8Array(1024);
+	let out = output ?? /* @__PURE__ */ new Uint8Array(1024);
 	if (!(input.length - inputIndex)) return out;
 	const payloadStart = gzipStart(input, inputIndex);
 	if (payloadStart === input.length - 8) return out;
@@ -2465,7 +2465,7 @@ function gunzip(input, output, inputIndex = 0, outputIndex = 0) {
 				const tl = hLiteral + bits(input, pos + 5, 31) + 1;
 				pos += 14;
 				const lengthDistanceTree = new Uint8Array(tl);
-				const codeLengthTree = new Uint8Array(19);
+				const codeLengthTree = /* @__PURE__ */ new Uint8Array(19);
 				for (let i = 0; i < hcLengths; ++i) codeLengthTree[codeLengthIndexMap[i]] = bits(input, pos + i * 3, 7);
 				pos += hcLengths * 3;
 				const codeLengthBits = Math.max(...codeLengthTree);

package/dist/_chunks/libs/icebird.mjs

@@ -962,7 +962,7 @@ function uuidToBytes(value, label) {
 	if (typeof value !== "string") throw new Error(`expected ${label}`);
 	const hex = value.toLowerCase().replace(/-/g, "");
 	if (!/^[0-9a-f]{32}$/.test(hex)) throw new Error(`expected ${label}`);
-	const bytes = new Uint8Array(16);
+	const bytes = /* @__PURE__ */ new Uint8Array(16);
 	for (let i = 0; i < bytes.length; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
 	return bytes;
 }
@@ -1074,7 +1074,7 @@ function bucketBytes(value, sourceType) {
 			else if (t === "timestamp" || t === "timestamptz") v = value instanceof Date ? BigInt(value.getTime()) * 1000n : BigInt(value);
 			else if (t === "timestamp_ns" || t === "timestamptz_ns") v = value instanceof Date ? BigInt(value.getTime()) * 1000n : BigInt(value) / 1000n;
 			else v = typeof value === "bigint" ? value : BigInt(value);
-			const out = new Uint8Array(8);
+			const out = /* @__PURE__ */ new Uint8Array(8);
 			new DataView(out.buffer).setBigInt64(0, v, true);
 			return out;
 		}
@@ -1835,7 +1835,7 @@ function avroWrite({ writer, schema, records, blockSize = 512, metadata }) {
 		writer.appendBytes(vb);
 	}
 	writer.appendVarInt(0);
-	const sync = new Uint8Array(16);
+	const sync = /* @__PURE__ */ new Uint8Array(16);
 	for (let i = 0; i < 16; i++) sync[i] = Math.random() * 256 | 0;
 	writer.appendBytes(sync);
 	for (let i = 0; i < records.length; i += blockSize) {
@@ -1940,7 +1940,7 @@ function appendZigZag64(writer, v) {
 function uuidStringToBytes$1(value) {
 	const hex = value.toLowerCase().replace(/-/g, "");
 	if (!/^[0-9a-f]{32}$/.test(hex)) throw new Error("expected uuid string");
-	const bytes = new Uint8Array(16);
+	const bytes = /* @__PURE__ */ new Uint8Array(16);
 	for (let i = 0; i < 16; i++) bytes[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
 	return bytes;
 }
@@ -2531,7 +2531,7 @@ function twosComplementMinBigEndian(value) {
 function uuidStringToBytes(s) {
 	const hex = s.replace(/-/g, "");
 	if (hex.length !== 32) return void 0;
-	const out = new Uint8Array(16);
+	const out = /* @__PURE__ */ new Uint8Array(16);
 	for (let i = 0; i < 16; i++) {
 		const byte = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
 		if (Number.isNaN(byte)) return void 0;
@@ -3286,7 +3286,7 @@ function resolveParquetCodec(value) {
 }
 function newSnapshotId(metadata) {
 	const used = new Set((metadata?.snapshots ?? []).map((s) => BigInt(s["snapshot-id"])));
-	const arr = new BigInt64Array(1);
+	const arr = /* @__PURE__ */ new BigInt64Array(1);
 	for (let attempt = 0; attempt < 32; attempt++) {
 		globalThis.crypto.getRandomValues(arr);
 		const masked = arr[0] & 9007199254740991n;
@@ -3319,24 +3319,38 @@ async function icebergManifests({ metadata, resolver, snapshotId, partitionFilte
 	});
 	return await fetchManifests(manifests, resolver);
 }
+const MANIFEST_FETCH_CONCURRENCY = 8;
+async function fetchOneManifest(manifest, resolver) {
+	const url = manifest.manifest_path;
+	const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
+	for (const entry of entries) {
+		entry.partition_spec_id = manifest.partition_spec_id ?? 0;
+		if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number ?? 0n;
+		if (entry.status === 1) {
+			if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number;
+			if (entry.file_sequence_number === void 0) entry.file_sequence_number = manifest.sequence_number;
+		} else if (entry.sequence_number === void 0 || entry.file_sequence_number === void 0) throw new Error("iceberg manifest entry missing sequence number");
+	}
+	assignFirstRowIds(manifest, entries);
+	return {
+		url,
+		entries
+	};
+}
 async function fetchManifests(manifests, resolver) {
-	return await Promise.all(manifests.map(async (manifest) => {
-		const url = manifest.manifest_path;
-		const entries = await fetchAvroRecords(url, resolver, Number(manifest.manifest_length));
-		for (const entry of entries) {
-			entry.partition_spec_id = manifest.partition_spec_id ?? 0;
-			if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number ?? 0n;
-			if (entry.status === 1) {
-				if (entry.sequence_number === void 0) entry.sequence_number = manifest.sequence_number;
-				if (entry.file_sequence_number === void 0) entry.file_sequence_number = manifest.sequence_number;
-			} else if (entry.sequence_number === void 0 || entry.file_sequence_number === void 0) throw new Error("iceberg manifest entry missing sequence number");
-		}
-		assignFirstRowIds(manifest, entries);
-		return {
-			url,
-			entries
-		};
-	}));
+	const results = new Array(manifests.length);
+	let next = 0;
+	async function worker() {
+		while (next < manifests.length) {
+			const i = next++;
+			results[i] = await fetchOneManifest(manifests[i], resolver);
+		}
+	}
+	const poolSize = Math.min(MANIFEST_FETCH_CONCURRENCY, manifests.length);
+	const workers = [];
+	for (let w = 0; w < poolSize; w++) workers.push(worker());
+	await Promise.all(workers);
+	return results;
 }
 function assignFirstRowIds(manifest, entries) {
 	if (manifest.content !== 0 || manifest.first_row_id == null) return;

package/dist/_chunks/pg-adapter.d.mts

@@ -13,7 +13,15 @@ declare const pgResolverAdapter: ResolverAdapter<PgTableKey>;
  * Single-use: build a fresh adapter per query. Cheap (no I/O) and avoids
  * accidental adapter caching that would lock in a stale `{{FILES}}` set.
  */
-declare function createParquetResolverAdapter(): ResolverAdapter<PgTableKey>;
+interface ResolverAdapterOptions {
+  /**
+   * Opt-in canonical-primary correctness: fold NULL/'' `query_canonical` back
+   * to the raw `query` so canonical is a total GROUP BY / join key. Default
+   * false preserves the legacy raw-column behaviour. See ADR-0018.
+   */
+  canonicalFallback?: boolean;
+}
+declare function createParquetResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
 /**
  * Multi-tenant pg-flavored adapter for the Iceberg / R2 SQL read path.
  * Identical SQL output to `pgResolverAdapter` except WHERE clauses inject
@@ -24,5 +32,5 @@ declare function createParquetResolverAdapter(): ResolverAdapter<PgTableKey>;
  * so callers must rewrite bare table names to their qualified form (e.g.
  * `${namespace}.pages`) before sending to R2 SQL.
  */
-declare function createIcebergResolverAdapter(): ResolverAdapter<PgTableKey>;
-export { PgTableKey, createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter };
+declare function createIcebergResolverAdapter(options?: ResolverAdapterOptions): ResolverAdapter<PgTableKey>;
+export { PgTableKey, ResolverAdapterOptions, createIcebergResolverAdapter, createParquetResolverAdapter, pgResolverAdapter };

package/dist/_chunks/resolver.mjs

@@ -180,7 +180,7 @@ function buildDimensionColumnMap(datasetToTableKey) {
 	return Object.fromEntries(entries);
 }
 function createSqlFragments(config) {
-	const { schema, datasetToTableKey, metricCast, regexPredicate, tableLabel, includeSiteId, includeSearchType, urlToPathExpr: urlToPathExprOverride, tableRef: tableRefOverride } = config;
+	const { schema, datasetToTableKey, metricCast, regexPredicate, tableLabel, includeSiteId, includeSearchType, urlToPathExpr: urlToPathExprOverride, tableRef: tableRefOverride, canonicalFallback = false } = config;
 	const DIM_COLUMN_MAP = buildDimensionColumnMap(datasetToTableKey);
 	function isMetricDimension(dim) {
 		return METRIC_NAMES.includes(dim);
@@ -217,6 +217,7 @@ function createSqlFragments(config) {
 	function dimExprSql(dim, tableKey) {
 		const colName = dimColumn(dim, tableKey);
 		if (dim === "page") return sql.raw(urlToPathExpr(colName));
+		if (canonicalFallback && dim === "queryCanonical") return sql`COALESCE(NULLIF(${colRef(tableKey, colName)}, ''), ${colRef(tableKey, "query")})`;
 		return colRef(tableKey, colName);
 	}
 	function metricSql(metric, tableKey) {
@@ -431,23 +432,37 @@ const pgResolverAdapter = createResolverAdapter({
 	...PG_BASE_CONFIG,
 	tableLabel: "pg-resolver-adapter"
 });
-function createParquetResolverAdapter() {
+function createParquetResolverAdapter(options = {}) {
 	return createResolverAdapter({
 		...PG_BASE_CONFIG,
 		tableLabel: "parquet-resolver-adapter",
+		canonicalFallback: options.canonicalFallback ?? false,
 		tableRef: (tk) => sql.raw(`read_parquet({{FILES}}, union_by_name = true) AS "${tk}"`)
 	});
 }
-function createIcebergResolverAdapter() {
+function createIcebergResolverAdapter(options = {}) {
 	return createResolverAdapter({
 		...PG_BASE_CONFIG,
 		schema: icebergSchema,
 		includeSiteId: true,
 		includeSearchType: true,
 		tableLabel: "iceberg-resolver-adapter",
+		canonicalFallback: options.canonicalFallback ?? false,
 		tableRef: (tk) => sql.raw(`"${tk}"`)
 	});
 }
+const ALLOWED_FILTER_DIMS = /* @__PURE__ */ new Set(["date", "queryCanonical"]);
+function planCoveredByCanonicalRollup(plan) {
+	if (plan.dataset !== "queries") return false;
+	if (plan.groupByDimensions.length !== 1 || plan.groupByDimensions[0] !== "queryCanonical") return false;
+	if (!plan.dimensionFilters.every((f) => ALLOWED_FILTER_DIMS.has(f.dimension))) return false;
+	if (plan.prefilters.length > 0) return false;
+	if (plan.specialFilters.topLevel) return false;
+	return true;
+}
+function canonicalRollupCovers(state, capabilities) {
+	return planCoveredByCanonicalRollup(buildLogicalPlan(state, capabilities));
+}
 const COMPARISON_FILTER_SQL = {
 	new: sql`AND COALESCE(p.impressions, 0) = 0 AND COALESCE(c.impressions, 0) > 0`,
 	lost: sql`AND COALESCE(p.impressions, 0) > 0 AND COALESCE(c.impressions, 0) = 0`,
@@ -726,7 +741,8 @@ function buildExtrasQueries(state, options) {
 	whereParts.push(sql`${adapter.dateColRef(queriesKey)} <= ${plan.dateRange.endDate}`);
 	const whereExpr = whereParts.length > 0 ? sql`WHERE ${joinAnd(whereParts)}` : sql``;
 	const outerQueryCol = sql.raw("query");
-	const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${t.query_canonical} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${t.query_canonical} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${t.query_canonical}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${t.query_canonical}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, GROUP_CONCAT(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
+	const canonKey = sql`COALESCE(NULLIF(${t.query_canonical}, ''), ${t.query})`;
+	const compiled = compileCollapsed(adapter, sql`WITH per_variant AS (SELECT ${canonKey} as joinKey, ${t.query} as query, SUM(${t.clicks}) as clicks, SUM(${t.impressions}) as impressions, SUM(${t.sum_position}) as sum_pos, ROW_NUMBER() OVER (PARTITION BY ${canonKey} ORDER BY SUM(${t.clicks}) DESC) as rn, COUNT(*) OVER (PARTITION BY ${canonKey}) as variantCount FROM ${table} ${whereExpr} GROUP BY ${canonKey}, ${t.query}) SELECT joinKey, MAX(variantCount) as variantCount, MAX(CASE WHEN rn = 1 THEN ${outerQueryCol} END) as canonicalName, GROUP_CONCAT(CASE WHEN rn <= 10 THEN ${outerQueryCol} || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') as variants FROM per_variant GROUP BY joinKey`);
 	extras.push({
 		key: "canonicalExtras",
 		sql: compiled.sql,
@@ -802,6 +818,22 @@ function mergeExtras(rows, extrasResults) {
 		return enriched;
 	});
 }
+const EXTRA_ROLLUP_IDS = { canonicalExtras: "query_canonical_variants" };
+function createRollupExtrasOverlay(readRollupRows) {
+	return async ({ key, ctx, dateRange }) => {
+		const id = EXTRA_ROLLUP_IDS[key];
+		if (id === void 0) return null;
+		return readRollupRows({
+			id,
+			ctx: {
+				userId: ctx.userId,
+				siteId: ctx.siteId
+			},
+			dateRange,
+			...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {}
+		});
+	};
+}
 function collectInternalFilters(filter) {
 	if (!filter || !("_filters" in filter)) return [];
 	const flat = filter._filters;
@@ -856,6 +888,9 @@ function matchesMetricFilter(row, filter) {
 function matchesTopLevelPage(row) {
 	return (normalizeUrl(dimensionValue(row, "page")).match(/\//g)?.length ?? 0) <= 1;
 }
+function canonicalSourceWithinCoverage(source, windowEnd) {
+	return source.coversThrough === void 0 || windowEnd <= source.coversThrough;
+}
 function runArgs(ctx, partitions) {
 	return {
 		ctx: {
@@ -870,9 +905,11 @@ function runArgs(ctx, partitions) {
 		...ctx.searchType !== void 0 ? { searchType: ctx.searchType } : {}
 	};
 }
-async function runOptimizedQuery(runSQL, ctx, state, dateRange) {
-	const adapter = createParquetResolverAdapter();
+async function runOptimizedQuery(runSQL, ctx, state, dateRange, options = {}) {
 	const base = runArgs(ctx, enumeratePartitions(dateRange.startDate, dateRange.endDate));
+	const probe = createParquetResolverAdapter({ canonicalFallback: options.canonicalFallback ?? false });
+	const useCanonicalSource = options.canonicalSource !== void 0 && (options.canonicalFallback ?? false) && canonicalSourceWithinCoverage(options.canonicalSource, dateRange.endDate) && canonicalRollupCovers(state, probe.capabilities);
+	const adapter = useCanonicalSource ? createParquetResolverAdapter({ canonicalFallback: false }) : probe;
 	const optimized = resolveToSQLOptimized(state, {
 		adapter,
 		siteId: void 0
@@ -881,15 +918,31 @@ async function runOptimizedQuery(runSQL, ctx, state, dateRange) {
 		adapter,
 		siteId: void 0
 	});
-	const [optRes, ...extrasRows] = await Promise.all([runSQL({
+	const mainArgs = useCanonicalSource ? {
 		...base,
+		fileSets: { FILES: {
+			table: ctx.table,
+			keys: options.canonicalSource.keys
+		} }
+	} : base;
+	const resolveExtra = options.resolveExtra;
+	const [optRes, ...extrasRows] = await Promise.all([runSQL({
+		...mainArgs,
 		sql: optimized.sql,
 		params: optimized.params
-	}), ...extras.map((e) => runSQL({
-		...base,
-		sql: e.sql,
-		params: e.params
-	}))]);
+	}), ...extras.map(async (e) => {
+		const overlaid = resolveExtra ? await resolveExtra({
+			key: e.key,
+			state,
+			ctx,
+			dateRange
+		}) : null;
+		return overlaid !== null ? { rows: overlaid } : runSQL({
+			...base,
+			sql: e.sql,
+			params: e.params
+		});
+	})]);
 	const firstRow = optRes.rows[0];
 	const totalCount = Number(firstRow?.totalCount ?? 0);
 	const totals = {
@@ -911,8 +964,10 @@ async function runOptimizedQuery(runSQL, ctx, state, dateRange) {
 		}))
 	};
 }
-async function runComparisonQuery(runSQL, ctx, current, previous, windows, filter) {
-	const adapter = createParquetResolverAdapter();
+async function runComparisonQuery(runSQL, ctx, current, previous, windows, filter, options = {}) {
+	const probe = createParquetResolverAdapter({ canonicalFallback: options.canonicalFallback ?? false });
+	const useCanonicalSource = options.canonicalSource !== void 0 && (options.canonicalFallback ?? false) && canonicalSourceWithinCoverage(options.canonicalSource, windows.current.endDate > windows.previous.endDate ? windows.current.endDate : windows.previous.endDate) && canonicalRollupCovers(current, probe.capabilities) && canonicalRollupCovers(previous, probe.capabilities);
+	const adapter = useCanonicalSource ? createParquetResolverAdapter({ canonicalFallback: false }) : probe;
 	const comparison = resolveComparisonSQL(current, previous, {
 		adapter,
 		siteId: void 0
@@ -921,7 +976,14 @@ async function runComparisonQuery(runSQL, ctx, current, previous, windows, filte
 		adapter,
 		siteId: void 0
 	});
-	const base = runArgs(ctx, enumeratePartitions(windows.current.startDate < windows.previous.startDate ? windows.current.startDate : windows.previous.startDate, windows.current.endDate > windows.previous.endDate ? windows.current.endDate : windows.previous.endDate));
+	const partitions = enumeratePartitions(windows.current.startDate < windows.previous.startDate ? windows.current.startDate : windows.previous.startDate, windows.current.endDate > windows.previous.endDate ? windows.current.endDate : windows.previous.endDate);
+	const base = useCanonicalSource ? {
+		...runArgs(ctx, partitions),
+		fileSets: { FILES: {
+			table: ctx.table,
+			keys: options.canonicalSource.keys
+		} }
+	} : runArgs(ctx, partitions);
 	const main = await runSQL({
 		...base,
 		sql: comparison.sql,
@@ -953,4 +1015,4 @@ function assertSchemaInSync(options) {
 		if (missing.length > 0 || extra.length > 0) throw new Error(`${label} drizzle schema for '${key}' drifted from SCHEMAS. Missing: [${missing.join(", ")}]. Extra: [${extra.join(", ")}].`);
 	}
 }
-export { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };
+export { DIMENSION_SURFACES, LOGICAL_DATASETS, UnresolvableDatasetError, assertDimensionsSupported, assertSchemaInSync, buildExtrasQueries, buildTotalsSql, canonicalRollupCovers, createIcebergResolverAdapter, createParquetResolverAdapter, createResolverAdapter, createRollupExtrasOverlay, createSqlFragments, dimensionColumn, dimensionValue, getDimensionFilters, getFilterDimensions, getInternalFilters, inferLogicalDataset, isDatasetResolvable, matchesDimensionFilter, matchesMetricFilter, matchesTopLevelPage, mergeExtras, metricValue, pgResolverAdapter, planCoveredByCanonicalRollup, resolveComparisonSQL, resolveToSQL, resolveToSQLOptimized, runComparisonQuery, runOptimizedQuery, supportsDimensionOnSurface };

package/dist/_chunks/schema.d.mts

@@ -1087,12 +1087,12 @@ declare const hourly_pages: import("drizzle-orm/pg-core").PgTableWithColumns<{
       identity: undefined;
       generated: undefined;
     }>;
-    hour: import("drizzle-orm/pg-core").PgBuildColumn<"hourly_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>>, {
+    hour: import("drizzle-orm/pg-core").PgBuildColumn<"hourly_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgIntegerBuilder>, {
       name: string;
       tableName: "hourly_pages";
-      dataType: "string";
-      data: string;
-      driverParam: string;
+      dataType: "number int32";
+      data: number;
+      driverParam: string | number;
       notNull: true;
       hasDefault: false;
       isPrimaryKey: false;
@@ -2193,12 +2193,12 @@ declare const drizzleSchema: {
         identity: undefined;
         generated: undefined;
       }>;
-      hour: import("drizzle-orm/pg-core").PgBuildColumn<"hourly_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgVarcharBuilder<[string, ...string[]]>>, {
+      hour: import("drizzle-orm/pg-core").PgBuildColumn<"hourly_pages", import("drizzle-orm/pg-core").SetNotNull<import("drizzle-orm/pg-core").PgIntegerBuilder>, {
         name: string;
         tableName: "hourly_pages";
-        dataType: "string";
-        data: string;
-        driverParam: string;
+        dataType: "number int32";
+        data: number;
+        driverParam: string | number;
         notNull: true;
         hasDefault: false;
         isPrimaryKey: false;
@@ -2236,6 +2236,13 @@ declare const TABLE_METADATA: Record<TableName, {
 declare const SCHEMAS: Record<TableName, TableSchema>;
 declare function currentSchemaVersion(table: TableName): number;
 declare function schemaFor(table: TableName): TableSchema;
+/**
+ * DATE column names for a table. The single schema-derived source every read
+ * path uses to build the legacy-VARCHAR date canonicalization (see
+ * `dateReplaceClause` in `./sql-fragments`), so the engine codec and the CLI
+ * `dump`/`export` commands agree on which columns to cast.
+ */
+declare function dateColumnsFor(table: TableName): string[];
 declare function allTables(): readonly TableName[];
 declare function inferTable(dimensions: readonly string[]): TableName;
 /**
@@ -2259,4 +2266,4 @@ declare function naturalKeyColumns(table: TableName): readonly string[];
  */
 declare function dedupeByNaturalKey(table: TableName, rows: readonly Row[]): Row[];
 declare function dimensionToColumn(dim: string, _table: TableName): string;
-export { type ColumnDef$1 as ColumnDef, type ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema$1 as TableSchema, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+export { type ColumnDef$1 as ColumnDef, type ColumnType, DrizzleSchema, SCHEMAS, TABLE_METADATA, type TableSchema$1 as TableSchema, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };

package/dist/_chunks/schema.mjs

@@ -74,7 +74,7 @@ const search_appearance_page_queries = pgTable("search_appearance_page_queries",
 });
 const hourly_pages = pgTable("hourly_pages", {
 	url: varchar("url").notNull(),
-	hour: varchar("hour").notNull(),
+	hour: integer("hour").notNull(),
 	date: dateCol(),
 	...metricCols()
 });
@@ -181,7 +181,7 @@ const TABLE_METADATA = {
 			"date",
 			"hour"
 		],
-		version: 1
+		version: 2
 	}
 };
 function pgSqlTypeToColumnType(sqlType) {
@@ -226,6 +226,9 @@ function currentSchemaVersion(table) {
 function schemaFor(table) {
 	return SCHEMAS[table];
 }
+function dateColumnsFor(table) {
+	return SCHEMAS[table].columns.filter((c) => c.type === "DATE").map((c) => c.name);
+}
 function allTables() {
 	return METRIC_TABLES;
 }
@@ -260,4 +263,4 @@ function dimensionToColumn(dim, _table) {
 	if (dim === "queryCanonical") return "query_canonical";
 	return dim;
 }
-export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };
+export { SCHEMAS, TABLE_METADATA, allTables, countries, currentSchemaVersion, dateColumnsFor, dates, dedupeByNaturalKey, dimensionToColumn, drizzleSchema, hourly_pages, inferTable, naturalKeyColumns, page_queries, pages, queries, schemaFor, search_appearance, search_appearance_page_queries, search_appearance_pages, search_appearance_queries };