@gscdump/engine 1.4.11 → 1.5.0

package/dist/rollups.mjs

@@ -1,1317 +1,8 @@
-import { engineErrors } from "./errors.mjs";
-import "./layout.mjs";
-import { inspectionParquetKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey } from "./entity-keys.mjs";
-import { encodeRowsToParquetFlex } from "./adapters/hyparquet.mjs";
-import { readOptional } from "./adapters/read-optional.mjs";
-import { decodeSitemapProjectionManifest, selectSitemapProjectionFiles } from "./sitemap-projection.mjs";
-import { createQueryDimStore } from "./query-dim.mjs";
-import { createIndexingMetadataStore, createSitemapReadStore } from "./entities.mjs";
-import { MS_PER_DAY } from "gscdump/dates";
-import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
-function rollupPrefix(ctx, searchType) {
-	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
-	return searchType !== void 0 && searchType !== "web" ? `${base}/${searchType}` : base;
-}
-function rollupKey(ctx, id, builtAt, searchType) {
-	return `${rollupPrefix(ctx, searchType)}/${id}__v${builtAt}.json`;
-}
-function rollupParquetKey(ctx, id, builtAt, searchType) {
-	return `${rollupPrefix(ctx, searchType)}/${id}__v${builtAt}.parquet`;
-}
-const ROLLUP_FILE_RE = /^(?<id>[a-z0-9_]+)__v(?<ts>\d+)\.json$/;
-async function readLatestRollup(bucket, ctx, id, searchType) {
-	const prefix = `${rollupPrefix(ctx, searchType)}/`;
-	let newest = null;
-	let cursor;
-	do {
-		const listing = await bucket.list({
-			prefix,
-			cursor
-		});
-		for (const obj of listing.objects) {
-			const m = ROLLUP_FILE_RE.exec(obj.key.slice(prefix.length));
-			if (!m?.groups || m.groups.id !== id) continue;
-			const ts = Number(m.groups.ts);
-			if (!newest || ts > newest.ts) newest = {
-				ts,
-				key: obj.key
-			};
-		}
-		cursor = listing.truncated ? listing.cursor : void 0;
-	} while (cursor !== void 0);
-	if (!newest) return null;
-	const obj = await bucket.get(newest.key);
-	if (!obj) return null;
-	return JSON.parse(await obj.text());
-}
-async function rebuildRollups(opts) {
-	const now = opts.now ?? (() => Date.now());
-	const dataEndMs = opts.dataEndDate !== void 0 ? isoDateToUtcMs(opts.dataEndDate) : null;
-	const results = [];
-	for (const def of opts.defs) {
-		const builtAt = now();
-		const windowAnchorMs = dataEndMs ?? builtAt;
-		const defSearchType = def.sliceOrthogonal === true ? void 0 : opts.searchType;
-		try {
-			const payload = await def.build({
-				engine: opts.engine,
-				ctx: opts.ctx,
-				dataSource: opts.dataSource,
-				windowAnchorMs,
-				...defSearchType !== void 0 ? { searchType: defSearchType } : {}
-			});
-			if (def.format === "parquet") {
-				if (!def.parquetColumns || def.parquetColumns.length === 0) throw new Error(`rollup '${def.id}' declared format='parquet' without parquetColumns`);
-				const rows = payload ?? [];
-				const parquetBytes = encodeRowsToParquetFlex(rows, {
-					columns: def.parquetColumns,
-					sortKey: def.parquetSortKey
-				});
-				const parquetKey = rollupParquetKey(opts.ctx, def.id, builtAt, defSearchType);
-				await opts.dataSource.write(parquetKey, parquetBytes);
-				const pointer = {
-					parquetKey,
-					rowCount: rows.length
-				};
-				const envelopeBytes = encodeJsonBigintSafe({
-					version: 1,
-					id: def.id,
-					builtAt,
-					windowDays: def.windowDays,
-					payload: pointer
-				});
-				const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
-				await opts.dataSource.write(key, envelopeBytes);
-				results.push({
-					id: def.id,
-					objectKey: key,
-					parquetKey,
-					bytes: envelopeBytes.byteLength,
-					parquetBytes: parquetBytes.byteLength,
-					builtAt
-				});
-				continue;
-			}
-			const bytes = encodeJsonBigintSafe({
-				version: 1,
-				id: def.id,
-				builtAt,
-				windowDays: def.windowDays,
-				payload
-			});
-			const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
-			await opts.dataSource.write(key, bytes);
-			results.push({
-				id: def.id,
-				objectKey: key,
-				bytes: bytes.byteLength,
-				builtAt
-			});
-		} catch (err) {
-			results.push({
-				id: def.id,
-				objectKey: "",
-				bytes: 0,
-				builtAt,
-				error: engineErrors.rollupBuildFailed(def.id, err)
-			});
-		}
-	}
-	return results;
-}
-function isoDateToUtcMs(iso) {
-	const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(iso);
-	if (!m) throw new Error(`dataEndDate must be ISO YYYY-MM-DD, got: ${iso}`);
-	return Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
-}
-function utcDateMinusDays(at, days) {
-	const d = new Date(at - days * MS_PER_DAY);
-	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
-}
-const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
-const ROLLUP_PAGE_ROWS = 5e4;
-const ROLLUP_PAGE_ROWS_WIDE = 2e4;
-const ROLLUP_PAGE_ROWS_DAILY = 7e4;
-const DAILY_MAX_WINDOW_DAYS = 7;
-const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
-const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
-const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
-const QUARTER_RE = /^quarterly\/(\d{4})-Q([1-4])$/;
-function isoDate(ms) {
-	const d = new Date(ms);
-	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
-}
-function partitionDaySpan(partition) {
-	const day = DAY_RE.exec(partition);
-	if (day) {
-		const ms = Date.UTC(Number(day[1]), Number(day[2]) - 1, Number(day[3]));
-		return {
-			startMs: ms,
-			endMs: ms
-		};
-	}
-	const week = WEEK_RE.exec(partition);
-	if (week) {
-		const ms = Date.UTC(Number(week[1]), Number(week[2]) - 1, Number(week[3]));
-		return {
-			startMs: ms,
-			endMs: ms + 6 * MS_PER_DAY
-		};
-	}
-	const month = MONTH_RE.exec(partition);
-	if (month) {
-		const y = Number(month[1]);
-		const m = Number(month[2]) - 1;
-		return {
-			startMs: Date.UTC(y, m, 1),
-			endMs: Date.UTC(y, m + 1, 1) - MS_PER_DAY
-		};
-	}
-	const quarter = QUARTER_RE.exec(partition);
-	if (quarter) {
-		const y = Number(quarter[1]);
-		const startMonth = (Number(quarter[2]) - 1) * 3;
-		return {
-			startMs: Date.UTC(y, startMonth, 1),
-			endMs: Date.UTC(y, startMonth + 3, 1) - MS_PER_DAY
-		};
-	}
-	return null;
-}
-function clamp(n, lo, hi) {
-	return Math.max(lo, Math.min(hi, n));
-}
-function planRollupWindows(parts, clampRange, maxWindowDays) {
-	const clampStartMs = clampRange ? Date.parse(`${clampRange.start}T00:00:00Z`) : void 0;
-	const clampEndMs = clampRange ? Date.parse(`${clampRange.end}T00:00:00Z`) : void 0;
-	const spans = [];
-	for (const p of parts) {
-		const span = partitionDaySpan(p.partition);
-		if (!span) continue;
-		if (clampStartMs !== void 0 && clampEndMs !== void 0) {
-			if (span.endMs < clampStartMs || span.startMs > clampEndMs) continue;
-		}
-		spans.push({
-			partition: p.partition,
-			bytes: p.bytes,
-			startMs: span.startMs,
-			endMs: span.endMs
-		});
-	}
-	if (spans.length === 0) return [];
-	let rangeStartMs = Number.POSITIVE_INFINITY;
-	let rangeEndMs = Number.NEGATIVE_INFINITY;
-	let totalBytes = 0;
-	for (const span of spans) {
-		if (span.startMs < rangeStartMs) rangeStartMs = span.startMs;
-		if (span.endMs > rangeEndMs) rangeEndMs = span.endMs;
-		totalBytes += span.bytes;
-	}
-	if (clampStartMs !== void 0) rangeStartMs = Math.max(rangeStartMs, clampStartMs);
-	if (clampEndMs !== void 0) rangeEndMs = Math.min(rangeEndMs, clampEndMs);
-	const spanDays = Math.floor((rangeEndMs - rangeStartMs) / MS_PER_DAY) + 1;
-	const bytesPerDay = Math.max(1, totalBytes / spanDays);
-	const byteWindowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
-	const windowDays = maxWindowDays != null ? Math.max(1, Math.min(byteWindowDays, maxWindowDays)) : byteWindowDays;
-	if (!Number.isFinite(windowDays) || !Number.isFinite(rangeStartMs) || !Number.isFinite(rangeEndMs) || rangeEndMs < rangeStartMs) return [];
-	const windowWidthMs = windowDays * MS_PER_DAY;
-	const windowCount = Math.floor((rangeEndMs - rangeStartMs) / windowWidthMs) + 1;
-	const partitionsByWindow = Array.from({ length: windowCount }, () => []);
-	for (const span of spans) {
-		const overlapStartMs = Math.max(span.startMs, rangeStartMs);
-		const overlapEndMs = Math.min(span.endMs, rangeEndMs);
-		if (overlapStartMs > overlapEndMs) continue;
-		const firstWindow = Math.max(0, Math.ceil((overlapStartMs - rangeStartMs - (windowDays - 1) * MS_PER_DAY) / windowWidthMs));
-		const lastWindow = Math.min(windowCount - 1, Math.floor((overlapEndMs - rangeStartMs) / windowWidthMs));
-		for (let index = firstWindow; index <= lastWindow; index++) partitionsByWindow[index].push(span.partition);
-	}
-	const windows = [];
-	for (let index = 0; index < windowCount; index++) {
-		const cursorMs = rangeStartMs + index * windowWidthMs;
-		const windowEndMs = Math.min(cursorMs + (windowDays - 1) * MS_PER_DAY, rangeEndMs);
-		const partitions = partitionsByWindow[index];
-		if (partitions.length > 0) windows.push({
-			start: isoDate(cursorMs),
-			end: isoDate(windowEndMs),
-			partitions
-		});
-	}
-	return windows;
-}
-function partitionsInRange(parts, start, end) {
-	const startMs = Date.parse(`${start}T00:00:00Z`);
-	const endMs = Date.parse(`${end}T00:00:00Z`);
-	const out = [];
-	for (const p of parts) {
-		const span = partitionDaySpan(p.partition);
-		if (!span) continue;
-		if (span.endMs >= startMs && span.startMs <= endMs) out.push(p.partition);
-	}
-	return out;
-}
-async function runPagedQuery(opts) {
-	const out = [];
-	for (let offset = 0;; offset += opts.pageRows) {
-		const result = await opts.engine.runSQL({
-			ctx: opts.ctx,
-			table: opts.table,
-			fileSets: opts.fileSets,
-			sql: `${opts.coreSql}\nORDER BY ${opts.orderBy}\nLIMIT ${opts.pageRows} OFFSET ${offset}`,
-			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
-		});
-		out.push(...result.rows);
-		if (result.rows.length < opts.pageRows) break;
-	}
-	return out;
-}
-async function runWindowed(opts) {
-	const windows = planRollupWindows(await opts.engine.listPartitions({
-		ctx: opts.ctx,
-		table: opts.table,
-		...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
-	}), void 0, opts.maxWindowDays);
-	const rows = [];
-	for (const w of windows) {
-		const fileSets = {
-			FILES: {
-				table: opts.table,
-				partitions: w.partitions
-			},
-			...opts.extraFileSets
-		};
-		if (opts.paginate) {
-			rows.push(...await runPagedQuery({
-				engine: opts.engine,
-				ctx: opts.ctx,
-				table: opts.table,
-				...opts.searchType !== void 0 ? { searchType: opts.searchType } : {},
-				fileSets,
-				coreSql: opts.sqlFor(w),
-				orderBy: opts.paginate.orderBy,
-				pageRows: opts.paginate.pageRows
-			}));
-			continue;
-		}
-		const result = await opts.engine.runSQL({
-			ctx: opts.ctx,
-			table: opts.table,
-			fileSets,
-			sql: opts.sqlFor(w),
-			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
-		});
-		rows.push(...result.rows);
-	}
-	return rows;
-}
-const dailyTotalsRollup = {
-	id: "daily_totals",
-	windowDays: null,
-	async build({ engine, ctx, searchType }) {
-		const pageRows = await runWindowed({
-			engine,
-			ctx,
-			table: "pages",
-			...searchType !== void 0 ? { searchType } : {},
-			sqlFor: (w) => `
-        SELECT
-          date,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${w.start}' AND date <= '${w.end}'
-        GROUP BY date
-        ORDER BY date
-      `
-		});
-		const queryRows = await runWindowed({
-			engine,
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {},
-			sqlFor: (w) => `
-        SELECT
-          date,
-          SUM(impressions)::BIGINT AS impressions
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${w.start}' AND date <= '${w.end}'
-        GROUP BY date
-      `
-		});
-		const pagesByDate = /* @__PURE__ */ new Map();
-		for (const r of pageRows) {
-			const date = String(r.date);
-			const cur = pagesByDate.get(date) ?? {
-				date,
-				clicks: BigInt(0),
-				impressions: BigInt(0),
-				sum_position: 0
-			};
-			cur.clicks += BigInt(r.clicks);
-			cur.impressions += BigInt(r.impressions);
-			cur.sum_position += Number(r.sum_position);
-			pagesByDate.set(date, cur);
-		}
-		const queryImpressionsByDate = /* @__PURE__ */ new Map();
-		for (const r of queryRows) {
-			const date = String(r.date);
-			queryImpressionsByDate.set(date, (queryImpressionsByDate.get(date) ?? BigInt(0)) + BigInt(r.impressions));
-		}
-		return Array.from(pagesByDate.values()).sort((a, b) => a.date < b.date ? -1 : 1).map((r) => {
-			const totalImpressions = BigInt(r.impressions);
-			const queryImpressions = queryImpressionsByDate.get(String(r.date)) ?? BigInt(0);
-			const anonymized = totalImpressions === BigInt(0) ? 0 : 1 - Number(queryImpressions) / Number(totalImpressions);
-			return {
-				date: r.date,
-				clicks: Number(r.clicks),
-				impressions: Number(r.impressions),
-				sum_position: Number(r.sum_position),
-				anonymizedImpressionsPct: Math.max(0, Math.min(1, anonymized))
-			};
-		});
-	}
-};
-const weeklyTotalsRollup = {
-	id: "weekly_totals",
-	windowDays: null,
-	async build({ engine, ctx, searchType }) {
-		const rows = await runWindowed({
-			engine,
-			ctx,
-			table: "pages",
-			...searchType !== void 0 ? { searchType } : {},
-			sqlFor: (w) => `
-        SELECT
-          strftime(date_trunc('week', date::DATE), '%Y-%m-%d') AS week,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${w.start}' AND date <= '${w.end}'
-        GROUP BY 1
-        ORDER BY 1
-      `
-		});
-		const byWeek = /* @__PURE__ */ new Map();
-		for (const r of rows) {
-			const week = String(r.week);
-			const cur = byWeek.get(week) ?? {
-				week,
-				clicks: 0,
-				impressions: 0,
-				sum_position: 0
-			};
-			cur.clicks += Number(r.clicks);
-			cur.impressions += Number(r.impressions);
-			cur.sum_position += Number(r.sum_position);
-			byWeek.set(week, cur);
-		}
-		return Array.from(byWeek.values()).sort((a, b) => a.week < b.week ? -1 : 1);
-	}
-};
-const topPages28dRollup = {
-	id: "top_pages_28d",
-	windowDays: 28,
-	async build({ engine, ctx, windowAnchorMs, searchType }) {
-		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
-			ctx,
-			table: "pages",
-			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
-		if (partitions.length === 0) return [];
-		return (await engine.runSQL({
-			ctx,
-			table: "pages",
-			fileSets: { FILES: {
-				table: "pages",
-				partitions
-			} },
-			...searchType !== void 0 ? { searchType } : {},
-			sql: `
-        SELECT
-          url,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${cutoff}'
-        GROUP BY url
-        ORDER BY clicks DESC
-        LIMIT 1000
-      `
-		})).rows.map((r) => ({
-			url: r.url,
-			clicks: Number(r.clicks),
-			impressions: Number(r.impressions),
-			sum_position: Number(r.sum_position)
-		}));
-	}
-};
-const topCountries28dRollup = {
-	id: "top_countries_28d",
-	windowDays: 28,
-	async build({ engine, ctx, windowAnchorMs, searchType }) {
-		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
-			ctx,
-			table: "countries",
-			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
-		if (partitions.length === 0) return [];
-		return (await engine.runSQL({
-			ctx,
-			table: "countries",
-			fileSets: { FILES: {
-				table: "countries",
-				partitions
-			} },
-			...searchType !== void 0 ? { searchType } : {},
-			sql: `
-        SELECT
-          country,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${cutoff}'
-        GROUP BY country
-        ORDER BY clicks DESC
-        LIMIT 250
-      `
-		})).rows.map((r) => ({
-			country: r.country,
-			clicks: Number(r.clicks),
-			impressions: Number(r.impressions),
-			sum_position: Number(r.sum_position)
-		}));
-	}
-};
-const topKeywords28dRollup = {
-	id: "top_keywords_28d",
-	windowDays: 28,
-	async build({ engine, ctx, windowAnchorMs, searchType }) {
-		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
-		if (partitions.length === 0) return [];
-		return (await engine.runSQL({
-			ctx,
-			table: "queries",
-			fileSets: { FILES: {
-				table: "queries",
-				partitions
-			} },
-			...searchType !== void 0 ? { searchType } : {},
-			sql: `
-        SELECT
-          query,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${cutoff}'
-        GROUP BY query
-        ORDER BY clicks DESC
-        LIMIT 1000
-      `
-		})).rows.map((r) => ({
-			query: r.query,
-			clicks: Number(r.clicks),
-			impressions: Number(r.impressions),
-			sum_position: Number(r.sum_position)
-		}));
-	}
-};
-const topKeywords28dParquetRollup = {
-	id: "top_keywords_28d_parquet",
-	windowDays: 28,
-	format: "parquet",
-	parquetColumns: [
-		{
-			name: "query",
-			type: "VARCHAR",
-			nullable: false
-		},
-		{
-			name: "clicks",
-			type: "BIGINT",
-			nullable: false
-		},
-		{
-			name: "impressions",
-			type: "BIGINT",
-			nullable: false
-		},
-		{
-			name: "sum_position",
-			type: "DOUBLE",
-			nullable: false
-		}
-	],
-	parquetSortKey: ["clicks"],
-	async build({ engine, ctx, windowAnchorMs, searchType }) {
-		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
-		const partitions = partitionsInRange(await engine.listPartitions({
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {}
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
-		if (partitions.length === 0) return [];
-		return (await engine.runSQL({
-			ctx,
-			table: "queries",
-			fileSets: { FILES: {
-				table: "queries",
-				partitions
-			} },
-			...searchType !== void 0 ? { searchType } : {},
-			sql: `
-        SELECT
-          query,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${cutoff}'
-        GROUP BY query
-        ORDER BY clicks DESC
-        LIMIT 1000
-      `
-		})).rows.map((r) => ({
-			query: String(r.query),
-			clicks: BigInt(r.clicks),
-			impressions: BigInt(r.impressions),
-			sum_position: Number(r.sum_position)
-		}));
-	}
-};
-const CANONICAL_VARIANT_LIMIT = 10;
-function retainCanonicalVariant(bucket, query, clicks, impressions, sumPos) {
-	bucket.count++;
-	let insertAt = 0;
-	while (insertAt < bucket.top.length) {
-		const existing = bucket.top[insertAt];
-		if (clicks > existing.clicks || clicks === existing.clicks && query.localeCompare(existing.query) < 0) break;
-		insertAt++;
-	}
-	if (insertAt >= CANONICAL_VARIANT_LIMIT) return;
-	bucket.top.splice(insertAt, 0, {
-		query,
-		clicks,
-		impressions,
-		sumPos
-	});
-	if (bucket.top.length > CANONICAL_VARIANT_LIMIT) bucket.top.pop();
-}
-const queryCanonicalVariantsRollup = {
-	id: "query_canonical_variants",
-	windowDays: null,
-	format: "parquet",
-	parquetColumns: [
-		{
-			name: "joinKey",
-			type: "VARCHAR",
-			nullable: false
-		},
-		{
-			name: "variantCount",
-			type: "BIGINT",
-			nullable: false
-		},
-		{
-			name: "canonicalName",
-			type: "VARCHAR",
-			nullable: true
-		},
-		{
-			name: "variants",
-			type: "VARCHAR",
-			nullable: true
-		}
-	],
-	parquetSortKey: ["joinKey"],
-	async build({ engine, ctx, dataSource, searchType }) {
-		const parts = await engine.listPartitions({
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {}
-		});
-		if (parts.length === 0) return [];
-		const partitions = parts.map((p) => p.partition);
-		const byCanonical = /* @__PURE__ */ new Map();
-		const dimStore = createQueryDimStore({ dataSource });
-		const useDim = await dimStore.loadMeta(ctx) !== null;
-		const canonExpr = useDim ? "COALESCE(qd.query_canonical, q.query)" : "q.query";
-		let cursor = null;
-		for (;;) {
-			const after = cursor === null ? "" : `AND q.query > '${cursor.replace(/'/g, "''")}'`;
-			const fileSets = { FILES: {
-				table: "queries",
-				partitions
-			} };
-			if (useDim) fileSets.QUERY_DIM = {
-				table: "queries",
-				keys: [dimStore.parquetKey(ctx)]
-			};
-			const { rows } = await engine.runSQL({
-				ctx,
-				table: "queries",
-				...searchType !== void 0 ? { searchType } : {},
-				fileSets,
-				sql: `
-          SELECT
-            ${canonExpr} AS joinKey,
-            q.query AS query,
-            SUM(q.clicks) AS clicks,
-            SUM(q.impressions) AS impressions,
-            SUM(q.sum_position) AS sum_pos
-          FROM read_parquet({{FILES}}, union_by_name = true) q
-          ${useDim ? "LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query" : ""}
-          WHERE q.query IS NOT NULL ${after}
-          GROUP BY ${canonExpr}, q.query
-          ORDER BY q.query
-          LIMIT ${ROLLUP_PAGE_ROWS_WIDE}
-        `
-			});
-			for (const r of rows) {
-				const joinKey = String(r.joinKey);
-				let bucket = byCanonical.get(joinKey);
-				if (!bucket) {
-					bucket = {
-						count: 0,
-						top: []
-					};
-					byCanonical.set(joinKey, bucket);
-				}
-				retainCanonicalVariant(bucket, String(r.query), Number(r.clicks), Number(r.impressions), Number(r.sum_pos));
-			}
-			if (rows.length < 2e4) break;
-			cursor = String(rows[rows.length - 1].query);
-		}
-		const out = [];
-		for (const [joinKey, bucket] of byCanonical) {
-			const canonicalName = bucket.top[0]?.query ?? null;
-			const top = bucket.top.filter((v) => v.impressions > 0);
-			const variantsStr = top.length === 0 ? null : top.map((v) => `${v.query}:::${v.clicks}:::${v.impressions}:::${(v.sumPos / v.impressions + 1).toFixed(1)}`).join("||");
-			out.push({
-				joinKey,
-				variantCount: BigInt(bucket.count),
-				canonicalName,
-				variants: variantsStr
-			});
-		}
-		return out;
-	}
-};
-const CANONICAL_DAILY_ROLLUP_FINAL_ID = "query_canonical_daily";
-const CANONICAL_DAILY_PART_STEM = "query_canonical_daily__part";
-const queryCanonicalDailyRollup = {
-	id: "query_canonical_daily",
-	windowDays: null,
-	format: "parquet",
-	parquetColumns: [
-		{
-			name: "query_canonical",
-			type: "VARCHAR",
-			nullable: false
-		},
-		{
-			name: "date",
-			type: "DATE",
-			nullable: false
-		},
-		{
-			name: "clicks",
-			type: "BIGINT",
-			nullable: false
-		},
-		{
-			name: "impressions",
-			type: "BIGINT",
-			nullable: false
-		},
-		{
-			name: "sum_position",
-			type: "DOUBLE",
-			nullable: false
-		}
-	],
-	parquetSortKey: ["date", "query_canonical"],
-	async build({ engine, ctx, dataSource, searchType }) {
-		const dimStore = createQueryDimStore({ dataSource });
-		const useDim = await dimStore.loadMeta(ctx) !== null;
-		const canonExpr = useDim ? `COALESCE(qd.query_canonical, q.query)` : `query`;
-		return (await runWindowed({
-			engine,
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {},
-			...useDim ? { extraFileSets: { QUERY_DIM: {
-				table: "queries",
-				keys: [dimStore.parquetKey(ctx)]
-			} } } : {},
-			paginate: {
-				orderBy: "date, query_canonical",
-				pageRows: ROLLUP_PAGE_ROWS_DAILY
-			},
-			maxWindowDays: DAILY_MAX_WINDOW_DAYS,
-			sqlFor: dailyWindowSqlFor(useDim, canonExpr)
-		})).map(mapDailyRow);
-	}
-};
-function canonicalDailyShardPredicate(queryExpr, shardIndex, shardCount) {
-	return shardCount <= 1 ? "" : ` AND (hash(${queryExpr}) % ${shardCount}) = ${shardIndex}`;
-}
-function dailyWindowSqlFor(useDim, canonExpr) {
-	return useDim ? (w, shard) => `
-        SELECT
-          ${canonExpr} AS query_canonical,
-          CAST(q.date AS VARCHAR) AS date,
-          SUM(q.clicks)::BIGINT AS clicks,
-          SUM(q.impressions)::BIGINT AS impressions,
-          SUM(q.sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true) q
-        LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query
-        WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'${canonicalDailyShardPredicate("q.query", shard?.index ?? 0, shard?.count ?? 1)}
-        GROUP BY ${canonExpr}, q.date
-      ` : (w, shard) => `
-        SELECT
-          ${canonExpr} AS query_canonical,
-          CAST(q.date AS VARCHAR) AS date,
-          SUM(q.clicks)::BIGINT AS clicks,
-          SUM(q.impressions)::BIGINT AS impressions,
-          SUM(q.sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true) q
-        WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'${canonicalDailyShardPredicate("q.query", shard?.index ?? 0, shard?.count ?? 1)}
-        GROUP BY ${canonExpr}, q.date
-      `;
-}
-function mapDailyRow(r) {
-	return {
-		query_canonical: String(r.query_canonical),
-		date: String(r.date),
-		clicks: BigInt(r.clicks),
-		impressions: BigInt(r.impressions),
-		sum_position: Number(r.sum_position)
-	};
-}
-async function rebuildCanonicalDailyResumable(opts) {
-	const { engine, ctx, dataSource, searchType, builtAt, windowOffset, deadlineMs } = opts;
-	const sType = searchType !== void 0 ? { searchType } : {};
-	const startPageOffset = opts.pageOffset ?? 0;
-	const pageRows = opts.pageRows ?? 7e4;
-	const maxWindowDays = opts.maxWindowDays ?? DAILY_MAX_WINDOW_DAYS;
-	const shardCount = Math.max(1, Math.floor(opts.shardCount ?? 1));
-	const windows = planRollupWindows((await engine.listPartitions({
-		ctx,
-		table: "queries",
-		...sType
-	})).map((p) => ({
-		partition: p.partition,
-		bytes: p.bytes
-	})), void 0, maxWindowDays);
-	const windowsTotal = windows.length;
-	const dimStore = createQueryDimStore({ dataSource });
-	const useDim = await dimStore.loadMeta(ctx) !== null;
-	const canonExpr = useDim ? `COALESCE(qd.query_canonical, q.query)` : `query`;
-	const extraFileSets = useDim ? { QUERY_DIM: {
-		table: "queries",
-		keys: [dimStore.parquetKey(ctx)]
-	} } : void 0;
-	const sqlFor = dailyWindowSqlFor(useDim, canonExpr);
-	const cols = queryCanonicalDailyRollup.parquetColumns;
-	const sortKey = queryCanonicalDailyRollup.parquetSortKey;
-	let i = windowOffset;
-	let page = startPageOffset;
-	let nextWindowOffset = windowsTotal;
-	let nextPageOffset = 0;
-	let rowsWritten = 0;
-	let pausedMidWindow = false;
-	const flushPage = async (windowIdx, pageOffset, rows) => {
-		if (rows.length === 0) return;
-		const key = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w${windowIdx}_p${pageOffset}`, builtAt, searchType);
-		await dataSource.write(key, encodeRowsToParquetFlex(rows, {
-			columns: cols,
-			sortKey
-		}));
-		rowsWritten += rows.length;
-	};
-	for (; i < windowsTotal; i++) {
-		const w = windows[i];
-		for (; page < shardCount;) {
-			const coreSql = sqlFor(w, {
-				index: page,
-				count: shardCount
-			});
-			const result = await engine.runSQL({
-				ctx,
-				table: "queries",
-				...sType,
-				fileSets: {
-					FILES: {
-						table: "queries",
-						partitions: w.partitions
-					},
-					...extraFileSets
-				},
-				sql: `${coreSql}\nORDER BY date, query_canonical\nLIMIT ${pageRows}`
-			});
-			if (result.rows.length >= pageRows) throw new Error(`query_canonical_daily shard overflow: window=${i} shard=${page}/${shardCount} returned >= ${pageRows} rows; increase shardCount or pageRows`);
-			await flushPage(i, page, result.rows.map(mapDailyRow));
-			page += 1;
-			if (Date.now() > deadlineMs) {
-				nextWindowOffset = i;
-				nextPageOffset = page;
-				pausedMidWindow = true;
-				break;
-			}
-		}
-		if (pausedMidWindow) break;
-		page = 0;
-		if (Date.now() > deadlineMs) {
-			nextWindowOffset = i + 1;
-			nextPageOffset = 0;
-			break;
-		}
-	}
-	if (nextWindowOffset < windowsTotal || nextPageOffset > 0) return {
-		done: false,
-		nextWindowOffset,
-		nextPageOffset,
-		windowsTotal,
-		windowsBuilt: nextWindowOffset - windowOffset,
-		rowsWritten
-	};
-	const partPrefixDir = rollupParquetKey(ctx, CANONICAL_DAILY_PART_STEM, builtAt, searchType).replace(/[^/]*$/, "");
-	const partKeys = (await dataSource.list(partPrefixDir)).filter((k) => k.includes(`${CANONICAL_DAILY_PART_STEM}__w`) && k.endsWith(`__v${builtAt}.parquet`)).sort();
-	if (partKeys.length === 0) {
-		const emptyKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w0_p0`, builtAt, searchType);
-		await dataSource.write(emptyKey, encodeRowsToParquetFlex([], {
-			columns: cols,
-			sortKey
-		}));
-		partKeys.push(emptyKey);
-	}
-	const envelope = {
-		version: 1,
-		id: CANONICAL_DAILY_ROLLUP_FINAL_ID,
-		builtAt,
-		windowDays: queryCanonicalDailyRollup.windowDays,
-		payload: {
-			parquetKey: partKeys[0],
-			parquetKeys: partKeys,
-			rowCount: 0
-		}
-	};
-	await dataSource.write(rollupKey(ctx, CANONICAL_DAILY_ROLLUP_FINAL_ID, builtAt, searchType), encodeJsonBigintSafe(envelope));
-	return {
-		done: true,
-		nextWindowOffset,
-		nextPageOffset,
-		windowsTotal,
-		windowsBuilt: nextWindowOffset - windowOffset,
-		rowsWritten
-	};
-}
-const indexingMetadataRollup = {
-	id: "indexing_metadata",
-	windowDays: null,
-	async build({ dataSource, ctx }) {
-		const index = await createIndexingMetadataStore({ dataSource }).loadIndex(ctx);
-		const records = Object.values(index.records);
-		const updatesByDay = /* @__PURE__ */ new Map();
-		const removesByDay = /* @__PURE__ */ new Map();
-		let totalUpdates = 0;
-		let totalRemoves = 0;
-		let latestUpdate;
-		let latestRemove;
-		for (const r of records) {
-			if (r.latestUpdateAt) {
-				totalUpdates++;
-				const day = r.latestUpdateAt.slice(0, 10);
-				updatesByDay.set(day, (updatesByDay.get(day) ?? 0) + 1);
-				if (!latestUpdate || r.latestUpdateAt > latestUpdate) latestUpdate = r.latestUpdateAt;
-			}
-			if (r.latestRemoveAt) {
-				totalRemoves++;
-				const day = r.latestRemoveAt.slice(0, 10);
-				removesByDay.set(day, (removesByDay.get(day) ?? 0) + 1);
-				if (!latestRemove || r.latestRemoveAt > latestRemove) latestRemove = r.latestRemoveAt;
-			}
-		}
-		const days = /* @__PURE__ */ new Set([...updatesByDay.keys(), ...removesByDay.keys()]);
-		const perDay = Array.from(days).sort().map((day) => ({
-			day,
-			updates: updatesByDay.get(day) ?? 0,
-			removes: removesByDay.get(day) ?? 0
-		}));
-		return {
-			totals: {
-				urls: records.length,
-				updates: totalUpdates,
-				removes: totalRemoves,
-				latestUpdateAt: latestUpdate ?? null,
-				latestRemoveAt: latestRemove ?? null
-			},
-			days: perDay
-		};
-	}
-};
-const indexingHealthRollup = {
-	id: "indexing_health",
-	windowDays: 90,
-	sliceOrthogonal: true,
-	async build({ engine, ctx, dataSource, windowAnchorMs }) {
-		const key = inspectionParquetKey(ctx);
-		if (!await dataSource.head?.(key)) return { days: [] };
-		const sql = `
-      SELECT
-        substr(CAST(inspectedAt AS VARCHAR), 1, 10) AS date,
-        COUNT(*)::BIGINT AS total_urls,
-        SUM(CASE WHEN CAST(indexStatus AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS indexed_count,
-        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'SOFT_404' THEN 1 ELSE 0 END)::BIGINT AS soft_404,
-        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'REDIRECT_ERROR' THEN 1 ELSE 0 END)::BIGINT AS redirect,
-        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'NOT_FOUND' THEN 1 ELSE 0 END)::BIGINT AS not_found,
-        SUM(CASE WHEN CAST(mobileUsabilityVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS mobile_passes,
-        SUM(CASE WHEN CAST(richResultsVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
-        SUM(CASE WHEN canonicalMismatchKind IN ('path', 'cross_domain') THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
-      FROM read_parquet({{INSPECTIONS}}, union_by_name = true)
-      WHERE substr(CAST(inspectedAt AS VARCHAR), 1, 10) >= '${utcDateMinusDays(windowAnchorMs, 90)}'
-      GROUP BY 1
-      ORDER BY 1
-    `;
-		return { days: (await engine.runSQL({
-			ctx,
-			table: "pages",
-			fileSets: { INSPECTIONS: {
-				table: "pages",
-				keys: [key]
-			} },
-			sql
-		})).rows.map((r) => ({
-			date: String(r.date),
-			total_urls: Number(r.total_urls),
-			indexed_count: Number(r.indexed_count),
-			soft_404: Number(r.soft_404),
-			redirect: Number(r.redirect),
-			not_found: Number(r.not_found),
-			mobile_passes: Number(r.mobile_passes),
-			rich_results_passes: Number(r.rich_results_passes),
-			canonical_mismatches: Number(r.canonical_mismatches)
-		})) };
-	}
-};
-function currentSitemapProjectionRelation(hasIndexes, hasDeltas) {
-	const sources = [];
-	if (hasIndexes) sources.push(`
-      SELECT
-        feedpath_hash,
-        url_hash,
-        loc,
-        removed_at,
-        greatest(
-          coalesce(removed_at, 0),
-          coalesce(last_seen_at, 0),
-          coalesce(first_seen_at, 0)
-        )::BIGINT AS observed_at,
-        0::INTEGER AS source_order
-      FROM read_parquet({{URLS_INDEX}}, union_by_name = true)
-    `);
-	if (hasDeltas) sources.push(`
-      SELECT
-        feedpath_hash,
-        url_hash,
-        loc,
-        CASE WHEN op = 'removed' THEN "at" ELSE NULL END AS removed_at,
-        "at"::BIGINT AS observed_at,
-        1::INTEGER AS source_order
-      FROM read_parquet({{URLS_DELTA}}, union_by_name = true)
-      WHERE op IN ('added', 'removed')
-    `);
-	return `(
-    WITH membership_events AS (
-      ${sources.join("\nUNION ALL\n")}
-    )
-    SELECT feedpath_hash, url_hash, loc, removed_at
-    FROM membership_events
-    QUALIFY row_number() OVER (
-      PARTITION BY feedpath_hash, url_hash
-      ORDER BY observed_at DESC, source_order DESC
-    ) = 1
-  )`;
-}
-const indexPercentRollup = {
-	id: "index_percent",
-	windowDays: 90,
-	sliceOrthogonal: true,
-	async build({ engine, ctx, dataSource, windowAnchorMs, searchType }) {
-		const [listedIndexKeys, listedDeltaKeys, manifestBytes] = await Promise.all([
-			dataSource.list(sitemapUrlsIndexPrefix(ctx)),
-			dataSource.list(`${sitemapUrlsPrefix(ctx)}/deltas/`),
-			readOptional(dataSource, sitemapUrlsProjectionManifestKey(ctx))
-		]);
-		const projection = selectSitemapProjectionFiles(listedIndexKeys, listedDeltaKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0);
-		if (projection.indexKeys.length === 0 && projection.deltaKeys.length === 0) return {
-			totalSitemapUrls: 0,
-			days: []
-		};
-		const sitemapFileSets = {
-			...projection.indexKeys.length > 0 ? { URLS_INDEX: {
-				table: "pages",
-				keys: projection.indexKeys
-			} } : {},
-			...projection.deltaKeys.length > 0 ? { URLS_DELTA: {
-				table: "pages",
-				keys: projection.deltaKeys
-			} } : {}
-		};
-		const currentMembership = currentSitemapProjectionRelation(projection.indexKeys.length > 0, projection.deltaKeys.length > 0);
-		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
-		const factSearchType = searchType ?? "web";
-		const pagesPartitions = partitionsInRange(await engine.listPartitions({
-			ctx,
-			table: "pages",
-			searchType: factSearchType
-		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
-		const numerator = await engine.runSQL({
-			ctx,
-			table: "pages",
-			fileSets: {
-				PAGES: {
-					table: "pages",
-					partitions: pagesPartitions
-				},
-				...sitemapFileSets
-			},
-			searchType: factSearchType,
-			sql: `
-        SELECT
-          p.date AS date,
-          COUNT(DISTINCT p.url)::BIGINT AS clicked_urls
-        FROM read_parquet({{PAGES}}, union_by_name = true) p
-        INNER JOIN ${currentMembership} s
-          ON s.loc = p.url AND s.removed_at IS NULL
-        WHERE p.clicks > 0 AND p.date >= '${cutoff}'
-        GROUP BY p.date
-        ORDER BY p.date
-      `
-		});
-		const denom = await engine.runSQL({
-			ctx,
-			table: "pages",
-			fileSets: sitemapFileSets,
-			sql: `
-        SELECT COUNT(DISTINCT loc)::BIGINT AS total
-        FROM ${currentMembership}
-        WHERE removed_at IS NULL
-      `
-		});
-		const total = Number(denom.rows[0]?.total ?? 0);
-		return {
-			totalSitemapUrls: total,
-			days: numerator.rows.map((r) => {
-				const clicked = Number(r.clicked_urls);
-				return {
-					date: String(r.date),
-					clicked_urls: clicked,
-					total_sitemap_urls: total,
-					ratio: total === 0 ? 0 : clicked / total
-				};
-			})
-		};
-	}
-};
-const sitemapHealthRollup = {
-	id: "sitemap_health",
-	windowDays: 90,
-	sliceOrthogonal: true,
-	async build({ dataSource, ctx, windowAnchorMs }) {
-		const index = await createSitemapReadStore({ dataSource }).loadIndex(ctx);
-		const records = Object.values(index.records);
-		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
-		const byDay = /* @__PURE__ */ new Map();
-		const feeds = [];
-		for (const r of records) {
-			const day = (r.capturedAt ?? r.lastDownloaded ?? "").slice(0, 10);
-			if (!day || day < cutoff) continue;
-			const errors = Number(r.errors ?? 0);
-			const warnings = Number(r.warnings ?? 0);
-			const urlCount = Number(r.urlCount ?? 0);
-			const bucket = byDay.get(day) ?? {
-				day,
-				feeds: 0,
-				total_urls: 0,
-				errors: 0,
-				warnings: 0
-			};
-			bucket.feeds += 1;
-			bucket.total_urls += urlCount;
-			bucket.errors += errors;
-			bucket.warnings += warnings;
-			byDay.set(day, bucket);
-			feeds.push({
-				path: r.path,
-				urlCount,
-				errors,
-				warnings,
-				contentHash: r.contentHash ?? null,
-				lastDownloaded: r.lastDownloaded ?? null,
-				capturedAt: r.capturedAt
-			});
-		}
-		return {
-			days: Array.from(byDay.values()).sort((a, b) => a.day < b.day ? -1 : 1),
-			feeds
-		};
-	}
-};
-const RECENT_SITEMAP_CHANGE_LIMIT = 200;
-function isLessRecent(a, b) {
-	return a.at < b.at || a.at === b.at && a.sequence > b.sequence;
-}
-function retainRecentSitemapChange(heap, change) {
-	if (heap.length < RECENT_SITEMAP_CHANGE_LIMIT) {
-		heap.push(change);
-		let index = heap.length - 1;
-		while (index > 0) {
-			const parent = index - 1 >> 1;
-			if (!isLessRecent(heap[index], heap[parent])) break;
-			const parentValue = heap[parent];
-			heap[parent] = heap[index];
-			heap[index] = parentValue;
-			index = parent;
-		}
-		return;
-	}
-	if (!isLessRecent(heap[0], change)) return;
-	heap[0] = change;
-	let index = 0;
-	for (;;) {
-		const left = index * 2 + 1;
-		const right = left + 1;
-		let leastRecent = index;
-		if (left < heap.length && isLessRecent(heap[left], heap[leastRecent])) leastRecent = left;
-		if (right < heap.length && isLessRecent(heap[right], heap[leastRecent])) leastRecent = right;
-		if (leastRecent === index) return;
-		const childValue = heap[leastRecent];
-		heap[leastRecent] = heap[index];
-		heap[index] = childValue;
-		index = leastRecent;
-	}
-}
-const sitemapChanges28dRollup = {
-	id: "sitemap_changes_28d",
-	windowDays: 28,
-	sliceOrthogonal: true,
-	async build({ dataSource, ctx, windowAnchorMs }) {
-		const store = createSitemapReadStore({ dataSource });
-		const from = utcDateMinusDays(windowAnchorMs, 28);
-		const to = utcDateMinusDays(windowAnchorMs, 0);
-		const counts = /* @__PURE__ */ new Map();
-		const addedTop = [];
-		const removedTop = [];
-		let sequence = 0;
-		function key(k) {
-			return `${k.day}\x00${k.feedpath}`;
-		}
-		for await (const d of store.loadEvents(ctx, {
-			from,
-			to
-		})) {
-			const day = new Date(d.observedAt).toISOString().slice(0, 10);
-			const k = key({
-				day,
-				feedpath: d.feedpath
-			});
-			const cur = counts.get(k) ?? {
-				day,
-				feedpath: d.feedpath,
-				added: 0,
-				removed: 0
-			};
-			const change = {
-				loc: d.loc,
-				feedpath: d.feedpath,
-				at: d.observedAt,
-				sequence: sequence++
-			};
-			if (d.op === "added") {
-				cur.added += 1;
-				retainRecentSitemapChange(addedTop, change);
-			} else {
-				cur.removed += 1;
-				retainRecentSitemapChange(removedTop, change);
-			}
-			counts.set(k, cur);
-		}
-		const days = Array.from(counts.values()).sort((a, b) => {
-			if (a.day !== b.day) return a.day < b.day ? -1 : 1;
-			return a.feedpath < b.feedpath ? -1 : 1;
-		});
-		const toRecentList = (heap) => heap.sort((a, b) => b.at - a.at || a.sequence - b.sequence).map(({ loc, feedpath, at }) => ({
-			loc,
-			feedpath,
-			at
-		}));
-		return {
-			days,
-			topAdded: toRecentList(addedTop),
-			topRemoved: toRecentList(removedTop)
-		};
-	}
-};
-async function rebuildDailyFromHourly(opts) {
-	const { engine, ctx, date, searchType } = opts;
-	const rows = (await engine.runSQL({
-		ctx,
-		table: "hourly_pages",
-		fileSets: { FILES: {
-			table: "hourly_pages",
-			partitions: [`hourly/${date}`]
-		} },
-		searchType,
-		sql: `
-      SELECT
-        url,
-        DATE '${date}' AS date,
-        SUM(clicks)::BIGINT AS clicks,
-        SUM(impressions)::BIGINT AS impressions,
-        SUM(sum_position)::DOUBLE AS sum_position
-      FROM read_parquet({{FILES}}, union_by_name = true)
-      WHERE date = '${date}'
-      GROUP BY url
-    `
-	})).rows.map((r) => ({
-		url: r.url,
-		date,
-		clicks: Number(r.clicks),
-		impressions: Number(r.impressions),
-		sum_position: Number(r.sum_position)
-	}));
-	await engine.writeDay({
-		userId: ctx.userId,
-		siteId: ctx.siteId,
-		table: "pages",
-		date,
-		searchType
-	}, rows);
-	return { rowsWritten: rows.length };
-}
-const DEFAULT_ROLLUPS = [
-	dailyTotalsRollup,
-	weeklyTotalsRollup,
-	topPages28dRollup,
-	topKeywords28dRollup,
-	topCountries28dRollup,
-	indexingMetadataRollup,
-	indexingHealthRollup,
-	indexPercentRollup,
-	sitemapHealthRollup,
-	sitemapChanges28dRollup
-];
-const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
+import { readLatestRollup, rebuildRollups, rollupKey, rollupParquetKey } from "./rollups/core.mjs";
+import { ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, planRollupWindows, runWindowed } from "./rollups/windows.mjs";
+import { queryCanonicalDailyRollup, queryCanonicalVariantsRollup, rebuildCanonicalDailyResumable } from "./rollups/canonical.mjs";
+import { indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, sitemapChanges28dRollup, sitemapHealthRollup } from "./rollups/indexing.mjs";
+import { dailyTotalsRollup, topCountries28dRollup, topKeywords28dParquetRollup, topPages28dRollup, weeklyTotalsRollup } from "./rollups/traffic.mjs";
+import { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS } from "./rollups/defaults.mjs";
+import { rebuildDailyFromHourly } from "./rollups/hourly.mjs";
 export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topPages28dRollup, weeklyTotalsRollup };