@gscdump/engine 1.4.11 → 1.5.0

package/dist/rollups/core.mjs

@@ -0,0 +1,116 @@
+import { engineErrors } from "../errors.mjs";
+import { encodeRowsToParquetFlex } from "../adapters/hyparquet.mjs";
+import { isoDateToUtcMs } from "./dates.mjs";
+import { encodeJsonBigintSafe } from "@gscdump/lakehouse/bigint";
+function rollupPrefix(ctx, searchType) {
+	const base = ctx.siteId ? `u_${ctx.userId}/${ctx.siteId}/rollups` : `u_${ctx.userId}/rollups`;
+	return searchType !== void 0 && searchType !== "web" ? `${base}/${searchType}` : base;
+}
+function rollupKey(ctx, id, builtAt, searchType) {
+	return `${rollupPrefix(ctx, searchType)}/${id}__v${builtAt}.json`;
+}
+function rollupParquetKey(ctx, id, builtAt, searchType) {
+	return `${rollupPrefix(ctx, searchType)}/${id}__v${builtAt}.parquet`;
+}
+const ROLLUP_FILE_RE = /^(?<id>[a-z0-9_]+)__v(?<ts>\d+)\.json$/;
+async function readLatestRollup(bucket, ctx, id, searchType) {
+	const prefix = `${rollupPrefix(ctx, searchType)}/`;
+	let newest = null;
+	let cursor;
+	do {
+		const listing = await bucket.list({
+			prefix,
+			cursor
+		});
+		for (const obj of listing.objects) {
+			const m = ROLLUP_FILE_RE.exec(obj.key.slice(prefix.length));
+			if (!m?.groups || m.groups.id !== id) continue;
+			const ts = Number(m.groups.ts);
+			if (!newest || ts > newest.ts) newest = {
+				ts,
+				key: obj.key
+			};
+		}
+		cursor = listing.truncated ? listing.cursor : void 0;
+	} while (cursor !== void 0);
+	if (!newest) return null;
+	const obj = await bucket.get(newest.key);
+	if (!obj) return null;
+	return JSON.parse(await obj.text());
+}
+async function rebuildRollups(opts) {
+	const now = opts.now ?? (() => Date.now());
+	const dataEndMs = opts.dataEndDate !== void 0 ? isoDateToUtcMs(opts.dataEndDate) : null;
+	const results = [];
+	for (const def of opts.defs) {
+		const builtAt = now();
+		const windowAnchorMs = dataEndMs ?? builtAt;
+		const defSearchType = def.sliceOrthogonal === true ? void 0 : opts.searchType;
+		try {
+			const payload = await def.build({
+				engine: opts.engine,
+				ctx: opts.ctx,
+				dataSource: opts.dataSource,
+				windowAnchorMs,
+				...defSearchType !== void 0 ? { searchType: defSearchType } : {}
+			});
+			if (def.format === "parquet") {
+				if (!def.parquetColumns || def.parquetColumns.length === 0) throw new Error(`rollup '${def.id}' declared format='parquet' without parquetColumns`);
+				const rows = payload ?? [];
+				const parquetBytes = encodeRowsToParquetFlex(rows, {
+					columns: def.parquetColumns,
+					sortKey: def.parquetSortKey
+				});
+				const parquetKey = rollupParquetKey(opts.ctx, def.id, builtAt, defSearchType);
+				await opts.dataSource.write(parquetKey, parquetBytes);
+				const pointer = {
+					parquetKey,
+					rowCount: rows.length
+				};
+				const envelopeBytes = encodeJsonBigintSafe({
+					version: 1,
+					id: def.id,
+					builtAt,
+					windowDays: def.windowDays,
+					payload: pointer
+				});
+				const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
+				await opts.dataSource.write(key, envelopeBytes);
+				results.push({
+					id: def.id,
+					objectKey: key,
+					parquetKey,
+					bytes: envelopeBytes.byteLength,
+					parquetBytes: parquetBytes.byteLength,
+					builtAt
+				});
+				continue;
+			}
+			const bytes = encodeJsonBigintSafe({
+				version: 1,
+				id: def.id,
+				builtAt,
+				windowDays: def.windowDays,
+				payload
+			});
+			const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
+			await opts.dataSource.write(key, bytes);
+			results.push({
+				id: def.id,
+				objectKey: key,
+				bytes: bytes.byteLength,
+				builtAt
+			});
+		} catch (err) {
+			results.push({
+				id: def.id,
+				objectKey: "",
+				bytes: 0,
+				builtAt,
+				error: engineErrors.rollupBuildFailed(def.id, err)
+			});
+		}
+	}
+	return results;
+}
+export { readLatestRollup, rebuildRollups, rollupKey, rollupParquetKey };

package/dist/rollups/dates.mjs

@@ -0,0 +1,11 @@
+import { MS_PER_DAY } from "gscdump/dates";
+function isoDateToUtcMs(iso) {
+	const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(iso);
+	if (!m) throw new Error(`dataEndDate must be ISO YYYY-MM-DD, got: ${iso}`);
+	return Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]));
+}
+function utcDateMinusDays(at, days) {
+	const d = new Date(at - days * MS_PER_DAY);
+	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
+}
+export { isoDateToUtcMs, utcDateMinusDays };

package/dist/rollups/defaults.d.mts

@@ -0,0 +1,11 @@
+import { RollupDef } from "./core.mjs";
+declare const DEFAULT_ROLLUPS: readonly RollupDef[];
+/**
+ * Canonical-primary rollups (ADR-0017 / ADR-0018). Opt-in — kept out of
+ * `DEFAULT_ROLLUPS` because they only pay off once the consumer queries by
+ * `queryCanonical` and wires the read seams (`resolveExtra` /
+ * `canonicalSource`). Hosts opt in by concatenating these onto their def list
+ * (CLI: `gscdump rollups --with-canonical`).
+ */
+declare const CANONICAL_ROLLUPS: readonly RollupDef[];
+export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS };

package/dist/rollups/defaults.mjs

@@ -0,0 +1,17 @@
+import { queryCanonicalDailyRollup, queryCanonicalVariantsRollup } from "./canonical.mjs";
+import { indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, sitemapChanges28dRollup, sitemapHealthRollup } from "./indexing.mjs";
+import { dailyTotalsRollup, topCountries28dRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup } from "./traffic.mjs";
+const DEFAULT_ROLLUPS = [
+	dailyTotalsRollup,
+	weeklyTotalsRollup,
+	topPages28dRollup,
+	topKeywords28dRollup,
+	topCountries28dRollup,
+	indexingMetadataRollup,
+	indexingHealthRollup,
+	indexPercentRollup,
+	sitemapHealthRollup,
+	sitemapChanges28dRollup
+];
+const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
+export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS };

package/dist/rollups/hourly.d.mts

@@ -0,0 +1,38 @@
+import { Row as Row$1 } from "../storage.mjs";
+import "../contracts.mjs";
+import { RollupEngine } from "./core.mjs";
+import { TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
+/**
+ * Aggregate one day's `hourly_pages` partition into the daily `pages` shape
+ * and write it to the daily Discover partition. After this runs for date D,
+ * the daily query path serves D from `pages/.../daily/D` and the `hourly/D`
+ * partition becomes read-only / GC-only.
+ *
+ * `(position - 1)` weighting matches the storage convention encoded by
+ * `toSumPosition`: `sum_position = SUM((position - 1) * impressions)`, so a
+ * downstream `SUM(sum_position) / SUM(impressions) + 1` recovers the mean.
+ *
+ * searchType-scoped: only call with `searchType: 'discover'`. The hourly
+ * partition lives under `hourly_pages` and the output lands under `pages` so
+ * existing dashboard queries (which read `pages`) see the rolled-up day
+ * transparently.
+ */
+interface RebuildDailyFromHourlyOptions {
+  engine: RollupEngine & {
+    writeDay: (scope: TenantCtx & {
+      table: TableTypeName;
+      date: string;
+      searchType?: SearchType;
+    }, rows: Row$1[]) => Promise<void>;
+  };
+  ctx: TenantCtx;
+  /** PT calendar day to roll up. */
+  date: string;
+  searchType: 'discover';
+}
+type TableTypeName = import('@gscdump/contracts').TableName;
+declare function rebuildDailyFromHourly(opts: RebuildDailyFromHourlyOptions): Promise<{
+  rowsWritten: number;
+}>;
+export { RebuildDailyFromHourlyOptions, rebuildDailyFromHourly };

package/dist/rollups/hourly.mjs

@@ -0,0 +1,38 @@
+async function rebuildDailyFromHourly(opts) {
+	const { engine, ctx, date, searchType } = opts;
+	const rows = (await engine.runSQL({
+		ctx,
+		table: "hourly_pages",
+		fileSets: { FILES: {
+			table: "hourly_pages",
+			partitions: [`hourly/${date}`]
+		} },
+		searchType,
+		sql: `
+      SELECT
+        url,
+        DATE '${date}' AS date,
+        SUM(clicks)::BIGINT AS clicks,
+        SUM(impressions)::BIGINT AS impressions,
+        SUM(sum_position)::DOUBLE AS sum_position
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date = '${date}'
+      GROUP BY url
+    `
+	})).rows.map((r) => ({
+		url: r.url,
+		date,
+		clicks: Number(r.clicks),
+		impressions: Number(r.impressions),
+		sum_position: Number(r.sum_position)
+	}));
+	await engine.writeDay({
+		userId: ctx.userId,
+		siteId: ctx.siteId,
+		table: "pages",
+		date,
+		searchType
+	}, rows);
+	return { rowsWritten: rows.length };
+}
+export { rebuildDailyFromHourly };

package/dist/rollups/indexing.d.mts

@@ -0,0 +1,46 @@
+import { RollupDef } from "./core.mjs";
+/**
+ * Aggregates the per-URL Indexing API metadata entity store (populated by
+ * `gscdump entities indexing snapshot`) into daily counts of `URL_UPDATED`
+ * and `URL_REMOVED` notifications. Covers the third entity-snapshot shape
+ * without needing its own parquet family — publish events are sparse and
+ * aggregate cleanly into a small JSON rollup.
+ *
+ * Safe no-op when the entity store is empty: returns `{ totals: {...}, days: [] }`
+ * so downstream readers don't have to special-case first-run sites.
+ */
+declare const indexingMetadataRollup: RollupDef;
+/**
+ * Indexing-API health by day: per `inspectedAt` date, counts of indexed,
+ * soft-404, redirect, not-found, mobile passes, rich-results passes, and
+ * canonical mismatches. Sourced from the inspections parquet sidecar
+ * (`InspectionStore.parquetUri`), which holds the latest record per URL.
+ *
+ * Empty-payload no-op when the sidecar URI is unavailable (in-memory
+ * `DataSource`, or before `materialize` has run).
+ */
+declare const indexingHealthRollup: RollupDef;
+/**
+ * Per-day index-percent: ratio of (sitemap URLs that received GSC clicks on
+ * that date) / (total live sitemap URLs). Uses a DuckDB JOIN between the
+ * sitemap urls parquet (`SitemapStore.urlsParquetUri`) and the `pages` fact
+ * parquet. Total denominator is the count of live URLs in the urls index;
+ * numerator is per-day distinct loc count where pages.clicks > 0.
+ */
+declare const indexPercentRollup: RollupDef;
+/**
+ * Sitemap-health per-day series materialized from the sitemap-store JSON
+ * index. Each `SitemapRecord` carries `urlCount`, `errors`, `warnings`,
+ * `contentHash`, and `lastDownloaded`. We bucket records by the day of their
+ * `capturedAt` (or `lastDownloaded` fallback) and emit per-day aggregates plus
+ * a snapshot of per-feed stats at the most recent capture.
+ */
+declare const sitemapHealthRollup: RollupDef;
+/**
+ * Trailing-28-day sitemap URL changes: per-day per-feedpath {added, removed}
+ * counts plus rolling top-200 added and removed URLs. Streams from
+ * retained `SitemapReadStore.loadEvents()` history. State compaction cannot
+ * erase analytics input; memory scales independently of total site state.
+ */
+declare const sitemapChanges28dRollup: RollupDef;
+export { indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, sitemapChanges28dRollup, sitemapHealthRollup };

package/dist/rollups/indexing.mjs

@@ -0,0 +1,357 @@
+import "../layout.mjs";
+import { inspectionParquetKey, sitemapUrlsIndexPrefix, sitemapUrlsPrefix, sitemapUrlsProjectionManifestKey } from "../entity-keys.mjs";
+import { readOptional } from "../adapters/read-optional.mjs";
+import { createIndexingMetadataStore } from "../entities/indexing-metadata.mjs";
+import { decodeSitemapProjectionManifest, selectSitemapProjectionFiles } from "../entities/sitemap-projection.mjs";
+import { createSitemapReadStore } from "../entities/sitemap.mjs";
+import "../entities.mjs";
+import { utcDateMinusDays } from "./dates.mjs";
+import { partitionsInRange } from "./windows.mjs";
+const indexingMetadataRollup = {
+	id: "indexing_metadata",
+	windowDays: null,
+	async build({ dataSource, ctx }) {
+		const index = await createIndexingMetadataStore({ dataSource }).loadIndex(ctx);
+		const records = Object.values(index.records);
+		const updatesByDay = /* @__PURE__ */ new Map();
+		const removesByDay = /* @__PURE__ */ new Map();
+		let totalUpdates = 0;
+		let totalRemoves = 0;
+		let latestUpdate;
+		let latestRemove;
+		for (const r of records) {
+			if (r.latestUpdateAt) {
+				totalUpdates++;
+				const day = r.latestUpdateAt.slice(0, 10);
+				updatesByDay.set(day, (updatesByDay.get(day) ?? 0) + 1);
+				if (!latestUpdate || r.latestUpdateAt > latestUpdate) latestUpdate = r.latestUpdateAt;
+			}
+			if (r.latestRemoveAt) {
+				totalRemoves++;
+				const day = r.latestRemoveAt.slice(0, 10);
+				removesByDay.set(day, (removesByDay.get(day) ?? 0) + 1);
+				if (!latestRemove || r.latestRemoveAt > latestRemove) latestRemove = r.latestRemoveAt;
+			}
+		}
+		const days = /* @__PURE__ */ new Set([...updatesByDay.keys(), ...removesByDay.keys()]);
+		const perDay = Array.from(days).sort().map((day) => ({
+			day,
+			updates: updatesByDay.get(day) ?? 0,
+			removes: removesByDay.get(day) ?? 0
+		}));
+		return {
+			totals: {
+				urls: records.length,
+				updates: totalUpdates,
+				removes: totalRemoves,
+				latestUpdateAt: latestUpdate ?? null,
+				latestRemoveAt: latestRemove ?? null
+			},
+			days: perDay
+		};
+	}
+};
+const indexingHealthRollup = {
+	id: "indexing_health",
+	windowDays: 90,
+	sliceOrthogonal: true,
+	async build({ engine, ctx, dataSource, windowAnchorMs }) {
+		const key = inspectionParquetKey(ctx);
+		if (!await dataSource.head?.(key)) return { days: [] };
+		const sql = `
+      SELECT
+        substr(CAST(inspectedAt AS VARCHAR), 1, 10) AS date,
+        COUNT(*)::BIGINT AS total_urls,
+        SUM(CASE WHEN CAST(indexStatus AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS indexed_count,
+        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'SOFT_404' THEN 1 ELSE 0 END)::BIGINT AS soft_404,
+        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'REDIRECT_ERROR' THEN 1 ELSE 0 END)::BIGINT AS redirect,
+        SUM(CASE WHEN CAST(pageFetchState AS VARCHAR) = 'NOT_FOUND' THEN 1 ELSE 0 END)::BIGINT AS not_found,
+        SUM(CASE WHEN CAST(mobileUsabilityVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS mobile_passes,
+        SUM(CASE WHEN CAST(richResultsVerdict AS VARCHAR) = 'PASS' THEN 1 ELSE 0 END)::BIGINT AS rich_results_passes,
+        SUM(CASE WHEN canonicalMismatchKind IN ('path', 'cross_domain') THEN 1 ELSE 0 END)::BIGINT AS canonical_mismatches
+      FROM read_parquet({{INSPECTIONS}}, union_by_name = true)
+      WHERE substr(CAST(inspectedAt AS VARCHAR), 1, 10) >= '${utcDateMinusDays(windowAnchorMs, 90)}'
+      GROUP BY 1
+      ORDER BY 1
+    `;
+		return { days: (await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: { INSPECTIONS: {
+				table: "pages",
+				keys: [key]
+			} },
+			sql
+		})).rows.map((r) => ({
+			date: String(r.date),
+			total_urls: Number(r.total_urls),
+			indexed_count: Number(r.indexed_count),
+			soft_404: Number(r.soft_404),
+			redirect: Number(r.redirect),
+			not_found: Number(r.not_found),
+			mobile_passes: Number(r.mobile_passes),
+			rich_results_passes: Number(r.rich_results_passes),
+			canonical_mismatches: Number(r.canonical_mismatches)
+		})) };
+	}
+};
+function currentSitemapProjectionRelation(hasIndexes, hasDeltas) {
+	const sources = [];
+	if (hasIndexes) sources.push(`
+      SELECT
+        feedpath_hash,
+        url_hash,
+        loc,
+        removed_at,
+        greatest(
+          coalesce(removed_at, 0),
+          coalesce(last_seen_at, 0),
+          coalesce(first_seen_at, 0)
+        )::BIGINT AS observed_at,
+        0::INTEGER AS source_order
+      FROM read_parquet({{URLS_INDEX}}, union_by_name = true)
+    `);
+	if (hasDeltas) sources.push(`
+      SELECT
+        feedpath_hash,
+        url_hash,
+        loc,
+        CASE WHEN op = 'removed' THEN "at" ELSE NULL END AS removed_at,
+        "at"::BIGINT AS observed_at,
+        1::INTEGER AS source_order
+      FROM read_parquet({{URLS_DELTA}}, union_by_name = true)
+      WHERE op IN ('added', 'removed')
+    `);
+	return `(
+    WITH membership_events AS (
+      ${sources.join("\nUNION ALL\n")}
+    )
+    SELECT feedpath_hash, url_hash, loc, removed_at
+    FROM membership_events
+    QUALIFY row_number() OVER (
+      PARTITION BY feedpath_hash, url_hash
+      ORDER BY observed_at DESC, source_order DESC
+    ) = 1
+  )`;
+}
+const indexPercentRollup = {
+	id: "index_percent",
+	windowDays: 90,
+	sliceOrthogonal: true,
+	async build({ engine, ctx, dataSource, windowAnchorMs, searchType }) {
+		const [listedIndexKeys, listedDeltaKeys, manifestBytes] = await Promise.all([
+			dataSource.list(sitemapUrlsIndexPrefix(ctx)),
+			dataSource.list(`${sitemapUrlsPrefix(ctx)}/deltas/`),
+			readOptional(dataSource, sitemapUrlsProjectionManifestKey(ctx))
+		]);
+		const projection = selectSitemapProjectionFiles(listedIndexKeys, listedDeltaKeys, manifestBytes ? decodeSitemapProjectionManifest(new TextDecoder().decode(manifestBytes)) : void 0);
+		if (projection.indexKeys.length === 0 && projection.deltaKeys.length === 0) return {
+			totalSitemapUrls: 0,
+			days: []
+		};
+		const sitemapFileSets = {
+			...projection.indexKeys.length > 0 ? { URLS_INDEX: {
+				table: "pages",
+				keys: projection.indexKeys
+			} } : {},
+			...projection.deltaKeys.length > 0 ? { URLS_DELTA: {
+				table: "pages",
+				keys: projection.deltaKeys
+			} } : {}
+		};
+		const currentMembership = currentSitemapProjectionRelation(projection.indexKeys.length > 0, projection.deltaKeys.length > 0);
+		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
+		const factSearchType = searchType ?? "web";
+		const pagesPartitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "pages",
+			searchType: factSearchType
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		const numerator = await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: {
+				PAGES: {
+					table: "pages",
+					partitions: pagesPartitions
+				},
+				...sitemapFileSets
+			},
+			searchType: factSearchType,
+			sql: `
+        SELECT
+          p.date AS date,
+          COUNT(DISTINCT p.url)::BIGINT AS clicked_urls
+        FROM read_parquet({{PAGES}}, union_by_name = true) p
+        INNER JOIN ${currentMembership} s
+          ON s.loc = p.url AND s.removed_at IS NULL
+        WHERE p.clicks > 0 AND p.date >= '${cutoff}'
+        GROUP BY p.date
+        ORDER BY p.date
+      `
+		});
+		const denom = await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: sitemapFileSets,
+			sql: `
+        SELECT COUNT(DISTINCT loc)::BIGINT AS total
+        FROM ${currentMembership}
+        WHERE removed_at IS NULL
+      `
+		});
+		const total = Number(denom.rows[0]?.total ?? 0);
+		return {
+			totalSitemapUrls: total,
+			days: numerator.rows.map((r) => {
+				const clicked = Number(r.clicked_urls);
+				return {
+					date: String(r.date),
+					clicked_urls: clicked,
+					total_sitemap_urls: total,
+					ratio: total === 0 ? 0 : clicked / total
+				};
+			})
+		};
+	}
+};
+const sitemapHealthRollup = {
+	id: "sitemap_health",
+	windowDays: 90,
+	sliceOrthogonal: true,
+	async build({ dataSource, ctx, windowAnchorMs }) {
+		const index = await createSitemapReadStore({ dataSource }).loadIndex(ctx);
+		const records = Object.values(index.records);
+		const cutoff = utcDateMinusDays(windowAnchorMs, 90);
+		const byDay = /* @__PURE__ */ new Map();
+		const feeds = [];
+		for (const r of records) {
+			const day = (r.capturedAt ?? r.lastDownloaded ?? "").slice(0, 10);
+			if (!day || day < cutoff) continue;
+			const errors = Number(r.errors ?? 0);
+			const warnings = Number(r.warnings ?? 0);
+			const urlCount = Number(r.urlCount ?? 0);
+			const bucket = byDay.get(day) ?? {
+				day,
+				feeds: 0,
+				total_urls: 0,
+				errors: 0,
+				warnings: 0
+			};
+			bucket.feeds += 1;
+			bucket.total_urls += urlCount;
+			bucket.errors += errors;
+			bucket.warnings += warnings;
+			byDay.set(day, bucket);
+			feeds.push({
+				path: r.path,
+				urlCount,
+				errors,
+				warnings,
+				contentHash: r.contentHash ?? null,
+				lastDownloaded: r.lastDownloaded ?? null,
+				capturedAt: r.capturedAt
+			});
+		}
+		return {
+			days: Array.from(byDay.values()).sort((a, b) => a.day < b.day ? -1 : 1),
+			feeds
+		};
+	}
+};
+const RECENT_SITEMAP_CHANGE_LIMIT = 200;
+function isLessRecent(a, b) {
+	return a.at < b.at || a.at === b.at && a.sequence > b.sequence;
+}
+function retainRecentSitemapChange(heap, change) {
+	if (heap.length < RECENT_SITEMAP_CHANGE_LIMIT) {
+		heap.push(change);
+		let index = heap.length - 1;
+		while (index > 0) {
+			const parent = index - 1 >> 1;
+			if (!isLessRecent(heap[index], heap[parent])) break;
+			const parentValue = heap[parent];
+			heap[parent] = heap[index];
+			heap[index] = parentValue;
+			index = parent;
+		}
+		return;
+	}
+	if (!isLessRecent(heap[0], change)) return;
+	heap[0] = change;
+	let index = 0;
+	for (;;) {
+		const left = index * 2 + 1;
+		const right = left + 1;
+		let leastRecent = index;
+		if (left < heap.length && isLessRecent(heap[left], heap[leastRecent])) leastRecent = left;
+		if (right < heap.length && isLessRecent(heap[right], heap[leastRecent])) leastRecent = right;
+		if (leastRecent === index) return;
+		const childValue = heap[leastRecent];
+		heap[leastRecent] = heap[index];
+		heap[index] = childValue;
+		index = leastRecent;
+	}
+}
+const sitemapChanges28dRollup = {
+	id: "sitemap_changes_28d",
+	windowDays: 28,
+	sliceOrthogonal: true,
+	async build({ dataSource, ctx, windowAnchorMs }) {
+		const store = createSitemapReadStore({ dataSource });
+		const from = utcDateMinusDays(windowAnchorMs, 28);
+		const to = utcDateMinusDays(windowAnchorMs, 0);
+		const counts = /* @__PURE__ */ new Map();
+		const addedTop = [];
+		const removedTop = [];
+		let sequence = 0;
+		function key(k) {
+			return `${k.day}\x00${k.feedpath}`;
+		}
+		for await (const d of store.loadEvents(ctx, {
+			from,
+			to
+		})) {
+			const day = new Date(d.observedAt).toISOString().slice(0, 10);
+			const k = key({
+				day,
+				feedpath: d.feedpath
+			});
+			const cur = counts.get(k) ?? {
+				day,
+				feedpath: d.feedpath,
+				added: 0,
+				removed: 0
+			};
+			const change = {
+				loc: d.loc,
+				feedpath: d.feedpath,
+				at: d.observedAt,
+				sequence: sequence++
+			};
+			if (d.op === "added") {
+				cur.added += 1;
+				retainRecentSitemapChange(addedTop, change);
+			} else {
+				cur.removed += 1;
+				retainRecentSitemapChange(removedTop, change);
+			}
+			counts.set(k, cur);
+		}
+		const days = Array.from(counts.values()).sort((a, b) => {
+			if (a.day !== b.day) return a.day < b.day ? -1 : 1;
+			return a.feedpath < b.feedpath ? -1 : 1;
+		});
+		const toRecentList = (heap) => heap.sort((a, b) => b.at - a.at || a.sequence - b.sequence).map(({ loc, feedpath, at }) => ({
+			loc,
+			feedpath,
+			at
+		}));
+		return {
+			days,
+			topAdded: toRecentList(addedTop),
+			topRemoved: toRecentList(removedTop)
+		};
+	}
+};
+export { indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, sitemapChanges28dRollup, sitemapHealthRollup };

package/dist/rollups/traffic.d.mts

@@ -0,0 +1,38 @@
+import { RollupDef } from "./core.mjs";
+/**
+ * Daily totals across the full history. One row per (date, table) with
+ * clicks + impressions + position. Powers sparklines and headline totals.
+ *
+ * Includes `anonymizedImpressionsPct` per day computed as
+ *   1 - sum(query_grained_impressions) / sum(page_grained_impressions)
+ * — surfaces GSC's anonymous-query gap so the dashboard can warn users not
+ * to trust query-grained breakdowns as comprehensive.
+ */
+declare const dailyTotalsRollup: RollupDef;
+/** Weekly totals, ISO week aligned. Cheap and stable for trend widgets. */
+declare const weeklyTotalsRollup: RollupDef;
+/**
+ * Top 1000 pages by clicks over the trailing 28-day window. JSON for v1;
+ * promote to parquet (`top_pages_28d.parquet`) when the dashboard needs
+ * server-side WHERE filtering on this rollup.
+ */
+declare const topPages28dRollup: RollupDef;
+/**
+ * Top 250 countries by clicks over the trailing 28-day window. Countries
+ * cardinality is bounded (~250 ISO codes), so the list fits in a tiny JSON
+ * payload regardless of traffic shape. Powers a geo-overview widget without
+ * spinning up DuckDB-WASM.
+ */
+declare const topCountries28dRollup: RollupDef;
+/**
+ * Parquet-format companion to `topKeywords28dRollup`. Same shape, but persists
+ * as a parquet object plus JSON sidecar pointer so widgets that need
+ * server-side WHERE (filter by prefix, by clicks threshold, paginate) can scan
+ * it directly with DuckDB-WASM instead of loading all 1000 rows into JS.
+ *
+ * Opt-in: include in the caller's rollup def list alongside (or instead of)
+ * the JSON variant; the runner treats the two as independent ids so they can
+ * coexist during a migration.
+ */
+declare const topKeywords28dParquetRollup: RollupDef;
+export { dailyTotalsRollup, topCountries28dRollup, topKeywords28dParquetRollup, topPages28dRollup, weeklyTotalsRollup };