@gscdump/engine 0.19.3 → 0.19.6

package/dist/_chunks/dispatch.mjs

@@ -17,13 +17,23 @@ function assertSatisfies(analyzer, source) {
 	if (missing.length > 0) throw new AnalyzerCapabilityError(analyzer.id, missing);
 }
 async function runAnalyzerFromSource(source, params, registry) {
-	const analyzer = registry.resolveAnalyzer(params.type, sourceHas(source, "executeSql"));
+	let analyzer = registry.resolveAnalyzer(params.type, sourceHas(source, "executeSql"));
 	if (!analyzer) throw new AnalyzerCapabilityError(params.type, ["executeSql"]);
 	assertSatisfies(analyzer, source);
-	const plan = analyzer.build(params, {
+	const buildCtx = {
 		adapter: source.adapter,
 		siteId: source.siteId
-	});
+	};
+	let plan;
+	try {
+		plan = analyzer.build(params, buildCtx);
+	} catch (err) {
+		const rowsVariant = err?.name === "UnresolvableDatasetError" ? registry.getAnalyzerVariants(params.type)?.rows : void 0;
+		if (!rowsVariant) throw err;
+		assertSatisfies(rowsVariant, source);
+		analyzer = rowsVariant;
+		plan = rowsVariant.build(params, buildCtx);
+	}
 	if (plan.kind === "rows") return runRowsPlanAgainstSource(source, analyzer, plan, params);
 	return runSqlPlanAgainstSource(source, analyzer, plan, params);
 }

package/dist/_chunks/schema.mjs

@@ -145,7 +145,7 @@ function inferTable(dimensions) {
 	if (dims.has("country")) return "countries";
 	if (dims.has("device")) return "devices";
 	if (dims.has("searchAppearance")) return "search_appearance";
-	return "devices";
+	return "pages";
 }
 function naturalKeyColumns(table) {
 	return TABLE_METADATA[table].sortKey;

package/dist/adapters/hyparquet.d.mts

@@ -1,5 +1,6 @@
 import { C as Row, M as TableName, i as DataSource, m as ParquetCodec, t as CodecCtx } from "../_chunks/storage.mjs";
 import { t as ColumnDef } from "../_chunks/schema.mjs";
+import { ParquetQueryFilter } from "hyparquet";
 declare function encodeRowsToParquet(table: TableName, rows: readonly Row[]): Uint8Array;
 interface EncodeFlexOptions {
   /** Columns defining the output schema + order. */
@@ -17,7 +18,18 @@ interface EncodeFlexOptions {
  * merges cleanly with fact-table reads.
  */
 declare function encodeRowsToParquetFlex(rows: readonly Row[], opts: EncodeFlexOptions): Uint8Array;
-declare function decodeParquetToRows(bytes: Uint8Array): Promise<Row[]>;
+interface DecodeParquetOptions {
+  /**
+   * Row filter pushed down into the parquet reader. hyparquet evaluates this
+   * per row group — pruning groups whose column statistics can't match and
+   * materialising only matching rows — so a filtered decode of a large file
+   * holds at most one row group plus the matches in memory, never the whole
+   * file. Use this whenever the caller needs a sub-slice of a big parquet
+   * (e.g. one feedpath out of a site-wide sitemap-urls index).
+   */
+  filter?: ParquetQueryFilter;
+}
+declare function decodeParquetToRows(bytes: Uint8Array, opts?: DecodeParquetOptions): Promise<Row[]>;
 interface HyparquetCodecOptions {
   /**
    * Override `readRows`. Useful when reads should be delegated to a faster
@@ -27,4 +39,4 @@ interface HyparquetCodecOptions {
   readRows?: (ctx: CodecCtx, key: string, dataSource: DataSource) => Promise<Row[]>;
 }
 declare function createHyparquetCodec(options?: HyparquetCodecOptions): ParquetCodec;
-export { EncodeFlexOptions, HyparquetCodecOptions, createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };
+export { DecodeParquetOptions, EncodeFlexOptions, HyparquetCodecOptions, createHyparquetCodec, decodeParquetToRows, encodeRowsToParquet, encodeRowsToParquetFlex };

package/dist/adapters/hyparquet.mjs

@@ -103,9 +103,12 @@ function asyncBufferFromBytes(bytes) {
 		}
 	};
 }
-async function decodeParquetToRows(bytes) {
+async function decodeParquetToRows(bytes, opts = {}) {
 	if (bytes.byteLength === 0) return [];
-	return await parquetReadObjects({ file: asyncBufferFromBytes(bytes) });
+	return await parquetReadObjects({
+		file: asyncBufferFromBytes(bytes),
+		...opts.filter ? { filter: opts.filter } : {}
+	});
 }
 function createHyparquetCodec(options = {}) {
 	return {

package/dist/entities.mjs

@@ -418,7 +418,7 @@ function createSitemapStore(opts) {
 			const fpHash = hash(feedpath);
 			const includeRemoved = opts?.includeRemoved ?? false;
 			const indexBytes = await ds.read(sitemapUrlsIndexKey(ctx)).catch(() => void 0);
-			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes) : [];
+			const indexRows = indexBytes ? await decodeParquetToRows(indexBytes, { filter: { feedpath_hash: { $eq: fpHash } } }) : [];
 			const deltaKeys = (await ds.list(`${sitemapUrlsPrefix(ctx)}/deltas/`)).sort();
 			const live = /* @__PURE__ */ new Map();
 			const removedMap = /* @__PURE__ */ new Map();

package/dist/rollups.d.mts

@@ -27,6 +27,21 @@ interface RollupEngine {
   }) => Promise<{
     rows: import('@gscdump/engine/contracts').Row[];
   }>;
+  /**
+   * Read the live manifest for a (tenant, table[, searchType]) cohort —
+   * cheap, no parquet decode. Builders use this to chunk a full-history scan
+   * into byte-bounded windows so a single `runSQL` call never has to ship
+   * more than ~14MB of decoded rows across the Workers service-binding RPC
+   * (32MiB hard cap).
+   */
+  listPartitions: (opts: {
+    ctx: TenantCtx;
+    table: import('@gscdump/engine/contracts').TableName;
+    searchType?: SearchType;
+  }) => Promise<Array<{
+    partition: string;
+    bytes: number;
+  }>>;
 }
 /**
  * One rollup definition. Build runs SQL over the tenant's facts and/or reads
@@ -150,8 +165,64 @@ interface RebuildRollupResult {
   /** Parquet payload byte size when `format === 'parquet'`. */
   parquetBytes?: number;
   builtAt: number;
+  /**
+   * Set when this def's build/encode/write failed. The runner records the
+   * failure and continues with the remaining defs so one bad rollup never
+   * aborts the rest. Successful defs have no `error`.
+   */
+  error?: string;
 }
 declare function rebuildRollups(opts: RebuildRollupsOptions): Promise<RebuildRollupResult[]>;
+/**
+ * Target decoded-bytes budget per window. Sits well under the 28MiB executor
+ *  guard so headroom remains for SQL + result rows.
+ */
+declare const WINDOW_BYTE_BUDGET: number;
+/**
+ * UTC day-aligned [startMs, endMs] span a partition covers. Returns null for
+ * `hourly/` partitions and anything unrecognised — those are excluded from
+ * windowed planning.
+ */
+declare function partitionDaySpan(partition: string): {
+  startMs: number;
+  endMs: number;
+} | null;
+/**
+ * Plan byte-bounded windows over a partition set. Each window names the
+ * partitions whose span intersects it; a coarse tier file can land in two
+ * windows, so every windowed SQL MUST also date-filter to the window bounds.
+ */
+declare function planRollupWindows(parts: Array<{
+  partition: string;
+  bytes: number;
+}>, clampRange?: {
+  start: string;
+  end: string;
+}): Array<{
+  start: string;
+  end: string;
+  partitions: string[];
+}>;
+/** Partition strings whose span intersects the inclusive [start, end] date range. */
+declare function partitionsInRange(parts: Array<{
+  partition: string;
+  bytes: number;
+}>, start: string, end: string): string[];
+/**
+ * Run a full-history aggregation in byte-bounded windows and concat the rows.
+ * Each window's SQL MUST date-filter to `[w.start, w.end]` (see `sqlFor`) so a
+ * tier file spanning a window boundary doesn't double-count calendar dates.
+ */
+declare function runWindowed(opts: {
+  engine: RollupEngine;
+  ctx: TenantCtx;
+  table: import('@gscdump/engine/contracts').TableName;
+  searchType?: SearchType;
+  sqlFor: (w: {
+    start: string;
+    end: string;
+  }) => string;
+}): Promise<Row$1[]>;
 /**
  * Daily totals across the full history. One row per (date, table) with
  * clicks + impressions + position. Powers sparklines and headline totals.
@@ -267,4 +338,4 @@ declare function rebuildDailyFromHourly(opts: RebuildDailyFromHourlyOptions): Pr
   rowsWritten: number;
 }>;
 declare const DEFAULT_ROLLUPS: readonly RollupDef[];
-export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -34,69 +34,76 @@ async function readLatestRollup(bucket, ctx, id, searchType) {
 async function rebuildRollups(opts) {
 	const now = opts.now ?? (() => Date.now());
 	const results = [];
-	const searchType = opts.searchType;
-	if (searchType !== void 0) {
-		for (const def of opts.defs) if (def.sliceOrthogonal === true) throw new Error(`rollup def '${def.id}' is slice-orthogonal; do not pass searchType`);
-	}
 	for (const def of opts.defs) {
 		const builtAt = now();
-		const payload = await def.build({
-			engine: opts.engine,
-			ctx: opts.ctx,
-			dataSource: opts.dataSource,
-			builtAt,
-			...searchType !== void 0 ? { searchType } : {}
-		});
-		if (def.format === "parquet") {
-			if (!def.parquetColumns || def.parquetColumns.length === 0) throw new Error(`rollup '${def.id}' declared format='parquet' without parquetColumns`);
-			const rows = payload ?? [];
-			const parquetBytes = encodeRowsToParquetFlex(rows, {
-				columns: def.parquetColumns,
-				sortKey: def.parquetSortKey
+		const defSearchType = def.sliceOrthogonal === true ? void 0 : opts.searchType;
+		try {
+			const payload = await def.build({
+				engine: opts.engine,
+				ctx: opts.ctx,
+				dataSource: opts.dataSource,
+				builtAt,
+				...defSearchType !== void 0 ? { searchType: defSearchType } : {}
 			});
-			const parquetKey = rollupParquetKey(opts.ctx, def.id, builtAt, searchType);
-			await opts.dataSource.write(parquetKey, parquetBytes);
-			const pointer = {
-				parquetKey,
-				rowCount: rows.length
-			};
+			if (def.format === "parquet") {
+				if (!def.parquetColumns || def.parquetColumns.length === 0) throw new Error(`rollup '${def.id}' declared format='parquet' without parquetColumns`);
+				const rows = payload ?? [];
+				const parquetBytes = encodeRowsToParquetFlex(rows, {
+					columns: def.parquetColumns,
+					sortKey: def.parquetSortKey
+				});
+				const parquetKey = rollupParquetKey(opts.ctx, def.id, builtAt, defSearchType);
+				await opts.dataSource.write(parquetKey, parquetBytes);
+				const pointer = {
+					parquetKey,
+					rowCount: rows.length
+				};
+				const envelope = {
+					version: 1,
+					id: def.id,
+					builtAt,
+					windowDays: def.windowDays,
+					payload: pointer
+				};
+				const envelopeBytes = new TextEncoder().encode(JSON.stringify(envelope));
+				const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
+				await opts.dataSource.write(key, envelopeBytes);
+				results.push({
+					id: def.id,
+					objectKey: key,
+					parquetKey,
+					bytes: envelopeBytes.byteLength,
+					parquetBytes: parquetBytes.byteLength,
+					builtAt
+				});
+				continue;
+			}
 			const envelope = {
 				version: 1,
 				id: def.id,
 				builtAt,
 				windowDays: def.windowDays,
-				payload: pointer
+				payload
 			};
-			const envelopeBytes = new TextEncoder().encode(JSON.stringify(envelope));
-			const key = rollupKey(opts.ctx, def.id, builtAt, searchType);
-			await opts.dataSource.write(key, envelopeBytes);
+			const json = JSON.stringify(envelope);
+			const bytes = new TextEncoder().encode(json);
+			const key = rollupKey(opts.ctx, def.id, builtAt, defSearchType);
+			await opts.dataSource.write(key, bytes);
 			results.push({
 				id: def.id,
 				objectKey: key,
-				parquetKey,
-				bytes: envelopeBytes.byteLength,
-				parquetBytes: parquetBytes.byteLength,
+				bytes: bytes.byteLength,
 				builtAt
 			});
-			continue;
+		} catch (err) {
+			results.push({
+				id: def.id,
+				objectKey: "",
+				bytes: 0,
+				builtAt,
+				error: err instanceof Error ? err.stack || err.message : String(err)
+			});
 		}
-		const envelope = {
-			version: 1,
-			id: def.id,
-			builtAt,
-			windowDays: def.windowDays,
-			payload
-		};
-		const json = JSON.stringify(envelope);
-		const bytes = new TextEncoder().encode(json);
-		const key = rollupKey(opts.ctx, def.id, builtAt, searchType);
-		await opts.dataSource.write(key, bytes);
-		results.push({
-			id: def.id,
-			objectKey: key,
-			bytes: bytes.byteLength,
-			builtAt
-		});
 	}
 	return results;
 }
@@ -104,42 +111,183 @@ function utcDateMinusDays(at, days) {
 	const d = new Date(at - days * MS_PER_DAY);
 	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
 }
+const WINDOW_BYTE_BUDGET = 14 * 1024 * 1024;
+const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
+const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
+const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
+const QUARTER_RE = /^quarterly\/(\d{4})-Q([1-4])$/;
+function isoDate(ms) {
+	const d = new Date(ms);
+	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
+}
+function partitionDaySpan(partition) {
+	const day = DAY_RE.exec(partition);
+	if (day) {
+		const ms = Date.UTC(Number(day[1]), Number(day[2]) - 1, Number(day[3]));
+		return {
+			startMs: ms,
+			endMs: ms
+		};
+	}
+	const week = WEEK_RE.exec(partition);
+	if (week) {
+		const ms = Date.UTC(Number(week[1]), Number(week[2]) - 1, Number(week[3]));
+		return {
+			startMs: ms,
+			endMs: ms + 6 * MS_PER_DAY
+		};
+	}
+	const month = MONTH_RE.exec(partition);
+	if (month) {
+		const y = Number(month[1]);
+		const m = Number(month[2]) - 1;
+		return {
+			startMs: Date.UTC(y, m, 1),
+			endMs: Date.UTC(y, m + 1, 1) - MS_PER_DAY
+		};
+	}
+	const quarter = QUARTER_RE.exec(partition);
+	if (quarter) {
+		const y = Number(quarter[1]);
+		const startMonth = (Number(quarter[2]) - 1) * 3;
+		return {
+			startMs: Date.UTC(y, startMonth, 1),
+			endMs: Date.UTC(y, startMonth + 3, 1) - MS_PER_DAY
+		};
+	}
+	return null;
+}
+function clamp(n, lo, hi) {
+	return Math.max(lo, Math.min(hi, n));
+}
+function planRollupWindows(parts, clampRange) {
+	const clampStartMs = clampRange ? Date.parse(`${clampRange.start}T00:00:00Z`) : void 0;
+	const clampEndMs = clampRange ? Date.parse(`${clampRange.end}T00:00:00Z`) : void 0;
+	const spans = [];
+	for (const p of parts) {
+		const span = partitionDaySpan(p.partition);
+		if (!span) continue;
+		if (clampStartMs !== void 0 && clampEndMs !== void 0) {
+			if (span.endMs < clampStartMs || span.startMs > clampEndMs) continue;
+		}
+		spans.push({
+			partition: p.partition,
+			bytes: p.bytes,
+			startMs: span.startMs,
+			endMs: span.endMs
+		});
+	}
+	if (spans.length === 0) return [];
+	let rangeStartMs = Math.min(...spans.map((s) => s.startMs));
+	let rangeEndMs = Math.max(...spans.map((s) => s.endMs));
+	if (clampStartMs !== void 0) rangeStartMs = Math.max(rangeStartMs, clampStartMs);
+	if (clampEndMs !== void 0) rangeEndMs = Math.min(rangeEndMs, clampEndMs);
+	const totalBytes = spans.reduce((a, s) => a + s.bytes, 0);
+	const spanDays = Math.floor((rangeEndMs - rangeStartMs) / MS_PER_DAY) + 1;
+	const bytesPerDay = Math.max(1, totalBytes / spanDays);
+	const windowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
+	const windows = [];
+	let cursorMs = rangeStartMs;
+	while (cursorMs <= rangeEndMs) {
+		const windowEndMs = Math.min(cursorMs + (windowDays - 1) * MS_PER_DAY, rangeEndMs);
+		const partitions = spans.filter((s) => s.endMs >= cursorMs && s.startMs <= windowEndMs).map((s) => s.partition);
+		if (partitions.length > 0) windows.push({
+			start: isoDate(cursorMs),
+			end: isoDate(windowEndMs),
+			partitions
+		});
+		cursorMs = windowEndMs + MS_PER_DAY;
+	}
+	return windows;
+}
+function partitionsInRange(parts, start, end) {
+	const startMs = Date.parse(`${start}T00:00:00Z`);
+	const endMs = Date.parse(`${end}T00:00:00Z`);
+	const out = [];
+	for (const p of parts) {
+		const span = partitionDaySpan(p.partition);
+		if (!span) continue;
+		if (span.endMs >= startMs && span.startMs <= endMs) out.push(p.partition);
+	}
+	return out;
+}
+async function runWindowed(opts) {
+	const windows = planRollupWindows(await opts.engine.listPartitions({
+		ctx: opts.ctx,
+		table: opts.table,
+		...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+	}));
+	const rows = [];
+	for (const w of windows) {
+		const result = await opts.engine.runSQL({
+			ctx: opts.ctx,
+			table: opts.table,
+			fileSets: { FILES: {
+				table: opts.table,
+				partitions: w.partitions
+			} },
+			sql: opts.sqlFor(w),
+			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+		});
+		rows.push(...result.rows);
+	}
+	return rows;
+}
 const dailyTotalsRollup = {
 	id: "daily_totals",
 	windowDays: null,
 	async build({ engine, ctx, searchType }) {
-		const pages = await engine.runSQL({
+		const pageRows = await runWindowed({
+			engine,
 			ctx,
 			table: "pages",
-			fileSets: { FILES: { table: "pages" } },
-			sql: `
+			...searchType !== void 0 ? { searchType } : {},
+			sqlFor: (w) => `
         SELECT
           date,
           SUM(clicks)::BIGINT AS clicks,
           SUM(impressions)::BIGINT AS impressions,
           SUM(sum_position)::DOUBLE AS sum_position
         FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
         GROUP BY date
         ORDER BY date
-      `,
-			...searchType !== void 0 ? { searchType } : {}
+      `
 		});
-		const keywords = await engine.runSQL({
+		const keywordRows = await runWindowed({
+			engine,
 			ctx,
 			table: "keywords",
-			fileSets: { FILES: { table: "keywords" } },
-			sql: `
+			...searchType !== void 0 ? { searchType } : {},
+			sqlFor: (w) => `
         SELECT
           date,
           SUM(impressions)::BIGINT AS impressions
         FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
         GROUP BY date
-      `,
-			...searchType !== void 0 ? { searchType } : {}
+      `
 		});
+		const pagesByDate = /* @__PURE__ */ new Map();
+		for (const r of pageRows) {
+			const date = String(r.date);
+			const cur = pagesByDate.get(date) ?? {
+				date,
+				clicks: BigInt(0),
+				impressions: BigInt(0),
+				sum_position: 0
+			};
+			cur.clicks += BigInt(r.clicks);
+			cur.impressions += BigInt(r.impressions);
+			cur.sum_position += Number(r.sum_position);
+			pagesByDate.set(date, cur);
+		}
 		const keywordImpressionsByDate = /* @__PURE__ */ new Map();
-		for (const r of keywords.rows) keywordImpressionsByDate.set(String(r.date), BigInt(r.impressions));
-		return pages.rows.map((r) => {
+		for (const r of keywordRows) {
+			const date = String(r.date);
+			keywordImpressionsByDate.set(date, (keywordImpressionsByDate.get(date) ?? BigInt(0)) + BigInt(r.impressions));
+		}
+		return Array.from(pagesByDate.values()).sort((a, b) => a.date < b.date ? -1 : 1).map((r) => {
 			const totalImpressions = BigInt(r.impressions);
 			const queryImpressions = keywordImpressionsByDate.get(String(r.date)) ?? BigInt(0);
 			const anonymized = totalImpressions === BigInt(0) ? 0 : 1 - Number(queryImpressions) / Number(totalImpressions);
@@ -157,27 +305,38 @@ const weeklyTotalsRollup = {
 	id: "weekly_totals",
 	windowDays: null,
 	async build({ engine, ctx, searchType }) {
-		return (await engine.runSQL({
+		const rows = await runWindowed({
+			engine,
 			ctx,
 			table: "pages",
-			fileSets: { FILES: { table: "pages" } },
 			...searchType !== void 0 ? { searchType } : {},
-			sql: `
+			sqlFor: (w) => `
         SELECT
           strftime(date_trunc('week', date::DATE), '%Y-%m-%d') AS week,
           SUM(clicks)::BIGINT AS clicks,
           SUM(impressions)::BIGINT AS impressions,
           SUM(sum_position)::DOUBLE AS sum_position
         FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
         GROUP BY 1
         ORDER BY 1
       `
-		})).rows.map((r) => ({
-			week: r.week,
-			clicks: Number(r.clicks),
-			impressions: Number(r.impressions),
-			sum_position: Number(r.sum_position)
-		}));
+		});
+		const byWeek = /* @__PURE__ */ new Map();
+		for (const r of rows) {
+			const week = String(r.week);
+			const cur = byWeek.get(week) ?? {
+				week,
+				clicks: 0,
+				impressions: 0,
+				sum_position: 0
+			};
+			cur.clicks += Number(r.clicks);
+			cur.impressions += Number(r.impressions);
+			cur.sum_position += Number(r.sum_position);
+			byWeek.set(week, cur);
+		}
+		return Array.from(byWeek.values()).sort((a, b) => a.week < b.week ? -1 : 1);
 	}
 };
 const topPages28dRollup = {
@@ -185,10 +344,19 @@ const topPages28dRollup = {
 	windowDays: 28,
 	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "pages",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(builtAt, 0));
+		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
 			table: "pages",
-			fileSets: { FILES: { table: "pages" } },
+			fileSets: { FILES: {
+				table: "pages",
+				partitions
+			} },
 			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
@@ -215,10 +383,19 @@ const topCountries28dRollup = {
 	windowDays: 28,
 	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "countries",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(builtAt, 0));
+		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
 			table: "countries",
-			fileSets: { FILES: { table: "countries" } },
+			fileSets: { FILES: {
+				table: "countries",
+				partitions
+			} },
 			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
@@ -245,10 +422,19 @@ const topKeywords28dRollup = {
 	windowDays: 28,
 	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "keywords",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(builtAt, 0));
+		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
 			table: "keywords",
-			fileSets: { FILES: { table: "keywords" } },
+			fileSets: { FILES: {
+				table: "keywords",
+				partitions
+			} },
 			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
@@ -299,10 +485,19 @@ const topKeywords28dParquetRollup = {
 	parquetSortKey: ["clicks"],
 	async build({ engine, ctx, builtAt, searchType }) {
 		const cutoff = utcDateMinusDays(builtAt, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "keywords",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(builtAt, 0));
+		if (partitions.length === 0) return [];
 		return (await engine.runSQL({
 			ctx,
 			table: "keywords",
-			fileSets: { FILES: { table: "keywords" } },
+			fileSets: { FILES: {
+				table: "keywords",
+				partitions
+			} },
 			...searchType !== void 0 ? { searchType } : {},
 			sql: `
         SELECT
@@ -423,11 +618,19 @@ const indexPercentRollup = {
 			days: []
 		};
 		const cutoff = utcDateMinusDays(builtAt, 90);
+		const pagesPartitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "pages",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(builtAt, 0));
 		const numerator = await engine.runSQL({
 			ctx,
 			table: "pages",
 			fileSets: {
-				PAGES: { table: "pages" },
+				PAGES: {
+					table: "pages",
+					partitions: pagesPartitions
+				},
 				URLS: {
 					table: "pages",
 					keys: [urlsKey]
@@ -626,4 +829,4 @@ const DEFAULT_ROLLUPS = [
 	sitemapHealthRollup,
 	sitemapChanges28dRollup
 ];
-export { DEFAULT_ROLLUPS, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { DEFAULT_ROLLUPS, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.19.3",
+  "version": "0.19.6",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -169,8 +169,8 @@
   "dependencies": {
     "drizzle-orm": "^0.45.2",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.19.3",
-    "gscdump": "0.19.3"
+    "gscdump": "0.19.6",
+    "@gscdump/contracts": "0.19.6"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",