@gscdump/engine 0.31.5 → 0.31.7

package/dist/rollups.d.mts

@@ -220,6 +220,8 @@ declare const WINDOW_BYTE_BUDGET: number;
  */
 declare const ROLLUP_PAGE_ROWS = 50000;
 declare const ROLLUP_PAGE_ROWS_WIDE = 20000;
+declare const ROLLUP_PAGE_ROWS_DAILY = 90000;
+declare const DAILY_MAX_WINDOW_DAYS = 7;
 /**
  * UTC day-aligned [startMs, endMs] span a partition covers. Returns null for
  * `hourly/` partitions and anything unrecognised — those are excluded from
@@ -240,7 +242,7 @@ declare function planRollupWindows(parts: Array<{
 }>, clampRange?: {
   start: string;
   end: string;
-}): Array<{
+}, maxWindowDays?: number): Array<{
   start: string;
   end: string;
   partitions: string[];
@@ -280,7 +282,8 @@ declare function runWindowed(opts: {
   paginate?: {
     orderBy: string;
     pageRows: number;
-  };
+  }; /** Cap each window's day span (output-cardinality bound). See `planRollupWindows`. */
+  maxWindowDays?: number;
 }): Promise<Row$1[]>;
 /**
  * Daily totals across the full history. One row per (date, table) with
@@ -446,4 +449,4 @@ declare const DEFAULT_ROLLUPS: readonly RollupDef[];
  * (CLI: `gscdump rollups --with-canonical`).
  */
 declare const CANONICAL_ROLLUPS: readonly RollupDef[];
-export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -129,6 +129,8 @@ function utcDateMinusDays(at, days) {
 const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
 const ROLLUP_PAGE_ROWS = 5e4;
 const ROLLUP_PAGE_ROWS_WIDE = 2e4;
+const ROLLUP_PAGE_ROWS_DAILY = 9e4;
+const DAILY_MAX_WINDOW_DAYS = 7;
 const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
 const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
 const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
@@ -177,7 +179,7 @@ function partitionDaySpan(partition) {
 function clamp(n, lo, hi) {
 	return Math.max(lo, Math.min(hi, n));
 }
-function planRollupWindows(parts, clampRange) {
+function planRollupWindows(parts, clampRange, maxWindowDays) {
 	const clampStartMs = clampRange ? Date.parse(`${clampRange.start}T00:00:00Z`) : void 0;
 	const clampEndMs = clampRange ? Date.parse(`${clampRange.end}T00:00:00Z`) : void 0;
 	const spans = [];
@@ -202,7 +204,8 @@ function planRollupWindows(parts, clampRange) {
 	const totalBytes = spans.reduce((a, s) => a + s.bytes, 0);
 	const spanDays = Math.floor((rangeEndMs - rangeStartMs) / MS_PER_DAY) + 1;
 	const bytesPerDay = Math.max(1, totalBytes / spanDays);
-	const windowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
+	const byteWindowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
+	const windowDays = maxWindowDays != null ? Math.max(1, Math.min(byteWindowDays, maxWindowDays)) : byteWindowDays;
 	const windows = [];
 	let cursorMs = rangeStartMs;
 	while (cursorMs <= rangeEndMs) {
@@ -248,7 +251,7 @@ async function runWindowed(opts) {
 		ctx: opts.ctx,
 		table: opts.table,
 		...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
-	}));
+	}), void 0, opts.maxWindowDays);
 	const rows = [];
 	for (const w of windows) {
 		const fileSets = {
@@ -603,44 +606,61 @@ const queryCanonicalVariantsRollup = {
 		});
 		if (parts.length === 0) return [];
 		const partitions = parts.map((p) => p.partition);
-		return (await runPagedQuery({
-			engine,
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {},
-			fileSets: { FILES: {
+		const byCanonical = /* @__PURE__ */ new Map();
+		let cursor = null;
+		for (;;) {
+			const after = cursor === null ? "" : `AND query > '${cursor.replace(/'/g, "''")}'`;
+			const { rows } = await engine.runSQL({
+				ctx,
 				table: "queries",
-				partitions
-			} },
-			orderBy: "joinKey",
-			pageRows: ROLLUP_PAGE_ROWS_WIDE,
-			coreSql: `
-        WITH per_variant AS (
+				...searchType !== void 0 ? { searchType } : {},
+				fileSets: { FILES: {
+					table: "queries",
+					partitions
+				} },
+				sql: `
           SELECT
             COALESCE(NULLIF(query_canonical, ''), query) AS joinKey,
             query AS query,
             SUM(clicks) AS clicks,
             SUM(impressions) AS impressions,
-            SUM(sum_position) AS sum_pos,
-            ROW_NUMBER() OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query) ORDER BY SUM(clicks) DESC) AS rn,
-            COUNT(*) OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query)) AS variantCount
+            SUM(sum_position) AS sum_pos
           FROM read_parquet({{FILES}}, union_by_name = true)
+          WHERE query IS NOT NULL ${after}
           GROUP BY COALESCE(NULLIF(query_canonical, ''), query), query
-        )
-        SELECT
-          joinKey,
-          MAX(variantCount)::BIGINT AS variantCount,
-          MAX(CASE WHEN rn = 1 THEN query END) AS canonicalName,
-          GROUP_CONCAT(CASE WHEN rn <= 10 THEN query || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') AS variants
-        FROM per_variant
-        GROUP BY joinKey
-      `
-		})).map((r) => ({
-			joinKey: String(r.joinKey),
-			variantCount: BigInt(r.variantCount),
-			canonicalName: r.canonicalName == null ? null : String(r.canonicalName),
-			variants: r.variants == null ? null : String(r.variants)
-		}));
+          ORDER BY query
+          LIMIT ${ROLLUP_PAGE_ROWS_WIDE}
+        `
+			});
+			for (const r of rows) {
+				const joinKey = String(r.joinKey);
+				const list = byCanonical.get(joinKey);
+				const v = {
+					query: String(r.query),
+					clicks: Number(r.clicks),
+					impressions: Number(r.impressions),
+					sumPos: Number(r.sum_pos)
+				};
+				if (list) list.push(v);
+				else byCanonical.set(joinKey, [v]);
+			}
+			if (rows.length < 2e4) break;
+			cursor = String(rows[rows.length - 1].query);
+		}
+		const out = [];
+		for (const [joinKey, variants] of byCanonical) {
+			variants.sort((a, b) => b.clicks - a.clicks || a.query.localeCompare(b.query));
+			const canonicalName = variants[0]?.query ?? null;
+			const top = variants.slice(0, 10).filter((v) => v.impressions > 0);
+			const variantsStr = top.length === 0 ? null : top.map((v) => `${v.query}:::${v.clicks}:::${v.impressions}:::${(v.sumPos / v.impressions + 1).toFixed(1)}`).join("||");
+			out.push({
+				joinKey,
+				variantCount: BigInt(variants.length),
+				canonicalName,
+				variants: variantsStr
+			});
+		}
+		return out;
 	}
 };
 const queryCanonicalDailyRollup = {
@@ -690,8 +710,9 @@ const queryCanonicalDailyRollup = {
 			} } } : {},
 			paginate: {
 				orderBy: "date, query_canonical",
-				pageRows: ROLLUP_PAGE_ROWS
+				pageRows: ROLLUP_PAGE_ROWS_DAILY
 			},
+			maxWindowDays: 7,
 			sqlFor: useDim ? (w) => `
           SELECT
             ${canonExpr} AS query_canonical,
@@ -1035,4 +1056,4 @@ const DEFAULT_ROLLUPS = [
 	sitemapChanges28dRollup
 ];
 const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
-export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.31.5",
+  "version": "0.31.7",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.31.5",
-    "@gscdump/contracts": "0.31.5"
+    "@gscdump/contracts": "0.31.7",
+    "gscdump": "0.31.7"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",