@gscdump/engine 0.31.5 → 0.31.6

package/dist/rollups.mjs

@@ -603,44 +603,61 @@ const queryCanonicalVariantsRollup = {
 		});
 		if (parts.length === 0) return [];
 		const partitions = parts.map((p) => p.partition);
-		return (await runPagedQuery({
-			engine,
-			ctx,
-			table: "queries",
-			...searchType !== void 0 ? { searchType } : {},
-			fileSets: { FILES: {
+		const byCanonical = /* @__PURE__ */ new Map();
+		let cursor = null;
+		for (;;) {
+			const after = cursor === null ? "" : `AND query > '${cursor.replace(/'/g, "''")}'`;
+			const { rows } = await engine.runSQL({
+				ctx,
 				table: "queries",
-				partitions
-			} },
-			orderBy: "joinKey",
-			pageRows: ROLLUP_PAGE_ROWS_WIDE,
-			coreSql: `
-        WITH per_variant AS (
+				...searchType !== void 0 ? { searchType } : {},
+				fileSets: { FILES: {
+					table: "queries",
+					partitions
+				} },
+				sql: `
           SELECT
             COALESCE(NULLIF(query_canonical, ''), query) AS joinKey,
             query AS query,
             SUM(clicks) AS clicks,
             SUM(impressions) AS impressions,
-            SUM(sum_position) AS sum_pos,
-            ROW_NUMBER() OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query) ORDER BY SUM(clicks) DESC) AS rn,
-            COUNT(*) OVER (PARTITION BY COALESCE(NULLIF(query_canonical, ''), query)) AS variantCount
+            SUM(sum_position) AS sum_pos
           FROM read_parquet({{FILES}}, union_by_name = true)
+          WHERE query IS NOT NULL ${after}
           GROUP BY COALESCE(NULLIF(query_canonical, ''), query), query
-        )
-        SELECT
-          joinKey,
-          MAX(variantCount)::BIGINT AS variantCount,
-          MAX(CASE WHEN rn = 1 THEN query END) AS canonicalName,
-          GROUP_CONCAT(CASE WHEN rn <= 10 THEN query || ':::' || clicks || ':::' || impressions || ':::' || CAST(ROUND(CAST(sum_pos AS REAL) / NULLIF(impressions, 0) + 1, 1) AS TEXT) END, '||') AS variants
-        FROM per_variant
-        GROUP BY joinKey
-      `
-		})).map((r) => ({
-			joinKey: String(r.joinKey),
-			variantCount: BigInt(r.variantCount),
-			canonicalName: r.canonicalName == null ? null : String(r.canonicalName),
-			variants: r.variants == null ? null : String(r.variants)
-		}));
+          ORDER BY query
+          LIMIT ${ROLLUP_PAGE_ROWS_WIDE}
+        `
+			});
+			for (const r of rows) {
+				const joinKey = String(r.joinKey);
+				const list = byCanonical.get(joinKey);
+				const v = {
+					query: String(r.query),
+					clicks: Number(r.clicks),
+					impressions: Number(r.impressions),
+					sumPos: Number(r.sum_pos)
+				};
+				if (list) list.push(v);
+				else byCanonical.set(joinKey, [v]);
+			}
+			if (rows.length < 2e4) break;
+			cursor = String(rows[rows.length - 1].query);
+		}
+		const out = [];
+		for (const [joinKey, variants] of byCanonical) {
+			variants.sort((a, b) => b.clicks - a.clicks || a.query.localeCompare(b.query));
+			const canonicalName = variants[0]?.query ?? null;
+			const top = variants.slice(0, 10).filter((v) => v.impressions > 0);
+			const variantsStr = top.length === 0 ? null : top.map((v) => `${v.query}:::${v.clicks}:::${v.impressions}:::${(v.sumPos / v.impressions + 1).toFixed(1)}`).join("||");
+			out.push({
+				joinKey,
+				variantCount: BigInt(variants.length),
+				canonicalName,
+				variants: variantsStr
+			});
+		}
+		return out;
 	}
 };
 const queryCanonicalDailyRollup = {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.31.5",
+  "version": "0.31.6",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.31.5",
-    "@gscdump/contracts": "0.31.5"
+    "gscdump": "0.31.6",
+    "@gscdump/contracts": "0.31.6"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",