@gscdump/engine 0.33.7 → 0.33.9

package/dist/rollups.d.mts

@@ -15,7 +15,7 @@ interface RollupEngine {
   runSQL: (opts: {
     ctx: TenantCtx;
     fileSets: Record<string, FileSetRef>;
-    table?: import('@gscdump/engine/contracts').TableName;
+    table?: TableName$1;
     sql: string;
     params?: unknown[];
     /**
@@ -26,7 +26,7 @@ interface RollupEngine {
      */
     searchType?: SearchType;
   }) => Promise<{
-    rows: import('@gscdump/engine/contracts').Row[];
+    rows: Row$1[];
   }>;
   /**
    * Read the live manifest for a (tenant, table[, searchType]) cohort —
@@ -37,7 +37,7 @@ interface RollupEngine {
    */
   listPartitions: (opts: {
     ctx: TenantCtx;
-    table: import('@gscdump/engine/contracts').TableName;
+    table: TableName$1;
     searchType?: SearchType;
   }) => Promise<Array<{
     partition: string;
@@ -385,14 +385,13 @@ declare const queryCanonicalDailyRollup: RollupDef;
  * returning `{ done:true }`. `builtAt` MUST be stable across the continuation chain
  * (it versions both the part keys and the final rollup key).
  *
- * INTRA-WINDOW resumability: the deadline is checked between OUTPUT PAGES, not just
- * between windows. A single high-cardinality window's paged aggregation can exceed
- * one 300s reservation on its own; checking only between windows let that window
- * run unbounded and stale-reservation-loop forever (huuto.net/comparaja.pt never
- * completed window 0). `pageOffset` lets a continuation resume the SAME window at
- * the next page, so no single invocation runs past the deadline by more than one
- * page. Parts are keyed by `(windowOffset, pageOffset)` so a mid-window pause and
- * its continuation write disjoint, non-colliding files.
+ * INTRA-WINDOW resumability: the deadline is checked between raw-query hash shards,
+ * not just between date windows. A single high-cardinality day can spend a full
+ * reservation inside one grouped/sorted aggregate before the deadline check gets
+ * control back. `pageOffset` is the next shard index for the current window, so a
+ * continuation resumes the SAME day at the next shard. Parts are keyed by
+ * `(windowOffset, pageOffset)`; multiple parts may contain the same canonical/date
+ * from different raw-query shards, and rollup reads sum over the union.
  */
 declare function rebuildCanonicalDailyResumable(opts: {
   engine: RollupEngine;
@@ -400,9 +399,11 @@ declare function rebuildCanonicalDailyResumable(opts: {
   dataSource: DataSource;
   searchType?: SearchType;
   builtAt: number;
-  windowOffset: number; /** Resume the `windowOffset` window at this output-page offset (0 = window start). */
+  windowOffset: number; /** Resume the `windowOffset` window at this shard offset (0 = window start). */
   pageOffset?: number; /** Output rows per page (default `ROLLUP_PAGE_ROWS_DAILY`). Injectable for tests. */
-  pageRows?: number;
+  pageRows?: number; /** Cap each input window's day span (default `DAILY_MAX_WINDOW_DAYS`). */
+  maxWindowDays?: number; /** Split each date window by raw-query hash before grouping (1 = no sharding). */
+  shardCount?: number;
   deadlineMs: number;
 }): Promise<{
   done: boolean;

package/dist/rollups.mjs

@@ -728,8 +728,11 @@ const queryCanonicalDailyRollup = {
 		})).map(mapDailyRow);
 	}
 };
+function canonicalDailyShardPredicate(queryExpr, shardIndex, shardCount) {
+	return shardCount <= 1 ? "" : ` AND (hash(${queryExpr}) % ${shardCount}) = ${shardIndex}`;
+}
 function dailyWindowSqlFor(useDim, canonExpr) {
-	return useDim ? (w) => `
+	return useDim ? (w, shard) => `
         SELECT
           ${canonExpr} AS query_canonical,
           CAST(q.date AS VARCHAR) AS date,
@@ -738,18 +741,18 @@ function dailyWindowSqlFor(useDim, canonExpr) {
           SUM(q.sum_position)::DOUBLE AS sum_position
         FROM read_parquet({{FILES}}, union_by_name = true) q
         LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query
-        WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'
+        WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'${canonicalDailyShardPredicate("q.query", shard?.index ?? 0, shard?.count ?? 1)}
         GROUP BY ${canonExpr}, q.date
-      ` : (w) => `
+      ` : (w, shard) => `
         SELECT
           ${canonExpr} AS query_canonical,
-          CAST(date AS VARCHAR) AS date,
-          SUM(clicks)::BIGINT AS clicks,
-          SUM(impressions)::BIGINT AS impressions,
-          SUM(sum_position)::DOUBLE AS sum_position
-        FROM read_parquet({{FILES}}, union_by_name = true)
-        WHERE date >= '${w.start}' AND date <= '${w.end}'
-        GROUP BY ${canonExpr}, date
+          CAST(q.date AS VARCHAR) AS date,
+          SUM(q.clicks)::BIGINT AS clicks,
+          SUM(q.impressions)::BIGINT AS impressions,
+          SUM(q.sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true) q
+        WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'${canonicalDailyShardPredicate("q.query", shard?.index ?? 0, shard?.count ?? 1)}
+        GROUP BY ${canonExpr}, q.date
       `;
 }
 function mapDailyRow(r) {
@@ -766,6 +769,8 @@ async function rebuildCanonicalDailyResumable(opts) {
 	const sType = searchType !== void 0 ? { searchType } : {};
 	const startPageOffset = opts.pageOffset ?? 0;
 	const pageRows = opts.pageRows ?? 7e4;
+	const maxWindowDays = opts.maxWindowDays ?? 7;
+	const shardCount = Math.max(1, Math.floor(opts.shardCount ?? 1));
 	const windows = planRollupWindows((await engine.listPartitions({
 		ctx,
 		table: "queries",
@@ -773,7 +778,7 @@ async function rebuildCanonicalDailyResumable(opts) {
 	})).map((p) => ({
 		partition: p.partition,
 		bytes: p.bytes
-	})), void 0, 7);
+	})), void 0, maxWindowDays);
 	const windowsTotal = windows.length;
 	const dimStore = createQueryDimStore({ dataSource });
 	const useDim = await dimStore.loadMeta(ctx) !== null;
@@ -802,8 +807,11 @@ async function rebuildCanonicalDailyResumable(opts) {
 	};
 	for (; i < windowsTotal; i++) {
 		const w = windows[i];
-		const coreSql = sqlFor(w);
-		for (;;) {
+		for (; page < shardCount;) {
+			const coreSql = sqlFor(w, {
+				index: page,
+				count: shardCount
+			});
 			const result = await engine.runSQL({
 				ctx,
 				table: "queries",
@@ -815,15 +823,11 @@ async function rebuildCanonicalDailyResumable(opts) {
 					},
 					...extraFileSets
 				},
-				sql: `${coreSql}\nORDER BY date, query_canonical\nLIMIT ${pageRows} OFFSET ${page}`
+				sql: `${coreSql}\nORDER BY date, query_canonical\nLIMIT ${pageRows}`
 			});
+			if (result.rows.length >= pageRows) throw new Error(`query_canonical_daily shard overflow: window=${i} shard=${page}/${shardCount} returned >= ${pageRows} rows; increase shardCount or pageRows`);
 			await flushPage(i, page, result.rows.map(mapDailyRow));
-			const windowDone = result.rows.length < pageRows;
-			page += pageRows;
-			if (windowDone) {
-				page = 0;
-				break;
-			}
+			page += 1;
 			if (Date.now() > deadlineMs) {
 				nextWindowOffset = i;
 				nextPageOffset = page;
@@ -832,6 +836,7 @@ async function rebuildCanonicalDailyResumable(opts) {
 			}
 		}
 		if (pausedMidWindow) break;
+		page = 0;
 		if (Date.now() > deadlineMs) {
 			nextWindowOffset = i + 1;
 			nextPageOffset = 0;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.33.7",
+  "version": "0.33.9",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.33.7",
-    "gscdump": "0.33.7"
+    "@gscdump/contracts": "0.33.9",
+    "gscdump": "0.33.9"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",