@gscdump/engine 0.33.3 → 0.33.5

package/dist/rollups.d.mts

@@ -377,14 +377,22 @@ declare const queryCanonicalDailyRollup: RollupDef;
  * Resumable, cross-invocation build of `query_canonical_daily` for a high-
  * cardinality site whose full windowed build exceeds one job reservation (300s).
  *
- * Each call builds the day-capped windows from `windowOffset` until `deadlineMs`,
- * writes that batch's rows to a STAGING partial parquet (windows are disjoint by
- * date, so partials never overlap), and returns `{ done:false, nextWindowOffset }`
- * for the caller to re-enqueue. When the last window is built it MERGES every
- * staging partial into the canonical `query_canonical_daily` parquet + envelope
- * (the read path is unchanged — still one rollup file) and deletes the staging,
+ * Each call builds from `(windowOffset, pageOffset)` until `deadlineMs`, writes
+ * that batch's rows to a PART parquet, and returns `{ done:false, nextWindowOffset,
+ * nextPageOffset }` for the caller to re-enqueue. When the last window is fully
+ * paged it publishes a multi-file envelope listing every part (parts are disjoint
+ * by `(query_canonical, date)`, so the read path just unions them — no merge),
  * returning `{ done:true }`. `builtAt` MUST be stable across the continuation chain
- * (it versions both the staging keys and the final rollup key).
+ * (it versions both the part keys and the final rollup key).
+ *
+ * INTRA-WINDOW resumability: the deadline is checked between OUTPUT PAGES, not just
+ * between windows. A single high-cardinality window's paged aggregation can exceed
+ * one 300s reservation on its own; checking only between windows let that window
+ * run unbounded and stale-reservation-loop forever (huuto.net/comparaja.pt never
+ * completed window 0). `pageOffset` lets a continuation resume the SAME window at
+ * the next page, so no single invocation runs past the deadline by more than one
+ * page. Parts are keyed by `(windowOffset, pageOffset)` so a mid-window pause and
+ * its continuation write disjoint, non-colliding files.
  */
 declare function rebuildCanonicalDailyResumable(opts: {
   engine: RollupEngine;
@@ -392,11 +400,14 @@ declare function rebuildCanonicalDailyResumable(opts: {
   dataSource: DataSource;
   searchType?: SearchType;
   builtAt: number;
-  windowOffset: number;
+  windowOffset: number; /** Resume the `windowOffset` window at this output-page offset (0 = window start). */
+  pageOffset?: number; /** Output rows per page (default `ROLLUP_PAGE_ROWS_DAILY`). Injectable for tests. */
+  pageRows?: number;
   deadlineMs: number;
 }): Promise<{
   done: boolean;
   nextWindowOffset: number;
+  nextPageOffset: number;
   windowsTotal: number;
   windowsBuilt: number;
   rowsWritten: number;

package/dist/rollups.mjs

@@ -764,6 +764,8 @@ function mapDailyRow(r) {
 async function rebuildCanonicalDailyResumable(opts) {
 	const { engine, ctx, dataSource, searchType, builtAt, windowOffset, deadlineMs } = opts;
 	const sType = searchType !== void 0 ? { searchType } : {};
+	const startPageOffset = opts.pageOffset ?? 0;
+	const pageRows = opts.pageRows ?? 7e4;
 	const windows = planRollupWindows((await engine.listPartitions({
 		ctx,
 		table: "queries",
@@ -783,54 +785,81 @@ async function rebuildCanonicalDailyResumable(opts) {
 	const sqlFor = dailyWindowSqlFor(useDim, canonExpr);
 	const cols = queryCanonicalDailyRollup.parquetColumns;
 	const sortKey = queryCanonicalDailyRollup.parquetSortKey;
-	const batchRows = [];
 	let i = windowOffset;
-	for (; i < windowsTotal; i++) {
+	let page = startPageOffset;
+	let nextWindowOffset = windowsTotal;
+	let nextPageOffset = 0;
+	let rowsWritten = 0;
+	const flushPage = async (windowIdx, pageOffset, rows) => {
+		if (rows.length === 0) return;
+		const key = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w${windowIdx}_p${pageOffset}`, builtAt, searchType);
+		await dataSource.write(key, encodeRowsToParquetFlex(rows, {
+			columns: cols,
+			sortKey
+		}));
+		rowsWritten += rows.length;
+	};
+	windowLoop: for (; i < windowsTotal; i++) {
 		const w = windows[i];
-		const winRows = await runPagedQuery({
-			engine,
-			ctx,
-			table: "queries",
-			...sType,
-			fileSets: {
-				FILES: {
-					table: "queries",
-					partitions: w.partitions
+		const coreSql = sqlFor(w);
+		for (;;) {
+			const result = await engine.runSQL({
+				ctx,
+				table: "queries",
+				...sType,
+				fileSets: {
+					FILES: {
+						table: "queries",
+						partitions: w.partitions
+					},
+					...extraFileSets
 				},
-				...extraFileSets
-			},
-			coreSql: sqlFor(w),
-			orderBy: "date, query_canonical",
-			pageRows: ROLLUP_PAGE_ROWS_DAILY
-		});
-		for (const r of winRows) batchRows.push(mapDailyRow(r));
+				sql: `${coreSql}\nORDER BY date, query_canonical\nLIMIT ${pageRows} OFFSET ${page}`
+			});
+			await flushPage(i, page, result.rows.map(mapDailyRow));
+			const windowDone = result.rows.length < pageRows;
+			page += pageRows;
+			if (windowDone) {
+				page = 0;
+				break;
+			}
+			if (Date.now() > deadlineMs) {
+				nextWindowOffset = i;
+				nextPageOffset = page;
+				break windowLoop;
+			}
+		}
 		if (Date.now() > deadlineMs) {
-			i++;
+			nextWindowOffset = i + 1;
+			nextPageOffset = 0;
 			break;
 		}
 	}
-	const nextWindowOffset = i;
-	const partKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w${windowOffset}`, builtAt, searchType);
-	await dataSource.write(partKey, encodeRowsToParquetFlex(batchRows, {
-		columns: cols,
-		sortKey
-	}));
-	if (nextWindowOffset < windowsTotal) return {
+	if (nextWindowOffset < windowsTotal || nextPageOffset > 0) return {
 		done: false,
 		nextWindowOffset,
+		nextPageOffset,
 		windowsTotal,
 		windowsBuilt: nextWindowOffset - windowOffset,
-		rowsWritten: batchRows.length
+		rowsWritten
 	};
 	const partPrefixDir = rollupParquetKey(ctx, CANONICAL_DAILY_PART_STEM, builtAt, searchType).replace(/[^/]*$/, "");
 	const partKeys = (await dataSource.list(partPrefixDir)).filter((k) => k.includes(`${CANONICAL_DAILY_PART_STEM}__w`) && k.endsWith(`__v${builtAt}.parquet`)).sort();
+	if (partKeys.length === 0) {
+		const emptyKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w0_p0`, builtAt, searchType);
+		await dataSource.write(emptyKey, encodeRowsToParquetFlex([], {
+			columns: cols,
+			sortKey
+		}));
+		partKeys.push(emptyKey);
+	}
 	const envelope = {
 		version: 1,
 		id: CANONICAL_DAILY_ROLLUP_FINAL_ID,
 		builtAt,
 		windowDays: queryCanonicalDailyRollup.windowDays,
 		payload: {
-			parquetKey: partKeys[0] ?? partKey,
+			parquetKey: partKeys[0],
 			parquetKeys: partKeys,
 			rowCount: 0
 		}
@@ -839,9 +868,10 @@ async function rebuildCanonicalDailyResumable(opts) {
 	return {
 		done: true,
 		nextWindowOffset,
+		nextPageOffset,
 		windowsTotal,
 		windowsBuilt: nextWindowOffset - windowOffset,
-		rowsWritten: batchRows.length
+		rowsWritten
 	};
 }
 const indexingMetadataRollup = {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.33.3",
+  "version": "0.33.5",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.33.3",
-    "gscdump": "0.33.3"
+    "gscdump": "0.33.5",
+    "@gscdump/contracts": "0.33.5"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",