@gscdump/engine 0.33.4 → 0.33.6

package/dist/rollups.mjs

@@ -785,12 +785,22 @@ async function rebuildCanonicalDailyResumable(opts) {
 	const sqlFor = dailyWindowSqlFor(useDim, canonExpr);
 	const cols = queryCanonicalDailyRollup.parquetColumns;
 	const sortKey = queryCanonicalDailyRollup.parquetSortKey;
-	const batchRows = [];
 	let i = windowOffset;
 	let page = startPageOffset;
 	let nextWindowOffset = windowsTotal;
 	let nextPageOffset = 0;
-	windowLoop: for (; i < windowsTotal; i++) {
+	let rowsWritten = 0;
+	let pausedMidWindow = false;
+	const flushPage = async (windowIdx, pageOffset, rows) => {
+		if (rows.length === 0) return;
+		const key = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w${windowIdx}_p${pageOffset}`, builtAt, searchType);
+		await dataSource.write(key, encodeRowsToParquetFlex(rows, {
+			columns: cols,
+			sortKey
+		}));
+		rowsWritten += rows.length;
+	};
+	for (; i < windowsTotal; i++) {
 		const w = windows[i];
 		const coreSql = sqlFor(w);
 		for (;;) {
@@ -807,7 +817,7 @@ async function rebuildCanonicalDailyResumable(opts) {
 				},
 				sql: `${coreSql}\nORDER BY date, query_canonical\nLIMIT ${pageRows} OFFSET ${page}`
 			});
-			for (const r of result.rows) batchRows.push(mapDailyRow(r));
+			await flushPage(i, page, result.rows.map(mapDailyRow));
 			const windowDone = result.rows.length < pageRows;
 			page += pageRows;
 			if (windowDone) {
@@ -817,37 +827,42 @@ async function rebuildCanonicalDailyResumable(opts) {
 			if (Date.now() > deadlineMs) {
 				nextWindowOffset = i;
 				nextPageOffset = page;
-				break windowLoop;
+				pausedMidWindow = true;
+				break;
 			}
 		}
+		if (pausedMidWindow) break;
 		if (Date.now() > deadlineMs) {
 			nextWindowOffset = i + 1;
 			nextPageOffset = 0;
 			break;
 		}
 	}
-	const partKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w${windowOffset}_p${startPageOffset}`, builtAt, searchType);
-	await dataSource.write(partKey, encodeRowsToParquetFlex(batchRows, {
-		columns: cols,
-		sortKey
-	}));
 	if (nextWindowOffset < windowsTotal || nextPageOffset > 0) return {
 		done: false,
 		nextWindowOffset,
 		nextPageOffset,
 		windowsTotal,
 		windowsBuilt: nextWindowOffset - windowOffset,
-		rowsWritten: batchRows.length
+		rowsWritten
 	};
 	const partPrefixDir = rollupParquetKey(ctx, CANONICAL_DAILY_PART_STEM, builtAt, searchType).replace(/[^/]*$/, "");
 	const partKeys = (await dataSource.list(partPrefixDir)).filter((k) => k.includes(`${CANONICAL_DAILY_PART_STEM}__w`) && k.endsWith(`__v${builtAt}.parquet`)).sort();
+	if (partKeys.length === 0) {
+		const emptyKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w0_p0`, builtAt, searchType);
+		await dataSource.write(emptyKey, encodeRowsToParquetFlex([], {
+			columns: cols,
+			sortKey
+		}));
+		partKeys.push(emptyKey);
+	}
 	const envelope = {
 		version: 1,
 		id: CANONICAL_DAILY_ROLLUP_FINAL_ID,
 		builtAt,
 		windowDays: queryCanonicalDailyRollup.windowDays,
 		payload: {
-			parquetKey: partKeys[0] ?? partKey,
+			parquetKey: partKeys[0],
 			parquetKeys: partKeys,
 			rowCount: 0
 		}
@@ -859,7 +874,7 @@ async function rebuildCanonicalDailyResumable(opts) {
 		nextPageOffset,
 		windowsTotal,
 		windowsBuilt: nextWindowOffset - windowOffset,
-		rowsWritten: batchRows.length
+		rowsWritten
 	};
 }
 const indexingMetadataRollup = {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.33.4",
+  "version": "0.33.6",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.33.4",
-    "gscdump": "0.33.4"
+    "@gscdump/contracts": "0.33.6",
+    "gscdump": "0.33.6"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",