@gscdump/engine 0.31.8 → 0.31.10

package/dist/rollups.d.mts

@@ -123,6 +123,15 @@ interface RollupEnvelope<T = unknown> {
 interface ParquetRollupPointer {
   parquetKey: string;
   rowCount: number;
+  /**
+   * MULTI-FILE rollup: when set, the rollup is the UNION of these parquet keys
+   * (disjoint by the grain's partition column, e.g. `date` for the resumable
+   * `query_canonical_daily` build). Readers MUST union all keys; `parquetKey`
+   * stays populated (the first part) for single-file readers. Avoids a JS
+   * merge/re-encode of the whole rollup — the scaling bottleneck for a
+   * cross-invocation resumable build.
+   */
+  parquetKeys?: string[];
 }
 declare function rollupKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string;
 declare function rollupParquetKey(ctx: TenantCtx, id: string, builtAt: number, searchType?: SearchType): string;
@@ -364,6 +373,34 @@ declare const queryCanonicalVariantsRollup: RollupDef;
  * `RunOptimizedQueryOptions.canonicalSource`).
  */
 declare const queryCanonicalDailyRollup: RollupDef;
+/**
+ * Resumable, cross-invocation build of `query_canonical_daily` for a high-
+ * cardinality site whose full windowed build exceeds one job reservation (300s).
+ *
+ * Each call builds the day-capped windows from `windowOffset` until `deadlineMs`,
+ * writes that batch's rows to a STAGING partial parquet (windows are disjoint by
+ * date, so partials never overlap), and returns `{ done:false, nextWindowOffset }`
+ * for the caller to re-enqueue. When the last window is built it MERGES every
+ * staging partial into the canonical `query_canonical_daily` parquet + envelope
+ * (the read path is unchanged — still one rollup file) and deletes the staging,
+ * returning `{ done:true }`. `builtAt` MUST be stable across the continuation chain
+ * (it versions both the staging keys and the final rollup key).
+ */
+declare function rebuildCanonicalDailyResumable(opts: {
+  engine: RollupEngine;
+  ctx: TenantCtx;
+  dataSource: DataSource;
+  searchType?: SearchType;
+  builtAt: number;
+  windowOffset: number;
+  deadlineMs: number;
+}): Promise<{
+  done: boolean;
+  nextWindowOffset: number;
+  windowsTotal: number;
+  windowsBuilt: number;
+  rowsWritten: number;
+}>;
 /**
  * Aggregates the per-URL Indexing API metadata entity store (populated by
  * `gscdump entities indexing snapshot`) into daily counts of `URL_UPDATED`
@@ -449,4 +486,4 @@ declare const DEFAULT_ROLLUPS: readonly RollupDef[];
  * (CLI: `gscdump rollups --with-canonical`).
  */
 declare const CANONICAL_ROLLUPS: readonly RollupDef[];
-export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -713,37 +713,128 @@ const queryCanonicalDailyRollup = {
 				pageRows: ROLLUP_PAGE_ROWS_DAILY
 			},
 			maxWindowDays: 7,
-			sqlFor: useDim ? (w) => `
-          SELECT
-            ${canonExpr} AS query_canonical,
-            CAST(q.date AS VARCHAR) AS date,
-            SUM(q.clicks)::BIGINT AS clicks,
-            SUM(q.impressions)::BIGINT AS impressions,
-            SUM(q.sum_position)::DOUBLE AS sum_position
-          FROM read_parquet({{FILES}}, union_by_name = true) q
-          LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query
-          WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'
-          GROUP BY ${canonExpr}, q.date
-        ` : (w) => `
-          SELECT
-            ${canonExpr} AS query_canonical,
-            CAST(date AS VARCHAR) AS date,
-            SUM(clicks)::BIGINT AS clicks,
-            SUM(impressions)::BIGINT AS impressions,
-            SUM(sum_position)::DOUBLE AS sum_position
-          FROM read_parquet({{FILES}}, union_by_name = true)
-          WHERE date >= '${w.start}' AND date <= '${w.end}'
-          GROUP BY ${canonExpr}, date
-        `
-		})).map((r) => ({
-			query_canonical: String(r.query_canonical),
-			date: String(r.date),
-			clicks: BigInt(r.clicks),
-			impressions: BigInt(r.impressions),
-			sum_position: Number(r.sum_position)
-		}));
+			sqlFor: dailyWindowSqlFor(useDim, canonExpr)
+		})).map(mapDailyRow);
 	}
 };
+function dailyWindowSqlFor(useDim, canonExpr) {
+	return useDim ? (w) => `
+        SELECT
+          ${canonExpr} AS query_canonical,
+          CAST(q.date AS VARCHAR) AS date,
+          SUM(q.clicks)::BIGINT AS clicks,
+          SUM(q.impressions)::BIGINT AS impressions,
+          SUM(q.sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true) q
+        LEFT JOIN read_parquet({{QUERY_DIM}}, union_by_name = true) qd ON q.query = qd.query
+        WHERE q.date >= '${w.start}' AND q.date <= '${w.end}'
+        GROUP BY ${canonExpr}, q.date
+      ` : (w) => `
+        SELECT
+          ${canonExpr} AS query_canonical,
+          CAST(date AS VARCHAR) AS date,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
+        GROUP BY ${canonExpr}, date
+      `;
+}
+function mapDailyRow(r) {
+	return {
+		query_canonical: String(r.query_canonical),
+		date: String(r.date),
+		clicks: BigInt(r.clicks),
+		impressions: BigInt(r.impressions),
+		sum_position: Number(r.sum_position)
+	};
+}
+async function rebuildCanonicalDailyResumable(opts) {
+	const { engine, ctx, dataSource, searchType, builtAt, windowOffset, deadlineMs } = opts;
+	const sType = searchType !== void 0 ? { searchType } : {};
+	const windows = planRollupWindows((await engine.listPartitions({
+		ctx,
+		table: "queries",
+		...sType
+	})).map((p) => ({
+		partition: p.partition,
+		bytes: p.bytes
+	})), void 0, 7);
+	const windowsTotal = windows.length;
+	const dimStore = createQueryDimStore({ dataSource });
+	const useDim = await dimStore.loadMeta(ctx) !== null;
+	const canonExpr = useDim ? `COALESCE(qd.query_canonical, NULLIF(q.query_canonical, ''), q.query)` : `COALESCE(NULLIF(query_canonical, ''), query)`;
+	const extraFileSets = useDim ? { QUERY_DIM: {
+		table: "queries",
+		keys: [dimStore.parquetKey(ctx)]
+	} } : void 0;
+	const sqlFor = dailyWindowSqlFor(useDim, canonExpr);
+	const cols = queryCanonicalDailyRollup.parquetColumns;
+	const sortKey = queryCanonicalDailyRollup.parquetSortKey;
+	const batchRows = [];
+	let i = windowOffset;
+	for (; i < windowsTotal; i++) {
+		const w = windows[i];
+		const winRows = await runPagedQuery({
+			engine,
+			ctx,
+			table: "queries",
+			...sType,
+			fileSets: {
+				FILES: {
+					table: "queries",
+					partitions: w.partitions
+				},
+				...extraFileSets
+			},
+			coreSql: sqlFor(w),
+			orderBy: "date, query_canonical",
+			pageRows: ROLLUP_PAGE_ROWS_DAILY
+		});
+		for (const r of winRows) batchRows.push(mapDailyRow(r));
+		if (Date.now() > deadlineMs) {
+			i++;
+			break;
+		}
+	}
+	const nextWindowOffset = i;
+	const partKey = rollupParquetKey(ctx, `${CANONICAL_DAILY_PART_STEM}__w${windowOffset}`, builtAt, searchType);
+	await dataSource.write(partKey, encodeRowsToParquetFlex(batchRows, {
+		columns: cols,
+		sortKey
+	}));
+	if (nextWindowOffset < windowsTotal) return {
+		done: false,
+		nextWindowOffset,
+		windowsTotal,
+		windowsBuilt: nextWindowOffset - windowOffset,
+		rowsWritten: batchRows.length
+	};
+	const partPrefixDir = rollupParquetKey(ctx, CANONICAL_DAILY_PART_STEM, builtAt, searchType).replace(/[^/]*$/, "");
+	const partKeys = (await dataSource.list(partPrefixDir)).filter((k) => k.includes(`${CANONICAL_DAILY_PART_STEM}__w`) && k.endsWith(`__v${builtAt}.parquet`)).sort();
+	const envelope = {
+		version: 1,
+		id: CANONICAL_DAILY_ROLLUP_FINAL_ID,
+		builtAt,
+		windowDays: queryCanonicalDailyRollup.windowDays,
+		payload: {
+			parquetKey: partKeys[0] ?? partKey,
+			parquetKeys: partKeys,
+			rowCount: 0
+		}
+	};
+	await dataSource.write(rollupKey(ctx, CANONICAL_DAILY_ROLLUP_FINAL_ID, builtAt, searchType), new TextEncoder().encode(JSON.stringify(envelope)));
+	return {
+		done: true,
+		nextWindowOffset,
+		windowsTotal,
+		windowsBuilt: nextWindowOffset - windowOffset,
+		rowsWritten: batchRows.length
+	};
+}
+const CANONICAL_DAILY_ROLLUP_FINAL_ID = "query_canonical_daily";
+const CANONICAL_DAILY_PART_STEM = "query_canonical_daily__part";
 const indexingMetadataRollup = {
 	id: "indexing_metadata",
 	windowDays: null,
@@ -1056,4 +1147,4 @@ const DEFAULT_ROLLUPS = [
 	sitemapChanges28dRollup
 ];
 const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
-export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildCanonicalDailyResumable, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.31.8",
+  "version": "0.31.10",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "@gscdump/contracts": "0.31.8",
-    "gscdump": "0.31.8"
+    "gscdump": "0.31.10",
+    "@gscdump/contracts": "0.31.10"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",