@gscdump/engine 0.31.3 → 0.31.4

package/dist/rollups.d.mts

@@ -1,4 +1,4 @@
-import { DataSource, FileSetRef, Row as Row$1 } from "./_chunks/storage.mjs";
+import { DataSource, FileSetRef, Row as Row$1, TableName as TableName$1 } from "./_chunks/storage.mjs";
 import { ColumnDef as ColumnDef$1 } from "./_chunks/schema.mjs";
 import { EngineError } from "./_chunks/errors.mjs";
 import { SearchType } from "gscdump/query";
@@ -200,6 +200,26 @@ declare function rebuildRollups(opts: RebuildRollupsOptions): Promise<RebuildRol
  * production and raise if headroom allows.
  */
 declare const WINDOW_BYTE_BUDGET: number;
+/**
+ * Per-page OUTPUT row cap for key-paginated rollups (`runWindowed({ paginate })`
+ * and `runPagedQuery`). `planRollupWindows` bounds the *input* parquet bytes a
+ * window scans, which is a fine proxy for output size on fact aggregations whose
+ * grain matches the input (one output row per input date). It is NOT a proxy for
+ * aggregations that COLLAPSE to a smaller-cardinality grain whose row count is
+ * driven by a high-cardinality GROUP key — `(query_canonical × date)` and
+ * `(query_canonical)` — where output rows scale with distinct canonicals, not
+ * input bytes. For those, each `runSQL` result (shipped as an Arrow IPC stream
+ * over the Workers service-binding RPC; 28MiB guard in `@gscdump/cloudflare`,
+ * duckdb-worker `assertResultBudget` at 24MiB / 100k rows) must be bounded by
+ * paging the OUTPUT, independent of how the input is windowed.
+ *
+ * Narrow rows — `(canonical, date, 3 metrics)` — page at 50k (≈16MiB at the
+ * worker's `cols×64` heuristic, well under both guards). WIDE rows carry a
+ * `GROUP_CONCAT` variants string (up to ~10 variants × ~60 chars) the heuristic
+ * under-counts, so they page smaller to keep the real IPC payload bounded.
+ */
+declare const ROLLUP_PAGE_ROWS = 50000;
+declare const ROLLUP_PAGE_ROWS_WIDE = 20000;
 /**
  * UTC day-aligned [startMs, endMs] span a partition covers. Returns null for
  * `hourly/` partitions and anything unrecognised — those are excluded from
@@ -234,11 +254,18 @@ declare function partitionsInRange(parts: Array<{
  * Run a full-history aggregation in byte-bounded windows and concat the rows.
  * Each window's SQL MUST date-filter to `[w.start, w.end]` (see `sqlFor`) so a
  * tier file spanning a window boundary doesn't double-count calendar dates.
+ *
+ * `paginate` additionally pages each window's OUTPUT (see `runPagedQuery`) so a
+ * window whose GROUP cardinality is high — `(query_canonical × date)` on a large
+ * site — can't ship an oversized result even though its input bytes fit a window.
+ * Date-windowing bounds the per-query scan; output paging bounds the IPC payload.
+ * The two are orthogonal and compose. When `paginate` is set, `sqlFor` MUST emit
+ * no trailing `ORDER BY`/`LIMIT` and `paginate.orderBy` MUST be a total order.
  */
 declare function runWindowed(opts: {
   engine: RollupEngine;
   ctx: TenantCtx;
-  table: import('@gscdump/engine/contracts').TableName;
+  table: TableName$1;
   searchType?: SearchType;
   sqlFor: (w: {
     start: string;
@@ -249,7 +276,11 @@ declare function runWindowed(opts: {
    * windowed `FILES`). Use to JOIN a non-windowed sidecar (e.g. the query
    * dimension parquet via `{ QUERY_DIM: { keys: [...] } }`) inside `sqlFor`.
    */
-  extraFileSets?: Record<string, FileSetRef>;
+  extraFileSets?: Record<string, FileSetRef>; /** Page each window's output by a total-order key. See `runPagedQuery`. */
+  paginate?: {
+    orderBy: string;
+    pageRows: number;
+  };
 }): Promise<Row$1[]>;
 /**
  * Daily totals across the full history. One row per (date, table) with
@@ -415,4 +446,4 @@ declare const DEFAULT_ROLLUPS: readonly RollupDef[];
  * (CLI: `gscdump rollups --with-canonical`).
  */
 declare const CANONICAL_ROLLUPS: readonly RollupDef[];
-export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ParquetRollupPointer, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -127,6 +127,8 @@ function utcDateMinusDays(at, days) {
 	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
 }
 const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
+const ROLLUP_PAGE_ROWS = 5e4;
+const ROLLUP_PAGE_ROWS_WIDE = 2e4;
 const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
 const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
 const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
@@ -226,6 +228,21 @@ function partitionsInRange(parts, start, end) {
 	}
 	return out;
 }
+async function runPagedQuery(opts) {
+	const out = [];
+	for (let offset = 0;; offset += opts.pageRows) {
+		const result = await opts.engine.runSQL({
+			ctx: opts.ctx,
+			table: opts.table,
+			fileSets: opts.fileSets,
+			sql: `${opts.coreSql}\nORDER BY ${opts.orderBy}\nLIMIT ${opts.pageRows} OFFSET ${offset}`,
+			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+		});
+		out.push(...result.rows);
+		if (result.rows.length < opts.pageRows) break;
+	}
+	return out;
+}
 async function runWindowed(opts) {
 	const windows = planRollupWindows(await opts.engine.listPartitions({
 		ctx: opts.ctx,
@@ -234,16 +251,30 @@ async function runWindowed(opts) {
 	}));
 	const rows = [];
 	for (const w of windows) {
+		const fileSets = {
+			FILES: {
+				table: opts.table,
+				partitions: w.partitions
+			},
+			...opts.extraFileSets
+		};
+		if (opts.paginate) {
+			rows.push(...await runPagedQuery({
+				engine: opts.engine,
+				ctx: opts.ctx,
+				table: opts.table,
+				...opts.searchType !== void 0 ? { searchType: opts.searchType } : {},
+				fileSets,
+				coreSql: opts.sqlFor(w),
+				orderBy: opts.paginate.orderBy,
+				pageRows: opts.paginate.pageRows
+			}));
+			continue;
+		}
 		const result = await opts.engine.runSQL({
 			ctx: opts.ctx,
 			table: opts.table,
-			fileSets: {
-				FILES: {
-					table: opts.table,
-					partitions: w.partitions
-				},
-				...opts.extraFileSets
-			},
+			fileSets,
 			sql: opts.sqlFor(w),
 			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
 		});
@@ -572,15 +603,18 @@ const queryCanonicalVariantsRollup = {
 		});
 		if (parts.length === 0) return [];
 		const partitions = parts.map((p) => p.partition);
-		return (await engine.runSQL({
+		return (await runPagedQuery({
+			engine,
 			ctx,
 			table: "queries",
+			...searchType !== void 0 ? { searchType } : {},
 			fileSets: { FILES: {
 				table: "queries",
 				partitions
 			} },
-			...searchType !== void 0 ? { searchType } : {},
-			sql: `
+			orderBy: "joinKey",
+			pageRows: ROLLUP_PAGE_ROWS_WIDE,
+			coreSql: `
         WITH per_variant AS (
           SELECT
             COALESCE(NULLIF(query_canonical, ''), query) AS joinKey,
@@ -601,7 +635,7 @@ const queryCanonicalVariantsRollup = {
         FROM per_variant
         GROUP BY joinKey
       `
-		})).rows.map((r) => ({
+		})).map((r) => ({
 			joinKey: String(r.joinKey),
 			variantCount: BigInt(r.variantCount),
 			canonicalName: r.canonicalName == null ? null : String(r.canonicalName),
@@ -654,6 +688,10 @@ const queryCanonicalDailyRollup = {
 				table: "queries",
 				keys: [dimStore.parquetKey(ctx)]
 			} } } : {},
+			paginate: {
+				orderBy: "date, query_canonical",
+				pageRows: ROLLUP_PAGE_ROWS
+			},
 			sqlFor: useDim ? (w) => `
           SELECT
             ${canonExpr} AS query_canonical,
@@ -997,4 +1035,4 @@ const DEFAULT_ROLLUPS = [
 	sitemapChanges28dRollup
 ];
 const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
-export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.31.3",
+  "version": "0.31.4",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -191,8 +191,8 @@
     "hyparquet": "^1.26.1",
     "hyparquet-writer": "^0.16.1",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.31.3",
-    "@gscdump/contracts": "0.31.3"
+    "gscdump": "0.31.4",
+    "@gscdump/contracts": "0.31.4"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",