@gscdump/engine 0.31.6 → 0.31.8

package/dist/rollups.d.mts

@@ -220,6 +220,8 @@ declare const WINDOW_BYTE_BUDGET: number;
  */
 declare const ROLLUP_PAGE_ROWS = 50000;
 declare const ROLLUP_PAGE_ROWS_WIDE = 20000;
+declare const ROLLUP_PAGE_ROWS_DAILY = 90000;
+declare const DAILY_MAX_WINDOW_DAYS = 7;
 /**
  * UTC day-aligned [startMs, endMs] span a partition covers. Returns null for
  * `hourly/` partitions and anything unrecognised — those are excluded from
@@ -240,7 +242,7 @@ declare function planRollupWindows(parts: Array<{
 }>, clampRange?: {
   start: string;
   end: string;
-}): Array<{
+}, maxWindowDays?: number): Array<{
   start: string;
   end: string;
   partitions: string[];
@@ -280,7 +282,8 @@ declare function runWindowed(opts: {
   paginate?: {
     orderBy: string;
     pageRows: number;
-  };
+  }; /** Cap each window's day span (output-cardinality bound). See `planRollupWindows`. */
+  maxWindowDays?: number;
 }): Promise<Row$1[]>;
 /**
  * Daily totals across the full history. One row per (date, table) with
@@ -446,4 +449,4 @@ declare const DEFAULT_ROLLUPS: readonly RollupDef[];
  * (CLI: `gscdump rollups --with-canonical`).
  */
 declare const CANONICAL_ROLLUPS: readonly RollupDef[];
-export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ParquetRollupPointer, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, RebuildDailyFromHourlyOptions, RebuildRollupResult, RebuildRollupsOptions, RollupBucket, RollupCtx, RollupDef, RollupEngine, RollupEnvelope, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups.mjs

@@ -129,6 +129,8 @@ function utcDateMinusDays(at, days) {
 const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
 const ROLLUP_PAGE_ROWS = 5e4;
 const ROLLUP_PAGE_ROWS_WIDE = 2e4;
+const ROLLUP_PAGE_ROWS_DAILY = 9e4;
+const DAILY_MAX_WINDOW_DAYS = 7;
 const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
 const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
 const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
@@ -177,7 +179,7 @@ function partitionDaySpan(partition) {
 function clamp(n, lo, hi) {
 	return Math.max(lo, Math.min(hi, n));
 }
-function planRollupWindows(parts, clampRange) {
+function planRollupWindows(parts, clampRange, maxWindowDays) {
 	const clampStartMs = clampRange ? Date.parse(`${clampRange.start}T00:00:00Z`) : void 0;
 	const clampEndMs = clampRange ? Date.parse(`${clampRange.end}T00:00:00Z`) : void 0;
 	const spans = [];
@@ -202,7 +204,8 @@ function planRollupWindows(parts, clampRange) {
 	const totalBytes = spans.reduce((a, s) => a + s.bytes, 0);
 	const spanDays = Math.floor((rangeEndMs - rangeStartMs) / MS_PER_DAY) + 1;
 	const bytesPerDay = Math.max(1, totalBytes / spanDays);
-	const windowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
+	const byteWindowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
+	const windowDays = maxWindowDays != null ? Math.max(1, Math.min(byteWindowDays, maxWindowDays)) : byteWindowDays;
 	const windows = [];
 	let cursorMs = rangeStartMs;
 	while (cursorMs <= rangeEndMs) {
@@ -248,7 +251,7 @@ async function runWindowed(opts) {
 		ctx: opts.ctx,
 		table: opts.table,
 		...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
-	}));
+	}), void 0, opts.maxWindowDays);
 	const rows = [];
 	for (const w of windows) {
 		const fileSets = {
@@ -707,8 +710,9 @@ const queryCanonicalDailyRollup = {
 			} } } : {},
 			paginate: {
 				orderBy: "date, query_canonical",
-				pageRows: ROLLUP_PAGE_ROWS
+				pageRows: ROLLUP_PAGE_ROWS_DAILY
 			},
+			maxWindowDays: 7,
 			sqlFor: useDim ? (w) => `
           SELECT
             ${canonExpr} AS query_canonical,
@@ -1052,4 +1056,4 @@ const DEFAULT_ROLLUPS = [
 	sitemapChanges28dRollup
 ];
 const CANONICAL_ROLLUPS = [queryCanonicalVariantsRollup, queryCanonicalDailyRollup];
-export { CANONICAL_ROLLUPS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };
+export { CANONICAL_ROLLUPS, DAILY_MAX_WINDOW_DAYS, DEFAULT_ROLLUPS, ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, dailyTotalsRollup, indexPercentRollup, indexingHealthRollup, indexingMetadataRollup, partitionDaySpan, partitionsInRange, planRollupWindows, queryCanonicalDailyRollup, queryCanonicalVariantsRollup, readLatestRollup, rebuildDailyFromHourly, rebuildRollups, rollupKey, rollupParquetKey, runWindowed, sitemapChanges28dRollup, sitemapHealthRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@gscdump/engine",
   "type": "module",
-  "version": "0.31.6",
+  "version": "0.31.8",
   "description": "Append-only Parquet/DuckDB storage engine + planner + adapters for the gscdump pipeline. Node + edge runtimes; opt-in heavy peers.",
   "author": {
     "name": "Harlan Wilton",
@@ -181,8 +181,8 @@
   "dependencies": {
     "drizzle-orm": "1.0.0-rc.3",
     "proper-lockfile": "^4.1.2",
-    "gscdump": "0.31.6",
-    "@gscdump/contracts": "0.31.6"
+    "@gscdump/contracts": "0.31.8",
+    "gscdump": "0.31.8"
   },
   "devDependencies": {
     "@duckdb/duckdb-wasm": "^1.32.0",