@gscdump/engine 1.4.11 → 1.5.0

package/dist/rollups/traffic.mjs

@@ -0,0 +1,289 @@
+import { utcDateMinusDays } from "./dates.mjs";
+import { partitionsInRange, runWindowed } from "./windows.mjs";
+const dailyTotalsRollup = {
+	id: "daily_totals",
+	windowDays: null,
+	async build({ engine, ctx, searchType }) {
+		const pageRows = await runWindowed({
+			engine,
+			ctx,
+			table: "pages",
+			...searchType !== void 0 ? { searchType } : {},
+			sqlFor: (w) => `
+        SELECT
+          date,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
+        GROUP BY date
+        ORDER BY date
+      `
+		});
+		const queryRows = await runWindowed({
+			engine,
+			ctx,
+			table: "queries",
+			...searchType !== void 0 ? { searchType } : {},
+			sqlFor: (w) => `
+        SELECT
+          date,
+          SUM(impressions)::BIGINT AS impressions
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
+        GROUP BY date
+      `
+		});
+		const pagesByDate = /* @__PURE__ */ new Map();
+		for (const r of pageRows) {
+			const date = String(r.date);
+			const cur = pagesByDate.get(date) ?? {
+				date,
+				clicks: BigInt(0),
+				impressions: BigInt(0),
+				sum_position: 0
+			};
+			cur.clicks += BigInt(r.clicks);
+			cur.impressions += BigInt(r.impressions);
+			cur.sum_position += Number(r.sum_position);
+			pagesByDate.set(date, cur);
+		}
+		const queryImpressionsByDate = /* @__PURE__ */ new Map();
+		for (const r of queryRows) {
+			const date = String(r.date);
+			queryImpressionsByDate.set(date, (queryImpressionsByDate.get(date) ?? BigInt(0)) + BigInt(r.impressions));
+		}
+		return Array.from(pagesByDate.values()).sort((a, b) => a.date < b.date ? -1 : 1).map((r) => {
+			const totalImpressions = BigInt(r.impressions);
+			const queryImpressions = queryImpressionsByDate.get(String(r.date)) ?? BigInt(0);
+			const anonymized = totalImpressions === BigInt(0) ? 0 : 1 - Number(queryImpressions) / Number(totalImpressions);
+			return {
+				date: r.date,
+				clicks: Number(r.clicks),
+				impressions: Number(r.impressions),
+				sum_position: Number(r.sum_position),
+				anonymizedImpressionsPct: Math.max(0, Math.min(1, anonymized))
+			};
+		});
+	}
+};
+const weeklyTotalsRollup = {
+	id: "weekly_totals",
+	windowDays: null,
+	async build({ engine, ctx, searchType }) {
+		const rows = await runWindowed({
+			engine,
+			ctx,
+			table: "pages",
+			...searchType !== void 0 ? { searchType } : {},
+			sqlFor: (w) => `
+        SELECT
+          strftime(date_trunc('week', date::DATE), '%Y-%m-%d') AS week,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${w.start}' AND date <= '${w.end}'
+        GROUP BY 1
+        ORDER BY 1
+      `
+		});
+		const byWeek = /* @__PURE__ */ new Map();
+		for (const r of rows) {
+			const week = String(r.week);
+			const cur = byWeek.get(week) ?? {
+				week,
+				clicks: 0,
+				impressions: 0,
+				sum_position: 0
+			};
+			cur.clicks += Number(r.clicks);
+			cur.impressions += Number(r.impressions);
+			cur.sum_position += Number(r.sum_position);
+			byWeek.set(week, cur);
+		}
+		return Array.from(byWeek.values()).sort((a, b) => a.week < b.week ? -1 : 1);
+	}
+};
+const topPages28dRollup = {
+	id: "top_pages_28d",
+	windowDays: 28,
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "pages",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		if (partitions.length === 0) return [];
+		return (await engine.runSQL({
+			ctx,
+			table: "pages",
+			fileSets: { FILES: {
+				table: "pages",
+				partitions
+			} },
+			...searchType !== void 0 ? { searchType } : {},
+			sql: `
+        SELECT
+          url,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY url
+        ORDER BY clicks DESC
+        LIMIT 1000
+      `
+		})).rows.map((r) => ({
+			url: r.url,
+			clicks: Number(r.clicks),
+			impressions: Number(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const topCountries28dRollup = {
+	id: "top_countries_28d",
+	windowDays: 28,
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "countries",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		if (partitions.length === 0) return [];
+		return (await engine.runSQL({
+			ctx,
+			table: "countries",
+			fileSets: { FILES: {
+				table: "countries",
+				partitions
+			} },
+			...searchType !== void 0 ? { searchType } : {},
+			sql: `
+        SELECT
+          country,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY country
+        ORDER BY clicks DESC
+        LIMIT 250
+      `
+		})).rows.map((r) => ({
+			country: r.country,
+			clicks: Number(r.clicks),
+			impressions: Number(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const topKeywords28dRollup = {
+	id: "top_keywords_28d",
+	windowDays: 28,
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "queries",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		if (partitions.length === 0) return [];
+		return (await engine.runSQL({
+			ctx,
+			table: "queries",
+			fileSets: { FILES: {
+				table: "queries",
+				partitions
+			} },
+			...searchType !== void 0 ? { searchType } : {},
+			sql: `
+        SELECT
+          query,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY query
+        ORDER BY clicks DESC
+        LIMIT 1000
+      `
+		})).rows.map((r) => ({
+			query: r.query,
+			clicks: Number(r.clicks),
+			impressions: Number(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+const topKeywords28dParquetRollup = {
+	id: "top_keywords_28d_parquet",
+	windowDays: 28,
+	format: "parquet",
+	parquetColumns: [
+		{
+			name: "query",
+			type: "VARCHAR",
+			nullable: false
+		},
+		{
+			name: "clicks",
+			type: "BIGINT",
+			nullable: false
+		},
+		{
+			name: "impressions",
+			type: "BIGINT",
+			nullable: false
+		},
+		{
+			name: "sum_position",
+			type: "DOUBLE",
+			nullable: false
+		}
+	],
+	parquetSortKey: ["clicks"],
+	async build({ engine, ctx, windowAnchorMs, searchType }) {
+		const cutoff = utcDateMinusDays(windowAnchorMs, 28);
+		const partitions = partitionsInRange(await engine.listPartitions({
+			ctx,
+			table: "queries",
+			...searchType !== void 0 ? { searchType } : {}
+		}), cutoff, utcDateMinusDays(windowAnchorMs, 0));
+		if (partitions.length === 0) return [];
+		return (await engine.runSQL({
+			ctx,
+			table: "queries",
+			fileSets: { FILES: {
+				table: "queries",
+				partitions
+			} },
+			...searchType !== void 0 ? { searchType } : {},
+			sql: `
+        SELECT
+          query,
+          SUM(clicks)::BIGINT AS clicks,
+          SUM(impressions)::BIGINT AS impressions,
+          SUM(sum_position)::DOUBLE AS sum_position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= '${cutoff}'
+        GROUP BY query
+        ORDER BY clicks DESC
+        LIMIT 1000
+      `
+		})).rows.map((r) => ({
+			query: String(r.query),
+			clicks: BigInt(r.clicks),
+			impressions: BigInt(r.impressions),
+			sum_position: Number(r.sum_position)
+		}));
+	}
+};
+export { dailyTotalsRollup, topCountries28dRollup, topKeywords28dParquetRollup, topKeywords28dRollup, topPages28dRollup, weeklyTotalsRollup };

package/dist/rollups/windows.d.mts

@@ -0,0 +1,90 @@
+import { FileSetRef, Row as Row$1, TableName as TableName$1 } from "../storage.mjs";
+import "../contracts.mjs";
+import { RollupEngine } from "./core.mjs";
+import { TenantCtx } from "@gscdump/contracts";
+import { SearchType } from "gscdump/query";
+/**
+ * Per-window budget, measured in *parquet* bytes (manifest `bytes`), used by
+ * `planRollupWindows` to chunk a full-history scan.
+ *
+ * The executor decodes a window's parquet and ships it as an Arrow IPC stream
+ * over the service binding; that IPC is hard-guarded at 28MiB
+ * (`IPC_PLACEHOLDER_BUDGET` in @gscdump/cloudflare). Parquet is compressed and
+ * the IPC stream is not, so a window inflates on the wire — keep this
+ * conservatively below the guard. Re-measure the parquet→IPC ratio against
+ * production and raise if headroom allows.
+ */
+declare const WINDOW_BYTE_BUDGET: number;
+/**
+ * Per-page OUTPUT row cap for key-paginated rollups (`runWindowed({ paginate })`
+ * and `runPagedQuery`). `planRollupWindows` bounds the *input* parquet bytes a
+ * window scans, which is a fine proxy for output size on fact aggregations whose
+ * grain matches the input (one output row per input date). It is NOT a proxy for
+ * aggregations that COLLAPSE to a smaller-cardinality grain whose row count is
+ * driven by a high-cardinality GROUP key — `(query_canonical × date)` and
+ * `(query_canonical)` — where output rows scale with distinct canonicals, not
+ * input bytes. For those, each `runSQL` result (shipped as an Arrow IPC stream
+ * over the Workers service-binding RPC; 28MiB guard in `@gscdump/cloudflare`,
+ * duckdb-worker `assertResultBudget` at 24MiB / 100k rows) must be bounded by
+ * paging the OUTPUT, independent of how the input is windowed.
+ *
+ * Narrow rows — `(canonical, date, 3 metrics)` — page at 50k (≈16MiB at the
+ * worker's `cols×64` heuristic, well under both guards). WIDE rows carry a
+ * `GROUP_CONCAT` variants string (up to ~10 variants × ~60 chars) the heuristic
+ * under-counts, so they page smaller to keep the real IPC payload bounded.
+ */
+declare const ROLLUP_PAGE_ROWS = 50000;
+declare const ROLLUP_PAGE_ROWS_WIDE = 20000;
+declare const ROLLUP_PAGE_ROWS_DAILY = 70000;
+/**
+ * Plan byte-bounded windows over a partition set. Each window names the
+ * partitions whose span intersects it; a coarse tier file can land in two
+ * windows, so every windowed SQL MUST also date-filter to the window bounds.
+ */
+declare function planRollupWindows(parts: Array<{
+  partition: string;
+  bytes: number;
+}>, clampRange?: {
+  start: string;
+  end: string;
+}, maxWindowDays?: number): Array<{
+  start: string;
+  end: string;
+  partitions: string[];
+}>;
+/**
+ * Run a full-history aggregation in byte-bounded windows and concat the rows.
+ * Each window's SQL MUST date-filter to `[w.start, w.end]` (see `sqlFor`) so a
+ * tier file spanning a window boundary doesn't double-count calendar dates.
+ *
+ * `paginate` additionally pages each window's OUTPUT (see `runPagedQuery`) so a
+ * window whose GROUP cardinality is high — `(query_canonical × date)` on a large
+ * site — can't ship an oversized result even though its input bytes fit a window.
+ * Date-windowing bounds the per-query scan; output paging bounds the IPC payload.
+ * The two are orthogonal and compose. When `paginate` is set, `sqlFor` MUST emit
+ * no trailing `ORDER BY`/`LIMIT` and `paginate.orderBy` MUST be a total order.
+ */
+declare function runWindowed(opts: {
+  engine: RollupEngine;
+  ctx: TenantCtx;
+  table: TableName$1;
+  searchType?: SearchType;
+  sqlFor: (w: {
+    start: string;
+    end: string;
+  }) => string;
+  /**
+   * Extra named file sets merged into every window's `runSQL` (alongside the
+   * windowed `FILES`). Use to JOIN a non-windowed sidecar (e.g. the query
+   * dimension parquet via `{ QUERY_DIM: { keys: [...] } }`) inside `sqlFor`.
+   */
+  extraFileSets?: Record<string, FileSetRef>;
+  /** Page each window's output by a total-order key. See `runPagedQuery`. */
+  paginate?: {
+    orderBy: string;
+    pageRows: number;
+  };
+  /** Cap each window's day span (output-cardinality bound). See `planRollupWindows`. */
+  maxWindowDays?: number;
+}): Promise<Row$1[]>;
+export { ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, planRollupWindows, runWindowed };

package/dist/rollups/windows.mjs

@@ -0,0 +1,176 @@
+import { MS_PER_DAY } from "gscdump/dates";
+const WINDOW_BYTE_BUDGET = 10 * 1024 * 1024;
+const ROLLUP_PAGE_ROWS = 5e4;
+const ROLLUP_PAGE_ROWS_WIDE = 2e4;
+const ROLLUP_PAGE_ROWS_DAILY = 7e4;
+const DAY_RE = /^daily\/(\d{4})-(\d{2})-(\d{2})$/;
+const WEEK_RE = /^weekly\/(\d{4})-(\d{2})-(\d{2})$/;
+const MONTH_RE = /^monthly\/(\d{4})-(\d{2})$/;
+const QUARTER_RE = /^quarterly\/(\d{4})-Q([1-4])$/;
+function isoDate(ms) {
+	const d = new Date(ms);
+	return `${d.getUTCFullYear()}-${String(d.getUTCMonth() + 1).padStart(2, "0")}-${String(d.getUTCDate()).padStart(2, "0")}`;
+}
+function partitionDaySpan(partition) {
+	const day = DAY_RE.exec(partition);
+	if (day) {
+		const ms = Date.UTC(Number(day[1]), Number(day[2]) - 1, Number(day[3]));
+		return {
+			startMs: ms,
+			endMs: ms
+		};
+	}
+	const week = WEEK_RE.exec(partition);
+	if (week) {
+		const ms = Date.UTC(Number(week[1]), Number(week[2]) - 1, Number(week[3]));
+		return {
+			startMs: ms,
+			endMs: ms + 6 * MS_PER_DAY
+		};
+	}
+	const month = MONTH_RE.exec(partition);
+	if (month) {
+		const y = Number(month[1]);
+		const m = Number(month[2]) - 1;
+		return {
+			startMs: Date.UTC(y, m, 1),
+			endMs: Date.UTC(y, m + 1, 1) - MS_PER_DAY
+		};
+	}
+	const quarter = QUARTER_RE.exec(partition);
+	if (quarter) {
+		const y = Number(quarter[1]);
+		const startMonth = (Number(quarter[2]) - 1) * 3;
+		return {
+			startMs: Date.UTC(y, startMonth, 1),
+			endMs: Date.UTC(y, startMonth + 3, 1) - MS_PER_DAY
+		};
+	}
+	return null;
+}
+function clamp(n, lo, hi) {
+	return Math.max(lo, Math.min(hi, n));
+}
+function planRollupWindows(parts, clampRange, maxWindowDays) {
+	const clampStartMs = clampRange ? Date.parse(`${clampRange.start}T00:00:00Z`) : void 0;
+	const clampEndMs = clampRange ? Date.parse(`${clampRange.end}T00:00:00Z`) : void 0;
+	const spans = [];
+	for (const p of parts) {
+		const span = partitionDaySpan(p.partition);
+		if (!span) continue;
+		if (clampStartMs !== void 0 && clampEndMs !== void 0) {
+			if (span.endMs < clampStartMs || span.startMs > clampEndMs) continue;
+		}
+		spans.push({
+			partition: p.partition,
+			bytes: p.bytes,
+			startMs: span.startMs,
+			endMs: span.endMs
+		});
+	}
+	if (spans.length === 0) return [];
+	let rangeStartMs = Number.POSITIVE_INFINITY;
+	let rangeEndMs = Number.NEGATIVE_INFINITY;
+	let totalBytes = 0;
+	for (const span of spans) {
+		if (span.startMs < rangeStartMs) rangeStartMs = span.startMs;
+		if (span.endMs > rangeEndMs) rangeEndMs = span.endMs;
+		totalBytes += span.bytes;
+	}
+	if (clampStartMs !== void 0) rangeStartMs = Math.max(rangeStartMs, clampStartMs);
+	if (clampEndMs !== void 0) rangeEndMs = Math.min(rangeEndMs, clampEndMs);
+	const spanDays = Math.floor((rangeEndMs - rangeStartMs) / MS_PER_DAY) + 1;
+	const bytesPerDay = Math.max(1, totalBytes / spanDays);
+	const byteWindowDays = clamp(Math.floor(WINDOW_BYTE_BUDGET / bytesPerDay), 7, 400);
+	const windowDays = maxWindowDays != null ? Math.max(1, Math.min(byteWindowDays, maxWindowDays)) : byteWindowDays;
+	if (!Number.isFinite(windowDays) || !Number.isFinite(rangeStartMs) || !Number.isFinite(rangeEndMs) || rangeEndMs < rangeStartMs) return [];
+	const windowWidthMs = windowDays * MS_PER_DAY;
+	const windowCount = Math.floor((rangeEndMs - rangeStartMs) / windowWidthMs) + 1;
+	const partitionsByWindow = Array.from({ length: windowCount }, () => []);
+	for (const span of spans) {
+		const overlapStartMs = Math.max(span.startMs, rangeStartMs);
+		const overlapEndMs = Math.min(span.endMs, rangeEndMs);
+		if (overlapStartMs > overlapEndMs) continue;
+		const firstWindow = Math.max(0, Math.ceil((overlapStartMs - rangeStartMs - (windowDays - 1) * MS_PER_DAY) / windowWidthMs));
+		const lastWindow = Math.min(windowCount - 1, Math.floor((overlapEndMs - rangeStartMs) / windowWidthMs));
+		for (let index = firstWindow; index <= lastWindow; index++) partitionsByWindow[index].push(span.partition);
+	}
+	const windows = [];
+	for (let index = 0; index < windowCount; index++) {
+		const cursorMs = rangeStartMs + index * windowWidthMs;
+		const windowEndMs = Math.min(cursorMs + (windowDays - 1) * MS_PER_DAY, rangeEndMs);
+		const partitions = partitionsByWindow[index];
+		if (partitions.length > 0) windows.push({
+			start: isoDate(cursorMs),
+			end: isoDate(windowEndMs),
+			partitions
+		});
+	}
+	return windows;
+}
+function partitionsInRange(parts, start, end) {
+	const startMs = Date.parse(`${start}T00:00:00Z`);
+	const endMs = Date.parse(`${end}T00:00:00Z`);
+	const out = [];
+	for (const p of parts) {
+		const span = partitionDaySpan(p.partition);
+		if (!span) continue;
+		if (span.endMs >= startMs && span.startMs <= endMs) out.push(p.partition);
+	}
+	return out;
+}
+async function runPagedQuery(opts) {
+	const out = [];
+	for (let offset = 0;; offset += opts.pageRows) {
+		const result = await opts.engine.runSQL({
+			ctx: opts.ctx,
+			table: opts.table,
+			fileSets: opts.fileSets,
+			sql: `${opts.coreSql}\nORDER BY ${opts.orderBy}\nLIMIT ${opts.pageRows} OFFSET ${offset}`,
+			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+		});
+		out.push(...result.rows);
+		if (result.rows.length < opts.pageRows) break;
+	}
+	return out;
+}
+async function runWindowed(opts) {
+	const windows = planRollupWindows(await opts.engine.listPartitions({
+		ctx: opts.ctx,
+		table: opts.table,
+		...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+	}), void 0, opts.maxWindowDays);
+	const rows = [];
+	for (const w of windows) {
+		const fileSets = {
+			FILES: {
+				table: opts.table,
+				partitions: w.partitions
+			},
+			...opts.extraFileSets
+		};
+		if (opts.paginate) {
+			rows.push(...await runPagedQuery({
+				engine: opts.engine,
+				ctx: opts.ctx,
+				table: opts.table,
+				...opts.searchType !== void 0 ? { searchType: opts.searchType } : {},
+				fileSets,
+				coreSql: opts.sqlFor(w),
+				orderBy: opts.paginate.orderBy,
+				pageRows: opts.paginate.pageRows
+			}));
+			continue;
+		}
+		const result = await opts.engine.runSQL({
+			ctx: opts.ctx,
+			table: opts.table,
+			fileSets,
+			sql: opts.sqlFor(w),
+			...opts.searchType !== void 0 ? { searchType: opts.searchType } : {}
+		});
+		rows.push(...result.rows);
+	}
+	return rows;
+}
+export { ROLLUP_PAGE_ROWS, ROLLUP_PAGE_ROWS_DAILY, ROLLUP_PAGE_ROWS_WIDE, WINDOW_BYTE_BUDGET, partitionsInRange, planRollupWindows, runWindowed };