@gscdump/analysis 3.7.0 → 4.0.0

package/README.md

@@ -114,9 +114,10 @@ After creating `source` above, run a Report supported by that Source:
 ```ts
 import { defaultReportRegistry, runReport } from '@gscdump/analysis/report'
 import { resolveWindow } from '@gscdump/engine/period'
+import { getLatestGscDate } from 'gscdump/dates'
 
 const report = defaultReportRegistry.getReport('movers')!
-const window = resolveWindow({ preset: 'last-28d', comparison: 'prev-period' })
+const window = resolveWindow({ preset: 'last-28d', anchor: getLatestGscDate(), comparison: 'prev-period' })
 const result = await runReport(report, {
   source,
   analyzers: defaultAnalyzerRegistry,
@@ -155,12 +156,25 @@ Analyzer support depends on the Source's SQL dialect and capabilities.
 ```ts
 import { resolveWindow } from '@gscdump/analysis'
 
-const window = resolveWindow({ preset: 'last-30d', comparison: 'yoy' })
+const window = resolveWindow({ preset: 'last-30d', anchor: '2026-08-28', comparison: 'yoy' })
 console.log(window.start, window.end, window.comparison)
 ```
 
-Presets: `last-7d`, `last-28d`, `last-30d`, `last-90d`, `last-180d`, `last-365d`, `mtd`, `ytd`, and `custom`.
-Comparisons: `none`, `prev-period`, and `yoy`.
+Every preset except `custom` needs an `anchor`: the last date the window includes.
+Pass the newest complete date in your data, such as the Store's newest synced day or `getLatestGscDate()` from `gscdump/dates`.
+`resolveWindow` never reads the clock.
+
+Presets: `last-7d`, `last-28d`, `last-30d`, `last-90d`, `last-180d`, `last-365d`, `mtd`, `qtd`, `ytd`, `last-quarter`, and `custom`.
+Comparisons: `none`, `prev-period`, and `yoy`. `yoy` shifts the window back 364 days, so each day compares with the same weekday.
+
+## Fetch budget and output limit
+
+`limit` caps the rows an Analyzer returns. It never caps the rows it reads.
+Row plans read at most `fetchBudget` rows per query: 25,000 (one GSC page) by default, up to 100,000.
+`meta.total` counts every match on every Analyzer with a row plan, not only the returned rows.
+The SQL-only Analyzers are the exception. They still report `meta.total` after `limit`: `bayesian-ctr`, `bipartite-pagerank`, `change-point`, `ctr-anomaly`, `intent-atlas`, `long-tail`, `query-migration`, `stl-decompose`, and `trends`.
+`meta.coverage` is `{ kind: 'complete' }`, or `{ kind: 'truncated', fetched }` when a query reached its fetch budget.
+A truncated run can miss rows, so treat its totals and rankings as partial.
 
 ## Public API
 

package/dist/analyzer/adapt-rows.mjs

@@ -1,12 +1,11 @@
 import { between, date, gsc, page, query } from "gscdump/query";
-const DEFAULT_LIMIT = 25e3;
-function queriesQueryState(period, limit = DEFAULT_LIMIT) {
-	return gsc.select(query, page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
+function queriesQueryState(period, budget) {
+	return gsc.select(query, page).where(between(date, period.startDate, period.endDate)).limit(budget).getState();
 }
-function pagesQueryState(period, limit = DEFAULT_LIMIT) {
-	return gsc.select(page).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
+function pagesQueryState(period, budget) {
+	return gsc.select(page).where(between(date, period.startDate, period.endDate)).limit(budget).getState();
 }
-function datesQueryState(period, limit = DEFAULT_LIMIT) {
-	return gsc.select(date).where(between(date, period.startDate, period.endDate)).limit(limit).getState();
+function datesQueryState(period, budget) {
+	return gsc.select(date).where(between(date, period.startDate, period.endDate)).limit(budget).getState();
 }
 export { datesQueryState, pagesQueryState, queriesQueryState };

package/dist/analyzer/paginate.mjs

@@ -82,4 +82,15 @@ function paginateSortedInMemory(rows, input, compare) {
 	if (invalidComparison) return fullSort();
 	return heap.slice(offset, offset + limit).map((entry) => entry.value);
 }
-export { clampLimit, clampOffset, paginateClause, paginateInMemory, paginateSortedInMemory };
+function resolveSort(input, allowed, defaults) {
+	return {
+		sortBy: input.sortBy && allowed.includes(input.sortBy) ? input.sortBy : defaults.sortBy,
+		sortDir: input.sortDir === "asc" || input.sortDir === "desc" ? input.sortDir : defaults.sortDir
+	};
+}
+const TOTAL_COUNT_SELECT = "CAST(COUNT(*) OVER () AS DOUBLE) AS totalCount";
+function totalCountOf(rows) {
+	const first = rows[0];
+	return first?.totalCount != null ? Number(first.totalCount) : rows.length;
+}
+export { TOTAL_COUNT_SELECT, clampLimit, clampOffset, paginateClause, paginateInMemory, paginateSortedInMemory, resolveSort, totalCountOf };

package/dist/analyzers/brand.mjs

@@ -1,7 +1,8 @@
 import { queriesQueryState } from "../analyzer/adapt-rows.mjs";
+import { TOTAL_COUNT_SELECT, paginateClause, paginateSortedInMemory, totalCountOf } from "../analyzer/paginate.mjs";
 import { rowString } from "../analyzer/row-values.mjs";
 import { analysisErrorToException, analysisErrors } from "../errors.mjs";
-import { num } from "@gscdump/engine/analysis-types";
+import { fetchBudgetOf, num } from "@gscdump/engine/analysis-types";
 import { defineAnalyzer } from "@gscdump/engine/analyzer";
 import { periodOf } from "@gscdump/engine/period";
 import { enumeratePartitions } from "@gscdump/engine/planner";
@@ -74,13 +75,26 @@ const brandAnalyzer = defineAnalyzer({
       WHERE date >= ? AND date <= ?
       GROUP BY query, url
       HAVING SUM(impressions) >= ?
+    ),
+    segmented AS (
+      SELECT
+        query, page, clicks, impressions, ctr, position,
+        CASE WHEN regexp_matches(LOWER(query), ?) THEN 'brand' ELSE 'non-brand' END AS segment
+      FROM agg
     )
     SELECT
-      query, page, clicks, impressions, ctr, position,
-      CASE WHEN regexp_matches(LOWER(query), ?) THEN 'brand' ELSE 'non-brand' END AS segment
-    FROM agg
+      query, page, clicks, impressions, ctr, position, segment,
+      SUM(CASE WHEN segment = 'brand' THEN clicks ELSE 0 END) OVER () AS brandClicks,
+      SUM(CASE WHEN segment = 'brand' THEN 0 ELSE clicks END) OVER () AS nonBrandClicks,
+      SUM(CASE WHEN segment = 'brand' THEN impressions ELSE 0 END) OVER () AS brandImpressions,
+      SUM(CASE WHEN segment = 'brand' THEN 0 ELSE impressions END) OVER () AS nonBrandImpressions,
+      ${TOTAL_COUNT_SELECT}
+    FROM segmented
     ORDER BY clicks DESC
-    LIMIT ${Number(limit)}
+    ${paginateClause({
+				limit,
+				offset: params.offset
+			})}
   `,
 			params: [
 				startDate,
@@ -95,7 +109,8 @@ const brandAnalyzer = defineAnalyzer({
 		};
 	},
 	reduceSql(rows) {
-		const normalized = (Array.isArray(rows) ? rows : []).map((r) => ({
+		const arr = Array.isArray(rows) ? rows : [];
+		const normalized = arr.map((r) => ({
 			query: rowString(r.query),
 			page: r.page == null ? void 0 : rowString(r.page),
 			clicks: num(r.clicks),
@@ -104,34 +119,27 @@ const brandAnalyzer = defineAnalyzer({
 			position: num(r.position),
 			segment: rowString(r.segment)
 		}));
-		let brandClicks = 0;
-		let nonBrandClicks = 0;
-		let brandImpressions = 0;
-		let nonBrandImpressions = 0;
-		for (const r of normalized) if (r.segment === "brand") {
-			brandClicks += r.clicks;
-			brandImpressions += r.impressions;
-		} else {
-			nonBrandClicks += r.clicks;
-			nonBrandImpressions += r.impressions;
-		}
+		const first = arr[0];
+		const brandClicks = num(first?.brandClicks);
+		const nonBrandClicks = num(first?.nonBrandClicks);
 		const totalClicks = brandClicks + nonBrandClicks;
 		return {
 			results: normalized,
 			meta: {
-				total: normalized.length,
+				total: totalCountOf(arr),
+				returned: normalized.length,
 				summary: {
 					brandClicks,
 					nonBrandClicks,
 					brandShare: totalClicks > 0 ? brandClicks / totalClicks : 0,
-					brandImpressions,
-					nonBrandImpressions
+					brandImpressions: num(first?.brandImpressions),
+					nonBrandImpressions: num(first?.nonBrandImpressions)
 				}
 			}
 		};
 	},
 	buildRows(params) {
-		return { queries: queriesQueryState(periodOf(params), params.limit) };
+		return { queries: queriesQueryState(periodOf(params), fetchBudgetOf(params)) };
 	},
 	reduceRows(rows, params) {
 		const brandTerms = requireBrandTerms(params.brandTerms);
@@ -139,15 +147,24 @@ const brandAnalyzer = defineAnalyzer({
 			brandTerms,
 			minImpressions: params.minImpressions
 		});
+		const segmented = [...result.brand.map((r) => ({
+			...r,
+			segment: "brand"
+		})), ...result.nonBrand.map((r) => ({
+			...r,
+			segment: "non-brand"
+		}))];
+		const paged = paginateSortedInMemory(segmented, {
+			limit: params.limit,
+			offset: params.offset
+		}, (left, right) => right.clicks - left.clicks);
 		return {
-			results: [...result.brand.map((r) => ({
-				...r,
-				segment: "brand"
-			})), ...result.nonBrand.map((r) => ({
-				...r,
-				segment: "non-brand"
-			}))],
-			meta: { summary: result.summary }
+			results: paged,
+			meta: {
+				total: segmented.length,
+				returned: paged.length,
+				summary: result.summary
+			}
 		};
 	}
 });

package/dist/analyzers/cannibalization.mjs

@@ -1,7 +1,8 @@
 import { queriesQueryState } from "../analyzer/adapt-rows.mjs";
+import { TOTAL_COUNT_SELECT, paginateInMemory, totalCountOf } from "../analyzer/paginate.mjs";
 import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
 import { createSorter } from "../types.mjs";
-import { num } from "@gscdump/engine/analysis-types";
+import { fetchBudgetOf, num } from "@gscdump/engine/analysis-types";
 import { defineAnalyzer } from "@gscdump/engine/analyzer";
 import { periodOf } from "@gscdump/engine/period";
 import { enumeratePartitions } from "@gscdump/engine/planner";
@@ -154,7 +155,10 @@ const cannibalizationAnalyzer = defineAnalyzer({
               * LEAST(LOG10(GREATEST(t.total_impressions, 10.0)) / 5.0, 1.0),
             1.0 / 3.0
           )
-        )) AS DOUBLE) AS severity
+        )) AS DOUBLE) AS severity,
+        SUM(e.stolen_clicks) OVER () AS allStolenClicks,
+        AVG(GREATEST(0.0, 1.0 - e.hhi / 10000.0)) OVER () AS allFragmentation,
+        ${TOTAL_COUNT_SELECT}
       FROM events e
       JOIN query_totals t USING (query)
       ORDER BY severity DESC, stolenClicks DESC
@@ -174,7 +178,8 @@ const cannibalizationAnalyzer = defineAnalyzer({
 		};
 	},
 	reduceSql(rows) {
-		const events = (Array.isArray(rows) ? rows : []).map((r) => ({
+		const arr = Array.isArray(rows) ? rows : [];
+		const events = arr.map((r) => ({
 			keyword: rowString(r.keyword),
 			totalImpressions: num(r.totalImpressions),
 			totalClicks: num(r.totalClicks),
@@ -234,12 +239,13 @@ const cannibalizationAnalyzer = defineAnalyzer({
 			queryCount: n.queries.size
 		}));
 		const edges = [...edgeAgg.values()];
-		const avgFragmentation = events.length > 0 ? events.reduce((s, e) => s + e.fragmentation, 0) / events.length : 0;
-		const totalStolenClicks = events.reduce((s, e) => s + e.stolenClicks, 0);
+		const avgFragmentation = arr[0] ? num(arr[0].allFragmentation) : 0;
+		const totalStolenClicks = arr[0] ? num(arr[0].allStolenClicks) : 0;
 		return {
 			results: events,
 			meta: {
-				total: events.length,
+				total: totalCountOf(arr),
+				returned: events.length,
 				totalStolenClicks,
 				avgFragmentation,
 				graph: {
@@ -250,7 +256,7 @@ const cannibalizationAnalyzer = defineAnalyzer({
 		};
 	},
 	buildRows(params) {
-		return { rows: queriesQueryState(periodOf(params), params.limit) };
+		return { rows: queriesQueryState(periodOf(params), fetchBudgetOf(params)) };
 	},
 	reduceRows(rows, params) {
 		const results = analyzeCannibalization(Array.isArray(rows) ? rows : [], {
@@ -258,9 +264,16 @@ const cannibalizationAnalyzer = defineAnalyzer({
 			maxPositionSpread: params.maxPositionSpread,
 			minPages: params.minPages
 		});
+		const paged = paginateInMemory(results, {
+			limit: params.limit,
+			offset: params.offset
+		});
 		return {
-			results,
-			meta: { total: results.length }
+			results: paged,
+			meta: {
+				total: results.length,
+				returned: paged.length
+			}
 		};
 	}
 });

package/dist/analyzers/clustering.mjs

@@ -1,6 +1,7 @@
 import { queriesQueryState } from "../analyzer/adapt-rows.mjs";
+import { paginateInMemory } from "../analyzer/paginate.mjs";
 import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
-import { num } from "@gscdump/engine/analysis-types";
+import { fetchBudgetOf, num } from "@gscdump/engine/analysis-types";
 import { defineAnalyzer } from "@gscdump/engine/analyzer";
 import { periodOf } from "@gscdump/engine/period";
 import { enumeratePartitions } from "@gscdump/engine/planner";
@@ -175,7 +176,7 @@ const clusteringAnalyzer = defineAnalyzer({
 			}
 		};
 	},
-	reduceSql(rows) {
+	reduceSql(rows, params) {
 		const clusters = (Array.isArray(rows) ? rows : []).map((r) => ({
 			clusterName: rowString(r.clusterName),
 			clusterType: rowString(r.clusterType),
@@ -191,16 +192,21 @@ const clusteringAnalyzer = defineAnalyzer({
 				position: num(k.position)
 			}))
 		}));
+		const paged = paginateInMemory(clusters, {
+			limit: params.limit,
+			offset: params.offset
+		});
 		return {
-			results: clusters,
+			results: paged,
 			meta: {
 				total: clusters.length,
+				returned: paged.length,
 				totalClusters: clusters.length
 			}
 		};
 	},
 	buildRows(params) {
-		return { queries: queriesQueryState(periodOf(params), params.limit) };
+		return { queries: queriesQueryState(periodOf(params), fetchBudgetOf(params)) };
 	},
 	reduceRows(rows, params) {
 		const result = analyzeClustering(Array.isArray(rows) ? rows : [], {
@@ -208,9 +214,17 @@ const clusteringAnalyzer = defineAnalyzer({
 			minClusterSize: params.minClusterSize,
 			minImpressions: params.minImpressions
 		});
+		const paged = paginateInMemory(result.clusters, {
+			limit: params.limit,
+			offset: params.offset
+		});
 		return {
-			results: result.clusters,
-			meta: { totalClusters: result.clusters.length }
+			results: paged,
+			meta: {
+				total: result.clusters.length,
+				returned: paged.length,
+				totalClusters: result.clusters.length
+			}
 		};
 	}
 });

package/dist/analyzers/concentration.mjs

@@ -1,6 +1,6 @@
 import { pagesQueryState, queriesQueryState } from "../analyzer/adapt-rows.mjs";
 import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
-import { num } from "@gscdump/engine/analysis-types";
+import { fetchBudgetOf, num } from "@gscdump/engine/analysis-types";
 import { defineAnalyzer } from "@gscdump/engine/analyzer";
 import { periodOf } from "@gscdump/engine/period";
 import { enumeratePartitions } from "@gscdump/engine/planner";
@@ -164,8 +164,8 @@ const concentrationAnalyzer = defineAnalyzer({
 		const dim = params.dimension || "pages";
 		const period = periodOf(params);
 		const out = {};
-		if (dim === "pages") out.pages = pagesQueryState(period, params.limit);
-		else out.queries = queriesQueryState(period, params.limit);
+		if (dim === "pages") out.pages = pagesQueryState(period, fetchBudgetOf(params));
+		else out.queries = queriesQueryState(period, fetchBudgetOf(params));
 		return out;
 	},
 	reduceRows(rows, params) {

package/dist/analyzers/decay.mjs

@@ -1,8 +1,8 @@
 import { pagesQueryState } from "../analyzer/adapt-rows.mjs";
+import { TOTAL_COUNT_SELECT, paginateClause, paginateInMemory, totalCountOf } from "../analyzer/paginate.mjs";
 import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
 import { buildPeriodMap, createMetricSorter } from "../types.mjs";
-import { paginateInMemory } from "../analyzer/paginate.mjs";
-import { num } from "@gscdump/engine/analysis-types";
+import { fetchBudgetOf, num } from "@gscdump/engine/analysis-types";
 import { defineAnalyzer } from "@gscdump/engine/analyzer";
 import { comparisonOf } from "@gscdump/engine/period";
 import { enumeratePartitions } from "@gscdump/engine/planner";
@@ -48,6 +48,7 @@ const decayAnalyzer = defineAnalyzer({
 		const { current: cur, previous: prev } = comparisonOf(params);
 		const minPreviousClicks = params.minPreviousClicks ?? 50;
 		const threshold = params.threshold ?? .2;
+		const limit = params.limit ?? 2e3;
 		return {
 			sql: `
       WITH cur AS (
@@ -107,11 +108,14 @@ const decayAnalyzer = defineAnalyzer({
         LEFT JOIN cur c ON p.url = c.url
         LEFT JOIN series_by_url s ON p.url = s.url
       )
-      SELECT *
+      SELECT *, ${TOTAL_COUNT_SELECT}
       FROM joined
       WHERE declinePercent >= ? AND lostClicks > 0
       ORDER BY lostClicks DESC
-      LIMIT 2000
+      ${paginateClause({
+				limit,
+				offset: params.offset
+			})}
     `,
 			params: [
 				cur.startDate,
@@ -135,8 +139,9 @@ const decayAnalyzer = defineAnalyzer({
 			}
 		};
 	},
-	reduceSql(rows, params) {
-		const mapped = (Array.isArray(rows) ? rows : []).map((r) => ({
+	reduceSql(rows) {
+		const arr = Array.isArray(rows) ? rows : [];
+		const mapped = arr.map((r) => ({
 			page: rowString(r.page),
 			currentClicks: num(r.currentClicks),
 			previousClicks: num(r.previousClicks),
@@ -152,18 +157,18 @@ const decayAnalyzer = defineAnalyzer({
 			}))
 		}));
 		return {
-			results: paginateInMemory(mapped, {
-				limit: params.limit ?? 2e3,
-				offset: params.offset
-			}),
-			meta: { total: mapped.length }
+			results: mapped,
+			meta: {
+				total: totalCountOf(arr),
+				returned: mapped.length
+			}
 		};
 	},
 	buildRows(params) {
 		const { current, previous } = comparisonOf(params);
 		return {
-			current: pagesQueryState(current, params.limit),
-			previous: pagesQueryState(previous, params.limit)
+			current: pagesQueryState(current, fetchBudgetOf(params)),
+			previous: pagesQueryState(previous, fetchBudgetOf(params))
 		};
 	},
 	reduceRows(rows, params) {
@@ -178,9 +183,16 @@ const decayAnalyzer = defineAnalyzer({
 			minPreviousClicks: params.minPreviousClicks,
 			threshold: params.threshold
 		});
+		const paged = paginateInMemory(results, {
+			limit: params.limit ?? 2e3,
+			offset: params.offset
+		});
 		return {
-			results,
-			meta: { total: results.length }
+			results: paged,
+			meta: {
+				total: results.length,
+				returned: paged.length
+			}
 		};
 	}
 });

package/dist/analyzers/movers.d.mts

@@ -1,12 +1,19 @@
 import { QueriesRow } from "../types.mjs";
-export type MoversSortMetric = 'clicks' | 'impressions' | 'clicksChange' | 'impressionsChange' | 'positionChange';
+/**
+ * Sort keys. `*Delta` sorts by the absolute change, `*DeltaPercent` by the
+ * percent change. Both sort by magnitude, so large drops rank with large gains.
+ */
+export type MoversSortMetric = 'clicks' | 'impressions' | 'clicksDelta' | 'clicksDeltaPercent' | 'impressionsDelta' | 'impressionsDeltaPercent' | 'positionDelta';
+export declare const MOVERS_SORT_METRICS: readonly MoversSortMetric[];
 export interface MoversOptions {
   /** Minimum change threshold to flag. Default: 0.2 (20%) */
   changeThreshold?: number;
-  /** Minimum impressions in recent period. Default: 50 */
+  /** Minimum impressions in the larger of the two periods. Default: 50 */
   minImpressions?: number;
-  /** Metric to sort results by. Default: clicksChange */
+  /** Metric to sort results by. Default: clicksDelta */
   sortBy?: MoversSortMetric;
+  /** `desc` puts the largest change first. Default: desc */
+  sortDir?: 'asc' | 'desc';
 }
 export interface MoversInput {
   current: QueriesRow[];
@@ -19,13 +26,15 @@ export interface MoverData {
   page: string | null;
   recentClicks: number;
   recentImpressions: number;
-  recentPosition: number;
+  /** `null` when the (query, page) pair has no current-period data (it vanished). */
+  recentPosition: number | null;
   baselineClicks: number;
   baselineImpressions: number;
-  /** `null` when the keyword has no previous-period data (a genuinely new query), not "position 0". */
+  /** `null` when the (query, page) pair has no previous-period data (it is new). */
   baselinePosition: number | null;
   clicksChange: number;
   clicksChangePercent: number;
+  impressionsChange: number;
   impressionsChangePercent: number;
   /** `null` unless both recentPosition and baselinePosition exist. */
   positionChange: number | null;
@@ -36,7 +45,10 @@ export interface MoversResult {
   stable: MoverData[];
 }
 /**
- * Pure helper: identify "movers and shakers" — keywords with significant
- * recent changes between two periods.
+ * Pure helper: identify "movers and shakers" between two periods. Rows join
+ * on the full `(query, page)` identity in both directions, so a pair that
+ * vanished from the current period reports as declining, and one query on
+ * two pages yields two movers. `minImpressions` applies to the larger of the
+ * two periods, so a drop below the floor still reports.
  */
 export declare function analyzeMovers(input: MoversInput, options?: MoversOptions): MoversResult;