@gscdump/analysis 1.4.0 → 1.4.2

package/dist/analyzers/movers.mjs

@@ -0,0 +1,283 @@
+import { percentDifference } from "../scoring.mjs";
+import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
+import { queriesQueryState } from "../analyzer/adapt-rows.mjs";
+import { buildPeriodMap } from "../types.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { comparisonOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
+import { num } from "@gscdump/engine/analysis-types";
+function withCanonicalFields(r) {
+	return {
+		...r,
+		clicks: r.recentClicks,
+		impressions: r.recentImpressions,
+		ctr: r.recentImpressions > 0 ? r.recentClicks / r.recentImpressions : 0,
+		position: r.recentPosition,
+		prevClicks: r.baselineClicks,
+		prevImpressions: r.baselineImpressions,
+		prevPosition: r.baselinePosition
+	};
+}
+function selectMoversDirection(rising, declining, stable, params) {
+	if (params.direction) {
+		const selected = params.direction === "rising" ? rising : declining;
+		const total = selected.length;
+		const offset = params.offset ?? 0;
+		const limit = params.limit ?? total;
+		return {
+			results: selected.slice(offset, offset + limit),
+			meta: {
+				total,
+				rising: rising.length,
+				declining: declining.length,
+				stable: stable.length
+			}
+		};
+	}
+	const combined = [...rising, ...declining];
+	return {
+		results: combined,
+		meta: {
+			total: combined.length,
+			rising: rising.length,
+			declining: declining.length,
+			stable: stable.length
+		}
+	};
+}
+function analyzeMovers(input, options = {}) {
+	const { changeThreshold = .2, minImpressions = 50, sortBy = "clicksChange" } = options;
+	const normFactor = input.normalizationFactor ?? 1;
+	const baselineMap = buildPeriodMap(input.previous, (r) => r.query, (r) => ({
+		clicks: num(r.clicks) / normFactor,
+		impressions: num(r.impressions) / normFactor,
+		position: num(r.position),
+		page: r.page ?? null
+	}));
+	const pageMap = /* @__PURE__ */ new Map();
+	for (const row of input.current) if (!pageMap.has(row.query) && row.page) pageMap.set(row.query, row.page);
+	for (const row of input.previous) if (!pageMap.has(row.query) && row.page) pageMap.set(row.query, row.page);
+	const rising = [];
+	const declining = [];
+	const stable = [];
+	for (const row of input.current) {
+		const impressions = num(row.impressions);
+		const clicks = num(row.clicks);
+		const position = num(row.position);
+		if (impressions < minImpressions) continue;
+		const baseline = baselineMap.get(row.query);
+		const baselineClicksRaw = baseline?.clicks ?? 0;
+		const baselineImpressionsRaw = baseline?.impressions ?? 0;
+		const baselinePosition = baseline?.position ?? null;
+		const clicksChangePercent = percentDifference(clicks, baselineClicksRaw);
+		const impressionsChangePercent = percentDifference(impressions, baselineImpressionsRaw);
+		const data = {
+			keyword: row.query,
+			page: pageMap.get(row.query) ?? null,
+			recentClicks: clicks,
+			recentImpressions: impressions,
+			recentPosition: position,
+			baselineClicks: Math.round(baselineClicksRaw),
+			baselineImpressions: Math.round(baselineImpressionsRaw),
+			baselinePosition,
+			clicksChange: clicks - Math.round(baselineClicksRaw),
+			clicksChangePercent,
+			impressionsChangePercent,
+			positionChange: baselinePosition != null ? position - baselinePosition : null
+		};
+		const absChange = Math.abs(clicksChangePercent / 100);
+		if (clicksChangePercent > 0 && absChange >= changeThreshold) rising.push(data);
+		else if (clicksChangePercent < 0 && absChange >= changeThreshold) declining.push(data);
+		else stable.push(data);
+	}
+	const sortFn = (a, b) => {
+		switch (sortBy) {
+			case "clicks": return b.recentClicks - a.recentClicks;
+			case "impressions": return b.recentImpressions - a.recentImpressions;
+			case "clicksChange": return Math.abs(b.clicksChangePercent) - Math.abs(a.clicksChangePercent);
+			case "impressionsChange": return Math.abs(b.impressionsChangePercent) - Math.abs(a.impressionsChangePercent);
+			case "positionChange": return Math.abs(b.positionChange ?? 0) - Math.abs(a.positionChange ?? 0);
+			default: return Math.abs(b.clicksChangePercent) - Math.abs(a.clicksChangePercent);
+		}
+	};
+	rising.sort(sortFn);
+	declining.sort(sortFn);
+	stable.sort((a, b) => b.recentClicks - a.recentClicks);
+	return {
+		rising,
+		declining,
+		stable
+	};
+}
+const moversAnalyzer = defineAnalyzer({
+	id: "movers",
+	buildSql(params) {
+		const { current: cur, previous: prev } = comparisonOf(params);
+		const minImpressions = params.minImpressions ?? 50;
+		const changeThreshold = params.changeThreshold ?? .2;
+		const limit = params.direction ? 5e3 : params.limit ?? 2e3;
+		return {
+			sql: `
+      WITH cur AS (
+        SELECT
+          query, url,
+          ${METRIC_EXPR.clicks} AS clicks,
+          ${METRIC_EXPR.impressions} AS impressions,
+          ${METRIC_EXPR.position} AS position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= ? AND date <= ?
+        GROUP BY query, url
+      ),
+      prev AS (
+        SELECT
+          query, url,
+          ${METRIC_EXPR.clicks} AS clicks,
+          ${METRIC_EXPR.impressions} AS impressions,
+          ${METRIC_EXPR.position} AS position
+        FROM read_parquet({{FILES_PREV}}, union_by_name = true)
+        WHERE date >= ? AND date <= ?
+        GROUP BY query, url
+      ),
+      weekly AS (
+        SELECT query, url, date_trunc('week', CAST(date AS DATE)) AS week,
+          ${METRIC_EXPR.clicks} AS clicks,
+          ${METRIC_EXPR.impressions} AS impressions
+        FROM (
+          SELECT query, url, date, clicks, impressions
+          FROM read_parquet({{FILES}}, union_by_name = true)
+          WHERE date >= ? AND date <= ?
+          UNION ALL
+          SELECT query, url, date, clicks, impressions
+          FROM read_parquet({{FILES_PREV}}, union_by_name = true)
+          WHERE date >= ? AND date <= ?
+        )
+        GROUP BY query, url, week
+      ),
+      series_by_entity AS (
+        SELECT query, url, to_json(list({
+          'week': strftime(week, '%Y-%m-%d'),
+          'clicks': clicks,
+          'impressions': impressions
+        } ORDER BY week)) AS seriesJson
+        FROM weekly GROUP BY query, url
+      ),
+      joined AS (
+        SELECT
+          c.query AS keyword,
+          c.url AS page,
+          c.clicks AS recentClicks,
+          c.impressions AS recentImpressions,
+          c.position AS recentPosition,
+          COALESCE(p.clicks, 0.0) AS baselineClicks,
+          COALESCE(p.impressions, 0.0) AS baselineImpressions,
+          p.position AS baselinePosition,
+          (c.clicks - COALESCE(p.clicks, 0.0)) AS clicksChange,
+          CASE
+            WHEN COALESCE(p.clicks, 0.0) = 0 THEN CASE WHEN c.clicks > 0 THEN 100.0 ELSE 0.0 END
+            ELSE (c.clicks - p.clicks) * 100.0 / p.clicks
+          END AS clicksChangePercent,
+          CASE
+            WHEN COALESCE(p.impressions, 0.0) = 0 THEN CASE WHEN c.impressions > 0 THEN 100.0 ELSE 0.0 END
+            ELSE (c.impressions - p.impressions) * 100.0 / p.impressions
+          END AS impressionsChangePercent,
+          CASE WHEN p.position IS NOT NULL THEN (c.position - p.position) ELSE NULL END AS positionChange,
+          s.seriesJson
+        FROM cur c
+        LEFT JOIN prev p ON c.query = p.query AND c.url = p.url
+        LEFT JOIN series_by_entity s ON c.query = s.query AND c.url = s.url
+        WHERE c.impressions >= ?
+      )
+      SELECT *,
+        CASE
+          WHEN clicksChangePercent > 0 AND ABS(clicksChangePercent) / 100.0 >= ? THEN 'rising'
+          WHEN clicksChangePercent < 0 AND ABS(clicksChangePercent) / 100.0 >= ? THEN 'declining'
+          ELSE 'stable'
+        END AS direction
+      FROM joined
+      ORDER BY ABS(clicksChangePercent) DESC
+      LIMIT ${Number(limit)}
+    `,
+			params: [
+				cur.startDate,
+				cur.endDate,
+				prev.startDate,
+				prev.endDate,
+				cur.startDate,
+				cur.endDate,
+				prev.startDate,
+				prev.endDate,
+				minImpressions,
+				changeThreshold,
+				changeThreshold
+			],
+			current: {
+				table: "page_queries",
+				partitions: enumeratePartitions(cur.startDate, cur.endDate)
+			},
+			previous: {
+				table: "page_queries",
+				partitions: enumeratePartitions(prev.startDate, prev.endDate)
+			}
+		};
+	},
+	reduceSql(rows, params) {
+		const normalized = (Array.isArray(rows) ? rows : []).map((r) => {
+			const recentClicks = num(r.recentClicks);
+			const recentImpressions = num(r.recentImpressions);
+			const recentPosition = num(r.recentPosition);
+			const baselineClicks = Math.round(num(r.baselineClicks));
+			const baselineImpressions = Math.round(num(r.baselineImpressions));
+			const baselinePosition = r.baselinePosition == null ? null : num(r.baselinePosition);
+			return {
+				keyword: rowString(r.keyword),
+				page: r.page == null ? null : rowString(r.page),
+				recentClicks,
+				recentImpressions,
+				recentPosition,
+				baselineClicks,
+				baselineImpressions,
+				baselinePosition,
+				clicksChange: num(r.clicksChange),
+				clicksChangePercent: num(r.clicksChangePercent),
+				impressionsChangePercent: num(r.impressionsChangePercent),
+				positionChange: r.positionChange == null ? null : num(r.positionChange),
+				direction: rowString(r.direction),
+				series: parseJsonRows(r.seriesJson).map((s) => ({
+					week: rowString(s.week),
+					clicks: num(s.clicks),
+					impressions: num(s.impressions)
+				}))
+			};
+		}).map(withCanonicalFields);
+		return selectMoversDirection(normalized.filter((r) => r.direction === "rising"), normalized.filter((r) => r.direction === "declining"), normalized.filter((r) => r.direction === "stable"), params);
+	},
+	buildRows(params) {
+		const { current, previous } = comparisonOf(params);
+		return {
+			current: queriesQueryState(current, params.limit),
+			previous: queriesQueryState(previous, params.limit)
+		};
+	},
+	reduceRows(rows, params) {
+		const map = rows && !Array.isArray(rows) ? rows : {
+			current: [],
+			previous: []
+		};
+		const result = analyzeMovers({
+			current: map.current ?? [],
+			previous: map.previous ?? []
+		}, {
+			changeThreshold: params.changeThreshold,
+			minImpressions: params.minImpressions
+		});
+		return selectMoversDirection(result.rising.map((r) => withCanonicalFields({
+			...r,
+			direction: "rising"
+		})), result.declining.map((r) => withCanonicalFields({
+			...r,
+			direction: "declining"
+		})), [], params);
+	}
+});
+export { analyzeMovers, moversAnalyzer };

package/dist/analyzers/opportunity.d.mts

@@ -0,0 +1,32 @@
+import { QueriesRow } from "../types.mjs";
+type OpportunitySortMetric = 'opportunityScore' | 'potentialClicks' | 'impressions' | 'position';
+interface OpportunityFactors {
+  positionScore: number;
+  impressionScore: number;
+  ctrGapScore: number;
+}
+interface OpportunityResult {
+  keyword: string;
+  page: string | null;
+  clicks: number;
+  impressions: number;
+  ctr: number;
+  position: number;
+  opportunityScore: number;
+  potentialClicks: number;
+  factors: OpportunityFactors;
+}
+interface OpportunityWeights {
+  position?: number;
+  impressions?: number;
+  ctrGap?: number;
+}
+interface OpportunityOptions {
+  minImpressions?: number;
+  weights?: OpportunityWeights;
+  sortBy?: OpportunitySortMetric;
+  limit?: number;
+}
+/** Pure opportunity scorer shared by hosted and package analyzer callers. */
+declare function analyzeOpportunity(keywords: QueriesRow[], options?: OpportunityOptions): OpportunityResult[];
+export { OpportunityFactors, OpportunityOptions, OpportunityResult, OpportunitySortMetric, OpportunityWeights, analyzeOpportunity };

package/dist/analyzers/opportunity.mjs

@@ -0,0 +1,212 @@
+import { queriesQueryState } from "../analyzer/adapt-rows.mjs";
+import { paginateClause, paginateSortedInMemory } from "../analyzer/paginate.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
+import { num } from "@gscdump/engine/analysis-types";
+const EXPECTED_CTR_BY_POSITION = {
+	1: .3,
+	2: .15,
+	3: .1,
+	4: .07,
+	5: .05,
+	6: .04,
+	7: .03,
+	8: .025,
+	9: .02,
+	10: .015
+};
+function getExpectedCtr(position) {
+	return EXPECTED_CTR_BY_POSITION[Math.round(Math.max(1, Math.min(position, 10)))] || .01;
+}
+function calculatePositionScore(position) {
+	if (position <= 3) return .2;
+	if (position > 50) return .1;
+	const distance = Math.abs(position - 11);
+	return Math.max(0, 1 - distance / 15);
+}
+function calculateImpressionScore(impressions) {
+	if (impressions <= 0) return 0;
+	return Math.min(Math.log10(impressions) / 5, 1);
+}
+function calculateCtrGapScore(actualCtr, position) {
+	const expectedCtr = getExpectedCtr(position);
+	if (actualCtr >= expectedCtr) return 0;
+	const gap = expectedCtr - actualCtr;
+	return Math.min(gap / expectedCtr, 1);
+}
+const SORT_DIR = {
+	opportunityScore: "desc",
+	potentialClicks: "desc",
+	impressions: "desc",
+	position: "asc"
+};
+function analyzeOpportunity(keywords, options = {}) {
+	const minImpressions = options.minImpressions ?? 100;
+	const positionWeight = options.weights?.position ?? 1;
+	const impressionsWeight = options.weights?.impressions ?? 1;
+	const ctrGapWeight = options.weights?.ctrGap ?? 1;
+	const totalWeight = positionWeight + impressionsWeight + ctrGapWeight;
+	const sortBy = options.sortBy ?? "opportunityScore";
+	const results = [];
+	for (const row of keywords) {
+		const impressions = num(row.impressions);
+		if (impressions < minImpressions) continue;
+		const position = num(row.position);
+		const ctr = num(row.ctr);
+		const clicks = num(row.clicks);
+		const positionScore = calculatePositionScore(position);
+		const impressionScore = calculateImpressionScore(impressions);
+		const ctrGapScore = calculateCtrGapScore(ctr, position);
+		const weightedProduct = positionScore ** positionWeight * impressionScore ** impressionsWeight * ctrGapScore ** ctrGapWeight;
+		const opportunityScore = Math.round(weightedProduct ** (1 / totalWeight) * 100);
+		const potentialClicks = Math.round(impressions * getExpectedCtr(Math.min(3, position)));
+		results.push({
+			keyword: row.query,
+			page: row.page ?? null,
+			clicks,
+			impressions,
+			ctr,
+			position,
+			opportunityScore,
+			potentialClicks,
+			factors: {
+				positionScore,
+				impressionScore,
+				ctrGapScore
+			}
+		});
+	}
+	const direction = SORT_DIR[sortBy];
+	results.sort((left, right) => direction === "asc" ? left[sortBy] - right[sortBy] : right[sortBy] - left[sortBy]);
+	return options.limit ? results.slice(0, options.limit) : results;
+}
+const opportunityAnalyzer = defineAnalyzer({
+	id: "opportunity",
+	buildSql(params) {
+		const { startDate, endDate } = periodOf(params);
+		const minImpressions = params.minImpressions ?? 100;
+		const w1 = 1;
+		const w2 = 1;
+		const w3 = 1;
+		const totalW = 3;
+		const limit = params.limit ?? 1e3;
+		return {
+			sql: `
+    WITH agg AS (
+      SELECT
+        query AS keyword,
+        url AS page,
+        ${METRIC_EXPR.clicks} AS clicks,
+        ${METRIC_EXPR.impressions} AS impressions,
+        ${METRIC_EXPR.ctr} AS ctr,
+        ${METRIC_EXPR.position} AS position
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+      GROUP BY query, url
+      HAVING SUM(impressions) >= ?
+    ),
+    scored AS (
+      SELECT
+        keyword, page, clicks, impressions, ctr, position,
+        CASE
+          WHEN position <= 3 THEN 0.2
+          WHEN position > 50 THEN 0.1
+          ELSE GREATEST(0.0, 1.0 - ABS(position - 11.0) / 15.0)
+        END AS positionScore,
+        CASE WHEN impressions <= 0 THEN 0.0 ELSE LEAST(LOG10(impressions) / 5.0, 1.0) END AS impressionScore,
+        CASE CAST(ROUND(GREATEST(LEAST(position, 10.0), 1.0)) AS INTEGER)
+          WHEN 1 THEN 0.30
+          WHEN 2 THEN 0.15
+          WHEN 3 THEN 0.10
+          WHEN 4 THEN 0.07
+          WHEN 5 THEN 0.05
+          WHEN 6 THEN 0.04
+          WHEN 7 THEN 0.03
+          WHEN 8 THEN 0.025
+          WHEN 9 THEN 0.02
+          WHEN 10 THEN 0.015
+          ELSE 0.01
+        END AS expectedCtr
+      FROM agg
+    ),
+    gapped AS (
+      SELECT
+        *,
+        CASE WHEN ctr >= expectedCtr THEN 0.0 ELSE LEAST((expectedCtr - ctr) / expectedCtr, 1.0) END AS ctrGapScore
+      FROM scored
+    )
+    SELECT
+      keyword, page, clicks, impressions, ctr, position,
+      CAST(ROUND(POWER(
+        POWER(positionScore, ${w1}) * POWER(impressionScore, ${w2}) * POWER(ctrGapScore, ${w3}),
+        1.0 / ${totalW}
+      ) * 100) AS DOUBLE) AS opportunityScore,
+      CAST(ROUND(impressions * (
+        CASE CAST(ROUND(GREATEST(LEAST(position, 3.0), 1.0)) AS INTEGER)
+          WHEN 1 THEN 0.30
+          WHEN 2 THEN 0.15
+          WHEN 3 THEN 0.10
+          ELSE 0.10
+        END
+      )) AS DOUBLE) AS potentialClicks,
+      positionScore, impressionScore, ctrGapScore
+    FROM gapped
+    ORDER BY opportunityScore DESC
+    ${paginateClause({
+				limit,
+				offset: params.offset
+			})}
+  `,
+			params: [
+				startDate,
+				endDate,
+				minImpressions
+			],
+			current: {
+				table: "page_queries",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows) {
+		const arr = Array.isArray(rows) ? rows : [];
+		return {
+			results: arr.map((r) => ({
+				keyword: r.keyword == null ? "" : String(r.keyword),
+				page: r.page == null ? null : String(r.page),
+				clicks: num(r.clicks),
+				impressions: num(r.impressions),
+				ctr: num(r.ctr),
+				position: num(r.position),
+				opportunityScore: num(r.opportunityScore),
+				potentialClicks: num(r.potentialClicks),
+				factors: {
+					positionScore: num(r.positionScore),
+					impressionScore: num(r.impressionScore),
+					ctrGapScore: num(r.ctrGapScore)
+				}
+			})),
+			meta: { total: arr.length }
+		};
+	},
+	buildRows(params) {
+		return { queries: queriesQueryState(periodOf(params), params.limit) };
+	},
+	reduceRows(rows, params) {
+		const results = analyzeOpportunity((Array.isArray(rows) ? rows : []) ?? [], { minImpressions: params.minImpressions });
+		const paged = paginateSortedInMemory(results, {
+			limit: params.limit,
+			offset: params.offset
+		}, (left, right) => right.opportunityScore - left.opportunityScore);
+		return {
+			results: paged,
+			meta: {
+				total: results.length,
+				returned: paged.length
+			}
+		};
+	}
+});
+export { analyzeOpportunity, opportunityAnalyzer };

package/dist/analyzers/position-distribution.mjs

@@ -0,0 +1,57 @@
+import { rowString } from "../analyzer/row-values.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { num } from "@gscdump/engine/analysis-types";
+const positionDistributionAnalyzer = defineAnalyzer({
+	id: "position-distribution",
+	buildSql(params) {
+		const { startDate, endDate } = periodOf(params);
+		return {
+			sql: `
+    WITH pos AS (
+      SELECT
+        date,
+        (sum_position / NULLIF(impressions, 0) + 1) AS avg_pos
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ? AND impressions > 0
+    )
+    SELECT
+      date,
+      CAST(SUM(CASE WHEN avg_pos <= 3 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_1_3,
+      CAST(SUM(CASE WHEN avg_pos > 3 AND avg_pos <= 10 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_4_10,
+      CAST(SUM(CASE WHEN avg_pos > 10 AND avg_pos <= 20 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_11_20,
+      CAST(SUM(CASE WHEN avg_pos > 20 THEN 1 ELSE 0 END) AS DOUBLE) AS pos_20_plus,
+      CAST(COUNT(*) AS DOUBLE) AS total
+    FROM pos
+    GROUP BY date
+    ORDER BY date ASC
+  `,
+			params: [startDate, endDate],
+			current: {
+				table: "queries",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows, params) {
+		const arr = Array.isArray(rows) ? rows : [];
+		const { startDate, endDate } = periodOf(params);
+		return {
+			results: arr.map((r) => ({
+				date: rowString(r.date),
+				pos_1_3: num(r.pos_1_3),
+				pos_4_10: num(r.pos_4_10),
+				pos_11_20: num(r.pos_11_20),
+				pos_20_plus: num(r.pos_20_plus),
+				total: num(r.total)
+			})),
+			meta: {
+				total: arr.length,
+				startDate,
+				endDate
+			}
+		};
+	}
+});
+export { positionDistributionAnalyzer };