@gscdump/analysis 1.4.0 → 1.4.2

package/dist/analyzers/dark-traffic.mjs

@@ -0,0 +1,130 @@
+import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { num } from "@gscdump/engine/analysis-types";
+const darkTrafficAnalyzer = defineAnalyzer({
+	id: "dark-traffic",
+	buildSql(params) {
+		const { startDate, endDate } = periodOf(params);
+		return {
+			sql: `
+    WITH page_totals AS (
+      SELECT SUM(clicks) AS total_clicks, SUM(impressions) AS total_impressions
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+    ),
+    kw_totals AS (
+      SELECT SUM(clicks) AS total_clicks, SUM(impressions) AS total_impressions
+      FROM read_parquet({{FILES_KEYWORDS}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+    ),
+    per_page AS (
+      SELECT url, SUM(clicks) AS page_clicks
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+      GROUP BY url
+      HAVING SUM(clicks) > 0
+    ),
+    per_page_kw AS (
+      SELECT url, SUM(clicks) AS attributed_clicks, COUNT(DISTINCT query) AS kw_count
+      FROM read_parquet({{FILES_PAGE_KEYWORDS}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+      GROUP BY url
+    ),
+    page_rows AS (
+      SELECT
+        p.url AS url,
+        CAST(p.page_clicks AS DOUBLE) AS totalClicks,
+        CAST(COALESCE(k.attributed_clicks, 0) AS DOUBLE) AS attributedClicks,
+        CAST(p.page_clicks - COALESCE(k.attributed_clicks, 0) AS DOUBLE) AS darkClicks,
+        CAST(p.page_clicks - COALESCE(k.attributed_clicks, 0) AS DOUBLE)
+          / NULLIF(p.page_clicks, 0) AS darkPercent,
+        CAST(COALESCE(k.kw_count, 0) AS DOUBLE) AS keywordCount
+      FROM per_page p
+      LEFT JOIN per_page_kw k ON p.url = k.url
+      WHERE p.page_clicks - COALESCE(k.attributed_clicks, 0) > 0
+      ORDER BY darkClicks DESC
+      LIMIT 50
+    )
+    SELECT
+      (SELECT to_json({
+        'totalClicks': CAST(total_clicks AS DOUBLE),
+        'totalImpressions': CAST(total_impressions AS DOUBLE)
+      }) FROM page_totals) AS page_totals_json,
+      (SELECT to_json({
+        'attributedClicks': CAST(total_clicks AS DOUBLE),
+        'attributedImpressions': CAST(total_impressions AS DOUBLE)
+      }) FROM kw_totals) AS kw_totals_json,
+      (SELECT to_json(list({
+        'url': url,
+        'totalClicks': totalClicks,
+        'attributedClicks': attributedClicks,
+        'darkClicks': darkClicks,
+        'darkPercent': darkPercent,
+        'keywordCount': keywordCount
+      })) FROM page_rows) AS pages_json
+  `,
+			params: [
+				startDate,
+				endDate,
+				startDate,
+				endDate,
+				startDate,
+				endDate,
+				startDate,
+				endDate
+			],
+			current: {
+				table: "pages",
+				partitions: enumeratePartitions(startDate, endDate)
+			},
+			extraFiles: {
+				KEYWORDS: {
+					table: "queries",
+					partitions: enumeratePartitions(startDate, endDate)
+				},
+				PAGE_KEYWORDS: {
+					table: "page_queries",
+					partitions: enumeratePartitions(startDate, endDate)
+				}
+			}
+		};
+	},
+	reduceSql(rows, params) {
+		const arr = Array.isArray(rows) ? rows : [];
+		const { startDate, endDate } = periodOf(params);
+		const row = arr[0] ?? {};
+		const pageTotals = typeof row.page_totals_json === "string" ? JSON.parse(row.page_totals_json) : row.page_totals_json ?? {};
+		const kwTotals = typeof row.kw_totals_json === "string" ? JSON.parse(row.kw_totals_json) : row.kw_totals_json ?? {};
+		const totalClicks = num(pageTotals.totalClicks);
+		const totalImpressions = num(pageTotals.totalImpressions);
+		const attributedClicks = num(kwTotals.attributedClicks);
+		const attributedImpressions = num(kwTotals.attributedImpressions);
+		const darkClicks = Math.max(0, totalClicks - attributedClicks);
+		const darkPercent = totalClicks > 0 ? darkClicks / totalClicks : 0;
+		return {
+			results: parseJsonRows(row.pages_json).map((r) => ({
+				url: rowString(r.url),
+				totalClicks: num(r.totalClicks),
+				attributedClicks: num(r.attributedClicks),
+				darkClicks: num(r.darkClicks),
+				darkPercent: num(r.darkPercent),
+				keywordCount: num(r.keywordCount)
+			})),
+			meta: {
+				summary: {
+					totalClicks,
+					attributedClicks,
+					darkClicks,
+					darkPercent,
+					totalImpressions,
+					attributedImpressions
+				},
+				startDate,
+				endDate
+			}
+		};
+	}
+});
+export { darkTrafficAnalyzer };

package/dist/analyzers/data-detail.mjs

@@ -0,0 +1,44 @@
+import { buildDataDetailPlan, buildDataDetailRows, shapeDataDetailRowResults, shapeDataDetailRows } from "../query/analyzers.mjs";
+import "../query/index.mjs";
+import { defineAnalyzer, requireAdapter } from "@gscdump/engine/analyzer";
+const dataDetailAnalyzer = defineAnalyzer({
+	id: "data-detail",
+	sqlRequires: [
+		"executeSql",
+		"attachedTables",
+		"adapter"
+	],
+	buildSql(params, ctx) {
+		const plan = buildDataDetailPlan(params, {
+			adapter: requireAdapter(ctx, "data-detail"),
+			siteId: ctx.siteId
+		});
+		return {
+			sql: plan.sql,
+			params: plan.params,
+			current: {
+				table: plan.tableKey,
+				partitions: []
+			},
+			extraQueries: plan.extraQueries
+		};
+	},
+	reduceSql(rows, params, ctx) {
+		const { results, meta } = shapeDataDetailRows(Array.isArray(rows) ? rows : [], params, ctx.extras);
+		return {
+			results,
+			meta
+		};
+	},
+	buildRows(params) {
+		return buildDataDetailRows(params);
+	},
+	reduceRows(rows, params) {
+		const { results, meta } = shapeDataDetailRowResults(Array.isArray(rows) ? {} : rows, params);
+		return {
+			results,
+			meta
+		};
+	}
+});
+export { dataDetailAnalyzer };

package/dist/analyzers/data-query.mjs

@@ -0,0 +1,44 @@
+import { buildDataQueryPlan, buildDataQueryRows, shapeDataQueryRowResults, shapeDataQueryRows } from "../query/analyzers.mjs";
+import "../query/index.mjs";
+import { defineAnalyzer, requireAdapter } from "@gscdump/engine/analyzer";
+const dataQueryAnalyzer = defineAnalyzer({
+	id: "data-query",
+	sqlRequires: [
+		"executeSql",
+		"attachedTables",
+		"adapter"
+	],
+	buildSql(params, ctx) {
+		const plan = buildDataQueryPlan(params, {
+			adapter: requireAdapter(ctx, "data-query"),
+			siteId: ctx.siteId
+		});
+		return {
+			sql: plan.sql,
+			params: plan.params,
+			current: {
+				table: plan.tableKey,
+				partitions: []
+			},
+			extraQueries: plan.extraQueries
+		};
+	},
+	reduceSql(rows, params, ctx) {
+		const { results, meta } = shapeDataQueryRows(Array.isArray(rows) ? rows : [], params, ctx.extras);
+		return {
+			results,
+			meta
+		};
+	},
+	buildRows(params) {
+		return buildDataQueryRows(params);
+	},
+	reduceRows(rows, params) {
+		const { results, meta } = shapeDataQueryRowResults(Array.isArray(rows) ? {} : rows, params);
+		return {
+			results,
+			meta
+		};
+	}
+});
+export { dataQueryAnalyzer };

package/dist/analyzers/decay.d.mts

@@ -0,0 +1,38 @@
+import { PageRow } from "../types.mjs";
+type DecaySortMetric = 'lostClicks' | 'declinePercent' | 'currentClicks';
+interface DecayOptions {
+  /** Minimum clicks in previous period to consider. Default: 50 */
+  minPreviousClicks?: number;
+  /** Minimum decline percentage (0-1). Default: 0.2 (20%) */
+  threshold?: number;
+  /** Metric to sort results by. Default: lostClicks */
+  sortBy?: DecaySortMetric;
+}
+interface DecayInput {
+  current: PageRow[];
+  previous: PageRow[];
+}
+interface DecaySeriesPoint {
+  week: string;
+  clicks: number;
+  impressions: number;
+}
+interface DecayResult {
+  page: string;
+  currentClicks: number;
+  previousClicks: number;
+  lostClicks: number;
+  declinePercent: number;
+  /** `null` when the page had no ranking data in the current period (not "position 0"). */
+  currentPosition: number | null;
+  previousPosition: number;
+  /** `null` unless both currentPosition and previousPosition exist. */
+  positionDrop: number | null;
+  series?: DecaySeriesPoint[];
+}
+/**
+ * Pure helper: identify "decaying" content — pages that have lost
+ * significant traffic between two periods.
+ */
+declare function analyzeDecay(input: DecayInput, options?: DecayOptions): DecayResult[];
+export { DecayInput, DecayOptions, DecayResult, DecaySeriesPoint, DecaySortMetric, analyzeDecay };

package/dist/analyzers/decay.mjs

@@ -0,0 +1,187 @@
+import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
+import { pagesQueryState } from "../analyzer/adapt-rows.mjs";
+import { buildPeriodMap, createMetricSorter } from "../types.mjs";
+import { paginateInMemory } from "../analyzer/paginate.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { comparisonOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
+import { num } from "@gscdump/engine/analysis-types";
+const sortResults = createMetricSorter("lostClicks", {
+	lostClicks: "desc",
+	declinePercent: "desc",
+	currentClicks: "asc"
+});
+function analyzeDecay(input, options = {}) {
+	const { minPreviousClicks = 50, threshold = .2, sortBy = "lostClicks" } = options;
+	const currentMap = buildPeriodMap(input.current, (r) => r.page, (r) => ({
+		clicks: num(r.clicks),
+		position: num(r.position)
+	}));
+	const previousMap = buildPeriodMap(input.previous, (r) => r.page, (r) => ({
+		clicks: num(r.clicks),
+		position: num(r.position)
+	}), (r) => num(r.clicks) >= minPreviousClicks);
+	const results = [];
+	for (const [page, prev] of previousMap) {
+		const curr = currentMap.get(page);
+		const currentClicks = curr?.clicks ?? 0;
+		const currentPosition = curr?.position ?? null;
+		const lostClicks = prev.clicks - currentClicks;
+		const declinePercent = prev.clicks > 0 ? lostClicks / prev.clicks : 0;
+		if (declinePercent >= threshold && lostClicks > 0) results.push({
+			page,
+			currentClicks,
+			previousClicks: prev.clicks,
+			lostClicks,
+			declinePercent,
+			currentPosition,
+			previousPosition: prev.position,
+			positionDrop: currentPosition != null ? currentPosition - prev.position : null
+		});
+	}
+	return sortResults(results, sortBy);
+}
+const decayAnalyzer = defineAnalyzer({
+	id: "decay",
+	buildSql(params) {
+		const { current: cur, previous: prev } = comparisonOf(params);
+		const minPreviousClicks = params.minPreviousClicks ?? 50;
+		const threshold = params.threshold ?? .2;
+		return {
+			sql: `
+      WITH cur AS (
+        SELECT
+          url,
+          ${METRIC_EXPR.clicks} AS clicks,
+          ${METRIC_EXPR.position} AS position
+        FROM read_parquet({{FILES}}, union_by_name = true)
+        WHERE date >= ? AND date <= ?
+        GROUP BY url
+      ),
+      prev AS (
+        SELECT
+          url,
+          ${METRIC_EXPR.clicks} AS clicks,
+          ${METRIC_EXPR.position} AS position
+        FROM read_parquet({{FILES_PREV}}, union_by_name = true)
+        WHERE date >= ? AND date <= ?
+        GROUP BY url
+        HAVING SUM(clicks) >= ?
+      ),
+      weekly AS (
+        SELECT url, date_trunc('week', CAST(date AS DATE)) AS week,
+          ${METRIC_EXPR.clicks} AS clicks,
+          ${METRIC_EXPR.impressions} AS impressions
+        FROM (
+          SELECT url, date, clicks, impressions
+          FROM read_parquet({{FILES}}, union_by_name = true)
+          WHERE date >= ? AND date <= ?
+          UNION ALL
+          SELECT url, date, clicks, impressions
+          FROM read_parquet({{FILES_PREV}}, union_by_name = true)
+          WHERE date >= ? AND date <= ?
+        )
+        GROUP BY url, week
+      ),
+      series_by_url AS (
+        SELECT url, to_json(list({
+          'week': strftime(week, '%Y-%m-%d'),
+          'clicks': clicks,
+          'impressions': impressions
+        } ORDER BY week)) AS seriesJson
+        FROM weekly GROUP BY url
+      ),
+      joined AS (
+        SELECT
+          p.url AS page,
+          COALESCE(c.clicks, 0.0) AS currentClicks,
+          p.clicks AS previousClicks,
+          (p.clicks - COALESCE(c.clicks, 0.0)) AS lostClicks,
+          (p.clicks - COALESCE(c.clicks, 0.0)) / NULLIF(p.clicks, 0) AS declinePercent,
+          c.position AS currentPosition,
+          p.position AS previousPosition,
+          CASE WHEN c.position IS NOT NULL THEN (c.position - p.position) ELSE NULL END AS positionDrop,
+          s.seriesJson
+        FROM prev p
+        LEFT JOIN cur c ON p.url = c.url
+        LEFT JOIN series_by_url s ON p.url = s.url
+      )
+      SELECT *
+      FROM joined
+      WHERE declinePercent >= ? AND lostClicks > 0
+      ORDER BY lostClicks DESC
+      LIMIT 2000
+    `,
+			params: [
+				cur.startDate,
+				cur.endDate,
+				prev.startDate,
+				prev.endDate,
+				minPreviousClicks,
+				cur.startDate,
+				cur.endDate,
+				prev.startDate,
+				prev.endDate,
+				threshold
+			],
+			current: {
+				table: "pages",
+				partitions: enumeratePartitions(cur.startDate, cur.endDate)
+			},
+			previous: {
+				table: "pages",
+				partitions: enumeratePartitions(prev.startDate, prev.endDate)
+			}
+		};
+	},
+	reduceSql(rows, params) {
+		const mapped = (Array.isArray(rows) ? rows : []).map((r) => ({
+			page: rowString(r.page),
+			currentClicks: num(r.currentClicks),
+			previousClicks: num(r.previousClicks),
+			lostClicks: num(r.lostClicks),
+			declinePercent: num(r.declinePercent),
+			currentPosition: r.currentPosition == null ? null : num(r.currentPosition),
+			previousPosition: num(r.previousPosition),
+			positionDrop: r.positionDrop == null ? null : num(r.positionDrop),
+			series: parseJsonRows(r.seriesJson).map((s) => ({
+				week: rowString(s.week),
+				clicks: num(s.clicks),
+				impressions: num(s.impressions)
+			}))
+		}));
+		return {
+			results: paginateInMemory(mapped, {
+				limit: params.limit ?? 2e3,
+				offset: params.offset
+			}),
+			meta: { total: mapped.length }
+		};
+	},
+	buildRows(params) {
+		const { current, previous } = comparisonOf(params);
+		return {
+			current: pagesQueryState(current, params.limit),
+			previous: pagesQueryState(previous, params.limit)
+		};
+	},
+	reduceRows(rows, params) {
+		const map = rows && !Array.isArray(rows) ? rows : {
+			current: [],
+			previous: []
+		};
+		const results = analyzeDecay({
+			current: map.current ?? [],
+			previous: map.previous ?? []
+		}, {
+			minPreviousClicks: params.minPreviousClicks,
+			threshold: params.threshold
+		});
+		return {
+			results,
+			meta: { total: results.length }
+		};
+	}
+});
+export { analyzeDecay, decayAnalyzer };

package/dist/analyzers/device-gap.mjs

@@ -0,0 +1,135 @@
+import { rowString } from "../analyzer/row-values.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { num } from "@gscdump/engine/analysis-types";
+const deviceGapAnalyzer = defineAnalyzer({
+	id: "device-gap",
+	buildSql(params) {
+		const { startDate, endDate } = periodOf(params);
+		return {
+			sql: `
+    WITH raw AS (
+      SELECT
+        date,
+        COALESCE(clicks_desktop, 0) AS c_desktop,
+        COALESCE(clicks_mobile, 0) AS c_mobile,
+        COALESCE(clicks_tablet, 0) AS c_tablet,
+        COALESCE(impressions_desktop, 0) AS i_desktop,
+        COALESCE(impressions_mobile, 0) AS i_mobile,
+        COALESCE(impressions_tablet, 0) AS i_tablet,
+        COALESCE(sum_position_desktop, 0) AS p_desktop,
+        COALESCE(sum_position_mobile, 0) AS p_mobile,
+        COALESCE(sum_position_tablet, 0) AS p_tablet
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+    ),
+    device_long AS (
+      SELECT date, 'DESKTOP' AS device, c_desktop AS clicks, i_desktop AS impressions, p_desktop AS sum_position FROM raw
+      UNION ALL
+      SELECT date, 'MOBILE', c_mobile, i_mobile, p_mobile FROM raw
+      UNION ALL
+      SELECT date, 'TABLET', c_tablet, i_tablet, p_tablet FROM raw
+    )
+    SELECT
+      date,
+      device,
+      CAST(SUM(clicks) AS DOUBLE) AS clicks,
+      CAST(SUM(impressions) AS DOUBLE) AS impressions,
+      CAST(SUM(clicks) AS DOUBLE) / NULLIF(SUM(impressions), 0) AS ctr,
+      SUM(sum_position) / NULLIF(SUM(impressions), 0) + 1 AS position
+    FROM device_long
+    GROUP BY date, device
+    ORDER BY date ASC
+  `,
+			params: [startDate, endDate],
+			current: {
+				table: "dates",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows, params) {
+		const arr = Array.isArray(rows) ? rows : [];
+		const { startDate, endDate } = periodOf(params);
+		const typed = arr.map((r) => ({
+			date: rowString(r.date),
+			device: rowString(r.device).toUpperCase(),
+			clicks: num(r.clicks),
+			impressions: num(r.impressions),
+			ctr: num(r.ctr),
+			position: num(r.position)
+		}));
+		const byDate = /* @__PURE__ */ new Map();
+		for (const r of typed) {
+			const entry = byDate.get(r.date) ?? {};
+			const metrics = {
+				clicks: r.clicks,
+				impressions: r.impressions,
+				ctr: r.ctr,
+				position: r.position
+			};
+			if (r.device === "DESKTOP") entry.desktop = metrics;
+			else if (r.device === "MOBILE") entry.mobile = metrics;
+			byDate.set(r.date, entry);
+		}
+		const zero = {
+			clicks: 0,
+			impressions: 0,
+			ctr: 0,
+			position: 0
+		};
+		const daily = [...byDate.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([date, sides]) => {
+			const d = sides.desktop ?? zero;
+			const m = sides.mobile ?? zero;
+			return {
+				date,
+				desktop: d,
+				mobile: m,
+				gaps: {
+					ctrGap: d.ctr - m.ctr,
+					positionGap: m.position - d.position
+				}
+			};
+		});
+		const weekly = (start, end) => {
+			const slice = daily.slice(start, end);
+			if (slice.length === 0) return {
+				ctr: 0,
+				pos: 0
+			};
+			const sum = slice.reduce((acc, d) => ({
+				ctr: acc.ctr + d.gaps.ctrGap,
+				pos: acc.pos + d.gaps.positionGap
+			}), {
+				ctr: 0,
+				pos: 0
+			});
+			return {
+				ctr: sum.ctr / slice.length,
+				pos: sum.pos / slice.length
+			};
+		};
+		const first = weekly(0, 7);
+		const last = weekly(Math.max(0, daily.length - 7), daily.length);
+		const classify = (firstVal, lastVal) => {
+			const diff = Math.abs(lastVal) - Math.abs(firstVal);
+			if (Math.abs(diff) < .005) return "stable";
+			return diff < 0 ? "improving" : "worsening";
+		};
+		return {
+			results: daily,
+			meta: {
+				summary: {
+					avgCtrGap: daily.reduce((s, d) => s + d.gaps.ctrGap, 0) / Math.max(1, daily.length),
+					avgPositionGap: daily.reduce((s, d) => s + d.gaps.positionGap, 0) / Math.max(1, daily.length),
+					ctrGapTrend: classify(first.ctr, last.ctr),
+					positionGapTrend: classify(first.pos, last.pos)
+				},
+				startDate,
+				endDate
+			}
+		};
+	}
+});
+export { deviceGapAnalyzer };