@gscdump/analysis 1.4.0 → 1.4.1

package/dist/analyzers/intent-atlas.mjs

@@ -0,0 +1,175 @@
+import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { defaultEndDate } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
+import { num } from "@gscdump/engine/analysis-types";
+import { daysAgoUtc } from "gscdump/dates";
+const INTENT_ATLAS_STOP_WORDS = [
+	"the",
+	"a",
+	"an",
+	"is",
+	"are",
+	"was",
+	"were",
+	"be",
+	"been",
+	"of",
+	"to",
+	"in",
+	"for",
+	"on",
+	"and",
+	"or",
+	"with",
+	"at",
+	"by",
+	"from",
+	"into",
+	"about",
+	"as",
+	"so",
+	"than",
+	"then",
+	"that",
+	"this",
+	"my",
+	"your",
+	"our",
+	"their",
+	"his",
+	"her",
+	"its",
+	"me",
+	"you",
+	"what",
+	"how",
+	"why",
+	"when",
+	"where",
+	"who",
+	"which",
+	"do",
+	"does"
+];
+const intentAtlasAnalyzer = defineAnalyzer({
+	id: "intent-atlas",
+	buildSql(params) {
+		const endDate = params.endDate ?? defaultEndDate();
+		const startDate = params.startDate ?? daysAgoUtc(90);
+		const minQueryImpressions = params.minImpressions ?? 20;
+		const minClusterSize = params.minClusterSize ?? 3;
+		const minTokenImpressions = 50;
+		const limit = params.limit ?? 200;
+		const stopList = INTENT_ATLAS_STOP_WORDS.map((w) => `'${w}'`).join(", ");
+		return {
+			sql: `
+    WITH queries AS (
+      SELECT
+        query,
+        ${METRIC_EXPR.impressions} AS impressions,
+        ${METRIC_EXPR.clicks} AS clicks,
+        ${METRIC_EXPR.position} AS position
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+        AND query IS NOT NULL AND query <> ''
+      GROUP BY query
+      HAVING SUM(impressions) >= ?
+    ),
+    tokens AS (
+      SELECT q.query, q.impressions, q.clicks, q.position,
+        LOWER(t.token) AS token
+      FROM queries q,
+        unnest(regexp_split_to_array(LOWER(q.query), '\\s+')) AS t(token)
+      WHERE LENGTH(t.token) >= 3
+        AND LOWER(t.token) NOT IN (${stopList})
+    ),
+    token_weights AS (
+      SELECT token,
+        SUM(impressions) AS token_impressions,
+        COUNT(DISTINCT query) AS query_count
+      FROM tokens
+      GROUP BY token
+      HAVING SUM(impressions) >= ${Number(minTokenImpressions)}
+    ),
+    ranked_tokens AS (
+      SELECT t.query, t.token, tw.token_impressions,
+        ROW_NUMBER() OVER (
+          PARTITION BY t.query
+          ORDER BY tw.token_impressions DESC, t.token ASC
+        ) AS rnk
+      FROM tokens t
+      JOIN token_weights tw USING (token)
+    ),
+    cluster_keys AS (
+      SELECT query,
+        array_to_string(list(token ORDER BY token), ' + ') AS cluster_key
+      FROM ranked_tokens
+      WHERE rnk <= 2
+      GROUP BY query
+      HAVING COUNT(*) >= 2
+    ),
+    clustered AS (
+      SELECT q.query, q.impressions, q.clicks, q.position, ck.cluster_key
+      FROM queries q
+      JOIN cluster_keys ck USING (query)
+    )
+    SELECT
+      cluster_key AS clusterKey,
+      COUNT(*) AS keywordCount,
+      SUM(impressions) AS totalImpressions,
+      SUM(clicks) AS totalClicks,
+      SUM(clicks) / NULLIF(SUM(impressions), 0) AS ctr,
+      SUM((position - 1) * impressions) / NULLIF(SUM(impressions), 0) + 1 AS avgPosition,
+      to_json(list({
+        'query': query,
+        'impressions': impressions,
+        'clicks': clicks,
+        'position': position
+      } ORDER BY impressions DESC)) AS keywords
+    FROM clustered
+    GROUP BY cluster_key
+    HAVING COUNT(*) >= ${Number(minClusterSize)}
+    ORDER BY totalImpressions DESC
+    LIMIT ${Number(limit)}
+  `,
+			params: [
+				startDate,
+				endDate,
+				minQueryImpressions
+			],
+			current: {
+				table: "queries",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows) {
+		const clusters = (Array.isArray(rows) ? rows : []).map((r) => ({
+			clusterKey: rowString(r.clusterKey),
+			keywordCount: num(r.keywordCount),
+			totalImpressions: num(r.totalImpressions),
+			totalClicks: num(r.totalClicks),
+			ctr: num(r.ctr),
+			avgPosition: num(r.avgPosition),
+			keywords: parseJsonRows(r.keywords).slice(0, 25).map((k) => ({
+				query: rowString(k.query),
+				impressions: num(k.impressions),
+				clicks: num(k.clicks),
+				position: num(k.position)
+			}))
+		}));
+		const totalImpressions = clusters.reduce((s, c) => s + c.totalImpressions, 0);
+		const totalKeywords = clusters.reduce((s, c) => s + c.keywordCount, 0);
+		return {
+			results: clusters,
+			meta: {
+				total: clusters.length,
+				totalImpressions,
+				totalKeywords
+			}
+		};
+	}
+});
+export { intentAtlasAnalyzer };

package/dist/analyzers/keyword-breadth.mjs

@@ -0,0 +1,115 @@
+import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
+import { num } from "@gscdump/engine/analysis-types";
+const keywordBreadthAnalyzer = defineAnalyzer({
+	id: "keyword-breadth",
+	buildSql(params) {
+		const { startDate, endDate } = periodOf(params);
+		return {
+			sql: `
+    WITH per_page AS (
+      SELECT
+        url,
+        CAST(COUNT(DISTINCT query) AS DOUBLE) AS keywordCount,
+        ${METRIC_EXPR.clicks} AS clicks,
+        ${METRIC_EXPR.impressions} AS impressions
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ? AND impressions > 0
+      GROUP BY url
+    ),
+    bucketed AS (
+      SELECT
+        CASE
+          WHEN keywordCount = 1 THEN '1'
+          WHEN keywordCount BETWEEN 2 AND 5 THEN '2-5'
+          WHEN keywordCount BETWEEN 6 AND 15 THEN '6-15'
+          WHEN keywordCount BETWEEN 16 AND 50 THEN '16-50'
+          ELSE '50+'
+        END AS bucket,
+        MIN(keywordCount) AS sort_key,
+        CAST(COUNT(*) AS DOUBLE) AS pageCount
+      FROM per_page
+      GROUP BY bucket
+    ),
+    fragile AS (
+      SELECT url, keywordCount, clicks, impressions
+      FROM per_page
+      WHERE keywordCount <= 2 AND clicks >= 5
+      ORDER BY clicks DESC
+      LIMIT 20
+    ),
+    authority AS (
+      SELECT url, keywordCount, clicks, impressions
+      FROM per_page
+      WHERE keywordCount >= 20
+      ORDER BY keywordCount DESC
+      LIMIT 20
+    ),
+    stats AS (
+      SELECT
+        CAST(COUNT(*) AS DOUBLE) AS totalPages,
+        CAST(AVG(keywordCount) AS DOUBLE) AS avgKeywordsPerPage,
+        CAST(SUM(CASE WHEN keywordCount <= 2 THEN 1 ELSE 0 END) AS DOUBLE) AS fragileCount,
+        CAST(SUM(CASE WHEN keywordCount >= 20 THEN 1 ELSE 0 END) AS DOUBLE) AS authorityCount
+      FROM per_page
+    )
+    SELECT
+      (SELECT to_json(list({ 'bucket': bucket, 'pageCount': pageCount, 'sortKey': sort_key })) FROM bucketed) AS distribution_json,
+      (SELECT to_json(list({ 'url': url, 'keywordCount': keywordCount, 'clicks': clicks, 'impressions': impressions })) FROM fragile) AS fragile_json,
+      (SELECT to_json(list({ 'url': url, 'keywordCount': keywordCount, 'clicks': clicks, 'impressions': impressions })) FROM authority) AS authority_json,
+      (SELECT to_json({
+        'totalPages': totalPages,
+        'avgKeywordsPerPage': avgKeywordsPerPage,
+        'fragileCount': fragileCount,
+        'authorityCount': authorityCount
+      }) FROM stats) AS stats_json
+  `,
+			params: [startDate, endDate],
+			current: {
+				table: "page_queries",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows, params) {
+		const arr = Array.isArray(rows) ? rows : [];
+		const { startDate, endDate } = periodOf(params);
+		const row = arr[0] ?? {};
+		const distribution = parseJsonRows(row.distribution_json).sort((a, b) => num(a.sortKey) - num(b.sortKey)).map((r) => ({
+			bucket: rowString(r.bucket),
+			pageCount: num(r.pageCount)
+		}));
+		const fragile = parseJsonRows(row.fragile_json).map((r) => ({
+			url: rowString(r.url),
+			keywordCount: num(r.keywordCount),
+			clicks: num(r.clicks),
+			impressions: num(r.impressions)
+		}));
+		const authority = parseJsonRows(row.authority_json).map((r) => ({
+			url: rowString(r.url),
+			keywordCount: num(r.keywordCount),
+			clicks: num(r.clicks),
+			impressions: num(r.impressions)
+		}));
+		const stats = typeof row.stats_json === "string" ? JSON.parse(row.stats_json) : row.stats_json ?? {};
+		return {
+			results: distribution,
+			meta: {
+				fragilePages: fragile,
+				authorityPages: authority,
+				summary: {
+					totalPages: num(stats.totalPages),
+					avgKeywordsPerPage: num(stats.avgKeywordsPerPage),
+					fragileCount: num(stats.fragileCount),
+					authorityCount: num(stats.authorityCount)
+				},
+				startDate,
+				endDate
+			}
+		};
+	}
+});
+export { keywordBreadthAnalyzer };

package/dist/analyzers/long-tail.mjs

@@ -0,0 +1,150 @@
+import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
+import { num } from "@gscdump/engine/analysis-types";
+function downsampleLogRank(points) {
+	const toPoint = (p) => ({
+		rank: num(p.rank),
+		impressions: num(p.impressions),
+		clicks: num(p.clicks),
+		query: rowString(p.query)
+	});
+	if (points.length <= 80) return points.map(toPoint);
+	const sampled = points.slice(0, 10).map(toPoint);
+	let nextThreshold = 1.15;
+	for (let i = 10; i < points.length; i++) {
+		const point = points[i];
+		if (num(point.rank) >= nextThreshold) {
+			sampled.push(toPoint(point));
+			nextThreshold *= 1.15;
+		}
+	}
+	return sampled;
+}
+const longTailAnalyzer = defineAnalyzer({
+	id: "long-tail",
+	buildSql(params) {
+		const { startDate, endDate } = periodOf(params);
+		const minQueries = 10;
+		const minQueryImpressions = params.minImpressions ?? 5;
+		const limit = params.limit ?? 100;
+		return {
+			sql: `
+    WITH pq AS (
+      SELECT
+        url AS page,
+        query,
+        ${METRIC_EXPR.impressions} AS impressions,
+        ${METRIC_EXPR.clicks} AS clicks
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+        AND query IS NOT NULL AND query <> ''
+        AND url IS NOT NULL AND url <> ''
+      GROUP BY url, query
+      HAVING SUM(impressions) >= ?
+    ),
+    ranked AS (
+      SELECT
+        page, query, impressions, clicks,
+        ROW_NUMBER() OVER (PARTITION BY page ORDER BY impressions DESC, query ASC) AS rnk
+      FROM pq
+    ),
+    log_space AS (
+      SELECT *,
+        LN(rnk) AS log_rank,
+        LN(impressions) AS log_impr
+      FROM ranked
+    ),
+    fit AS (
+      SELECT
+        page,
+        COUNT(*) AS query_count,
+        SUM(impressions) AS total_impressions,
+        SUM(clicks) AS total_clicks,
+        REGR_SLOPE(log_impr, log_rank) AS slope,
+        REGR_INTERCEPT(log_impr, log_rank) AS intercept,
+        REGR_R2(log_impr, log_rank) AS r2,
+        MAX(impressions) AS head_impressions,
+        MAX(CASE WHEN rnk = 1 THEN impressions END) / NULLIF(SUM(impressions), 0) AS head_share
+      FROM log_space
+      GROUP BY page
+      HAVING COUNT(*) >= ${Number(minQueries)}
+    ),
+    scatter AS (
+      SELECT
+        l.page,
+        to_json(list({
+          'rank': l.rnk,
+          'impressions': l.impressions,
+          'clicks': l.clicks,
+          'query': l.query
+        } ORDER BY l.rnk)) AS pointsJson
+      FROM log_space l
+      JOIN fit f USING (page)
+      GROUP BY l.page
+    )
+    SELECT
+      f.page,
+      f.query_count AS queryCount,
+      f.total_impressions AS totalImpressions,
+      f.total_clicks AS totalClicks,
+      f.slope AS slope,
+      f.intercept AS intercept,
+      f.r2 AS r2,
+      f.head_impressions AS headImpressions,
+      f.head_share AS headShare,
+      s.pointsJson AS pointsJson,
+      CASE
+        WHEN f.slope > -0.6 THEN 'flat-tail'
+        WHEN f.slope > -1.2 THEN 'balanced'
+        ELSE 'head-heavy'
+      END AS fingerprint
+    FROM fit f
+    LEFT JOIN scatter s USING (page)
+    ORDER BY f.total_impressions DESC
+    LIMIT ${Number(limit)}
+  `,
+			params: [
+				startDate,
+				endDate,
+				minQueryImpressions
+			],
+			current: {
+				table: "page_queries",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows) {
+		const results = (Array.isArray(rows) ? rows : []).map((r) => ({
+			page: rowString(r.page),
+			queryCount: num(r.queryCount),
+			totalImpressions: num(r.totalImpressions),
+			totalClicks: num(r.totalClicks),
+			slope: num(r.slope),
+			intercept: num(r.intercept),
+			r2: num(r.r2),
+			headImpressions: num(r.headImpressions),
+			headShare: num(r.headShare),
+			fingerprint: rowString(r.fingerprint),
+			points: downsampleLogRank(parseJsonRows(r.pointsJson))
+		}));
+		const counts = {
+			"flat-tail": 0,
+			"balanced": 0,
+			"head-heavy": 0
+		};
+		for (const r of results) counts[r.fingerprint]++;
+		return {
+			results,
+			meta: {
+				total: results.length,
+				fingerprints: counts,
+				avgSlope: results.length > 0 ? results.reduce((s, r) => s + r.slope, 0) / results.length : 0
+			}
+		};
+	}
+});
+export { longTailAnalyzer };

package/dist/analyzers/movers.d.mts

@@ -0,0 +1,43 @@
+import { QueriesRow } from "../types.mjs";
+type MoversSortMetric = 'clicks' | 'impressions' | 'clicksChange' | 'impressionsChange' | 'positionChange';
+interface MoversOptions {
+  /** Minimum change threshold to flag. Default: 0.2 (20%) */
+  changeThreshold?: number;
+  /** Minimum impressions in recent period. Default: 50 */
+  minImpressions?: number;
+  /** Metric to sort results by. Default: clicksChange */
+  sortBy?: MoversSortMetric;
+}
+interface MoversInput {
+  current: QueriesRow[];
+  previous: QueriesRow[];
+  /** If periods have different lengths, provide normalization factor (previous/current) */
+  normalizationFactor?: number;
+}
+interface MoverData {
+  keyword: string;
+  page: string | null;
+  recentClicks: number;
+  recentImpressions: number;
+  recentPosition: number;
+  baselineClicks: number;
+  baselineImpressions: number;
+  /** `null` when the keyword has no previous-period data (a genuinely new query), not "position 0". */
+  baselinePosition: number | null;
+  clicksChange: number;
+  clicksChangePercent: number;
+  impressionsChangePercent: number;
+  /** `null` unless both recentPosition and baselinePosition exist. */
+  positionChange: number | null;
+}
+interface MoversResult {
+  rising: MoverData[];
+  declining: MoverData[];
+  stable: MoverData[];
+}
+/**
+ * Pure helper: identify "movers and shakers" — keywords with significant
+ * recent changes between two periods.
+ */
+declare function analyzeMovers(input: MoversInput, options?: MoversOptions): MoversResult;
+export { MoverData, MoversInput, MoversOptions, MoversResult, MoversSortMetric, analyzeMovers };