@gscdump/analysis 1.4.0 → 1.4.2

package/dist/analyzers/bipartite-pagerank.mjs

@@ -0,0 +1,243 @@
+import { parseJsonRows, rowString } from "../analyzer/row-values.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { num } from "@gscdump/engine/analysis-types";
+const BIPARTITE_PAGERANK_ITERATIONS = 25;
+const BIPARTITE_PAGERANK_DAMPING = .85;
+const bipartitePagerankAnalyzer = defineAnalyzer({
+	id: "bipartite-pagerank",
+	buildSql(params) {
+		const { startDate, endDate } = periodOf(params);
+		const minImpressions = params.minImpressions ?? 50;
+		const topQueries = 1e3;
+		const topUrls = 500;
+		const limit = params.limit ?? 50;
+		const bridgingEdgeThreshold = .05;
+		const anchoringEdgeThreshold = .05;
+		const iterations = BIPARTITE_PAGERANK_ITERATIONS;
+		const d = BIPARTITE_PAGERANK_DAMPING;
+		const iterCtes = [];
+		for (let i = 1; i <= iterations; i++) iterCtes.push(`
+    ranks_${i} AS (
+      SELECT
+        'q' AS kind,
+        e.qid AS id,
+        (1.0 - ${d}) / (SELECT n FROM query_count)
+          + ${d} * SUM(e.w_u_to_q * r.rank) AS rank
+      FROM u_to_q_weights e
+      JOIN ranks_${i - 1} r ON r.kind = 'u' AND r.id = e.uid
+      GROUP BY e.qid
+      UNION ALL
+      SELECT
+        'u' AS kind,
+        e.uid AS id,
+        (1.0 - ${d}) / (SELECT n FROM url_count)
+          + ${d} * SUM(e.w_q_to_u * r.rank) AS rank
+      FROM q_to_u_weights e
+      JOIN ranks_${i - 1} r ON r.kind = 'q' AND r.id = e.qid
+      GROUP BY e.uid
+    )`);
+		const deltaParts = [];
+		for (let i = 1; i <= iterations; i++) deltaParts.push(`
+      SELECT ${i} AS step,
+        (SELECT COALESCE(SUM(ABS(a.rank - b.rank)), 0.0)
+         FROM ranks_${i} a
+         JOIN ranks_${i - 1} b USING (kind, id)) AS l1`);
+		return {
+			sql: `
+    WITH edges0 AS (
+      SELECT
+        query AS qid,
+        url   AS uid,
+        CAST(SUM(impressions) AS DOUBLE) AS impressions
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+        AND query IS NOT NULL AND query <> ''
+        AND url   IS NOT NULL AND url   <> ''
+      GROUP BY query, url
+      HAVING SUM(impressions) >= ?
+    ),
+    -- Top-N caps per side keep the iteration tractable.
+    query_totals AS (
+      SELECT qid, SUM(impressions) AS tot
+      FROM edges0 GROUP BY qid
+    ),
+    url_totals AS (
+      SELECT uid, SUM(impressions) AS tot
+      FROM edges0 GROUP BY uid
+    ),
+    top_queries AS (
+      SELECT qid FROM query_totals
+      ORDER BY tot DESC, qid ASC LIMIT ${Number(topQueries)}
+    ),
+    top_urls AS (
+      SELECT uid FROM url_totals
+      ORDER BY tot DESC, uid ASC LIMIT ${Number(topUrls)}
+    ),
+    edges AS (
+      SELECT e.qid, e.uid, e.impressions
+      FROM edges0 e
+      JOIN top_queries tq USING (qid)
+      JOIN top_urls    tu USING (uid)
+    ),
+    query_nodes AS (SELECT DISTINCT qid FROM edges),
+    url_nodes   AS (SELECT DISTINCT uid FROM edges),
+    query_count AS (SELECT GREATEST(COUNT(*), 1) AS n FROM query_nodes),
+    url_count   AS (SELECT GREATEST(COUNT(*), 1) AS n FROM url_nodes),
+    -- Row-stochastic transition weights in each direction. For q->u the
+    -- weights out of a query sum to 1; symmetric for u->q.
+    q_out AS (SELECT qid, SUM(impressions) AS s FROM edges GROUP BY qid),
+    u_out AS (SELECT uid, SUM(impressions) AS s FROM edges GROUP BY uid),
+    q_to_u_weights AS (
+      SELECT e.qid, e.uid,
+        e.impressions / NULLIF(q.s, 0) AS w_q_to_u
+      FROM edges e JOIN q_out q USING (qid)
+    ),
+    u_to_q_weights AS (
+      SELECT e.qid, e.uid,
+        e.impressions / NULLIF(u.s, 0) AS w_u_to_q
+      FROM edges e JOIN u_out u USING (uid)
+    ),
+    -- Seed: uniform distribution per side. Total mass = 2 (one unit per side).
+    ranks_0 AS (
+      SELECT 'q' AS kind, q.qid AS id, 1.0 / (SELECT n FROM query_count) AS rank
+      FROM query_nodes q
+      UNION ALL
+      SELECT 'u' AS kind, u.uid AS id, 1.0 / (SELECT n FROM url_count) AS rank
+      FROM url_nodes u
+    ),
+    ${iterCtes.join(",\n")},
+    final_ranks AS (SELECT * FROM ranks_${iterations}),
+    -- Hub/anchor diagnostics computed from raw edge mass (not rank). A
+    -- query "bridges" URLs it sends >= ${bridgingEdgeThreshold} of its mass
+    -- to; a URL "anchors" queries that contribute >= ${anchoringEdgeThreshold}
+    -- of its incoming mass.
+    q_bridging AS (
+      SELECT qid, COUNT(*) AS bridging
+      FROM q_to_u_weights
+      WHERE w_q_to_u >= ${bridgingEdgeThreshold}
+      GROUP BY qid
+    ),
+    u_anchoring AS (
+      SELECT uid, COUNT(*) AS anchoring
+      FROM u_to_q_weights
+      WHERE w_u_to_q >= ${anchoringEdgeThreshold}
+      GROUP BY uid
+    ),
+    q_degree AS (
+      SELECT qid, COUNT(*) AS degree, SUM(impressions) AS impressions
+      FROM edges GROUP BY qid
+    ),
+    u_degree AS (
+      SELECT uid, COUNT(*) AS degree, SUM(impressions) AS impressions
+      FROM edges GROUP BY uid
+    ),
+    deltas AS (
+      ${deltaParts.join("\n      UNION ALL\n")}
+    ),
+    query_rows AS (
+      SELECT
+        'query' AS kind, f.id, f.rank,
+        COALESCE(b.bridging, 0)      AS bridging,
+        0                             AS anchoring,
+        COALESCE(qd.degree, 0)       AS degree,
+        COALESCE(qd.impressions, 0)  AS impressions
+      FROM final_ranks f
+      LEFT JOIN q_bridging b ON b.qid = f.id
+      LEFT JOIN q_degree   qd ON qd.qid = f.id
+      WHERE f.kind = 'q'
+      ORDER BY f.rank DESC
+      LIMIT ${Number(limit)}
+    ),
+    url_rows AS (
+      SELECT
+        'url' AS kind, f.id, f.rank,
+        0                             AS bridging,
+        COALESCE(a.anchoring, 0)     AS anchoring,
+        COALESCE(ud.degree, 0)       AS degree,
+        COALESCE(ud.impressions, 0)  AS impressions
+      FROM final_ranks f
+      LEFT JOIN u_anchoring a ON a.uid = f.id
+      LEFT JOIN u_degree   ud ON ud.uid = f.id
+      WHERE f.kind = 'u'
+      ORDER BY f.rank DESC
+      LIMIT ${Number(limit)}
+    ),
+    nodes AS (
+      SELECT * FROM query_rows
+      UNION ALL
+      SELECT * FROM url_rows
+    ),
+    counts AS (
+      SELECT
+        (SELECT n FROM query_count) AS q_count,
+        (SELECT n FROM url_count)   AS u_count
+    ),
+    deltas_json AS (
+      SELECT to_json(list({ 'step': step, 'l1': l1 } ORDER BY step)) AS dj
+      FROM deltas
+    )
+    SELECT
+      n.kind,
+      n.id,
+      n.rank,
+      n.bridging,
+      n.anchoring,
+      n.degree,
+      n.impressions,
+      c.q_count AS queryCount,
+      c.u_count AS urlCount,
+      dj.dj     AS deltasJson
+    FROM nodes n
+    CROSS JOIN counts c
+    CROSS JOIN deltas_json dj
+    ORDER BY n.kind, n.rank DESC
+  `,
+			params: [
+				startDate,
+				endDate,
+				minImpressions
+			],
+			current: {
+				table: "page_queries",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows) {
+		const arr = Array.isArray(rows) ? rows : [];
+		const iterations = BIPARTITE_PAGERANK_ITERATIONS;
+		const d = BIPARTITE_PAGERANK_DAMPING;
+		const results = arr.map((r) => ({
+			kind: rowString(r.kind),
+			id: rowString(r.id),
+			rank: num(r.rank),
+			bridging: num(r.bridging),
+			anchoring: num(r.anchoring),
+			degree: num(r.degree),
+			impressions: num(r.impressions)
+		}));
+		const first = arr[0] ?? {};
+		const queryCount = num(first.queryCount);
+		const urlCount = num(first.urlCount);
+		const deltas = parseJsonRows(first.deltasJson).map((e) => ({
+			step: num(e.step),
+			l1: num(e.l1)
+		}));
+		const convergenceDelta = deltas.length > 0 ? deltas[deltas.length - 1].l1 : 0;
+		return {
+			results,
+			meta: {
+				total: results.length,
+				convergenceDelta,
+				iterations,
+				damping: d,
+				queryCount,
+				urlCount,
+				deltas
+			}
+		};
+	}
+});
+export { bipartitePagerankAnalyzer };

package/dist/analyzers/brand.d.mts

@@ -0,0 +1,33 @@
+import { QueriesRow } from "../types.mjs";
+interface BrandSegmentationOptions {
+  /** Brand terms to match against keywords (case-insensitive) */
+  brandTerms: string[];
+  /** Minimum impressions for a keyword to be included. Default: 10 */
+  minImpressions?: number;
+}
+interface BrandSegmentationRow {
+  query?: string;
+  keyword?: string;
+  variants?: readonly string[];
+  clicks?: number | null;
+  impressions?: number | null;
+}
+interface BrandSummary {
+  brandClicks: number;
+  nonBrandClicks: number;
+  brandShare: number;
+  brandImpressions: number;
+  nonBrandImpressions: number;
+}
+interface BrandSegmentationResult<T extends BrandSegmentationRow = QueriesRow> {
+  brand: T[];
+  nonBrand: T[];
+  summary: BrandSummary;
+}
+/**
+ * Pure helper: segment keywords into brand and non-brand based on provided
+ * brand terms. Re-exported from `@gscdump/analysis` for portable callers.
+ */
+declare function analyzeBrandSegmentation(keywords: QueriesRow[], options: BrandSegmentationOptions): BrandSegmentationResult<QueriesRow>;
+declare function analyzeBrandSegmentation<T extends BrandSegmentationRow>(keywords: T[], options: BrandSegmentationOptions): BrandSegmentationResult<T>;
+export { BrandSegmentationOptions, BrandSegmentationResult, BrandSegmentationRow, BrandSummary, analyzeBrandSegmentation };

package/dist/analyzers/brand.mjs

@@ -0,0 +1,154 @@
+import { rowString } from "../analyzer/row-values.mjs";
+import { queriesQueryState } from "../analyzer/adapt-rows.mjs";
+import { analysisErrorToException, analysisErrors } from "../errors.mjs";
+import { defineAnalyzer } from "@gscdump/engine/analyzer";
+import { periodOf } from "@gscdump/engine/period";
+import { enumeratePartitions } from "@gscdump/engine/planner";
+import { METRIC_EXPR } from "@gscdump/engine/sql-fragments";
+import { num } from "@gscdump/engine/analysis-types";
+import { err, ok, unwrapResult } from "gscdump/result";
+function escapeRegexAlt(s) {
+	return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function requireBrandTermsResult(brandTerms) {
+	return brandTerms?.length ? ok(brandTerms) : err(analysisErrors.missingBrandTerms());
+}
+function requireBrandTerms(brandTerms) {
+	return unwrapResult(requireBrandTermsResult(brandTerms), analysisErrorToException);
+}
+function analyzeBrandSegmentation(keywords, options) {
+	const { brandTerms, minImpressions = 10 } = options;
+	const lowerBrandTerms = brandTerms.map((t) => t.toLowerCase());
+	const brand = [];
+	const nonBrand = [];
+	let brandClicks = 0;
+	let nonBrandClicks = 0;
+	let brandImpressions = 0;
+	let nonBrandImpressions = 0;
+	for (const row of keywords) {
+		const impressions = num(row.impressions);
+		if (impressions < minImpressions) continue;
+		const clicks = num(row.clicks);
+		if ((row.variants?.length ? row.variants : [row.query ?? row.keyword ?? ""]).some((query) => lowerBrandTerms.some((term) => query.toLowerCase().includes(term)))) {
+			brand.push(row);
+			brandClicks += clicks;
+			brandImpressions += impressions;
+		} else {
+			nonBrand.push(row);
+			nonBrandClicks += clicks;
+			nonBrandImpressions += impressions;
+		}
+	}
+	const totalClicks = brandClicks + nonBrandClicks;
+	return {
+		brand,
+		nonBrand,
+		summary: {
+			brandClicks,
+			nonBrandClicks,
+			brandShare: totalClicks > 0 ? brandClicks / totalClicks : 0,
+			brandImpressions,
+			nonBrandImpressions
+		}
+	};
+}
+const brandAnalyzer = defineAnalyzer({
+	id: "brand",
+	buildSql(params) {
+		const brandTerms = requireBrandTerms(params.brandTerms);
+		const { startDate, endDate } = periodOf(params);
+		const minImpressions = params.minImpressions ?? 10;
+		const limit = params.limit ?? 1e4;
+		const regex = `(${brandTerms.map((t) => escapeRegexAlt(t.toLowerCase())).join("|")})`;
+		return {
+			sql: `
+    WITH agg AS (
+      SELECT
+        query,
+        url AS page,
+        ${METRIC_EXPR.clicks} AS clicks,
+        ${METRIC_EXPR.impressions} AS impressions,
+        ${METRIC_EXPR.ctr} AS ctr,
+        ${METRIC_EXPR.position} AS position
+      FROM read_parquet({{FILES}}, union_by_name = true)
+      WHERE date >= ? AND date <= ?
+      GROUP BY query, url
+      HAVING SUM(impressions) >= ?
+    )
+    SELECT
+      query, page, clicks, impressions, ctr, position,
+      CASE WHEN regexp_matches(LOWER(query), ?) THEN 'brand' ELSE 'non-brand' END AS segment
+    FROM agg
+    ORDER BY clicks DESC
+    LIMIT ${Number(limit)}
+  `,
+			params: [
+				startDate,
+				endDate,
+				minImpressions,
+				regex
+			],
+			current: {
+				table: "page_queries",
+				partitions: enumeratePartitions(startDate, endDate)
+			}
+		};
+	},
+	reduceSql(rows) {
+		const normalized = (Array.isArray(rows) ? rows : []).map((r) => ({
+			query: rowString(r.query),
+			page: r.page == null ? void 0 : rowString(r.page),
+			clicks: num(r.clicks),
+			impressions: num(r.impressions),
+			ctr: num(r.ctr),
+			position: num(r.position),
+			segment: rowString(r.segment)
+		}));
+		let brandClicks = 0;
+		let nonBrandClicks = 0;
+		let brandImpressions = 0;
+		let nonBrandImpressions = 0;
+		for (const r of normalized) if (r.segment === "brand") {
+			brandClicks += r.clicks;
+			brandImpressions += r.impressions;
+		} else {
+			nonBrandClicks += r.clicks;
+			nonBrandImpressions += r.impressions;
+		}
+		const totalClicks = brandClicks + nonBrandClicks;
+		return {
+			results: normalized,
+			meta: {
+				total: normalized.length,
+				summary: {
+					brandClicks,
+					nonBrandClicks,
+					brandShare: totalClicks > 0 ? brandClicks / totalClicks : 0,
+					brandImpressions,
+					nonBrandImpressions
+				}
+			}
+		};
+	},
+	buildRows(params) {
+		return { queries: queriesQueryState(periodOf(params), params.limit) };
+	},
+	reduceRows(rows, params) {
+		const brandTerms = requireBrandTerms(params.brandTerms);
+		const result = analyzeBrandSegmentation(Array.isArray(rows) ? rows : [], {
+			brandTerms,
+			minImpressions: params.minImpressions
+		});
+		return {
+			results: [...result.brand.map((r) => ({
+				...r,
+				segment: "brand"
+			})), ...result.nonBrand.map((r) => ({
+				...r,
+				segment: "non-brand"
+			}))],
+			meta: { summary: result.summary }
+		};
+	}
+});
+export { analyzeBrandSegmentation, brandAnalyzer };

package/dist/analyzers/cannibalization.d.mts

@@ -0,0 +1,58 @@
+import { QueryPageRow, SortOrder } from "../types.mjs";
+type CannibalizationSortMetric = 'clicks' | 'impressions' | 'positionSpread' | 'pageCount';
+interface CannibalizationOptions {
+  /** Minimum impressions for a query to be considered. Default: 10 */
+  minImpressions?: number;
+  /** Maximum position spread to flag as cannibalization. Default: 10 */
+  maxPositionSpread?: number;
+  /** Minimum number of pages ranking for same query. Default: 2 */
+  minPages?: number;
+  /** Sort metric. Default: clicks */
+  sortBy?: CannibalizationSortMetric;
+  /** Sort order. Default: desc */
+  sortOrder?: SortOrder;
+}
+interface CannibalizationPage {
+  page: string;
+  clicks: number;
+  impressions: number;
+  ctr: number;
+  position: number;
+}
+interface CannibalizationResult {
+  query: string;
+  pages: CannibalizationPage[];
+  totalClicks: number;
+  totalImpressions: number;
+  positionSpread: number;
+}
+interface CannibalizationCompetitor {
+  url: string;
+  clicks: number;
+  impressions: number;
+  ctr: number;
+  position: number;
+  share: number;
+  rank: number;
+}
+interface CannibalizationEvent {
+  keyword: string;
+  totalImpressions: number;
+  totalClicks: number;
+  competitorCount: number;
+  leaderUrl: string;
+  leaderCtr: number;
+  leaderPosition: number;
+  hhi: number;
+  fragmentation: number;
+  stolenClicks: number;
+  severity: number;
+  competitors: CannibalizationCompetitor[];
+}
+/**
+ * Pure helper: detects keyword cannibalization from row data — queries
+ * ranking for multiple pages within a small position spread. Re-exported
+ * from `@gscdump/analysis` for portable callers.
+ */
+declare function analyzeCannibalization(rows: QueryPageRow[], options?: CannibalizationOptions): CannibalizationResult[];
+export { CannibalizationCompetitor, CannibalizationEvent, CannibalizationOptions, CannibalizationPage, CannibalizationResult, CannibalizationSortMetric, analyzeCannibalization };