gsc-cli 2.0.2 → 2.2.0

data/lib/gsc/cannibalization_analyzer.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+module GSC
+  class CannibalizationAnalyzer
+    def self.analyze(rows, min_imp: 10, min_secondary_ratio: 0.15)
+      grouped = Hash.new { |h, k| h[k] = [] }
+
+      rows.each do |r|
+        query = r[:query] || r['query'] || r.dig('keys', 0)
+        page  = r[:page]  || r['page']  || r.dig('keys', 1)
+        clicks = r[:clicks] || r['clicks'] || 0
+        imp    = r[:impressions] || r['impressions'] || 0
+        pos    = (r[:position] || r['position'] || 100.0).to_f.round(1)
+
+        next unless query && page
+
+        grouped[query] << {
+          page: page,
+          clicks: clicks,
+          impressions: imp,
+          position: pos
+        }
+      end
+
+      conflicts = []
+      total_wasted_imp = 0
+
+      grouped.each do |query, pages|
+        next if pages.size < 2
+
+        total_imp = pages.sum { |p| p[:impressions] }
+        next if total_imp < min_imp
+
+        sorted_pages = pages.sort_by { |p| -p[:impressions] }
+        primary = sorted_pages[0]
+        secondary = sorted_pages[1]
+
+        sec_ratio = secondary[:impressions].to_f / total_imp
+        next if sec_ratio < min_secondary_ratio
+
+        severity = calculate_severity(primary, secondary, total_imp, sec_ratio)
+        remedy = determine_remedy(primary, secondary, query)
+
+        # Wasted / split impressions from secondary+ competing pages
+        diluted_imp = pages[1..].sum { |p| p[:impressions] }
+        total_wasted_imp += diluted_imp
+
+        conflicts << {
+          query: query,
+          severity: severity,
+          total_impressions: total_imp,
+          total_clicks: pages.sum { |p| p[:clicks] },
+          competing_pages_count: pages.size,
+          diluted_impressions: diluted_imp,
+          remedy: remedy,
+          pages: sorted_pages.map do |p|
+            share = ((p[:impressions].to_f / total_imp) * 100).round(1)
+            p.merge(
+              impression_share: share,
+              impression_share_str: "#{share}%"
+            )
+          end
+        }
+      end
+
+      # Sort by severity (CRITICAL > HIGH > MODERATE > LOW) then total impressions
+      severity_order = { 'CRITICAL' => 4, 'HIGH' => 3, 'MODERATE' => 2, 'LOW' => 1 }
+      conflicts.sort_by! { |c| [-severity_order[c[:severity]], -c[:total_impressions]] }
+
+      critical_count = conflicts.count { |c| c[:severity] == 'CRITICAL' }
+      health_score = [100 - (conflicts.size * 10 + critical_count * 15), 0].max
+
+      grade = case health_score
+              when 90..100 then 'A'
+              when 80..89  then 'B'
+              when 70..79  then 'C'
+              when 60..69  then 'D'
+              else 'F'
+              end
+
+      {
+        total_queries_evaluated: grouped.size,
+        conflicts_count: conflicts.size,
+        critical_count: critical_count,
+        total_diluted_impressions: total_wasted_imp,
+        health_score: health_score,
+        health_grade: grade,
+        conflicts: conflicts
+      }
+    end
+
+    def self.calculate_severity(primary, secondary, total_imp, sec_ratio)
+      # If both rank on Page 1 or 2 and share is closely split
+      if (primary[:position] <= 20.0 || secondary[:position] <= 20.0) && (sec_ratio >= 0.35 || total_imp >= 50)
+        'CRITICAL'
+      elsif sec_ratio >= 0.25 || total_imp >= 25
+        'HIGH'
+      elsif sec_ratio >= 0.15
+        'MODERATE'
+      else
+        'LOW'
+      end
+    end
+
+    def self.determine_remedy(primary, secondary, _query)
+      p_path = primary[:page].sub(%r{^https?://[^/]+}, '')
+      p_path = '/' if p_path.empty?
+      s_path = secondary[:page].sub(%r{^https?://[^/]+}, '')
+      s_path = '/' if s_path.empty?
+
+      # Case 1: Secondary page actually ranks higher than primary!
+      if secondary[:position] < (primary[:position] - 1.5)
+        {
+          action: 'FLIP_FLOP_CONSOLIDATION',
+          recommendation: "Googlebot prefers secondary page (#{s_path} ranks Pos #{secondary[:position]} vs #{primary[:position]}). Set canonical from #{p_path} pointing to #{s_path}, or 301 redirect #{p_path} to #{s_path}."
+        }
+      # Case 2: Inverted or duplicate slugs in same directory
+      elsif slug_similarity(p_path, s_path) > 0.7
+        {
+          action: '301_REDIRECT',
+          recommendation: "Near-identical slug intent. 301 redirect secondary #{s_path} into primary #{p_path} to merge PageRank."
+        }
+      # Case 3: Distinct topics accidentally colliding
+      else
+        {
+          action: 'CANONICAL_OR_LINK_DE_OPTIMIZE',
+          recommendation: "Differentiate content angles: set canonical pointing to #{p_path}, and remove \"#{_query}\" keyword from secondary #{s_path}'s H1/title."
+        }
+      end
+    end
+
+    def self.slug_similarity(path1, path2)
+      tokens1 = path1.downcase.split(/[^a-z0-9]+/).reject { |t| t.length < 2 }
+      tokens2 = path2.downcase.split(/[^a-z0-9]+/).reject { |t| t.length < 2 }
+      return 0.0 if tokens1.empty? || tokens2.empty?
+
+      common = (tokens1 & tokens2).size
+      common.to_f / [tokens1.size, tokens2.size].max
+    end
+  end
+end

data/lib/gsc/canonical_chains.rb

@@ -0,0 +1,367 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'set'
+require 'time'
+
+module GSC
+  class CanonicalChains
+    EQUITY_DECAY_PER_HOP = 0.15 # 15% estimated equity loss per redirect hop
+
+    attr_reader :max_hops, :timeout, :user_agent
+
+    def initialize(max_hops: 10, timeout: 8, user_agent: nil)
+      @max_hops = max_hops
+      @timeout = timeout
+      @user_agent = user_agent || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) gsc-cli/2.1'
+    end
+
+    # Analyze a single URL through redirect hops and canonical declarations
+    def audit_url(url, gsc_rows: nil)
+      start_url = normalize_url(url)
+      hops = []
+      visited_urls = []
+      loop_detected = false
+      loop_type = nil
+      current_url = start_url
+
+      @max_hops.times do |hop_index|
+        if visited_urls.include?(current_url)
+          loop_detected = true
+          loop_type = :redirect_loop
+          hops << {
+            hop: hop_index + 1,
+            url: current_url,
+            status_code: 0,
+            duration_ms: 0,
+            error: "Redirect loop detected: '#{current_url}' previously visited in chain"
+          }
+          break
+        end
+
+        visited_urls << current_url
+        hop_data = fetch_hop(current_url, hop_index + 1)
+        hops << hop_data
+
+        break if hop_data[:error]
+
+        # Check for HTTP redirect
+        if hop_data[:is_redirect] && hop_data[:redirect_target]
+          next_url = hop_data[:redirect_target]
+          current_url = next_url
+        else
+          # Final destination reached, inspect canonical
+          break
+        end
+      end
+
+      final_hop = hops.reject { |h| h[:error] && h[:status_code] == 0 }.last || hops.last || {}
+      final_url = final_hop[:url] || current_url
+      final_canonical = final_hop[:canonical_url]
+
+      # Check for Canonical Loop: Final page's canonical points back to an earlier hop in the chain
+      canonical_loop = false
+      if final_canonical && normalize_url(final_canonical) != normalize_url(final_url) && visited_urls.include?(normalize_url(final_canonical))
+        canonical_loop = true
+        loop_detected = true
+        loop_type = :mixed_canonical_loop
+      end
+
+      # Canonical target status check: if canonical points to a different URL, check if that canonical itself redirects
+      canonical_status = nil
+      canonical_redirects = false
+      if final_canonical && normalize_url(final_canonical) != normalize_url(final_url)
+        canonical_info = check_canonical_target(final_canonical)
+        canonical_status = canonical_info[:status_code]
+        canonical_redirects = canonical_info[:is_redirect]
+      end
+
+      redirect_hops_count = hops.count { |h| h[:is_redirect] }
+      total_duration_ms = hops.sum { |h| h[:duration_ms] || 0 }.round(1)
+
+      # Equity calculation
+      # Base equity = 1.0 (100%). Each redirect hop loses ~15%. Canonical loops destroy equity completely.
+      equity_retention = if canonical_loop || loop_type == :redirect_loop
+                           0.0
+                         elsif redirect_hops_count == 0
+                           1.0
+                         else
+                           ((1.0 - EQUITY_DECAY_PER_HOP)**redirect_hops_count).round(4)
+                         end
+      equity_loss_pct = ((1.0 - equity_retention) * 100).round(1)
+
+      # Severity classification
+      issues = []
+      severity = :healthy
+
+      if loop_detected
+        severity = :critical
+        issues << {
+          code: :loop,
+          type: loop_type,
+          message: loop_type == :mixed_canonical_loop ?
+            "Mixed Canonical Loop: Final URL '#{final_url}' canonicalizes to '#{final_canonical}', which redirects back into chain" :
+            "Infinite Redirect Loop detected at '#{current_url}'"
+        }
+      end
+
+      if redirect_hops_count >= 2
+        severity = :warning if severity == :healthy
+        issues << {
+          code: :redirect_chain,
+          message: "Multi-hop redirect chain detected: #{redirect_hops_count} hops (#{hops.map { |h| h[:status_code] }.compact.join(' -> ')})"
+        }
+      end
+
+      if canonical_redirects
+        severity = :critical
+        issues << {
+          code: :canonical_redirects,
+          message: "Target canonical URL '#{final_canonical}' issues a redirect (#{canonical_status}). Google ignores redirecting canonicals."
+        }
+      end
+
+      if final_canonical.nil? && final_hop[:status_code] == 200
+        issues << {
+          code: :missing_canonical,
+          message: "Final destination '#{final_url}' is missing a rel=\"canonical\" tag"
+        }
+      end
+
+      # Cross-reference GSC traffic if rows provided
+      traffic_at_risk = extract_gsc_traffic(visited_urls + [final_canonical].compact, gsc_rows)
+
+      # Generate server collapse rewrite rules
+      server_rules = generate_collapse_rules(start_url, final_canonical || final_url)
+
+      {
+        start_url: start_url,
+        final_url: final_url,
+        canonical_url: final_canonical,
+        redirect_hops: redirect_hops_count,
+        total_hops: hops.length,
+        total_duration_ms: total_duration_ms,
+        equity_retention_pct: (equity_retention * 100).round(1),
+        equity_loss_pct: equity_loss_pct,
+        loop_detected: loop_detected,
+        loop_type: loop_type,
+        canonical_loop: canonical_loop,
+        canonical_status: canonical_status,
+        canonical_redirects: canonical_redirects,
+        severity: severity,
+        issues: issues,
+        traffic_at_risk: traffic_at_risk,
+        server_rules: server_rules,
+        hops: hops
+      }
+    end
+
+    # Batch audit an array of URLs or top pages
+    def audit_batch(urls, gsc_rows: nil, concurrency: 4)
+      clean_urls = urls.map { |u| normalize_url(u) }.uniq
+      results = []
+
+      # Thread pool for faster batch audits
+      queue = Queue.new
+      clean_urls.each { |u| queue << u }
+      mutex = Mutex.new
+
+      workers = [concurrency, clean_urls.size].min
+      workers = 1 if workers < 1
+
+      threads = Array.new(workers) do
+        Thread.new do
+          until queue.empty?
+            url = begin
+                    queue.pop(true)
+                  rescue ThreadError
+                    nil
+                  end
+            break unless url
+
+            res = audit_url(url, gsc_rows: gsc_rows)
+            mutex.synchronize { results << res }
+          end
+        end
+      end
+      threads.each(&:join)
+
+      # Summary metrics
+      total_audited = results.size
+      chains = results.select { |r| r[:redirect_hops] >= 2 }
+      loops = results.select { |r| r[:loop_detected] }
+      canonical_issues = results.select { |r| r[:canonical_redirects] || r[:canonical_loop] }
+      total_equity_lost = results.sum { |r| r[:equity_loss_pct] }
+      avg_equity_retention = total_audited > 0 ? (results.sum { |r| r[:equity_retention_pct] } / total_audited).round(1) : 100.0
+      total_clicks_at_risk = results.sum { |r| r.dig(:traffic_at_risk, :total_clicks) || 0 }
+      total_impressions_at_risk = results.sum { |r| r.dig(:traffic_at_risk, :total_impressions) || 0 }
+
+      {
+        total_audited: total_audited,
+        redirect_chains_count: chains.size,
+        loops_count: loops.size,
+        canonical_issues_count: canonical_issues.size,
+        avg_equity_retention_pct: avg_equity_retention,
+        total_clicks_at_risk: total_clicks_at_risk,
+        total_impressions_at_risk: total_impressions_at_risk,
+        results: results.sort_by { |r| [r[:loop_detected] ? 0 : 1, -r[:redirect_hops], -r[:equity_loss_pct]] }
+      }
+    end
+
+    # Generate server redirect collapse rules (Nginx, Apache, Cloudflare, Vercel/Netlify)
+    def generate_collapse_rules(from_url, to_url)
+      from_uri = URI.parse(from_url) rescue nil
+      to_uri = URI.parse(to_url) rescue nil
+      return {} unless from_uri && to_uri
+
+      from_path = from_uri.path.empty? ? '/' : from_uri.path
+      from_path += "?#{from_uri.query}" if from_uri.query
+
+      to_target = to_url
+
+      nginx_rule = "rewrite ^#{Regexp.escape(from_uri.path)}/?$ #{to_target} permanent;"
+      apache_rule = "RewriteRule ^#{from_uri.path.sub(%r{^/}, '')}/?$ #{to_target} [R=301,L]"
+      cloudflare_expr = "(http.request.full_uri eq \"#{from_url}\") -> 301 to #{to_target}"
+      vercel_netlify = "#{from_path} #{to_target} 301!"
+
+      {
+        nginx: nginx_rule,
+        apache: apache_rule,
+        cloudflare: cloudflare_expr,
+        vercel_netlify: vercel_netlify
+      }
+    end
+
+    private
+
+    def normalize_url(url)
+      u = url.to_s.strip
+      u = "https://#{u}" unless u =~ %r{^https?://}
+      u
+    end
+
+    def fetch_hop(url, hop_num)
+      uri = URI.parse(url) rescue nil
+      return { hop: hop_num, url: url, error: "Invalid URI: #{url}", duration_ms: 0, status_code: 0 } unless uri && uri.host
+
+      start_t = Time.now
+      res = nil
+      body_content = ''
+
+      begin
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: (uri.scheme == 'https'), open_timeout: @timeout, read_timeout: @timeout) do |http|
+          req = Net::HTTP::Get.new(uri.request_uri)
+          req['User-Agent'] = @user_agent
+          req['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+          res = http.request(req)
+          body_content = res.body.to_s[0..25_000] if res.body # read first 25KB for head tags
+        end
+      rescue StandardError => e
+        return {
+          hop: hop_num,
+          url: url,
+          status_code: 0,
+          duration_ms: ((Time.now - start_t) * 1000).round(1),
+          error: e.message
+        }
+      end
+
+      duration_ms = ((Time.now - start_t) * 1000).round(1)
+      status_code = res.code.to_i
+      location = res['location']
+      redirect_target = nil
+
+      if [301, 302, 303, 307, 308].include?(status_code) && location
+        redirect_target = URI.join(url, location).to_s rescue location
+      end
+
+      # Extract canonical from header and HTML body
+      canonical_header = extract_canonical_header(res['link'])
+      canonical_html = extract_canonical_html(body_content, url)
+      canonical_url = canonical_header || canonical_html
+
+      {
+        hop: hop_num,
+        url: url,
+        status_code: status_code,
+        duration_ms: duration_ms,
+        is_redirect: !redirect_target.nil?,
+        redirect_target: redirect_target,
+        canonical_url: canonical_url,
+        canonical_source: canonical_header ? :http_header : (canonical_html ? :html_tag : nil),
+        x_robots_tag: res['x-robots-tag'],
+        server: res['server'],
+        content_type: res['content-type']
+      }
+    end
+
+    def extract_canonical_header(link_header)
+      return nil unless link_header
+      if link_header =~ /<([^>]+)>;\s*rel=["']?canonical["']?/i
+        $1.strip
+      end
+    end
+
+    def extract_canonical_html(html, base_url)
+      return nil unless html && !html.empty?
+      # Case-insensitive scan for <link ... rel="canonical" ... href="..." >
+      if html =~ /<link\b[^>]*\brel=["']canonical["'][^>]*>/i
+        tag = $&
+        if tag =~ /href=["']([^"']+)["']/i
+          raw_href = $1.strip
+          URI.join(base_url, raw_href).to_s rescue raw_href
+        end
+      elsif html =~ /<link\b[^>]*\bhref=["']([^"']+)["'][^>]*\brel=["']canonical["'][^>]*>/i
+        raw_href = $1.strip
+        URI.join(base_url, raw_href).to_s rescue raw_href
+      end
+    end
+
+    def check_canonical_target(canonical_url)
+      uri = URI.parse(canonical_url) rescue nil
+      return { status_code: 0, is_redirect: false } unless uri && uri.host
+
+      begin
+        Net::HTTP.start(uri.hostname, uri.port, use_ssl: (uri.scheme == 'https'), open_timeout: 4, read_timeout: 5) do |http|
+          req = Net::HTTP::Head.new(uri.request_uri)
+          req['User-Agent'] = @user_agent
+          res = http.request(req)
+          code = res.code.to_i
+          return { status_code: code, is_redirect: [301, 302, 303, 307, 308].include?(code) }
+        end
+      rescue StandardError
+        { status_code: 0, is_redirect: false }
+      end
+    end
+
+    def extract_gsc_traffic(urls, gsc_rows)
+      return { total_clicks: 0, total_impressions: 0, queries: [] } unless gsc_rows && gsc_rows.is_a?(Array)
+
+      clean_set = urls.map { |u| normalize_url(u).downcase }.to_set
+      matched_clicks = 0
+      matched_impressions = 0
+      matched_queries = []
+
+      gsc_rows.each do |r|
+        keys = r['keys'] || []
+        page = keys.find { |k| k =~ %r{^https?://} }
+        next unless page && clean_set.include?(normalize_url(page).downcase)
+
+        clicks = (r['clicks'] || 0).to_i
+        impressions = (r['impressions'] || 0).to_i
+        matched_clicks += clicks
+        matched_impressions += impressions
+        query = keys.find { |k| k !~ %r{^https?://} }
+        matched_queries << { query: query, clicks: clicks, impressions: impressions } if query
+      end
+
+      {
+        total_clicks: matched_clicks,
+        total_impressions: matched_impressions,
+        queries: matched_queries.sort_by { |q| -q[:clicks] }.first(5)
+      }
+    end
+  end
+end