gsc-cli 2.1.0 → 2.2.2

data/lib/gsc/internal_links.rb

@@ -3,22 +3,30 @@
 
 require 'uri'
 require 'set'
+require 'thread'
+require 'time'
 
 module GSC
   class InternalLinks
-    attr_reader :base_url, :pages, :graph, :orphans, :depths
+    attr_reader :base_url, :pages, :graph, :orphans, :depths, :concurrency
 
-    def initialize(base_url, limit: 50)
+    def initialize(base_url, limit: 50, concurrency: 5)
       @base_url = base_url.to_s.strip
       @base_url = "https://#{@base_url}" unless @base_url =~ %r{^https?://}
       @base_uri = URI.parse(@base_url)
-      @limit = limit
-      @graph = Hash.new { |h, k| h[k] = Set.new } # target_url => Set of source_urls
-      @out_links = Hash.new { |h, k| h[k] = Set.new } # source_url => Set of target_urls
+      @limit = limit.to_i > 0 ? limit.to_i : 50
+      @concurrency = [[concurrency.to_i, 1].max, 20].min
+      @graph = Hash.new { |h, k| h[k] = Set.new }      # target_url => Set of source_urls
+      @out_links = Hash.new { |h, k| h[k] = Set.new }  # source_url => Set of target_urls
+      @page_titles = {}                                # url => String
+      @page_anchors = Hash.new { |h, k| h[k] = {} }    # target_url => { source_url => anchor_text }
       @all_discovered = Set.new
+      @mutex = Mutex.new
     end
 
-    def audit(sitemap_urls = nil)
+    def audit(sitemap_urls = nil, &progress_block)
+      start_time = Time.now
+
       urls_to_crawl = if sitemap_urls && !sitemap_urls.empty?
                         sitemap_urls.first(@limit)
                       else
@@ -29,59 +37,138 @@ module GSC
         @all_discovered << normalize_url(url)
       end
 
-      # Crawl each URL and extract internal links
-      urls_to_crawl.each do |url|
-        pa = GSC::PageAnalyzer.new(url)
-        pa.load_content! rescue next
-        dom = pa.analyze_dom rescue next
-
-        norm_source = normalize_url(url)
-        links = dom.dig(:links, :all) || []
+      # Multi-threaded concurrent crawling
+      crawl_urls_concurrently(urls_to_crawl, &progress_block)
 
-        links.each do |link_obj|
-          href = link_obj[:href]
-          target_url = resolve_internal_url(href)
-          next unless target_url
+      crawl_duration = (Time.now - start_time).round(2)
 
-          norm_target = normalize_url(target_url)
-          next if norm_target == norm_source
+      # Calculate click depths via BFS from root
+      normalized_root = normalize_url(@base_url)
+      depths = calculate_depths(normalized_root)
 
-          @graph[norm_target] << norm_source
-          @out_links[norm_source] << norm_target
-        end
-      end
+      # Top hub pages (highest out-links and in-links)
+      top_hubs = top_linked_pages(10)
+      hub_urls = top_hubs.map { |h| h[:url] }.reject { |u| u == normalized_root }
 
-      # Calculate Orphans (pages in sitemap/discovered with 0 incoming internal links)
+      # Calculate Orphans and Weak Pages
       orphans = []
-      weak_pages = [] # only 1 internal link
+      orphan_details = []
+      weak_pages = []
+      deep_pages = []
 
       @all_discovered.each do |url|
+        next if url == normalized_root
+
         in_degree = @graph[url].size
-        if in_degree == 0 && url != normalize_url(@base_url)
+        depth = depths[url]
+
+        if depth && depth >= 4
+          deep_pages << { url: url, depth: depth, in_degree: in_degree }
+        end
+
+        if in_degree == 0
           orphans << url
+          orphan_details << {
+            url: url,
+            depth: depth || 'Unreachable via internal links (∞)',
+            suggested_rescues: generate_rescue_suggestions(url, hub_urls, normalized_root)
+          }
         elsif in_degree == 1
-          weak_pages << { url: url, source: @graph[url].first }
+          source = @graph[url].first
+          anchor = @page_anchors.dig(url, source) || 'View page'
+          weak_pages << {
+            url: url,
+            source: source,
+            anchor: anchor,
+            depth: depth,
+            suggested_rescues: generate_rescue_suggestions(url, hub_urls, normalized_root)
+          }
         end
       end
 
-      # Calculate click depths via BFS from root
-      depths = calculate_depths(normalize_url(@base_url))
+      # Link Equity Health Score (0–100)
+      health_score = [100 - (orphans.size * 12) - (weak_pages.size * 3) - (deep_pages.size * 4), 0].max
+      health_grade = case health_score
+                     when 90..100 then 'A'
+                     when 80..89  then 'B'
+                     when 70..79  then 'C'
+                     when 60..69  then 'D'
+                     else              'F'
+                     end
+
+      # Depth distribution
+      depth_dist = Hash.new(0)
+      depths.each_value { |d| depth_dist[d] += 1 }
 
       {
         base_url: @base_url,
         total_pages: @all_discovered.size,
+        crawl_duration_s: crawl_duration,
+        health_score: health_score,
+        health_grade: health_grade,
         orphans: orphans,
+        orphan_details: orphan_details,
         weak_pages: weak_pages,
-        top_linked: top_linked_pages(10),
-        depths: depths
+        deep_pages: deep_pages,
+        top_linked: top_hubs,
+        depths: depths,
+        depth_distribution: depth_dist.sort.to_h
       }
     end
 
     private
 
+    def crawl_urls_concurrently(urls, &_block)
+      queue = Queue.new
+      urls.each { |u| queue << u }
+
+      workers = (1..@concurrency).map do
+        Thread.new do
+          while !queue.empty? && (url = queue.pop(true) rescue nil)
+            crawl_single_page(url)
+          end
+        end
+      end
+
+      workers.each(&:join)
+    end
+
+    def crawl_single_page(url)
+      pa = GSC::PageAnalyzer.new(url)
+      pa.load_content!
+      dom = pa.analyze_dom
+      return unless dom
+
+      norm_source = normalize_url(url)
+      title = dom.dig(:title, :text).to_s.strip
+      links = dom.dig(:links, :all) || []
+
+      @mutex.synchronize do
+        @page_titles[norm_source] = title unless title.empty?
+      end
+
+      links.each do |link_obj|
+        href = link_obj[:href]
+        target_url = resolve_internal_url(href)
+        next unless target_url
+
+        norm_target = normalize_url(target_url)
+        next if norm_target == norm_source
+
+        anchor = link_obj[:anchor].to_s.strip
+
+        @mutex.synchronize do
+          @graph[norm_target] << norm_source
+          @out_links[norm_source] << norm_target
+          @page_anchors[norm_target][norm_source] = anchor unless anchor.empty?
+        end
+      end
+    rescue StandardError
+      # resilient worker execution
+    end
+
     def discover_urls
-      loader = GSC::SitemapLoader.new(@base_url)
-      urls = loader.load
+      urls = GSC::SitemapLoader.resolve_urls(@base_url, @base_url, quiet: true)
       urls.empty? ? [@base_url] : urls.first(@limit)
     rescue StandardError
       [@base_url]
@@ -101,7 +188,7 @@ module GSC
 
     def normalize_url(url)
       u = url.to_s.strip.sub(%r{/$}, '')
-      u
+      u.empty? ? @base_url : u
     end
 
     def calculate_depths(root_url)
@@ -125,8 +212,38 @@ module GSC
 
     def top_linked_pages(limit)
       @graph.map do |url, sources|
-        { url: url, incoming_count: sources.size }
+        {
+          url: url,
+          title: @page_titles[url] || '',
+          incoming_count: sources.size,
+          outgoing_count: (@out_links[url] || []).size
+        }
       end.sort_by { |item| -item[:incoming_count] }.first(limit)
     end
+
+    def generate_rescue_suggestions(orphan_url, hub_urls, root_url)
+      # Extract keyword tokens from slug
+      path = URI.parse(orphan_url).path.to_s rescue ''
+      slug = path.split('/').reject(&:empty?).last || ''
+      keyword = slug.gsub(/[-_]+/, ' ').strip
+      keyword_title = keyword.split.map(&:capitalize).join(' ')
+
+      # Pick 2 best rescue sources: 1 high-equity hub, 1 root/pillar
+      sources = []
+      if !hub_urls.empty?
+        # Pick first hub that isn't the orphan itself
+        hub_source = hub_urls.find { |h| h != orphan_url }
+        sources << hub_source if hub_source
+      end
+      sources << root_url unless sources.include?(root_url)
+
+      sources.first(2).map do |src|
+        {
+          source_url: src,
+          recommended_anchor: keyword.empty? ? 'Explore this guide' : keyword_title,
+          action: "Add in-content link from #{src} pointing to #{orphan_url} with anchor \"#{keyword.empty? ? 'Learn more' : keyword_title}\""
+        }
+      end
+    end
   end
 end

data/lib/gsc/keyword_value.rb

@@ -0,0 +1,191 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'date'
+require_relative 'ctr_curve'
+require_relative 'intent_shift'
+
+module GSC
+  class KeywordValue
+    DEFAULT_AOV = 75.0
+    DEFAULT_CONV_RATE = 0.025 # 2.5%
+    DEFAULT_MARGIN = 0.60    # 60% gross margin
+
+    attr_reader :options, :api, :domain, :aov, :conv_rate, :margin, :target_pos
+
+    def initialize(options = {}, api = nil, domain = nil)
+      @options = options
+      @api = api
+      @domain = domain.to_s.strip
+      @aov = (options[:aov] || DEFAULT_AOV).to_f
+      @conv_rate = (options[:conv_rate] || DEFAULT_CONV_RATE).to_f
+      @margin = (options[:margin] || DEFAULT_MARGIN).to_f
+      @target_pos = (options[:target_pos] || 3).to_i
+    end
+
+    def self.analyze(options = {}, api = nil, domain = nil)
+      new(options, api, domain).analyze
+    end
+
+    def analyze
+      rows = fetch_rows
+      target_ctr = GSC::CtrCurve.benchmark_for(@target_pos)
+
+      analyzed_keywords = rows.map do |row|
+        analyze_row(row, target_ctr)
+      end
+
+      # Sort by monthly revenue upside descending
+      analyzed_keywords.sort_by! { |k| -k[:monthly_revenue_upside] }
+
+      synthesize_portfolio(analyzed_keywords)
+    end
+
+    private
+
+    def analyze_row(row, target_ctr)
+      query = row[:query]
+      imp = row[:impressions] || 0
+      clicks = row[:clicks] || 0
+      pos = (row[:position] || 100.0).round(1)
+      ctr = row[:ctr] || (imp > 0 ? ((clicks.to_f / imp) * 100.0).round(2) : 0.0)
+
+      # Classify intent to apply dynamic conversion multipliers
+      intent = GSC::IntentShift.classify_query(query)
+      intent_multiplier = case intent
+                          when :transactional then 2.0
+                          when :commercial    then 1.2
+                          when :navigational  then 1.5
+                          else 0.6 # informational
+                          end
+
+      effective_conv_rate = (@conv_rate * intent_multiplier).round(4)
+
+      # Current realized monthly revenue
+      current_orders = (clicks * effective_conv_rate).round(1)
+      current_revenue = (current_orders * @aov).round(2)
+      current_gross_profit = (current_revenue * @margin).round(2)
+
+      # Projected revenue at target position (e.g. Top 3)
+      if pos <= @target_pos
+        projected_clicks = clicks
+        incremental_clicks = 0
+        potential_revenue = current_revenue
+        monthly_upside = 0.0
+      else
+        projected_clicks = ((target_ctr / 100.0) * imp).round
+        incremental_clicks = [projected_clicks - clicks, 0].max
+        potential_orders = (projected_clicks * effective_conv_rate).round(1)
+        potential_revenue = (potential_orders * @aov).round(2)
+        monthly_upside = [potential_revenue - current_revenue, 0.0].max.round(2)
+      end
+      annual_upside = (monthly_upside * 12).round(2)
+
+      # Value Index score (0-100)
+      # High score if: in striking distance (pos 4-15), high upside, high intent
+      pos_weight = if pos.between?(4.0, 10.0) then 1.0
+                   elsif pos.between?(10.1, 20.0) then 0.8
+                   elsif pos <= 3.0 then 0.3 # already top 3
+                   else 0.4
+                   end
+
+      raw_score = (pos_weight * 40) + ([monthly_upside / 50.0, 40].min) + (intent_multiplier * 10)
+      value_score = [[raw_score.round, 100].min, 1].max
+
+      action = if pos <= 3.0
+                 "👑 Top 3 Defend: Protect ranking with fresh content updates & internal hub equity."
+               elsif pos.between?(4.0, 10.0)
+                 "🚀 Page-1 Striking Distance: +$#{format_currency(monthly_upside)}/mo upside! Optimize title CTR & add FAQ schema."
+               elsif pos.between?(10.1, 20.0)
+                 "🎯 Page-2 Striking Query: +$#{format_currency(monthly_upside)}/mo upside. Expand content depth & earn 2 backlinks."
+               else
+                 "🌱 Deep Discovery: Expand keyword topic cluster to lift organic visibility."
+               end
+
+      {
+        query: query,
+        intent: intent,
+        position: pos,
+        impressions: imp,
+        clicks: clicks,
+        ctr: ctr,
+        effective_conv_rate: (effective_conv_rate * 100.0).round(2),
+        current_orders: current_orders,
+        current_monthly_revenue: current_revenue,
+        current_gross_profit: current_gross_profit,
+        potential_clicks: projected_clicks,
+        incremental_clicks: incremental_clicks,
+        potential_monthly_revenue: potential_revenue,
+        monthly_revenue_upside: monthly_upside,
+        annual_revenue_upside: annual_upside,
+        value_score: value_score,
+        action: action
+      }
+    end
+
+    def synthesize_portfolio(keywords)
+      total_current_rev = keywords.sum { |k| k[:current_monthly_revenue] }.round(2)
+      total_potential_rev = keywords.sum { |k| k[:potential_monthly_revenue] }.round(2)
+      total_monthly_upside = keywords.sum { |k| k[:monthly_revenue_upside] }.round(2)
+      total_annual_upside = (total_monthly_upside * 12).round(2)
+
+      striking_count = keywords.count { |k| k[:position].between?(4.0, 20.0) }
+      high_intent_count = keywords.count { |k| k[:intent] == :transactional || k[:intent] == :commercial }
+
+      {
+        domain: @domain,
+        parameters: {
+          aov: @aov,
+          conversion_rate_pct: (@conv_rate * 100.0).round(2),
+          profit_margin_pct: (@margin * 100.0).round(2),
+          target_position: @target_pos
+        },
+        total_keywords_analyzed: keywords.size,
+        striking_distance_keywords: striking_count,
+        high_commercial_intent_keywords: high_intent_count,
+        financials: {
+          current_monthly_revenue: total_current_rev,
+          current_annual_run_rate: (total_current_rev * 12).round(2),
+          potential_monthly_revenue: total_potential_rev,
+          unlocked_monthly_upside: total_monthly_upside,
+          unlocked_annual_pipeline_upside: total_annual_upside
+        },
+        keywords: keywords
+      }
+    end
+
+    def fetch_rows
+      if @api
+        begin
+          days = (@options[:days] || 28).to_i
+          end_date = (Date.today - 2).strftime('%Y-%m-%d')
+          start_date = (Date.today - 2 - days).strftime('%Y-%m-%d')
+          res = @api.search_analytics(@domain, start_date: start_date, end_date: end_date, dimensions: %w[query])
+          rows = res['rows'] || []
+          if rows.any?
+            return rows.first(25).map do |r|
+              {
+                query: r['keys'][0],
+                clicks: r['clicks'] || 0,
+                impressions: r['impressions'] || 0,
+                ctr: ((r['ctr'] || 0) * 100.0).round(2),
+                position: (r['position'] || 0).round(1)
+              }
+            end
+          end
+        rescue StandardError
+          # GSC query failed
+        end
+      end
+
+      []
+    end
+
+    def format_currency(val)
+      parts = sprintf('%.2f', val.to_f).split('.')
+      parts[0] = parts[0].reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+      parts.join('.')
+    end
+  end
+end

data/lib/gsc/landing_roi.rb

@@ -0,0 +1,195 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'uri'
+
+module GSC
+  class LandingRoi
+    DEFAULT_AOV = 75.0            # Average Order Value / Customer LTV ($)
+    DEFAULT_CONV_RATE = 0.025     # 2.5% Conversion rate of engaged visitors
+    DEFAULT_BENCHMARK_BOUNCE = 45.0 # 45% standard healthy bounce rate
+    DEFAULT_CPC = 1.50            # Replacement PPC cost per click
+
+    attr_reader :options, :aov, :conv_rate, :benchmark_bounce, :cpc
+
+    def initialize(options = {})
+      @options = options
+      @aov = (options[:aov] || DEFAULT_AOV).to_f
+      @conv_rate = (options[:conv_rate] || DEFAULT_CONV_RATE).to_f
+      @benchmark_bounce = (options[:benchmark_bounce] || DEFAULT_BENCHMARK_BOUNCE).to_f
+      @cpc = (options[:cpc] || DEFAULT_CPC).to_f
+    end
+
+    # Analyzes combined GSC pre-click and GA4 post-click data
+    def analyze_pages(pages_data)
+      analyzed = pages_data.map do |page|
+        analyze_single_page(page)
+      end
+
+      # Sort by highest monthly revenue leak by default
+      analyzed.sort_by! { |p| -(p[:monthly_revenue_leak] || 0.0) }
+
+      total_clicks = analyzed.sum { |p| p[:clicks] || 0 }
+      total_sessions = analyzed.sum { |p| p[:sessions] || 0 }
+      total_leak = analyzed.sum { |p| p[:monthly_revenue_leak] || 0.0 }
+      total_actual_rev = analyzed.sum { |p| p[:actual_revenue_est] || 0.0 }
+      total_lost_visitors = analyzed.sum { |p| p[:lost_visitors] || 0 }
+
+      quadrant_summary = {
+        cash_cows: analyzed.count { |p| p[:quadrant] == :cash_cow },
+        revenue_leakers: analyzed.count { |p| p[:quadrant] == :revenue_leaker },
+        hidden_gems: analyzed.count { |p| p[:quadrant] == :hidden_gem },
+        zombies: analyzed.count { |p| p[:quadrant] == :zombie }
+      }
+
+      {
+        total_pages: analyzed.size,
+        total_clicks: total_clicks,
+        total_sessions: total_sessions,
+        total_monthly_leak: total_leak.round(2),
+        total_annual_leak: (total_leak * 12.0).round(2),
+        total_actual_revenue_est: total_actual_rev.round(2),
+        total_lost_visitors: total_lost_visitors,
+        quadrants: quadrant_summary,
+        assumptions: {
+          aov: @aov,
+          conv_rate_pct: (@conv_rate * 100.0).round(2),
+          benchmark_bounce_pct: @benchmark_bounce,
+          estimated_cpc: @cpc
+        },
+        pages: analyzed
+      }
+    end
+
+    def analyze_single_page(page)
+      url = page[:url] || page['url'] || page[:path] || page['path'] || '/'
+      clicks = (page[:clicks] || page['clicks'] || 0).to_i
+      impressions = (page[:impressions] || page['impressions'] || 0).to_i
+      position = (page[:position] || page['position'] || 0.0).to_f.round(1)
+      ctr = (page[:ctr] || page['ctr'] || 0.0).to_f.round(2)
+      sessions = (page[:sessions] || page['sessions'] || clicks).to_i
+      bounce_rate = (page[:bounce_rate] || page['bounce_rate'] || estimate_bounce_rate(clicks, position)).to_f.round(1)
+      duration = (page[:duration] || page['duration'] || page[:duration_seconds] || 60.0).to_f.round(1)
+
+      # Economic Calculations
+      excess_bounce = [0.0, bounce_rate - @benchmark_bounce].max
+      lost_visitors = (clicks * (excess_bounce / 100.0)).round
+      monthly_revenue_leak = (lost_visitors * @conv_rate * @aov).round(2)
+      annual_revenue_leak = (monthly_revenue_leak * 12.0).round(2)
+
+      engaged_clicks = [0, clicks - lost_visitors].max
+      actual_revenue_est = (engaged_clicks * @conv_rate * @aov).round(2)
+      traffic_asset_value = (clicks * @cpc).round(2)
+
+      # Page Economic Health Index (PEHI 0-100)
+      # High clicks + low bounce + high duration = 100
+      pehi_score = calculate_pehi(clicks, bounce_rate, duration)
+
+      # Strategic Quadrant
+      quadrant = classify_quadrant(clicks, bounce_rate, duration, position)
+
+      # Conversion Prescriptions & Fixes
+      prescriptions = generate_cro_prescriptions(bounce_rate, duration, clicks, position)
+
+      {
+        url: url,
+        clicks: clicks,
+        impressions: impressions,
+        position: position,
+        ctr: ctr,
+        sessions: sessions,
+        bounce_rate: bounce_rate,
+        duration_seconds: duration,
+        pehi_score: pehi_score,
+        quadrant: quadrant,
+        lost_visitors: lost_visitors,
+        monthly_revenue_leak: monthly_revenue_leak,
+        annual_revenue_leak: annual_revenue_leak,
+        actual_revenue_est: actual_revenue_est,
+        traffic_asset_value: traffic_asset_value,
+        prescriptions: prescriptions
+      }
+    end
+
+    private
+
+    def calculate_pehi(clicks, bounce_rate, duration)
+      score = 50.0
+
+      # Bounce Rate Component (max +/- 30pts)
+      if bounce_rate <= 35.0
+        score += 30.0
+      elsif bounce_rate <= 45.0
+        score += 15.0
+      elsif bounce_rate >= 75.0
+        score -= 30.0
+      elsif bounce_rate >= 60.0
+        score -= 15.0
+      end
+
+      # Duration Component (max +/- 15pts)
+      if duration >= 120.0
+        score += 15.0
+      elsif duration >= 60.0
+        score += 8.0
+      elsif duration <= 25.0
+        score -= 15.0
+      elsif duration <= 45.0
+        score -= 8.0
+      end
+
+      # Volume Bonus (max +5pts)
+      score += 5.0 if clicks >= 100
+
+      score.clamp(0.0, 100.0).round
+    end
+
+    def classify_quadrant(clicks, bounce_rate, duration, position)
+      if clicks >= 25 && bounce_rate <= @benchmark_bounce && duration >= 45.0
+        :cash_cow
+      elsif clicks >= 25 && bounce_rate > @benchmark_bounce
+        :revenue_leaker
+      elsif clicks < 25 && bounce_rate <= @benchmark_bounce && duration >= 60.0
+        :hidden_gem
+      else
+        :zombie
+      end
+    end
+
+    def generate_cro_prescriptions(bounce_rate, duration, clicks, position)
+      fixes = []
+
+      if bounce_rate >= 70.0
+        fixes << "🚨 HIGH BOUNCE TRAP: Place sticky, high-contrast CTA button above 450px fold."
+        fixes << "🛡️ ADD SOCIAL PROOF: Insert customer rating stars, logos, or guarantee badge in the top viewport."
+      elsif bounce_rate > 55.0
+        fixes << "⚡ REDUCE BOUNCE: Add interactive jump-links / Table of Contents to lower drop-off."
+      end
+
+      if duration < 30.0
+        fixes << "⏱️ LOW TIME ON PAGE: Searchers are not finding answers instantly; add a 2-sentence executive summary under H1."
+      end
+
+      if clicks >= 50 && bounce_rate > 60.0
+        fixes << "💰 HIGH TRAFFIC BLEED: Priority CRO target. Run A/B test on hero headline and primary action."
+      end
+
+      if position >= 7.0 && bounce_rate <= 40.0
+        fixes << "⭐ HIDDEN HIGH CONVERTER: Searchers love this page (low bounce). Build 3 internal links to push from pos #{position} to Top 3."
+      end
+
+      fixes << "✅ HEALTHY ENGAGEMENT: Maintain current content structure and monitor core queries." if fixes.empty?
+      fixes
+    end
+
+    def estimate_bounce_rate(clicks, position)
+      # Fallback heuristic when GA4 is not linked:
+      # Informational / deeper SERP positions generally experience higher bounce rates
+      base = 50.0
+      base += 5.0 if position > 5.0
+      base += 8.0 if position > 10.0
+      base.clamp(35.0, 80.0)
+    end
+  end
+end