gsc-cli 2.0.2 → 2.2.0

data/lib/gsc/intent_shift.rb

@@ -0,0 +1,188 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'date'
+
+module GSC
+  class IntentShift
+    TRANSACTIONAL_MODIFIERS = %w[
+      buy order purchase discount coupon price pricing cost cheap deal shop store subscription checkout hire
+    ].freeze
+
+    COMMERCIAL_MODIFIERS = %w[
+      best top review reviews vs versus compare comparison alternative alternatives recommended software tool platform
+    ].freeze
+
+    INFORMATIONAL_MODIFIERS = %w[
+      how what why when where who guide tutorial tips steps learn ideas strategy example examples template explain
+    ].freeze
+
+    NAVIGATIONAL_MODIFIERS = %w[
+      login log-in signin sign-in portal account dashboard support helpdesk download
+    ].freeze
+
+    attr_reader :options, :api, :domain
+
+    def initialize(options = {}, api = nil, domain = nil)
+      @options = options
+      @api = api
+      @domain = domain.to_s.strip
+    end
+
+    def self.analyze(options = {}, api = nil, domain = nil)
+      new(options, api, domain).analyze
+    end
+
+    def analyze
+      rows = collect_rows
+      shifts = detect_intent_shifts(rows)
+      summarize_portfolio(shifts)
+    end
+
+    def self.classify_query(query, brand = nil)
+      q = query.to_s.downcase.strip
+
+      if brand && !brand.empty? && q.include?(brand.downcase)
+        return :navigational if NAVIGATIONAL_MODIFIERS.any? { |m| q.include?(m) }
+      end
+
+      return :navigational if NAVIGATIONAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+      return :transactional if TRANSACTIONAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+      return :commercial if COMMERCIAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+      return :informational if INFORMATIONAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+
+      :informational
+    end
+
+    def self.classify_page(url)
+      u = url.to_s.downcase
+      if u =~ %r{/(?:cart|checkout|pricing|buy|products?|shop|store|orders?)(?:/|$|\?|#)}
+        :transactional
+      elsif u =~ %r{/(?:blog|guides?|tutorials?|learn|how-to|articles?|docs|knowledge-base)(?:/|$|\?|#)}
+        :informational
+      elsif u =~ %r{/(?:comparison|compare|vs|alternatives?|best|reviews?)(?:/|$|\?|#)}
+        :commercial
+      elsif u =~ %r{/(?:login|portal|accounts?|dashboard|signin)(?:/|$|\?|#)}
+        :navigational
+      else
+        :hybrid
+      end
+    end
+
+    private
+
+    def collect_rows
+      if @api
+        begin
+          days = (@options[:days] || 28).to_i
+          end_date = (Date.today - 2).strftime('%Y-%m-%d')
+          start_date = (Date.today - 2 - days).strftime('%Y-%m-%d')
+          res = @api.search_analytics(@domain, start_date: start_date, end_date: end_date, dimensions: %w[query page])
+          raw_rows = res['rows'] || []
+          if raw_rows.any?
+            return raw_rows.map do |r|
+              {
+                query: r['keys'][0],
+                page: r['keys'][1],
+                clicks: r['clicks'] || 0,
+                impressions: r['impressions'] || 0,
+                ctr: ((r['ctr'] || 0) * 100.0).round(2),
+                position: (r['position'] || 0).round(1)
+              }
+            end
+          end
+        rescue StandardError
+          # GSC query failed
+        end
+      end
+
+      []
+    end
+
+    def detect_intent_shifts(rows)
+      brand = @options[:brand] || @domain.split('.').first
+
+      shifts = []
+
+      rows.each do |row|
+        q_intent = self.class.classify_query(row[:query], brand)
+        p_intent = self.class.classify_page(row[:page])
+
+        mismatch = false
+        risk_level = :none
+        diagnosis = nil
+        prescription = nil
+
+        # Scenario 1: Informational/Commercial query ranking on a purely Transactional page
+        if (q_intent == :informational || q_intent == :commercial) && p_intent == :transactional
+          mismatch = true
+          risk_level = row[:position] > 10.0 ? :high : :medium
+          diagnosis = "Google expects #{q_intent.to_s.upcase} research content, but ranking URL is a TRANSACTIONAL #{File.basename(row[:page])} page."
+          prescription = "Publish a dedicated #{q_intent} guide/comparison landing page to capture Top 3 SERP intent instead of sending users to checkout/product page."
+
+        # Scenario 2: Transactional query ranking on an Informational blog post
+        elsif q_intent == :transactional && p_intent == :informational
+          mismatch = true
+          risk_level = :medium
+          diagnosis = "Users have HIGH BUYING INTENT (#{row[:query]}), but are landing on an INFORMATIONAL blog article."
+          prescription = "Embed prominent 1-click checkout widgets, pricing tables, and product CTAs directly above the fold in this blog post."
+
+        # Scenario 3: Commercial comparison query landing on generic pricing page
+        elsif q_intent == :commercial && p_intent == :hybrid
+          mismatch = true
+          risk_level = :low
+          diagnosis = "Comparison query landing on generic page."
+          prescription = "Deploy a structured vs/comparison matrix table."
+        end
+
+        shifts << {
+          query: row[:query],
+          page: row[:page],
+          clicks: row[:clicks],
+          impressions: row[:impressions],
+          position: row[:position],
+          ctr: row[:ctr],
+          query_intent: q_intent,
+          page_intent: p_intent,
+          has_mismatch: mismatch,
+          risk_level: risk_level,
+          diagnosis: diagnosis,
+          prescription: prescription
+        }
+      end
+
+      shifts
+    end
+
+    def summarize_portfolio(shifts)
+      total = shifts.size
+      mismatched = shifts.select { |s| s[:has_mismatch] }
+      high_risk  = shifts.select { |s| s[:risk_level] == :high }
+
+      intent_counts = {
+        informational: shifts.count { |s| s[:query_intent] == :informational },
+        transactional: shifts.count { |s| s[:query_intent] == :transactional },
+        commercial: shifts.count { |s| s[:query_intent] == :commercial },
+        navigational: shifts.count { |s| s[:query_intent] == :navigational }
+      }
+
+      volatility_score = ((mismatched.size.to_f / [total, 1].max) * 100.0).round(1)
+
+      prescriptions = mismatched.map { |m| m[:prescription] }.compact.uniq
+      grade = total.zero? ? 'N/A' : (volatility_score > 40.0 ? 'HIGH RISK' : (volatility_score > 20.0 ? 'MODERATE' : 'OPTIMAL'))
+
+      {
+        domain: @domain,
+        total_queries_analyzed: total,
+        intent_distribution: intent_counts,
+        mismatched_queries_count: mismatched.size,
+        high_risk_shifts_count: high_risk.size,
+        portfolio_volatility_pct: volatility_score,
+        risk_grade: grade,
+        prescriptions: prescriptions,
+        shifts: shifts
+      }
+    end
+  end
+end

data/lib/gsc/internal_links.rb

@@ -0,0 +1,249 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'uri'
+require 'set'
+require 'thread'
+require 'time'
+
+module GSC
+  class InternalLinks
+    attr_reader :base_url, :pages, :graph, :orphans, :depths, :concurrency
+
+    def initialize(base_url, limit: 50, concurrency: 5)
+      @base_url = base_url.to_s.strip
+      @base_url = "https://#{@base_url}" unless @base_url =~ %r{^https?://}
+      @base_uri = URI.parse(@base_url)
+      @limit = limit.to_i > 0 ? limit.to_i : 50
+      @concurrency = [[concurrency.to_i, 1].max, 20].min
+      @graph = Hash.new { |h, k| h[k] = Set.new }      # target_url => Set of source_urls
+      @out_links = Hash.new { |h, k| h[k] = Set.new }  # source_url => Set of target_urls
+      @page_titles = {}                                # url => String
+      @page_anchors = Hash.new { |h, k| h[k] = {} }    # target_url => { source_url => anchor_text }
+      @all_discovered = Set.new
+      @mutex = Mutex.new
+    end
+
+    def audit(sitemap_urls = nil, &progress_block)
+      start_time = Time.now
+
+      urls_to_crawl = if sitemap_urls && !sitemap_urls.empty?
+                        sitemap_urls.first(@limit)
+                      else
+                        discover_urls
+                      end
+
+      urls_to_crawl.each do |url|
+        @all_discovered << normalize_url(url)
+      end
+
+      # Multi-threaded concurrent crawling
+      crawl_urls_concurrently(urls_to_crawl, &progress_block)
+
+      crawl_duration = (Time.now - start_time).round(2)
+
+      # Calculate click depths via BFS from root
+      normalized_root = normalize_url(@base_url)
+      depths = calculate_depths(normalized_root)
+
+      # Top hub pages (highest out-links and in-links)
+      top_hubs = top_linked_pages(10)
+      hub_urls = top_hubs.map { |h| h[:url] }.reject { |u| u == normalized_root }
+
+      # Calculate Orphans and Weak Pages
+      orphans = []
+      orphan_details = []
+      weak_pages = []
+      deep_pages = []
+
+      @all_discovered.each do |url|
+        next if url == normalized_root
+
+        in_degree = @graph[url].size
+        depth = depths[url]
+
+        if depth && depth >= 4
+          deep_pages << { url: url, depth: depth, in_degree: in_degree }
+        end
+
+        if in_degree == 0
+          orphans << url
+          orphan_details << {
+            url: url,
+            depth: depth || 'Unreachable via internal links (∞)',
+            suggested_rescues: generate_rescue_suggestions(url, hub_urls, normalized_root)
+          }
+        elsif in_degree == 1
+          source = @graph[url].first
+          anchor = @page_anchors.dig(url, source) || 'View page'
+          weak_pages << {
+            url: url,
+            source: source,
+            anchor: anchor,
+            depth: depth,
+            suggested_rescues: generate_rescue_suggestions(url, hub_urls, normalized_root)
+          }
+        end
+      end
+
+      # Link Equity Health Score (0–100)
+      health_score = [100 - (orphans.size * 12) - (weak_pages.size * 3) - (deep_pages.size * 4), 0].max
+      health_grade = case health_score
+                     when 90..100 then 'A'
+                     when 80..89  then 'B'
+                     when 70..79  then 'C'
+                     when 60..69  then 'D'
+                     else              'F'
+                     end
+
+      # Depth distribution
+      depth_dist = Hash.new(0)
+      depths.each_value { |d| depth_dist[d] += 1 }
+
+      {
+        base_url: @base_url,
+        total_pages: @all_discovered.size,
+        crawl_duration_s: crawl_duration,
+        health_score: health_score,
+        health_grade: health_grade,
+        orphans: orphans,
+        orphan_details: orphan_details,
+        weak_pages: weak_pages,
+        deep_pages: deep_pages,
+        top_linked: top_hubs,
+        depths: depths,
+        depth_distribution: depth_dist.sort.to_h
+      }
+    end
+
+    private
+
+    def crawl_urls_concurrently(urls, &_block)
+      queue = Queue.new
+      urls.each { |u| queue << u }
+
+      workers = (1..@concurrency).map do
+        Thread.new do
+          while !queue.empty? && (url = queue.pop(true) rescue nil)
+            crawl_single_page(url)
+          end
+        end
+      end
+
+      workers.each(&:join)
+    end
+
+    def crawl_single_page(url)
+      pa = GSC::PageAnalyzer.new(url)
+      pa.load_content!
+      dom = pa.analyze_dom
+      return unless dom
+
+      norm_source = normalize_url(url)
+      title = dom.dig(:title, :text).to_s.strip
+      links = dom.dig(:links, :all) || []
+
+      @mutex.synchronize do
+        @page_titles[norm_source] = title unless title.empty?
+      end
+
+      links.each do |link_obj|
+        href = link_obj[:href]
+        target_url = resolve_internal_url(href)
+        next unless target_url
+
+        norm_target = normalize_url(target_url)
+        next if norm_target == norm_source
+
+        anchor = link_obj[:anchor].to_s.strip
+
+        @mutex.synchronize do
+          @graph[norm_target] << norm_source
+          @out_links[norm_source] << norm_target
+          @page_anchors[norm_target][norm_source] = anchor unless anchor.empty?
+        end
+      end
+    rescue StandardError
+      # resilient worker execution
+    end
+
+    def discover_urls
+      urls = GSC::SitemapLoader.resolve_urls(@base_url, @base_url, quiet: true)
+      urls.empty? ? [@base_url] : urls.first(@limit)
+    rescue StandardError
+      [@base_url]
+    end
+
+    def resolve_internal_url(href)
+      return nil if href.nil? || href.strip.empty?
+      return nil if href =~ /^(mailto|tel|javascript|#):/i
+
+      uri = URI.join(@base_url, href) rescue nil
+      return nil unless uri && uri.scheme =~ /^https?$/i
+      return nil unless uri.host.downcase == @base_uri.host.downcase
+
+      uri.fragment = nil
+      uri.to_s
+    end
+
+    def normalize_url(url)
+      u = url.to_s.strip.sub(%r{/$}, '')
+      u.empty? ? @base_url : u
+    end
+
+    def calculate_depths(root_url)
+      depths = { root_url => 0 }
+      queue = [root_url]
+
+      until queue.empty?
+        curr = queue.shift
+        curr_depth = depths[curr]
+
+        (@out_links[curr] || []).each do |neighbor|
+          next if depths.key?(neighbor)
+
+          depths[neighbor] = curr_depth + 1
+          queue << neighbor
+        end
+      end
+
+      depths
+    end
+
+    def top_linked_pages(limit)
+      @graph.map do |url, sources|
+        {
+          url: url,
+          title: @page_titles[url] || '',
+          incoming_count: sources.size,
+          outgoing_count: (@out_links[url] || []).size
+        }
+      end.sort_by { |item| -item[:incoming_count] }.first(limit)
+    end
+
+    def generate_rescue_suggestions(orphan_url, hub_urls, root_url)
+      # Extract keyword tokens from slug
+      path = URI.parse(orphan_url).path.to_s rescue ''
+      slug = path.split('/').reject(&:empty?).last || ''
+      keyword = slug.gsub(/[-_]+/, ' ').strip
+      keyword_title = keyword.split.map(&:capitalize).join(' ')
+
+      # Pick 2 best rescue sources: 1 high-equity hub, 1 root/pillar
+      sources = []
+      if !hub_urls.empty?
+        # Pick first hub that isn't the orphan itself
+        hub_source = hub_urls.find { |h| h != orphan_url }
+        sources << hub_source if hub_source
+      end
+      sources << root_url unless sources.include?(root_url)
+
+      sources.first(2).map do |src|
+        {
+          source_url: src,
+          recommended_anchor: keyword.empty? ? 'Explore this guide' : keyword_title,
+          action: "Add in-content link from #{src} pointing to #{orphan_url} with anchor \"#{keyword.empty? ? 'Learn more' : keyword_title}\""
+        }
+      end
+    end
+  end
+end

data/lib/gsc/keyword_value.rb

@@ -0,0 +1,191 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'date'
+require_relative 'ctr_curve'
+require_relative 'intent_shift'
+
+module GSC
+  class KeywordValue
+    DEFAULT_AOV = 75.0
+    DEFAULT_CONV_RATE = 0.025 # 2.5%
+    DEFAULT_MARGIN = 0.60    # 60% gross margin
+
+    attr_reader :options, :api, :domain, :aov, :conv_rate, :margin, :target_pos
+
+    def initialize(options = {}, api = nil, domain = nil)
+      @options = options
+      @api = api
+      @domain = domain.to_s.strip
+      @aov = (options[:aov] || DEFAULT_AOV).to_f
+      @conv_rate = (options[:conv_rate] || DEFAULT_CONV_RATE).to_f
+      @margin = (options[:margin] || DEFAULT_MARGIN).to_f
+      @target_pos = (options[:target_pos] || 3).to_i
+    end
+
+    def self.analyze(options = {}, api = nil, domain = nil)
+      new(options, api, domain).analyze
+    end
+
+    def analyze
+      rows = fetch_rows
+      target_ctr = GSC::CtrCurve.benchmark_for(@target_pos)
+
+      analyzed_keywords = rows.map do |row|
+        analyze_row(row, target_ctr)
+      end
+
+      # Sort by monthly revenue upside descending
+      analyzed_keywords.sort_by! { |k| -k[:monthly_revenue_upside] }
+
+      synthesize_portfolio(analyzed_keywords)
+    end
+
+    private
+
+    def analyze_row(row, target_ctr)
+      query = row[:query]
+      imp = row[:impressions] || 0
+      clicks = row[:clicks] || 0
+      pos = (row[:position] || 100.0).round(1)
+      ctr = row[:ctr] || (imp > 0 ? ((clicks.to_f / imp) * 100.0).round(2) : 0.0)
+
+      # Classify intent to apply dynamic conversion multipliers
+      intent = GSC::IntentShift.classify_query(query)
+      intent_multiplier = case intent
+                          when :transactional then 2.0
+                          when :commercial    then 1.2
+                          when :navigational  then 1.5
+                          else 0.6 # informational
+                          end
+
+      effective_conv_rate = (@conv_rate * intent_multiplier).round(4)
+
+      # Current realized monthly revenue
+      current_orders = (clicks * effective_conv_rate).round(1)
+      current_revenue = (current_orders * @aov).round(2)
+      current_gross_profit = (current_revenue * @margin).round(2)
+
+      # Projected revenue at target position (e.g. Top 3)
+      if pos <= @target_pos
+        projected_clicks = clicks
+        incremental_clicks = 0
+        potential_revenue = current_revenue
+        monthly_upside = 0.0
+      else
+        projected_clicks = ((target_ctr / 100.0) * imp).round
+        incremental_clicks = [projected_clicks - clicks, 0].max
+        potential_orders = (projected_clicks * effective_conv_rate).round(1)
+        potential_revenue = (potential_orders * @aov).round(2)
+        monthly_upside = [potential_revenue - current_revenue, 0.0].max.round(2)
+      end
+      annual_upside = (monthly_upside * 12).round(2)
+
+      # Value Index score (0-100)
+      # High score if: in striking distance (pos 4-15), high upside, high intent
+      pos_weight = if pos.between?(4.0, 10.0) then 1.0
+                   elsif pos.between?(10.1, 20.0) then 0.8
+                   elsif pos <= 3.0 then 0.3 # already top 3
+                   else 0.4
+                   end
+
+      raw_score = (pos_weight * 40) + ([monthly_upside / 50.0, 40].min) + (intent_multiplier * 10)
+      value_score = [[raw_score.round, 100].min, 1].max
+
+      action = if pos <= 3.0
+                 "👑 Top 3 Defend: Protect ranking with fresh content updates & internal hub equity."
+               elsif pos.between?(4.0, 10.0)
+                 "🚀 Page-1 Striking Distance: +$#{format_currency(monthly_upside)}/mo upside! Optimize title CTR & add FAQ schema."
+               elsif pos.between?(10.1, 20.0)
+                 "🎯 Page-2 Striking Query: +$#{format_currency(monthly_upside)}/mo upside. Expand content depth & earn 2 backlinks."
+               else
+                 "🌱 Deep Discovery: Expand keyword topic cluster to lift organic visibility."
+               end
+
+      {
+        query: query,
+        intent: intent,
+        position: pos,
+        impressions: imp,
+        clicks: clicks,
+        ctr: ctr,
+        effective_conv_rate: (effective_conv_rate * 100.0).round(2),
+        current_orders: current_orders,
+        current_monthly_revenue: current_revenue,
+        current_gross_profit: current_gross_profit,
+        potential_clicks: projected_clicks,
+        incremental_clicks: incremental_clicks,
+        potential_monthly_revenue: potential_revenue,
+        monthly_revenue_upside: monthly_upside,
+        annual_revenue_upside: annual_upside,
+        value_score: value_score,
+        action: action
+      }
+    end
+
+    def synthesize_portfolio(keywords)
+      total_current_rev = keywords.sum { |k| k[:current_monthly_revenue] }.round(2)
+      total_potential_rev = keywords.sum { |k| k[:potential_monthly_revenue] }.round(2)
+      total_monthly_upside = keywords.sum { |k| k[:monthly_revenue_upside] }.round(2)
+      total_annual_upside = (total_monthly_upside * 12).round(2)
+
+      striking_count = keywords.count { |k| k[:position].between?(4.0, 20.0) }
+      high_intent_count = keywords.count { |k| k[:intent] == :transactional || k[:intent] == :commercial }
+
+      {
+        domain: @domain,
+        parameters: {
+          aov: @aov,
+          conversion_rate_pct: (@conv_rate * 100.0).round(2),
+          profit_margin_pct: (@margin * 100.0).round(2),
+          target_position: @target_pos
+        },
+        total_keywords_analyzed: keywords.size,
+        striking_distance_keywords: striking_count,
+        high_commercial_intent_keywords: high_intent_count,
+        financials: {
+          current_monthly_revenue: total_current_rev,
+          current_annual_run_rate: (total_current_rev * 12).round(2),
+          potential_monthly_revenue: total_potential_rev,
+          unlocked_monthly_upside: total_monthly_upside,
+          unlocked_annual_pipeline_upside: total_annual_upside
+        },
+        keywords: keywords
+      }
+    end
+
+    def fetch_rows
+      if @api
+        begin
+          days = (@options[:days] || 28).to_i
+          end_date = (Date.today - 2).strftime('%Y-%m-%d')
+          start_date = (Date.today - 2 - days).strftime('%Y-%m-%d')
+          res = @api.search_analytics(@domain, start_date: start_date, end_date: end_date, dimensions: %w[query])
+          rows = res['rows'] || []
+          if rows.any?
+            return rows.first(25).map do |r|
+              {
+                query: r['keys'][0],
+                clicks: r['clicks'] || 0,
+                impressions: r['impressions'] || 0,
+                ctr: ((r['ctr'] || 0) * 100.0).round(2),
+                position: (r['position'] || 0).round(1)
+              }
+            end
+          end
+        rescue StandardError
+          # GSC query failed
+        end
+      end
+
+      []
+    end
+
+    def format_currency(val)
+      parts = sprintf('%.2f', val.to_f).split('.')
+      parts[0] = parts[0].reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse
+      parts.join('.')
+    end
+  end
+end