gsc-cli 2.1.0 → 2.2.2

data/lib/gsc/watchdog.rb

@@ -0,0 +1,235 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'net/http'
+require 'uri'
+require 'date'
+
+module GSC
+  class Watchdog
+    DEFAULT_DROP_THRESHOLD = 2.0 # 2+ rank position drop
+    DEFAULT_CTR_THRESHOLD  = 20.0 # 20%+ CTR decrease
+
+    attr_reader :options, :api, :domain
+
+    def initialize(options = {}, api = nil, domain = nil)
+      @options = options
+      @api = api
+      @domain = domain.to_s.strip
+    end
+
+    def self.check(options = {}, api = nil, domain = nil)
+      new(options, api, domain).check
+    end
+
+    def check
+      telemetry = fetch_telemetry
+      alerts = detect_anomalies(telemetry)
+      summary = synthesize_summary(telemetry, alerts)
+
+      if @options[:webhook] || @options[:slack]
+        webhook_res = dispatch_webhook(summary, @options[:webhook] || @options[:slack])
+        summary[:webhook_dispatched] = webhook_res
+      end
+
+      summary
+    end
+
+    def generate_crontab_entry
+      bin = current_binary_path
+      hours = (@options[:hours] || 6).to_i
+      webhook_flag = @options[:webhook] ? " --webhook #{@options[:webhook]}" : ""
+      "0 */#{hours} * * * #{bin} watch #{@domain} --once#{webhook_flag} >> ~/.config/gsc/watch.log 2>&1"
+    end
+
+    def generate_launchd_plist
+      bin = current_binary_path
+      interval = (@options[:interval] || 21600).to_i
+      label = "com.gsc.watchdog.#{@domain.gsub(/[^a-zA-Z0-9]/, '_')}"
+
+      <<~XML
+        <?xml version="1.0" encoding="UTF-8"?>
+        <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+        <plist version="1.0">
+        <dict>
+            <key>Label</key>
+            <string>#{label}</string>
+            <key>ProgramArguments</key>
+            <array>
+                <string>#{bin}</string>
+                <string>watch</string>
+                <string>#{@domain}</string>
+                <string>--once</string>
+            </array>
+            <key>StartInterval</key>
+            <integer>#{interval}</integer>
+            <key>StandardOutPath</key>
+            <string>/tmp/gsc_watch_#{@domain}.log</string>
+            <key>StandardErrorPath</key>
+            <string>/tmp/gsc_watch_#{@domain}_err.log</string>
+        </dict>
+        </plist>
+      XML
+    end
+
+    def generate_systemd_unit
+      bin = current_binary_path
+      <<~INI
+        [Unit]
+        Description=GSC Continuous SEO Rank & CTR Volatility Watchdog for #{@domain}
+        After=network.target
+
+        [Service]
+        Type=oneshot
+        ExecStart=#{bin} watch #{@domain} --once
+        StandardOutput=journal
+        StandardError=journal
+
+        [Install]
+        WantedBy=multi-user.target
+      INI
+    end
+
+    private
+
+    def current_binary_path
+      ENV['GSC_BIN_PATH'] || File.expand_path('~/.local/bin/gsc')
+    end
+
+    def fetch_telemetry
+      if @api
+        begin
+          curr_end = (Date.today - 2).strftime('%Y-%m-%d')
+          curr_start = (Date.today - 9).strftime('%Y-%m-%d')
+          prev_end = (Date.today - 10).strftime('%Y-%m-%d')
+          prev_start = (Date.today - 17).strftime('%Y-%m-%d')
+
+          curr_res = @api.search_analytics(@domain, start_date: curr_start, end_date: curr_end, dimensions: %w[query])
+          prev_res = @api.search_analytics(@domain, start_date: prev_start, end_date: prev_end, dimensions: %w[query])
+
+          curr_rows = (curr_res['rows'] || []).each_with_object({}) { |r, h| h[r['keys'][0]] = r }
+          prev_rows = (prev_res['rows'] || []).each_with_object({}) { |r, h| h[r['keys'][0]] = r }
+
+          all_queries = (curr_rows.keys + prev_rows.keys).uniq
+
+          if all_queries.any?
+            return all_queries.first(25).map do |q|
+              cr = curr_rows[q]
+              pr = prev_rows[q]
+
+              curr_pos = cr ? (cr['position'] || 0).round(1) : 100.0
+              prev_pos = pr ? (pr['position'] || 0).round(1) : 100.0
+              curr_clicks = cr ? (cr['clicks'] || 0) : 0
+              prev_clicks = pr ? (pr['clicks'] || 0) : 0
+              curr_ctr = cr ? ((cr['ctr'] || 0) * 100.0).round(2) : 0.0
+              prev_ctr = pr ? ((pr['ctr'] || 0) * 100.0).round(2) : 0.0
+
+              {
+                query: q,
+                current_position: curr_pos,
+                previous_position: prev_pos,
+                current_clicks: curr_clicks,
+                previous_clicks: prev_clicks,
+                current_ctr: curr_ctr,
+                previous_ctr: prev_ctr
+              }
+            end
+          end
+        rescue StandardError
+          # return empty
+        end
+      end
+
+      []
+    end
+
+    def detect_anomalies(telemetry)
+      alerts = []
+      pos_threshold = (@options[:threshold] || DEFAULT_DROP_THRESHOLD).to_f
+      ctr_threshold = DEFAULT_CTR_THRESHOLD
+
+      telemetry.each do |row|
+        pos_delta = (row[:current_position] - row[:previous_position]).round(1) # positive means dropped rank
+        ctr_drop_pct = row[:previous_ctr] > 0 ? (((row[:previous_ctr] - row[:current_ctr]) / row[:previous_ctr]) * 100.0).round(1) : 0.0
+
+        if pos_delta >= pos_threshold
+          severity = pos_delta >= 4.0 ? :critical : :warning
+          alerts << {
+            query: row[:query],
+            type: :position_drop,
+            severity: severity,
+            message: "Rank dropped #{pos_delta} positions (Pos #{row[:previous_position]} ➔ Pos #{row[:current_position]})",
+            current_position: row[:current_position],
+            previous_position: row[:previous_position],
+            current_clicks: row[:current_clicks],
+            previous_clicks: row[:previous_clicks]
+          }
+        elsif ctr_drop_pct >= ctr_threshold && row[:current_position] <= 10.0
+          alerts << {
+            query: row[:query],
+            type: :ctr_anomaly,
+            severity: :warning,
+            message: "CTR dropped #{ctr_drop_pct}% (from #{row[:previous_ctr]}% to #{row[:current_ctr]}%) despite stable rank (Pos #{row[:current_position]}). Likely Zero-Click AI Overview suppression.",
+            current_position: row[:current_position],
+            previous_position: row[:previous_position],
+            current_clicks: row[:current_clicks],
+            previous_clicks: row[:previous_clicks]
+          }
+        end
+      end
+
+      alerts
+    end
+
+    def synthesize_summary(telemetry, alerts)
+      critical_count = alerts.count { |a| a[:severity] == :critical }
+      warning_count  = alerts.count { |a| a[:severity] == :warning }
+
+      status = if critical_count > 0
+                 'CRITICAL ANOMALIES'
+               elsif warning_count > 0
+                 'WARNINGS DETECTED'
+               else
+                 'SERP HEALTH OPTIMAL'
+               end
+
+      {
+        domain: @domain,
+        monitored_queries_count: telemetry.size,
+        status: status,
+        critical_alerts_count: critical_count,
+        warning_alerts_count: warning_count,
+        total_alerts_count: alerts.size,
+        checked_at: Time.now.strftime('%Y-%m-%d %H:%M:%S %Z'),
+        alerts: alerts,
+        telemetry: telemetry
+      }
+    end
+
+    def dispatch_webhook(summary, webhook_url)
+      uri = URI.parse(webhook_url)
+      payload = {
+        text: "🚨 [GSC Rank Watchdog] #{summary[:total_alerts_count]} Organic SERP Alerts Detected for #{summary[:domain]} (#{summary[:status]})",
+        domain: summary[:domain],
+        status: summary[:status],
+        critical_count: summary[:critical_alerts_count],
+        warnings_count: summary[:warning_alerts_count],
+        checked_at: summary[:checked_at],
+        alerts: summary[:alerts].map { |a| { query: a[:query], message: a[:message], severity: a[:severity] } }
+      }
+
+      req = Net::HTTP::Post.new(uri)
+      req['Content-Type'] = 'application/json'
+      req.body = JSON.generate(payload)
+
+      res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 5, read_timeout: 5) do |http|
+        http.request(req)
+      end
+
+      res.code.to_i >= 200 && res.code.to_i < 300
+    rescue StandardError => e
+      false
+    end
+  end
+end

data/lib/gsc/zombie_purger.rb

@@ -0,0 +1,366 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'uri'
+
+module GSC
+  class ZombiePurger
+    DEFAULT_DAYS = 90
+    DEFAULT_THRESHOLD = 0
+    EST_MONTHLY_CRAWLS_PER_ZOMBIE = 6
+
+    # Common utility/legal URL path fragments that should remain accessible to users but noindexed
+    UTILITY_PATHS = %w[
+      /privacy /terms /disclaimer /legal /contact /about-us /about
+      /account /login /signin /signup /register /cart /checkout
+      /order /thank-you /confirmation /search /sitemap
+    ].freeze
+
+    # Common taxonomy, archive, or thin content fragments suited for 410 purge
+    THIN_PATTERNS = [
+      %r{/page/\d+},
+      %r{/tag/},
+      %r{/tags/},
+      %r{/category/},
+      %r{/archive/},
+      %r{/author/},
+      %r{/attachment/},
+      %r{\?replytocom=},
+      %r{\?amp=1},
+      %r{/feed/?$}
+    ].freeze
+
+    STOPWORDS = %w[the a an and of to in for with on at by from is are was were this that].freeze
+
+    attr_reader :options, :days, :threshold
+
+    def initialize(options = {})
+      @options = options
+      @days = (options[:days] || DEFAULT_DAYS).to_i
+      @threshold = (options[:threshold] || DEFAULT_THRESHOLD).to_i
+    end
+
+    # Convenience class method for auditing inventory against GSC pages
+    def self.audit(inventory_urls, gsc_pages, options = {})
+      new(options).audit(inventory_urls, gsc_pages)
+    end
+
+    # inventory_urls: Array of URL strings
+    # gsc_pages: Hash of { clean_url => { impressions:, clicks:, position: } } OR Array of row hashes
+    def audit(inventory_urls, gsc_pages)
+      urls = normalize_url_list(inventory_urls)
+      active_map = normalize_gsc_pages(gsc_pages)
+
+      active_items = []
+      zombie_items = []
+
+      # List of all active clean URLs for candidate pillar matching
+      active_candidates = active_map.keys
+
+      urls.each do |raw_url|
+        clean_key = clean_url_key(raw_url)
+        page_stat = active_map[clean_key]
+        impressions = page_stat ? (page_stat[:impressions] || 0) : 0
+        clicks = page_stat ? (page_stat[:clicks] || 0) : 0
+        position = page_stat ? (page_stat[:position] || 0.0) : 0.0
+
+        if impressions > @threshold
+          active_items << {
+            url: raw_url,
+            clean_url: clean_key,
+            impressions: impressions,
+            clicks: clicks,
+            position: position
+          }
+        else
+          # Triage this zombie URL
+          triage = classify_zombie(raw_url, active_candidates)
+          zombie_items << {
+            url: raw_url,
+            clean_url: clean_key,
+            impressions: impressions,
+            clicks: clicks,
+            action: triage[:action],
+            action_label: format_action_label(triage[:action]),
+            target_url: triage[:target_url],
+            rationale: triage[:rationale],
+            path: uri_path(raw_url)
+          }
+        end
+      end
+
+      total_inventory = urls.size
+      zombie_count = zombie_items.size
+      active_count = active_items.size
+
+      zombie_ratio = total_inventory > 0 ? ((zombie_count.to_f / total_inventory) * 100.0).round(1) : 0.0
+      cedi = zombie_ratio # Crawl Equity Dilution Index (0-100, lower is better)
+      health_score = [0.0, (100.0 - cedi)].max.round(1)
+      grade = compute_grade(cedi)
+
+      wasted_crawls_monthly = zombie_count * EST_MONTHLY_CRAWLS_PER_ZOMBIE
+      wasted_crawls_annually = wasted_crawls_monthly * 12
+
+      action_breakdown = {
+        purge_410: zombie_items.count { |z| z[:action] == :purge_410 },
+        redirect_301: zombie_items.count { |z| z[:action] == :redirect_301 },
+        consolidate: zombie_items.count { |z| z[:action] == :consolidate },
+        noindex: zombie_items.count { |z| z[:action] == :noindex }
+      }
+
+      # Filter if specific action requested
+      filtered_zombies = if @options[:action]
+                           target_act = @options[:action].to_s.downcase.sub('301', '').sub('410', '').to_sym
+                           zombie_items.select { |z| z[:action].to_s.include?(target_act.to_s) }
+                         else
+                           zombie_items
+                         end
+
+      limit = (@options[:limit] || 25).to_i
+      displayed_zombies = filtered_zombies.first(limit)
+
+      {
+        total_inventory: total_inventory,
+        active_count: active_count,
+        zombie_count: zombie_count,
+        zombie_percentage: zombie_ratio,
+        cedi: cedi,
+        health_score: health_score,
+        grade: grade,
+        days: @days,
+        threshold: @threshold,
+        wasted_crawls_monthly: wasted_crawls_monthly,
+        wasted_crawls_annually: wasted_crawls_annually,
+        action_breakdown: action_breakdown,
+        zombies: displayed_zombies,
+        all_zombies_count: filtered_zombies.size,
+        server_rules: {
+          nginx: generate_nginx(displayed_zombies),
+          htaccess: generate_htaccess(displayed_zombies),
+          redirects: generate_redirects(displayed_zombies),
+          meta_robots: generate_meta_robots
+        }
+      }
+    end
+
+    # Classifies a single zombie URL into a strategic action
+    def classify_zombie(url, active_candidates)
+      path = uri_path(url).downcase
+
+      # 1. Check if it's a utility or legal page that should be noindexed
+      if UTILITY_PATHS.any? { |up| path.start_with?(up) || path == up }
+        return {
+          action: :noindex,
+          target_url: nil,
+          rationale: "Utility/legal page necessary for user experience; noindex to preserve crawl equity."
+        }
+      end
+
+      # 2. Check for thin taxonomy, pagination, or feed loops
+      if THIN_PATTERNS.any? { |pat| path =~ pat }
+        return {
+          action: :purge_410,
+          target_url: nil,
+          rationale: "Thin taxonomy/archive/pagination loop; return 410 Gone to immediately drop from Googlebot index."
+        }
+      end
+
+      # 3. Check for candidate active pillar target to redirect equity
+      candidate = find_candidate_target(url, active_candidates)
+      if candidate
+        return {
+          action: :redirect_301,
+          target_url: candidate,
+          rationale: "Topical slug overlap with active pillar '#{uri_path(candidate)}'; 301 redirect to consolidate equity."
+        }
+      end
+
+      # 4. Check for dated content or parameter duplicates
+      if path.match?(%r{/\d{4}/\d{2}/}) || path.include?('?')
+        return {
+          action: :consolidate,
+          target_url: nil,
+          rationale: "Dated archive or parameter URL; consolidate or rewrite into evergreen canonical."
+        }
+      end
+
+      # 5. Default dead zombie: Purge 410
+      {
+        action: :purge_410,
+        target_url: nil,
+        rationale: "Zero impressions over #{@days} days with no active pillar match; return 410 Gone to reclaim crawl budget."
+      }
+    end
+
+    # Finds an active page with substantial slug keyword overlap
+    def find_candidate_target(zombie_url, active_candidates)
+      zombie_tokens = extract_slug_tokens(zombie_url)
+      return nil if zombie_tokens.empty?
+
+      best_candidate = nil
+      best_score = 0.0
+
+      active_candidates.each do |act_url|
+        act_tokens = extract_slug_tokens(act_url)
+        next if act_tokens.empty?
+
+        common = zombie_tokens & act_tokens
+        next if common.empty?
+
+        # Score based on token overlap
+        overlap_ratio = common.size.to_f / [zombie_tokens.size, act_tokens.size].max
+        if overlap_ratio > best_score && common.size >= 2
+          best_score = overlap_ratio
+          best_candidate = act_url
+        end
+      end
+
+      best_score >= 0.4 ? best_candidate : nil
+    end
+
+    # Server directive generators
+    def generate_nginx(zombies)
+      lines = [
+        "# ====================================================================",
+        "# Nginx Zombie Content Purge Directives (gsc-cli)",
+        "# Reclaims Googlebot crawl budget and stops 404 crawl waste",
+        "# ===================================================================="
+      ]
+
+      zombies.each do |z|
+        path = uri_path(z[:url])
+        case z[:action]
+        when :purge_410
+          lines << "location = #{path} { return 410; }"
+        when :redirect_301
+          target_path = uri_path(z[:target_url] || '/')
+          lines << "location = #{path} { return 301 #{target_path}; }"
+        end
+      end
+      lines.join("\n")
+    end
+
+    def generate_htaccess(zombies)
+      lines = [
+        "# ====================================================================",
+        "# Apache .htaccess Zombie Content Purge Directives (gsc-cli)",
+        "# ===================================================================="
+      ]
+
+      zombies.each do |z|
+        path = uri_path(z[:url])
+        case z[:action]
+        when :purge_410
+          lines << "RedirectGone #{path}"
+        when :redirect_301
+          target_path = uri_path(z[:target_url] || '/')
+          lines << "Redirect 301 #{path} #{target_path}"
+        end
+      end
+      lines.join("\n")
+    end
+
+    def generate_redirects(zombies)
+      lines = [
+        "# ====================================================================",
+        "# Netlify / Cloudflare Pages _redirects Rules (gsc-cli)",
+        "# ===================================================================="
+      ]
+
+      zombies.each do |z|
+        path = uri_path(z[:url])
+        case z[:action]
+        when :purge_410
+          lines << "#{path}   410!"
+        when :redirect_301
+          target_path = uri_path(z[:target_url] || '/')
+          lines << "#{path}   #{target_path}   301"
+        end
+      end
+      lines.join("\n")
+    end
+
+    def generate_meta_robots
+      '<meta name="robots" content="noindex, follow">'
+    end
+
+    private
+
+    def compute_grade(cedi)
+      case cedi
+      when 0..10.0 then 'A'
+      when 10.1..25.0 then 'B'
+      when 25.1..45.0 then 'C'
+      when 45.1..70.0 then 'D'
+      else 'F'
+      end
+    end
+
+    def format_action_label(action)
+      case action
+      when :purge_410 then 'PURGE (410 GONE)'
+      when :redirect_301 then 'REDIRECT (301)'
+      when :consolidate then 'CONSOLIDATE'
+      when :noindex then 'NOINDEX'
+      else action.to_s.upcase
+      end
+    end
+
+    def extract_slug_tokens(url)
+      path = uri_path(url)
+      clean = path.sub(/\.(html?|php|aspx?)$/i, '')
+      tokens = clean.split(/[\/\-_]/).map(&:downcase).reject do |t|
+        t.empty? || t.match?(/^\d+$/) || STOPWORDS.include?(t)
+      end
+      tokens.uniq
+    end
+
+    def uri_path(url_str)
+      return '/' if url_str.nil? || url_str.empty?
+      u = URI.parse(url_str)
+      u.path.nil? || u.path.empty? ? '/' : u.path
+    rescue URI::InvalidURIError
+      url_str.sub(%r{^https?://[^/]+}, '')
+    end
+
+    def clean_url_key(url_str)
+      url_str.to_s.strip.downcase.sub(%r{/$}, '')
+    end
+
+    def normalize_url_list(urls)
+      Array(urls).map(&:to_s).map(&:strip).reject(&:empty?).uniq
+    end
+
+    def normalize_gsc_pages(gsc_pages)
+      map = {}
+      if gsc_pages.is_a?(Hash)
+        gsc_pages.each do |k, v|
+          clean_k = clean_url_key(k)
+          if v.is_a?(Hash)
+            map[clean_k] = {
+              clicks: (v[:clicks] || v['clicks'] || 0).to_i,
+              impressions: (v[:impressions] || v['impressions'] || 0).to_i,
+              position: (v[:position] || v['position'] || 0.0).to_f
+            }
+          else
+            map[clean_k] = { clicks: 0, impressions: v.to_i, position: 0.0 }
+          end
+        end
+      elsif gsc_pages.is_a?(Array)
+        gsc_pages.each do |row|
+          next unless row.is_a?(Hash)
+          keys = row['keys'] || row[:keys]
+          next unless keys && keys.first
+          clean_k = clean_url_key(keys.first)
+          map[clean_k] = {
+            clicks: (row['clicks'] || row[:clicks] || 0).to_i,
+            impressions: (row['impressions'] || row[:impressions] || 0).to_i,
+            position: (row['position'] || row[:position] || 0.0).to_f
+          }
+        end
+      end
+      map
+    end
+  end
+end