gsc-cli 2.1.0 → 2.2.0

data/lib/gsc/sitemap_loader.rb

@@ -4,9 +4,18 @@ require 'net/http'
 require 'uri'
 require 'zlib'
 require 'stringio'
+require 'cgi'
 
 module GSC
   class SitemapLoader
+    def initialize(path_or_url = nil)
+      @path_or_url = path_or_url
+    end
+
+    def load(default_origin = nil)
+      self.class.resolve_urls(@path_or_url, default_origin, quiet: true)
+    end
+
     def self.fetch_content(path_or_url)
       if path_or_url.start_with?('http://', 'https://')
         uri = URI(path_or_url)
@@ -61,14 +70,15 @@ module GSC
       end
 
       xml = fetch_content(target_input)
+      clean_xml = xml.to_s.gsub(/<!\[CDATA\[(.*?)\]\]>/m, '\1')
       urls = []
 
       # Parse child sitemaps if this is a sitemap index
-      sitemap_locs = xml.scan(/<sitemap>\s*<loc>([^<]+)<\/loc>/m).flatten
+      sitemap_locs = clean_xml.scan(/<sitemap>\s*<loc>([^<]+)<\/loc>/m).flatten
       if sitemap_locs.any?
         puts "📑 Found #{sitemap_locs.size} nested sitemaps in index..." unless quiet
         sitemap_locs.each do |child_url|
-          child_url = child_url.strip
+          child_url = CGI.unescapeHTML(child_url.strip)
           unless child_url.start_with?('http://', 'https://')
             puts Color.c("   ⚠️ Skipping non-HTTP child sitemap location: #{child_url}", Color::YELLOW) unless quiet
             next
@@ -76,13 +86,14 @@ module GSC
           puts "   ↳ Loading child sitemap: #{child_url}" unless quiet
           begin
             child_xml = fetch_content(child_url)
-            urls.concat(child_xml.scan(/<url>\s*<loc>([^<]+)<\/loc>/m).flatten.map(&:strip))
+            clean_child = child_xml.to_s.gsub(/<!\[CDATA\[(.*?)\]\]>/m, '\1')
+            urls.concat(clean_child.scan(/<url>\s*<loc>([^<]+)<\/loc>/m).flatten.map { |u| CGI.unescapeHTML(u.strip) })
           rescue StandardError => e
             puts Color.c("   ⚠️ Warning: Could not load child sitemap #{child_url}: #{e.message}", Color::YELLOW) unless quiet
           end
         end
       else
-        urls.concat(xml.scan(/<loc>([^<]+)<\/loc>/m).flatten.map(&:strip))
+        urls.concat(clean_xml.scan(/<loc>([^<]+)<\/loc>/m).flatten.map { |u| CGI.unescapeHTML(u.strip) })
       end
 
       urls.uniq

data/lib/gsc/sitemap_tree.rb

@@ -0,0 +1,301 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'zlib'
+require 'stringio'
+require 'time'
+require 'date'
+require 'cgi'
+
+module GSC
+  class SitemapTree
+    MAX_URLS_PER_SITEMAP = 50_000
+    MAX_BYTES_PER_SITEMAP = 50 * 1024 * 1024 # 50 MB
+    STALE_DAYS_THRESHOLD = 180
+
+    attr_reader :root_source, :options, :tree, :all_urls, :violations
+
+    def initialize(root_source, options = {})
+      @root_source = root_source.to_s.strip
+      @options = options
+      @tree = {}
+      @all_urls = []
+      @violations = []
+    end
+
+    def audit(gsc_pages = [])
+      content, size_bytes = fetch_raw_content(@root_source)
+      is_index = content.include?('<sitemapindex')
+
+      @tree = if is_index
+                parse_sitemap_index(@root_source, content, size_bytes)
+              else
+                parse_single_urlset(@root_source, content, size_bytes)
+              end
+
+      # Collect all unique URLs from tree
+      @all_urls = collect_urls_from_tree(@tree).uniq
+
+      # Coverage analysis against GSC pages if provided
+      coverage = analyze_coverage(gsc_pages, @all_urls)
+
+      # Global compliance health score (0-100)
+      health_score = calculate_health_score
+
+      {
+        root: @root_source,
+        is_index: is_index,
+        total_sitemaps: count_sitemaps(@tree),
+        total_urls: @all_urls.size,
+        total_size_bytes: sum_sizes(@tree),
+        tree: @tree,
+        all_urls: @all_urls,
+        violations: @violations,
+        coverage: coverage,
+        health_score: health_score
+      }
+    end
+
+    private
+
+    def parse_sitemap_index(source_url, xml, size_bytes)
+      sub_sitemaps = []
+      now = Time.now
+
+      # Extract <sitemap> blocks
+      xml.scan(/<sitemap\b[^>]*>(.*?)<\/sitemap>/im).each do |match|
+        block = match[0]
+        loc = block.match(/<loc\b[^>]*>(.*?)<\/loc>/i)&.[](1)&.strip
+        next unless loc
+        loc = loc.sub(/^<!\[CDATA\[/i, '').sub(/\]\]>$/i, '').strip
+        loc = CGI.unescapeHTML(loc)
+
+        lastmod_str = block.match(/<lastmod\b[^>]*>(.*?)<\/lastmod>/i)&.[](1)&.strip
+        lastmod_time, lastmod_valid = parse_and_validate_timestamp(lastmod_str, loc)
+
+        # Recursively fetch child sitemap if requested or level 1
+        child_node = begin
+                       child_content, child_size = fetch_raw_content(loc)
+                       if child_content.include?('<sitemapindex')
+                         parse_sitemap_index(loc, child_content, child_size)
+                       else
+                         parse_single_urlset(loc, child_content, child_size)
+                       end
+                     rescue => e
+                       @violations << { type: :fetch_error, sitemap: loc, message: e.message }
+                       {
+                         url: loc,
+                         type: :error,
+                         error: e.message,
+                         url_count: 0,
+                         urls: []
+                       }
+                     end
+
+        child_node[:declared_lastmod] = lastmod_str
+        child_node[:lastmod_time] = lastmod_time
+        child_node[:lastmod_valid] = lastmod_valid
+
+        sub_sitemaps << child_node
+      end
+
+      {
+        url: source_url,
+        type: :sitemapindex,
+        size_bytes: size_bytes,
+        children: sub_sitemaps,
+        url_count: sub_sitemaps.sum { |s| s[:url_count] || 0 }
+      }
+    end
+
+    def parse_single_urlset(source_url, xml, size_bytes)
+      urls = []
+      now = Time.now
+      latest_lastmod = nil
+
+      # Check file size limit (50MB)
+      if size_bytes > MAX_BYTES_PER_SITEMAP
+        @violations << {
+          type: :size_overflow,
+          sitemap: source_url,
+          size_bytes: size_bytes,
+          message: "Exceeds Google 50MB limit (#{format('%.2f', size_bytes / (1024.0 * 1024.0))} MB)"
+        }
+      end
+
+      xml.scan(/<url\b[^>]*>(.*?)<\/url>/im).each do |match|
+        block = match[0]
+        loc = block.match(/<loc\b[^>]*>(.*?)<\/loc>/i)&.[](1)&.strip
+        next unless loc
+        loc = loc.sub(/^<!\[CDATA\[/i, '').sub(/\]\]>$/i, '').strip
+        loc = CGI.unescapeHTML(loc)
+
+        lastmod_str = block.match(/<lastmod\b[^>]*>(.*?)<\/lastmod>/i)&.[](1)&.strip
+        changefreq = block.match(/<changefreq\b[^>]*>(.*?)<\/changefreq>/i)&.[](1)&.strip
+        priority = block.match(/<priority\b[^>]*>(.*?)<\/priority>/i)&.[](1)&.strip
+
+        lastmod_time, lastmod_valid = parse_and_validate_timestamp(lastmod_str, source_url)
+        if lastmod_time && (latest_lastmod.nil? || lastmod_time > latest_lastmod)
+          latest_lastmod = lastmod_time
+        end
+
+        urls << {
+          loc: loc,
+          lastmod: lastmod_str,
+          changefreq: changefreq,
+          priority: priority
+        }
+      end
+
+      # Check URL count limit (50,000)
+      if urls.size > MAX_URLS_PER_SITEMAP
+        @violations << {
+          type: :url_overflow,
+          sitemap: source_url,
+          count: urls.size,
+          message: "Exceeds Google 50,000 URLs per sitemap limit (#{urls.size} URLs)"
+        }
+      end
+
+      # Check date drift / staleness
+      drift_days = latest_lastmod ? ((now - latest_lastmod) / 86400.0).round : nil
+      is_stale = drift_days && drift_days > STALE_DAYS_THRESHOLD
+
+      {
+        url: source_url,
+        type: :urlset,
+        size_bytes: size_bytes,
+        url_count: urls.size,
+        latest_lastmod: latest_lastmod ? latest_lastmod.strftime('%Y-%m-%d') : nil,
+        drift_days: drift_days,
+        is_stale: is_stale,
+        urls: urls
+      }
+    end
+
+    def parse_and_validate_timestamp(ts_str, sitemap_url)
+      return [nil, true] if ts_str.nil? || ts_str.empty?
+
+      # ISO-8601 regex test
+      is_iso = ts_str =~ /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)?$/
+      unless is_iso
+        @violations << {
+          type: :invalid_timestamp,
+          sitemap: sitemap_url,
+          timestamp: ts_str,
+          message: "Invalid non-ISO-8601 timestamp: '#{ts_str}'"
+        }
+        return [nil, false]
+      end
+
+      t = Time.parse(ts_str) rescue nil
+      if t && t > (Time.now + 86400) # Allow 1 day clock drift
+        @violations << {
+          type: :future_timestamp,
+          sitemap: sitemap_url,
+          timestamp: ts_str,
+          message: "Future timestamp detected: #{ts_str}"
+        }
+        return [t, false]
+      end
+
+      [t, true]
+    end
+
+    def collect_urls_from_tree(node)
+      if node[:type] == :sitemapindex && node[:children]
+        node[:children].flat_map { |c| collect_urls_from_tree(c) }
+      elsif node[:type] == :urlset && node[:urls]
+        node[:urls].map { |u| u[:loc] }
+      else
+        []
+      end
+    end
+
+    def count_sitemaps(node)
+      if node[:type] == :sitemapindex && node[:children]
+        1 + node[:children].sum { |c| count_sitemaps(c) }
+      else
+        1
+      end
+    end
+
+    def sum_sizes(node)
+      size = node[:size_bytes] || 0
+      if node[:type] == :sitemapindex && node[:children]
+        size += node[:children].sum { |c| sum_sizes(c) }
+      end
+      size
+    end
+
+    def analyze_coverage(gsc_pages, sitemap_urls)
+      return { status: :no_gsc_data } if gsc_pages.nil? || gsc_pages.empty?
+
+      sitemap_set = sitemap_urls.map { |u| normalize_url(u) }.to_set
+      gsc_urls = gsc_pages.map { |p| normalize_url(p[:url] || p['url'] || p) }
+
+      included = gsc_urls.select { |u| sitemap_set.include?(u) }
+      missing = gsc_urls.reject { |u| sitemap_set.include?(u) }
+
+      coverage_pct = gsc_urls.any? ? ((included.size.to_f / gsc_urls.size) * 100.0).round(1) : 100.0
+
+      {
+        total_gsc_pages: gsc_urls.size,
+        included_in_sitemap: included.size,
+        missing_from_sitemap: missing.size,
+        coverage_pct: coverage_pct,
+        missing_sample: missing.first(10)
+      }
+    end
+
+    def calculate_health_score
+      score = 100
+      @violations.each do |v|
+        case v[:type]
+        when :fetch_error then score -= 25
+        when :size_overflow, :url_overflow then score -= 20
+        when :future_timestamp then score -= 15
+        when :invalid_timestamp then score -= 10
+        end
+      end
+      [0, score].max
+    end
+
+    def fetch_raw_content(source)
+      if source =~ %r{^https?://}
+        uri = URI.parse(source)
+        req = Net::HTTP::Get.new(uri)
+        req['User-Agent'] = 'Mozilla/5.0 (compatible; GSC-SitemapTree-Auditor/1.0)'
+        req['Accept-Encoding'] = 'gzip'
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == 'https')
+        http.open_timeout = 8
+        http.read_timeout = 15
+
+        res = http.request(req)
+        raise "HTTP #{res.code} on #{source}" unless res.is_a?(Net::HTTPSuccess)
+
+        raw = res.body || ''
+        decompressed = if (res['content-encoding'] =~ /gzip/i || source.end_with?('.gz')) && !raw.empty?
+                         Zlib::GzipReader.new(StringIO.new(raw)).read
+                       else
+                         raw
+                       end
+        content = decompressed.to_s.dup.force_encoding('UTF-8').scrub
+        [content, content.bytesize]
+      else
+        raise "Local sitemap file not found: #{source}" unless File.exist?(source)
+        content = File.read(source, encoding: 'UTF-8').scrub
+        [content, content.bytesize]
+      end
+    end
+
+    def normalize_url(url)
+      u = url.to_s.strip.downcase.chomp('/')
+      u.sub(%r{^https?://(www\.)?}, '')
+    end
+  end
+end

data/lib/gsc/skill_pack.rb

@@ -0,0 +1,195 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+
+module GSC
+  class SkillPack
+    DEFAULT_SKILL_NAME = 'gsc'
+
+    attr_reader :options, :target_dir
+
+    def initialize(options = {}, target_dir = nil)
+      @options = options
+      @target_dir = target_dir || options[:dir]
+    end
+
+    def self.package(options = {}, target_dir = nil)
+      new(options, target_dir).package
+    end
+
+    def package
+      content_md = generate_skill_markdown
+      recipes = generate_recipes_json
+
+      destinations = resolve_destinations
+
+      installed_paths = []
+
+      unless @options[:dry_run]
+        destinations.each do |dest|
+          FileUtils.mkdir_p(dest[:dir])
+          skill_path = File.join(dest[:dir], 'SKILL.md')
+          recipe_path = File.join(dest[:dir], 'recipes.json')
+
+          File.write(skill_path, content_md, encoding: 'UTF-8')
+          File.write(recipe_path, JSON.pretty_generate(recipes), encoding: 'UTF-8')
+
+          installed_paths << {
+            platform: dest[:platform],
+            skill_path: skill_path,
+            recipe_path: recipe_path
+          }
+        end
+      end
+
+      # Agent verification check
+      verification = verify_agent_environment
+
+      {
+        skill_name: DEFAULT_SKILL_NAME,
+        version: GSC::VERSION,
+        dry_run: !!@options[:dry_run],
+        destinations_count: destinations.size,
+        installed_locations: installed_paths,
+        verification: verification,
+        recipes_count: recipes[:recipes].size,
+        skill_preview: content_md[0..400] + "..."
+      }
+    end
+
+    def generate_skill_markdown
+      <<~MARKDOWN
+        ---
+        name: gsc
+        description: Automates Google Search Console, Google Indexing API, live URL index inspection, sitemap batch submission, search ranking analytics, E-E-A-T audits, image optimization, schema testing, and conversion-weighted revenue opportunity matrices.
+        ---
+
+        # Google Search Console (GSC) CLI Agent Skill
+
+        This skill provides full-featured, zero-dependency autonomous search engine intelligence directly inside AI developer coding environments (Google Antigravity, Claude Code, Cursor, Windsurf).
+
+        ## Core Capabilities & Quick Reference
+
+        ### 1. Organic Search Performance & Analytics
+        - `gsc performance [domain] --json`: Overview clicks, impressions, CTR, average position.
+        - `gsc top-queries [domain] --limit 50 --json`: Top ranking search queries.
+        - `gsc top-pages [domain] --limit 50 --json`: Top ranking landing pages.
+        - `gsc opportunities [domain] --json`: High-opportunity queries in striking distance (positions 4–20).
+        - `gsc strike [domain] --json`: Tactical striking-distance keyword playbook.
+        - `gsc brand [domain] --json`: Brand vs non-brand organic search segmentation.
+
+        ### 2. Live URL Indexing & Googlebot Inspection
+        - `gsc inspect <url> --json`: Real-time Googlebot crawl verdict and indexing status.
+        - `gsc index <url> --json`: Submit URL to Google Indexing API for rapid re-crawl.
+        - `gsc index-batch [sitemap] --json`: Bulk submit URLs from sitemap to Indexing API.
+
+        ### 3. SEO Diagnostics & Technical Audits
+        - `gsc audit [domain] --json`: Comprehensive 360-degree organic health audit.
+        - `gsc report [domain] --html --json`: Generate single-file executive audit dashboard.
+        - `gsc image-seo <url|file> --json`: Audit WebP/AVIF formats, missing alt text, and CLS image dimensions.
+        - `gsc hreflang-check <url|file> --json`: International hreflang reciprocity and ISO syntax verification.
+        - `gsc eeat <url|file> --json`: Author E-E-A-T, credentials, reviewer attribution, and Knowledge Graph signals.
+        - `gsc rich-results <url|file> --json`: Test JSON-LD against Google Rich Results API rules.
+        - `gsc mobile-parity [domain] --json`: Cross-device Desktop vs Mobile position gap and responsive penalty audit.
+
+        ### 4. Growth, Revenue & Strategy
+        - `gsc kw-value [domain] --json`: Conversion-weighted keyword revenue matrix ($ upside to Top 3).
+        - `gsc intent-shift [domain] --json`: Search intent drift & landing page mismatch detection.
+        - `gsc zombies [sitemap] --json`: Identify dead crawl-waste pages and generate 410/301 rules.
+        - `gsc watch [domain] --once --json`: Background rank drop and CTR crash monitoring watchdog.
+        - `gsc cite-sim <url> <query> --json`: AI Search Overviews citation probability simulator.
+
+        ## Autonomous Execution Recipes for AI Agents
+
+        ### Recipe 1: Diagnose Sudden Organic Traffic Drop
+        1. Run `gsc watch [domain] --once --json` to detect queries with severe rank drops (>= 2 positions).
+        2. Run `gsc mobile-parity [domain] --json` to check if drops are concentrated on mobile devices.
+        3. For dropped URLs, run `gsc inspect <url> --json` to check for Googlebot indexing failures.
+
+        ### Recipe 2: High-ROI Striking Distance Acceleration
+        1. Run `gsc kw-value [domain] --json` to locate queries in positions 4–15 with highest monthly $ upside.
+        2. Run `gsc intent-shift [domain] --json` to verify searcher intent matches the landing page type.
+        3. Run `gsc rich-results <url> --json` to check if adding FAQ or Product schema can capture SERP real estate.
+      MARKDOWN
+    end
+
+    def generate_recipes_json
+      {
+        skill: DEFAULT_SKILL_NAME,
+        version: GSC::VERSION,
+        recipes: [
+          {
+            name: "diagnose_traffic_drop",
+            trigger: "User reports traffic drop or lost search rankings",
+            steps: [
+              { command: "gsc watch {{domain}} --once --json", extract: "alerts" },
+              { command: "gsc mobile-parity {{domain}} --json", extract: "disparities" }
+            ]
+          },
+          {
+            name: "optimize_striking_distance",
+            trigger: "User asks how to increase revenue or capture high-ROI search keywords",
+            steps: [
+              { command: "gsc kw-value {{domain}} --json", extract: "keywords" },
+              { command: "gsc intent-shift {{domain}} --json", extract: "shifts" }
+            ]
+          },
+          {
+            name: "technical_page_audit",
+            trigger: "User asks to audit an individual page or template",
+            steps: [
+              { command: "gsc rich-results {{url}} --json", extract: "schemas" },
+              { command: "gsc image-seo {{url}} --json", extract: "images" },
+              { command: "gsc eeat {{url}} --json", extract: "signals" }
+            ]
+          }
+        ]
+      }
+    end
+
+    private
+
+    def resolve_destinations
+      destinations = []
+
+      # If explicit target directory provided
+      if @target_dir && !@target_dir.empty?
+        destinations << { platform: 'Custom Directory', dir: File.expand_path(@target_dir) }
+        return destinations
+      end
+
+      target = (@options[:target] || 'all').to_s.downcase
+
+      if target == 'all' || target == 'antigravity' || target == 'gemini'
+        gemini_dir = File.expand_path('~/.gemini/config/skills/gsc')
+        destinations << { platform: 'Google Antigravity / Gemini', dir: gemini_dir }
+      end
+
+      if target == 'all' || target == 'claude'
+        claude_dir = File.expand_path('~/.claude/skills/gsc')
+        destinations << { platform: 'Claude Code', dir: claude_dir }
+      end
+
+      if target == 'all' || target == 'workspace' || target == 'local'
+        local_dir = File.expand_path('.agents/skills/gsc', Dir.pwd)
+        destinations << { platform: 'Universal Workspace (.agents)', dir: local_dir }
+      end
+
+      destinations
+    end
+
+    def verify_agent_environment
+      bin = ENV['GSC_BIN_PATH'] || File.expand_path('~/.local/bin/gsc')
+      bin_exists = File.exist?(bin) && File.executable?(bin)
+
+      {
+        binary_path: bin,
+        binary_available: bin_exists,
+        ruby_version: RUBY_VERSION,
+        status: bin_exists ? 'READY_FOR_AI_AGENTS' : 'BINARY_NEEDS_INSTALL'
+      }
+    end
+  end
+end