gsc-cli 2.0.2 → 2.2.0

data/lib/gsc/client.rb

@@ -55,12 +55,27 @@ module GSC
       # Transparent gzip decompression
       raw_body = response.body
       body_str = if response['content-encoding'] =~ /gzip/i && raw_body && !raw_body.empty?
-                   Zlib::GzipReader.new(StringIO.new(raw_body)).read
+                   begin
+                     Zlib::GzipReader.new(StringIO.new(raw_body)).read
+                   rescue Zlib::GzipFile::Error, Zlib::Error
+                     raw_body
+                   end
                  else
                    raw_body
                  end
 
-      body = body_str ? (JSON.parse(body_str) rescue body_str) : nil
+      body_str = body_str.to_s.dup.force_encoding('UTF-8').scrub if body_str
+
+      body = if body_str && !body_str.strip.empty?
+               begin
+                 parsed = JSON.parse(body_str)
+                 parsed.is_a?(Hash) || parsed.is_a?(Array) ? parsed : { 'value' => parsed }
+               rescue JSON::ParserError, Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
+                 { 'error' => { 'message' => body_str }, 'raw' => body_str }
+               end
+             else
+               {}
+             end
 
       {
         ok: response.is_a?(Net::HTTPSuccess),

data/lib/gsc/color.rb

@@ -16,7 +16,22 @@ module GSC
     GRAY      = "\e[90m"
 
     def self.c(text, *styles)
-      "#{styles.join}#{text}#{RESET}"
+      t = text.to_s.dup.force_encoding('UTF-8').scrub
+      "#{styles.join}#{t}#{RESET}"
+    end
+
+    def self.cyan(str); c(str, CYAN); end
+    def self.yellow(str); c(str, YELLOW); end
+    def self.green(str); c(str, GREEN); end
+    def self.red(str); c(str, RED); end
+    def self.bold(str); c(str, BOLD); end
+    def self.gray(str); c(str, GRAY); end
+    def self.magenta(str); c(str, MAGENTA); end
+    def self.white(str); c(str, WHITE); end
+    def self.dim(str); c(str, DIM); end
+
+    def self.strip_ansi(str)
+      str.to_s.gsub(/\e\[[0-9;]*[mK]/, '')
     end
   end
 end

data/lib/gsc/command_registry.rb

@@ -6,9 +6,9 @@ COMMAND_REGISTRY = [
   {
     category: "Google Trends & Keyword Demand",
     commands: [
-{ name: "ke <seed|file>", shortcut: "ke", desc: "Keywords Everywhere: Exact monthly volume, CPC, competition & GSC correlation", flags: ["--country", "--limit", "--json"] },
-{ name: "ke-credits", shortcut: "ke-credits", desc: "Check remaining Keywords Everywhere account API credits", flags: ["--json"] },
-{ name: "connect ke [key]", shortcut: "connect ke", desc: "Connect Keywords Everywhere API key and save to config.json", flags: [] },
+      { name: "ke <seed|file>", shortcut: "ke", desc: "Keywords Everywhere: Exact monthly volume, CPC, competition & GSC correlation", flags: ["--country", "--limit", "--json"] },
+      { name: "ke-credits", shortcut: "ke-credits", desc: "Check remaining Keywords Everywhere account API credits", flags: ["--json"] },
+      { name: "connect ke [key]", shortcut: "connect ke", desc: "Connect Keywords Everywhere API key and save to config.json", flags: [] },
       { name: "trends <query>", shortcut: "tr", desc: "Live Google Trends: 5yr/1yr demand trajectory, velocity & breakout queries", flags: ["--geo", "--time", "--json"] },
       { name: "planner <seed>", shortcut: "kp", desc: "Free keyword planner: intent expansion & GSC ranking correlation", flags: ["--country", "--limit", "--json"] },
       { name: "planner-import <file>", shortcut: "pi", desc: "Import Google Ads CSV or Markdown table to score & rank opportunities", flags: ["--limit", "--json"] }
@@ -18,12 +18,18 @@ COMMAND_REGISTRY = [
     category: "Search Console Intelligence",
     commands: [
       { name: "performance", shortcut: "p", desc: "Executive dashboard: Clicks, Impressions, CTR, Position", flags: ["--days", "--limit", "--csv"] },
-      { name: "top-queries", shortcut: "t", desc: "Top search queries, rankings, and CTR", flags: ["--days", "--limit", "-s", "--order", "--csv"] },
+      { name: "top-queries", shortcut: "t", desc: "Top search queries, rankings, and CTR", flags: ["--days", "--limit", "-s", "--order", "--brand", "--non-brand", "--brand-name", "--csv"] },
+      { name: "brand", shortcut: "b", desc: "Brand vs. non-brand search query segmentation & share", flags: ["--days", "--limit", "--brand-name", "--json"] },
+      { name: "ctr-curve", shortcut: nil, desc: "Expected SERP CTR curve & click gain simulator", flags: ["--target-pos", "--min-imp", "--limit", "--days", "--csv"] },
       { name: "top-pages", shortcut: nil, desc: "Top landing pages driving organic search clicks", flags: ["--days", "--limit", "-s", "--order", "--csv"] },
       { name: "opportunities", shortcut: "o", desc: "Striking-distance queries (Pos 7-20) to push to Top 3", flags: ["--days", "--min-imp", "--min-pos", "--max-pos"] },
+      { name: "strike", shortcut: "sp", desc: "Striking-distance tactical playbook: Title rewrites, heading recipes & internal link anchors (Pos 7-20)", flags: ["--days", "--min-imp", "--min-pos", "--max-pos", "--limit", "--csv", "--json"] },
       { name: "underperformers", shortcut: "u", desc: "High-ranking queries (Top 10) with low CTR (title tag wins)", flags: ["--days", "--min-imp"] },
+      { name: "low-ctr [query/url]", shortcut: "lc", desc: "Low-CTR High-Impression Title Tag Rewriter: 3 SERP-safe hook titles (<560px), click hemorrhage & revenue recovery", flags: ["--min-imp", "--cpc", "--limit", "--brand", "--csv", "--json"] },
       { name: "cannibalization", shortcut: "c", desc: "Detect internal URLs competing for the same keywords", flags: ["--days", "--min-imp"] },
+      { name: "questions-harvest [page]", shortcut: "qh", desc: "Harvest question queries from GSC and generate FAQPage JSON-LD schema", flags: ["--days", "--min-imp", "--limit", "--generate", "--csv", "--json"] },
       { name: "decay", shortcut: "d", desc: "Period-over-period decay detection (decaying vs surging)", flags: ["--compare"] },
+      { name: "sparklines [query|page]", shortcut: "spark", desc: "Terminal ASCII Sparklines & Multi-Horizon Trajectory Graphs (Clicks, Imp, CTR, Pos)", flags: ["--days", "--metric", "--height", "--data", "--csv", "--json"] },
       { name: "trends", shortcut: nil, desc: "Compare current vs prior period search trends", flags: ["--compare"] },
       { name: "devices", shortcut: nil, desc: "Desktop vs Mobile vs Tablet search traffic share", flags: ["--days"] },
       { name: "countries", shortcut: nil, desc: "Geographic search demand by country", flags: ["--days", "--limit"] },
@@ -52,27 +58,59 @@ COMMAND_REGISTRY = [
     commands: [
       { name: "inspect <url>", shortcut: nil, desc: "Live Google index check (coverage, canonical, date, robots.txt)", flags: ["-d"] },
       { name: "index <url>", shortcut: nil, desc: "Notify Googlebot to crawl/index URL immediately (URL_UPDATED)", flags: ["--dry-run"] },
+      { name: "index-batch [action]", shortcut: "ib", desc: "Automated 200 URL/day quota queue manager (add, run, status, clear)", flags: ["--batch-size", "--dry-run", "--json"] },
       { name: "remove <url>", shortcut: nil, desc: "Notify Googlebot a page has been deleted (URL_DELETED)", flags: ["--dry-run"] },
       { name: "status <url>", shortcut: nil, desc: "Check Google Indexing API notification metadata", flags: [] },
       { name: "sitemaps-list", shortcut: nil, desc: "List registered XML sitemaps in Search Console", flags: ["-d"] },
       { name: "sitemaps-submit <url>", shortcut: nil, desc: "Submit/register XML sitemap with Search Console", flags: ["-d"] },
       { name: "index-sitemap <file/url>", shortcut: nil, desc: "Batch notify Googlebot to index all sitemap URLs", flags: ["--delay", "--dry-run"] },
-      { name: "inspect-sitemap <file/url>", shortcut: nil, desc: "Bulk inspect indexation status for all sitemap URLs", flags: ["--delay"] }
+      { name: "inspect-sitemap <file/url>", shortcut: nil, desc: "Bulk inspect indexation status for all sitemap URLs", flags: ["--delay"] },
+      { name: "indexnow <url>", shortcut: "in", desc: "Instant indexing via IndexNow protocol (Bing, Yandex, Seznam, Naver)", flags: ["--key", "--json"] },
+      { name: "indexnow-sitemap <sitemap>", shortcut: "ins", desc: "Batch submit all sitemap URLs to IndexNow", flags: ["--key", "--limit", "--json"] },
+      { name: "connect indexnow [key]", shortcut: nil, desc: "Configure IndexNow API key and display verification file details", flags: [] }
     ]
   },
   {
     category: "Health, Diagnostics & Setup",
     commands: [
+      { name: "geo [url]", shortcut: "aeo", desc: "Generative Engine Optimization: Audit AI bot access & citability score (ChatGPT, Perplexity, Claude)", flags: ["--json"] },
+      { name: "cite-sim <url|file> [query]", shortcut: "cite", desc: "AI Search Citation Simulator: Emulates ChatGPT, Perplexity & Claude citations, verifiable quote extraction & CLS score", flags: ["--query", "--model", "--csv", "--json"] },
+      { name: "answer <query>", shortcut: "ans", desc: "Synthesize high-citability 40-60w direct answers & info-gain snippets (Featured Snippets & Perplexity)", flags: ["--brand", "--type", "--generate", "--json"] },
+      { name: "entity [url]", shortcut: "kg", desc: "Knowledge Graph entity & sameAs disambiguation audit (Wikidata, Wikipedia, LinkedIn)", flags: ["--generate", "--json"] },
+      { name: "serp <query|url>", shortcut: "preview", desc: "Live SERP Feature Detector (AI Overviews, Snippets, PAA, Local) & Pixel Width Simulator", flags: ["--live", "--preview", "--title", "--desc", "--url", "--json"] },
+      { name: "serp-features <query>", shortcut: "sf", desc: "Scan live Google SERP features, Zero-Click Threat index & Rich Snippet capture playbook", flags: ["--json"] },
       { name: "page <url|file>", shortcut: "pg", desc: "Detailed On-Page DOM + Off-Page GSC Performance Audit", flags: ["--check-links", "--json"] },
+      { name: "headings <url|file>", shortcut: "ht", desc: "Heading hierarchy tree validator: H1-H6 nesting, depth, empty tags & health score", flags: ["--keyword", "--json"] },
       { name: "site-audit [sitemap]", shortcut: "crl", desc: "Full site crawl, broken links (404/500), image alts & AI report", flags: ["--check-links", "--report", "--json"] },
+      { name: "orphans [sitemap|url]", shortcut: "il", desc: "Internal link equity & orphan page rescue: Detect 0-link pages, click depth & suggested rescue sources", flags: ["--limit", "--concurrency", "--csv", "--json"] },
+      { name: "speed-correlate [url]", shortcut: "cwv", desc: "Correlate Core Web Vitals (LCP/CLS) directly with Search Console impressions & projected traffic surge", flags: ["--days", "--strategy", "-d", "--json"] },
+      { name: "firewall [url]", shortcut: "ai-bots", desc: "Scan /robots.txt & edge WAF/Cloudflare bot challenges for ChatGPT, Claude & Perplexity", flags: ["--user-agent", "--json"] },
+      { name: "security [url]", shortcut: "sec", desc: "Security Headers & Mixed Content Auditor: HSTS preload, CSP, clickjacking & SSL/TLS certificate expiry", flags: ["--json"] },
+      { name: "landing-roi [url]", shortcut: "roi", desc: "Landing Page Economic ROI & Revenue Hemorrhage: Merge GSC clicks with GA4 bounce rates to calculate dollar revenue leaks", flags: ["--aov", "--conv-rate", "--benchmark", "--clicks", "--bounce", "--csv", "--json"] },
+      { name: "schema-generate [url]", shortcut: "schema-gen", desc: "Structured Data Rich Snippet Generator: Auto-synthesizes valid Schema.org JSON-LD (Product, FAQ, HowTo, Article, Software) with Google validation", flags: ["--type", "--price", "--currency", "--rating", "--format", "--open", "--json"] },
+      { name: "sitemap-tree [path|url]", shortcut: "smt", desc: "Sitemap Index Hierarchy & Coverage Auditor: Recursive sub-sitemap tree, Google 50k URL / 50MB compliance & GSC coverage ratio", flags: ["--csv", "--json"] },
+      { name: "titles [sitemap|url]", shortcut: "title-opt", desc: "Batch audit title pixel width (>580px) & synthesize non-truncating alternatives", flags: ["--overflow-only", "--limit", "--concurrency", "--csv", "--json"] },
+      { name: "image-seo <url|file>", shortcut: "img", desc: "Image SEO & Next-Gen Format Auditor: WebP/AVIF formats, missing alt text, explicit width/height for CLS & LCP preloading", flags: ["--check-size", "--limit", "--csv", "--json"] },
+      { name: "hreflang-check <url|file>", shortcut: "hreflang", desc: "International Hreflang Reciprocity Validator: Bidirectional link graph, ISO 639-1 / 3166-1 codes & x-default fallback", flags: ["--no-reciprocity", "--csv", "--json"] },
+      { name: "eeat <url|file>", shortcut: "author", desc: "Author E-E-A-T & Credential Auditor: Person schema, credentials, editorial reviews, dates & Knowledge Graph links", flags: ["--csv", "--json"] },
       { name: "audit", shortcut: "a", desc: "360-degree Comprehensive SEO & GA4 health audit", flags: ["--days", "-d"] },
-      { name: "zombies <sitemap>", shortcut: nil, desc: "Find zero-impression crawl waste pages over 90 days", flags: [] },
-      { name: "use <domain or 1-9>", shortcut: nil, desc: "Switch active default domain", flags: [] },
+      { name: "report [domain]", shortcut: "rep", desc: "Automated Executive SEO & AI Search Report Generator: Single-file HTML dashboard & Markdown reports", flags: ["--html", "--md", "--days", "--title", "--json"] },
+      { name: "intent-shift [domain]", shortcut: "intent", desc: "Search Intent Drift & Landing Page Mismatch Tracker: Detect transactional/informational intent divergence & compute Portfolio Volatility Index", flags: ["--days", "--brand", "--csv", "--json"] },
+      { name: "rich-results <url|file>", shortcut: "rich-test", desc: "Google Rich Results Testing & Validation Suite: Disqualifying errors, recommended enhancements, 1-click JSON-LD fixes & browser testing", flags: ["--open", "--patch", "--csv", "--json"] },
+      { name: "watch [domain]", shortcut: "mon", desc: "Continuous SERP Watchdog & Rank Drift Monitor: Real-time rank drop alerts, CTR crash detection, webhooks & automated cron/launchd daemon", flags: ["--once", "--daemon", "--interval", "--threshold", "--webhook", "--slack", "--cron", "--launchd", "--systemd", "--csv", "--json"] },
+      { name: "kw-value [domain]", shortcut: "kwval", desc: "Conversion-Weighted Keyword Opportunity Matrix: Calculate revenue per keyword, simulate Top 3 pipeline upside & prioritize high-buyer-intent terms", flags: ["--aov", "--conv-rate", "--margin", "--target-pos", "--days", "--csv", "--json"] },
+      { name: "mobile-parity [domain]", shortcut: "mobile", desc: "Mobile vs. Desktop SERP Parity Auditor: Cross-device rank gaps, responsive suppression penalties & mobile CTR loss diagnosis", flags: ["--days", "--min-imp", "--gap-threshold", "--csv", "--json"] },
+      { name: "aio-hunter [query|domain]", shortcut: "aio", desc: "Google AI Overview (AIO) Opportunity Hunter: Detects AIO SERP presence, extracts cited sources, quantifies citation gaps & generates snippet capture recipes", flags: ["--limit", "--min-imp", "--days", "--csv", "--json"] },
+      { name: "skill-pack [dir]", shortcut: "agent-pack", desc: "Autonomous AI Agent Skill Packaging Engine: Synthesizes production SKILL.md, recipes.json & integration hooks for Antigravity, Claude Code & Cursor", flags: ["--target", "--dir", "--dry-run", "--json"] },
+      { name: "zombies [sitemap|file]", shortcut: "z", desc: "Zombie Content Purger & Crawl Equity Optimizer: CEDI score (0-100), automated 410/301 triage & 1-click server rules", flags: ["--days", "--threshold", "--action", "--format", "--inventory", "--csv", "--json"] },
+      { name: "soft-404 [url|file]", shortcut: "404", desc: "404 & Soft-404 Crawl Error Diagnostic: Detects thin/empty pages returning 200 OK, calculates crawl waste & synthesizes smart 301/410 redirect rules", flags: ["--concurrency", "--threshold", "--format", "--csv", "--json"] },
+      { name: "switch <domain>", shortcut: "use", desc: "Instant agency domain & credentials switcher (domain, alias, or index #)", flags: ["--json"] },
+      { name: "vault [action]", shortcut: nil, desc: "Agency Credential Vault: AES-256-GCM encrypted store for 50+ service accounts (list, add, remove, status)", flags: ["--domain", "--alias", "--property", "--json"] },
       { name: "domains", shortcut: nil, desc: "List verified domains and GA4 property links", flags: [] },
+      { name: "doctor", shortcut: "doc", desc: "Zero-Dependency Architecture Certification & System Health: Verifies 100% pure Ruby stdlib, cold-start latency (<100ms), OpenSSL cipher support & config permissions", flags: ["--strict", "--fix", "--json"] },
       { name: "where", shortcut: nil, desc: "Inspect installation path, active key, and config file", flags: [] },
       { name: "connect", shortcut: nil, desc: "1-Click Setup Wizard: auto-detects key or drag & drop", flags: [] },
       { name: "connect-ga4", shortcut: nil, desc: "Interactive GA4 linking wizard", flags: [] },
-      { name: "open", shortcut: nil, desc: "Reveal configuration directory (~/.config/gsc) in Finder", flags: [] },
       { name: "prompts [id]", shortcut: "pb", desc: "25 Autonomous AI SEO Playbooks & ready-to-paste prompts", flags: ["--copy", "--json"] },
       { name: "skills [install|show]", shortcut: nil, desc: "Inspect or auto-install AI Agent Skill", flags: [] },
       { name: "commands", shortcut: nil, desc: "List all commands (human-readable or JSON with --json)", flags: ["--json"] }
@@ -267,7 +305,7 @@ COMMAND_REGISTRY = [
       2. **Optimal Length**: Keep `<title>` between **50 and 60 characters** (maximum 580px width) so Google does not truncate with `...`.
       3. **Emotional Hook / CTR Multiplier**: Include brackets `[Free Calculator]`, actionable numbers (`10 Best`, `2026 Checklist`), or primary value props.
       4. **Brand Suffix**: Always append ` | BrandName` at the end.
-      5. **Meta Description**: 130–155 characters summarizing the page benefit with a clear call-to-action (e.g. *"Calculate exact moving box counts by room, size, and weight. Free instant estimator."*).
+      5. **Meta Description**: 130–155 characters summarizing the page benefit with a clear call-to-action (e.g. *"Inspect technical SEO health, Core Web Vitals, and index coverage instantly. Free diagnostic audit."*).
       6. **Heading Polish**: Ensure `<h1>` matches search intent and **never ends with a period (`.`)**.
       
       #### Step 4: Immediately Trigger Googlebot Indexing

data/lib/gsc/config.rb

@@ -9,6 +9,15 @@ module GSC
     CONFIG_DIR  = File.expand_path('~/.config/gsc')
     CONFIG_FILE = File.join(CONFIG_DIR, 'config.json')
 
+def self.get(key)
+  load[key.to_s]
+end
+
+def self.set(key, value)
+  save(key.to_s => value)
+  value
+end
+
     def self.load
       return {} unless File.exist?(CONFIG_FILE)
       JSON.parse(File.read(CONFIG_FILE))
@@ -144,7 +153,7 @@ def self.load_saved_keywords(domain, identifier)
 
   target_file = if identifier.to_s =~ /^\d+$/
                   idx = identifier.to_i - 1
-                  list[idx]&.fetch('path', nil)
+                  idx >= 0 ? list[idx]&.fetch('path', nil) : nil
                 else
                   found = list.find { |item| item['file'] == identifier || item['seed'] == identifier || item['file'].include?(identifier.to_s) }
                   found ? found['path'] : nil
@@ -162,7 +171,7 @@ def self.delete_saved_keywords(domain, identifier)
 
   target_file = if identifier.to_s =~ /^\d+$/
                   idx = identifier.to_i - 1
-                  list[idx]&.fetch('path', nil)
+                  idx >= 0 ? list[idx]&.fetch('path', nil) : nil
                 else
                   found = list.find { |item| item['file'] == identifier || item['file'].include?(identifier.to_s) }
                   found ? found['path'] : nil

data/lib/gsc/content_gap.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'set'
+
+module GSC
+  class ContentGap
+    STOP_WORDS = Set.new(%w[
+      a about above after again against all am an and any are aren't as at be because been before being below
+      between both but by can't cannot could couldn't did didn't do does doesn't doing don't down during each
+      few for from further had hadn't has hasn't have haven't having he he'd he'll he's her here here's hers
+      herself him himself his how how's i i'd i'll i'm i've if in into is isn't it it's its itself let's me
+      more most mustn't my myself no nor not of off on once only or other ought our ours ourselves out over own
+      same shan't she she'd she'll she's should shouldn't so some such than that that's the their theirs them
+      themselves then there there's these they they'd they'll they're they've this those through to too under until
+      up very was wasn't we we'd we'll we're we've were weren't what what's when when's where where's which while
+      who who's whom why why's with won't would wouldn't you you'd you'll you're you've your yours yourself yourselves
+    ])
+
+    attr_reader :url1, :url2
+
+    def initialize(url1, url2)
+      @url1 = url1
+      @url2 = url2
+    end
+
+    def analyze
+      pa1 = GSC::PageAnalyzer.new(@url1)
+      pa2 = GSC::PageAnalyzer.new(@url2)
+
+      data1 = pa1.fetch_and_analyze
+      data2 = pa2.fetch_and_analyze
+
+      text1 = extract_clean_text(pa1.html || '')
+      text2 = extract_clean_text(pa2.html || '')
+
+      tokens1 = tokenize(text1)
+      tokens2 = tokenize(text2)
+
+      unigrams1 = ngrams(tokens1, 1)
+      unigrams2 = ngrams(tokens2, 1)
+
+      bigrams1 = ngrams(tokens1, 2)
+      bigrams2 = ngrams(tokens2, 2)
+
+      trigrams1 = ngrams(tokens1, 3)
+      trigrams2 = ngrams(tokens2, 3)
+
+      # Competitor terms that occur at least 2 times, but occur 0 times in page1
+      missing_unigrams = term_gap(unigrams1, unigrams2, min_count: 2)
+      missing_bigrams  = term_gap(bigrams1, bigrams2, min_count: 2)
+      missing_trigrams = term_gap(trigrams1, trigrams2, min_count: 2)
+
+      # Heading gaps (H2/H3 in page2 that have no match in page1)
+      h1 = (data1.dig(:headings, :h2) || []) + (data1.dig(:headings, :h3) || [])
+      h2 = (data2.dig(:headings, :h2) || []) + (data2.dig(:headings, :h3) || [])
+
+      missing_headings = h2.reject do |heading|
+        h1.any? { |my_h| my_h.downcase.include?(heading.downcase[0..15]) }
+      end
+
+      {
+        page1: { url: @url1, word_count: tokens1.length },
+        page2: { url: @url2, word_count: tokens2.length },
+        missing_unigrams: missing_unigrams.first(15),
+        missing_bigrams: missing_bigrams.first(15),
+        missing_trigrams: missing_trigrams.first(10),
+        missing_headings: missing_headings.first(10)
+      }
+    end
+
+    private
+
+    def extract_clean_text(html)
+      text = html.dup
+      text.gsub!(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/i, ' ')
+      text.gsub!(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/i, ' ')
+      text.gsub!(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/i, ' ')
+      text.gsub!(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/i, ' ')
+      text.gsub!(/<[^>]+>/, ' ')
+      text.gsub!(/&[a-z]+;/i, ' ')
+      text.gsub!(/\s+/, ' ')
+      text.strip
+    end
+
+    def tokenize(text)
+      text.downcase.scan(/[a-z0-9]+/).reject { |w| w.length < 3 || STOP_WORDS.include?(w) }
+    end
+
+    def ngrams(tokens, n)
+      counts = Hash.new(0)
+      return counts if tokens.length < n
+
+      (0..(tokens.length - n)).each do |i|
+        gram = tokens[i, n].join(' ')
+        counts[gram] += 1
+      end
+      counts
+    end
+
+    def term_gap(counts1, counts2, min_count: 2)
+      gap = []
+      counts2.each do |term, count2|
+        count1 = counts1[term] || 0
+        if count2 >= min_count && count1 == 0
+          gap << { term: term, competitor_count: count2, your_count: count1 }
+        end
+      end
+      gap.sort_by { |item| -item[:competitor_count] }
+    end
+  end
+end

data/lib/gsc/ctr_curve.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module GSC
+  class CtrCurve
+    # Empirical Google Organic SERP CTR Curve benchmark (Ahrefs / Backlinko / Advanced Web Ranking composite)
+    BENCHMARK_CTR = {
+      1 => 28.0,
+      2 => 15.5,
+      3 => 11.0,
+      4 => 8.0,
+      5 => 6.0,
+      6 => 4.5,
+      7 => 3.5,
+      8 => 2.8,
+      9 => 2.2,
+      10 => 1.8,
+      11 => 1.4,
+      12 => 1.2,
+      13 => 1.0,
+      14 => 0.9,
+      15 => 0.8,
+      16 => 0.7,
+      17 => 0.6,
+      18 => 0.5,
+      19 => 0.4,
+      20 => 0.4
+    }.freeze
+
+    DEFAULT_PAGE2_CTR = 0.5
+    DEFAULT_PAGE3_CTR = 0.2
+
+    def self.benchmark_for(position)
+      pos = position.to_f.round
+      return BENCHMARK_CTR[1] if pos <= 1
+      return BENCHMARK_CTR[pos] if BENCHMARK_CTR.key?(pos)
+      return DEFAULT_PAGE2_CTR if pos <= 20
+
+      DEFAULT_PAGE3_CTR
+    end
+
+    def self.project(rows, target_pos: 3, min_imp: 10)
+      target_pos = target_pos.to_i
+      target_pos = 3 if target_pos <= 0
+      target_ctr = benchmark_for(target_pos)
+
+      analyzed = []
+      total_incremental = 0
+      underperforming_queries = []
+
+      rows.each do |row|
+        imp = row[:impressions] || row['impressions'] || 0
+        next if imp < min_imp
+
+        clicks = row[:clicks] || row['clicks'] || 0
+        actual_ctr = if imp > 0
+                       (clicks.to_f / imp * 100.0).round(2)
+                     elsif (row[:ctr] || row['ctr'])
+                       (row[:ctr] || row['ctr']).to_f.round(2)
+                     else
+                       0.0
+                     end
+        pos = (row[:position] || row['position'] || 100.0).to_f.round(1)
+        expected_ctr = benchmark_for(pos)
+
+        # Gain projection if ranking moved to target_pos
+        projected_clicks = ((target_ctr / 100.0) * imp).round
+        incremental_clicks = [projected_clicks - clicks, 0].max
+
+        # Priority score: Higher incremental clicks and closer to top page 1 = higher leverage
+        proximity_factor = [1.0, (25.0 - [pos, 25.0].min) / 20.0].max
+        opportunity_score = (incremental_clicks * proximity_factor).round(1)
+
+        # Flag title tag optimization wins: ranking on Page 1 (<=10) but CTR is < 50% of benchmark
+        is_underperforming = pos <= 10.0 && actual_ctr < (expected_ctr * 0.6) && imp >= 20
+
+        data = {
+          query: row[:query] || row['query'],
+          current_position: pos,
+          target_position: target_pos,
+          impressions: imp,
+          current_clicks: clicks,
+          current_ctr: actual_ctr,
+          expected_ctr: expected_ctr,
+          target_ctr: target_ctr,
+          projected_clicks: projected_clicks,
+          simulated_clicks: projected_clicks, # alias for backward compatibility
+          incremental_clicks: incremental_clicks,
+          opportunity_score: opportunity_score,
+          underperforming: is_underperforming
+        }
+
+        analyzed << data
+        total_incremental += incremental_clicks
+        underperforming_queries << data if is_underperforming
+      end
+
+      # Sort by highest opportunity score
+      analyzed.sort_by! { |r| -r[:opportunity_score] }
+      underperforming_queries.sort_by! { |r| -r[:impressions] }
+
+      {
+        target_position: target_pos,
+        target_ctr: target_ctr,
+        total_queries_analyzed: analyzed.size,
+        total_incremental_clicks: total_incremental,
+        opportunities: analyzed,
+        underperformers: underperforming_queries
+      }
+    end
+
+    class << self
+      alias simulate project
+    end
+  end
+end