gsc-cli 2.0.2 → 2.1.0

data/lib/gsc/content_gap.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'set'
+
+module GSC
+  class ContentGap
+    STOP_WORDS = Set.new(%w[
+      a about above after again against all am an and any are aren't as at be because been before being below
+      between both but by can't cannot could couldn't did didn't do does doesn't doing don't down during each
+      few for from further had hadn't has hasn't have haven't having he he'd he'll he's her here here's hers
+      herself him himself his how how's i i'd i'll i'm i've if in into is isn't it it's its itself let's me
+      more most mustn't my myself no nor not of off on once only or other ought our ours ourselves out over own
+      same shan't she she'd she'll she's should shouldn't so some such than that that's the their theirs them
+      themselves then there there's these they they'd they'll they're they've this those through to too under until
+      up very was wasn't we we'd we'll we're we've were weren't what what's when when's where where's which while
+      who who's whom why why's with won't would wouldn't you you'd you'll you're you've your yours yourself yourselves
+    ])
+
+    attr_reader :url1, :url2
+
+    def initialize(url1, url2)
+      @url1 = url1
+      @url2 = url2
+    end
+
+    def analyze
+      pa1 = GSC::PageAnalyzer.new(@url1)
+      pa2 = GSC::PageAnalyzer.new(@url2)
+
+      data1 = pa1.fetch_and_analyze
+      data2 = pa2.fetch_and_analyze
+
+      text1 = extract_clean_text(pa1.html || '')
+      text2 = extract_clean_text(pa2.html || '')
+
+      tokens1 = tokenize(text1)
+      tokens2 = tokenize(text2)
+
+      unigrams1 = ngrams(tokens1, 1)
+      unigrams2 = ngrams(tokens2, 1)
+
+      bigrams1 = ngrams(tokens1, 2)
+      bigrams2 = ngrams(tokens2, 2)
+
+      trigrams1 = ngrams(tokens1, 3)
+      trigrams2 = ngrams(tokens2, 3)
+
+      # Competitor terms that occur at least 2 times, but occur 0 times in page1
+      missing_unigrams = term_gap(unigrams1, unigrams2, min_count: 2)
+      missing_bigrams  = term_gap(bigrams1, bigrams2, min_count: 2)
+      missing_trigrams = term_gap(trigrams1, trigrams2, min_count: 2)
+
+      # Heading gaps (H2/H3 in page2 that have no match in page1)
+      h1 = (data1.dig(:headings, :h2) || []) + (data1.dig(:headings, :h3) || [])
+      h2 = (data2.dig(:headings, :h2) || []) + (data2.dig(:headings, :h3) || [])
+
+      missing_headings = h2.reject do |heading|
+        h1.any? { |my_h| my_h.downcase.include?(heading.downcase[0..15]) }
+      end
+
+      {
+        page1: { url: @url1, word_count: tokens1.length },
+        page2: { url: @url2, word_count: tokens2.length },
+        missing_unigrams: missing_unigrams.first(15),
+        missing_bigrams: missing_bigrams.first(15),
+        missing_trigrams: missing_trigrams.first(10),
+        missing_headings: missing_headings.first(10)
+      }
+    end
+
+    private
+
+    def extract_clean_text(html)
+      text = html.dup
+      text.gsub!(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/i, ' ')
+      text.gsub!(/<style\b[^<]*(?:(?!<\/style>)<[^<]*)*<\/style>/i, ' ')
+      text.gsub!(/<nav\b[^<]*(?:(?!<\/nav>)<[^<]*)*<\/nav>/i, ' ')
+      text.gsub!(/<footer\b[^<]*(?:(?!<\/footer>)<[^<]*)*<\/footer>/i, ' ')
+      text.gsub!(/<[^>]+>/, ' ')
+      text.gsub!(/&[a-z]+;/i, ' ')
+      text.gsub!(/\s+/, ' ')
+      text.strip
+    end
+
+    def tokenize(text)
+      text.downcase.scan(/[a-z0-9]+/).reject { |w| w.length < 3 || STOP_WORDS.include?(w) }
+    end
+
+    def ngrams(tokens, n)
+      counts = Hash.new(0)
+      return counts if tokens.length < n
+
+      (0..(tokens.length - n)).each do |i|
+        gram = tokens[i, n].join(' ')
+        counts[gram] += 1
+      end
+      counts
+    end
+
+    def term_gap(counts1, counts2, min_count: 2)
+      gap = []
+      counts2.each do |term, count2|
+        count1 = counts1[term] || 0
+        if count2 >= min_count && count1 == 0
+          gap << { term: term, competitor_count: count2, your_count: count1 }
+        end
+      end
+      gap.sort_by { |item| -item[:competitor_count] }
+    end
+  end
+end

data/lib/gsc/google_suggest.rb

@@ -0,0 +1,109 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+
+module GSC
+  class GoogleSuggest
+    SUGGEST_URL = 'https://suggestqueries.google.com/complete/search'
+
+    attr_reader :query, :options
+
+    def initialize(query, options = {})
+      @query = query.to_s.strip
+      @options = options
+    end
+
+    def fetch(alphabet: false, questions: false)
+      if questions
+        fetch_questions
+      elsif alphabet
+        fetch_alphabet_soup
+      else
+        fetch_single(@query)
+      end
+    end
+
+    def fetch_single(search_term)
+      uri = URI("#{SUGGEST_URL}?client=chrome&q=#{URI.encode_www_form_component(search_term)}")
+      req = Net::HTTP::Get.new(uri)
+      req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36'
+      req['Accept'] = 'application/json, text/javascript, */*; q=0.01'
+
+      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, open_timeout: 5, read_timeout: 10) do |http|
+        http.request(req)
+      end
+
+      return [] unless res.code == '200'
+
+      parsed = JSON.parse(res.body.force_encoding('UTF-8'))
+      terms = parsed[1] || []
+      types = parsed[4] ? parsed[4]['google:suggesttype'] || [] : []
+
+      terms.map.with_index do |term, idx|
+        {
+          term: term,
+          type: types[idx] || 'QUERY',
+          root: search_term
+        }
+      end
+    rescue StandardError => e
+      []
+    end
+
+    def fetch_alphabet_soup
+      results = {}
+      base = @query.strip
+
+      # Root query first
+      results['root'] = fetch_single(base)
+
+      # A-Z permutations
+      ('a'..'z').each do |letter|
+        term = "#{base} #{letter}"
+        items = fetch_single(term)
+        results[letter] = items unless items.empty?
+        sleep(0.05) # Polite throttle
+      end
+
+      # 0-9 permutations if requested
+      if @options[:numbers]
+        ('0'..'9').each do |num|
+          term = "#{base} #{num}"
+          items = fetch_single(term)
+          results[num] = items unless items.empty?
+          sleep(0.05)
+        end
+      end
+
+      results
+    end
+
+    def fetch_questions
+      prefixes = [
+        'how to',
+        'how do',
+        'why do',
+        'why does',
+        'what is',
+        'what are',
+        'can you',
+        'best',
+        'where to',
+        'which'
+      ]
+
+      results = {}
+      prefixes.each do |pfx|
+        term = "#{pfx} #{@query}"
+        items = fetch_single(term)
+        results[pfx] = items unless items.empty?
+        sleep(0.05)
+      end
+
+      results
+    end
+  end
+end

data/lib/gsc/internal_links.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'uri'
+require 'set'
+
+module GSC
+  class InternalLinks
+    attr_reader :base_url, :pages, :graph, :orphans, :depths
+
+    def initialize(base_url, limit: 50)
+      @base_url = base_url.to_s.strip
+      @base_url = "https://#{@base_url}" unless @base_url =~ %r{^https?://}
+      @base_uri = URI.parse(@base_url)
+      @limit = limit
+      @graph = Hash.new { |h, k| h[k] = Set.new } # target_url => Set of source_urls
+      @out_links = Hash.new { |h, k| h[k] = Set.new } # source_url => Set of target_urls
+      @all_discovered = Set.new
+    end
+
+    def audit(sitemap_urls = nil)
+      urls_to_crawl = if sitemap_urls && !sitemap_urls.empty?
+                        sitemap_urls.first(@limit)
+                      else
+                        discover_urls
+                      end
+
+      urls_to_crawl.each do |url|
+        @all_discovered << normalize_url(url)
+      end
+
+      # Crawl each URL and extract internal links
+      urls_to_crawl.each do |url|
+        pa = GSC::PageAnalyzer.new(url)
+        pa.load_content! rescue next
+        dom = pa.analyze_dom rescue next
+
+        norm_source = normalize_url(url)
+        links = dom.dig(:links, :all) || []
+
+        links.each do |link_obj|
+          href = link_obj[:href]
+          target_url = resolve_internal_url(href)
+          next unless target_url
+
+          norm_target = normalize_url(target_url)
+          next if norm_target == norm_source
+
+          @graph[norm_target] << norm_source
+          @out_links[norm_source] << norm_target
+        end
+      end
+
+      # Calculate Orphans (pages in sitemap/discovered with 0 incoming internal links)
+      orphans = []
+      weak_pages = [] # only 1 internal link
+
+      @all_discovered.each do |url|
+        in_degree = @graph[url].size
+        if in_degree == 0 && url != normalize_url(@base_url)
+          orphans << url
+        elsif in_degree == 1
+          weak_pages << { url: url, source: @graph[url].first }
+        end
+      end
+
+      # Calculate click depths via BFS from root
+      depths = calculate_depths(normalize_url(@base_url))
+
+      {
+        base_url: @base_url,
+        total_pages: @all_discovered.size,
+        orphans: orphans,
+        weak_pages: weak_pages,
+        top_linked: top_linked_pages(10),
+        depths: depths
+      }
+    end
+
+    private
+
+    def discover_urls
+      loader = GSC::SitemapLoader.new(@base_url)
+      urls = loader.load
+      urls.empty? ? [@base_url] : urls.first(@limit)
+    rescue StandardError
+      [@base_url]
+    end
+
+    def resolve_internal_url(href)
+      return nil if href.nil? || href.strip.empty?
+      return nil if href =~ /^(mailto|tel|javascript|#):/i
+
+      uri = URI.join(@base_url, href) rescue nil
+      return nil unless uri && uri.scheme =~ /^https?$/i
+      return nil unless uri.host.downcase == @base_uri.host.downcase
+
+      uri.fragment = nil
+      uri.to_s
+    end
+
+    def normalize_url(url)
+      u = url.to_s.strip.sub(%r{/$}, '')
+      u
+    end
+
+    def calculate_depths(root_url)
+      depths = { root_url => 0 }
+      queue = [root_url]
+
+      until queue.empty?
+        curr = queue.shift
+        curr_depth = depths[curr]
+
+        (@out_links[curr] || []).each do |neighbor|
+          next if depths.key?(neighbor)
+
+          depths[neighbor] = curr_depth + 1
+          queue << neighbor
+        end
+      end
+
+      depths
+    end
+
+    def top_linked_pages(limit)
+      @graph.map do |url, sources|
+        { url: url, incoming_count: sources.size }
+      end.sort_by { |item| -item[:incoming_count] }.first(limit)
+    end
+  end
+end

data/lib/gsc/llms_generator.rb

@@ -0,0 +1,104 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'uri'
+
+module GSC
+  class LlmsGenerator
+    attr_reader :base_url, :pages
+
+    def initialize(base_url)
+      @base_url = base_url.to_s.strip
+      @base_url = "https://#{@base_url}" unless @base_url =~ %r{^https?://}
+    end
+
+    def generate_llms_txt(title: nil, summary: nil)
+      loader = GSC::SitemapLoader.new(@base_url)
+      urls = loader.load rescue [@base_url]
+      urls = [@base_url] if urls.empty?
+
+      site_title = title || URI.parse(@base_url).host.sub(/^www\./, '').capitalize
+      site_summary = summary || "Official documentation and product guides for #{site_title}."
+
+      out = []
+      out << "# #{site_title}"
+      out << ""
+      out << "> #{site_summary}"
+      out << ""
+      out << "## Core Documentation"
+      out << ""
+
+      # Sample first 15 key URLs and fetch metadata
+      urls.first(15).each do |url|
+        pa = GSC::PageAnalyzer.new(url)
+        pa.load_content! rescue next
+        dom = pa.analyze_dom rescue next
+
+        page_title = dom.dig(:title, :text) || url
+        page_desc = dom.dig(:meta_description, :text) || "Documentation page."
+
+        out << "- [#{page_title}](#{url}): #{page_desc}"
+      end
+
+      out << ""
+      out << "## Optional"
+      out << ""
+      out << "- [Full Documentation](#{@base_url}/llms-full.txt): Complete consolidated knowledge base for LLM context ingestion."
+      out << ""
+
+      out.join("\n")
+    end
+
+    def audit_ai_readability(url)
+      pa = GSC::PageAnalyzer.new(url)
+      data = pa.fetch_and_analyze
+
+      # Check criteria:
+      # 1. Clear H1 presence
+      # 2. Table presence (LLMs love tables)
+      # 3. Schema presence (structured data)
+      # 4. Definition / Bullet presence
+      html = pa.html || ''
+      has_tables = html.include?('<table')
+      has_lists = html.include?('<ul') || html.include?('<ol')
+      schemas = data.dig(:structured_data, :schemas) || []
+      h1_count = (data.dig(:headings, :h1) || []).length
+
+      score = 100
+      issues = []
+
+      if h1_count != 1
+        score -= 20
+        issues << "H1 count is #{h1_count} (Must be exactly 1 for clean LLM hierarchy)"
+      end
+
+      unless has_tables
+        score -= 15
+        issues << "No <table> found (tables increase LLM citation and fact extraction by 3x)"
+      end
+
+      unless has_lists
+        score -= 15
+        issues << "No bullet lists (<ul> or <ol>) found for quick entity consumption"
+      end
+
+      if schemas.empty?
+        score -= 20
+        issues << "No JSON-LD schemas detected (structured data accelerates AI knowledge graph inclusion)"
+      end
+
+      {
+        url: url,
+        ai_readability_score: [score, 0].max,
+        grade: score >= 80 ? 'A (Excellent)' : (score >= 60 ? 'B (Acceptable)' : 'C (Needs Work)'),
+        issues: issues,
+        features: {
+          has_tables: has_tables,
+          has_lists: has_lists,
+          schemas_found: schemas.length,
+          h1_count: h1_count
+        }
+      }
+    end
+  end
+end

data/lib/gsc/network_tracer.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+
+module GSC
+  class NetworkTracer
+    attr_reader :start_url, :max_hops
+
+    def initialize(start_url, max_hops: 10)
+      @start_url = start_url.to_s.strip
+      @start_url = "http://#{@start_url}" unless @start_url =~ %r{^https?://}
+      @max_hops = max_hops
+    end
+
+    def trace
+      current_url = @start_url
+      hops = []
+      visited = Set.new
+
+      @max_hops.times do |hop_idx|
+        break if visited.include?(current_url)
+        visited << current_url
+
+        uri = URI.parse(current_url) rescue nil
+        break unless uri && uri.host
+
+        start_t = Time.now
+        res = nil
+
+        begin
+          Net::HTTP.start(uri.hostname, uri.port, use_ssl: (uri.scheme == 'https'), open_timeout: 5, read_timeout: 10) do |http|
+            req = Net::HTTP::Get.new(uri.request_uri)
+            req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) gsc-cli/2.1'
+            res = http.request(req)
+          end
+        rescue StandardError => e
+          hops << {
+            hop: hop_idx + 1,
+            url: current_url,
+            error: e.message
+          }
+          break
+        end
+
+        duration_ms = ((Time.now - start_t) * 1000).round(1)
+        status_code = res.code.to_i
+        location = res['location']
+
+        hop_info = {
+          hop: hop_idx + 1,
+          url: current_url,
+          status_code: status_code,
+          duration_ms: duration_ms,
+          x_robots_tag: res['x-robots-tag'],
+          canonical_header: res['link'] =~ /rel="canonical"/i ? res['link'] : nil,
+          hsts: res['strict-transport-security'],
+          content_type: res['content-type'],
+          server: res['server']
+        }
+
+        hops << hop_info
+
+        if [301, 302, 303, 307, 308].include?(status_code) && location
+          current_url = URI.join(current_url, location).to_s
+        else
+          break
+        end
+      end
+
+      total_time = hops.sum { |h| h[:duration_ms] || 0 }.round(1)
+      final_hop = hops.last || {}
+      is_redirect_chain = hops.length > 2
+
+      {
+        start_url: @start_url,
+        final_url: final_hop[:url],
+        total_hops: hops.length,
+        total_duration_ms: total_time,
+        is_redirect_chain: is_redirect_chain,
+        hops: hops
+      }
+    end
+  end
+end

data/lib/gsc/open_page_rank.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+
+module GSC
+  class OpenPageRank
+    API_URL = 'https://openpagerank.com/api/v1.0/getPageRank'
+
+    attr_reader :api_key
+
+    def initialize(api_key = nil)
+      @api_key = api_key || ENV['OPENPAGERANK_API_KEY'] || GSC::Config.get('opr_api_key')
+    end
+
+    def configured?
+      !@api_key.nil? && !@api_key.strip.empty?
+    end
+
+    def check_domains(domains)
+      domains = Array(domains).map { |d| clean_domain(d) }.reject(&:empty?).uniq
+      return { error: 'No valid domains provided' } if domains.empty?
+      return { error: 'OpenPageRank API key not configured. Set via `gsc config set opr_api_key <key>` or OPENPAGERANK_API_KEY env (Get free 300k calls/mo at openpagerank.com)' } unless configured?
+
+      # Construct query params: domains[0]=a.com&domains[1]=b.com
+      params = domains.map.with_index { |d, idx| "domains%5B#{idx}%5D=#{URI.encode_www_form_component(d)}" }.join('&')
+      uri = URI("#{API_URL}?#{params}")
+
+      req = Net::HTTP::Get.new(uri)
+      req['API-OPR'] = @api_key
+      req['User-Agent'] = 'gsc-cli/2.1'
+
+      res = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true, open_timeout: 5, read_timeout: 15) do |http|
+        http.request(req)
+      end
+
+      if res.code == '200'
+        data = JSON.parse(res.body)
+        records = (data['response'] || []).map do |r|
+          {
+            domain: r['domain'],
+            page_rank_decimal: r['page_rank_decimal'] || 0.0,
+            page_rank_integer: r['page_rank_integer'] || 0,
+            rank: r['rank'],
+            status_code: r['status_code']
+          }
+        end
+        {
+          status: 'success',
+          status_code: data['status_code'],
+          records: records
+        }
+      else
+        {
+          status: 'error',
+          code: res.code.to_i,
+          message: res.body
+        }
+      end
+    rescue StandardError => e
+      { status: 'error', message: e.message }
+    end
+
+    def clean_domain(input)
+      d = input.to_s.strip.downcase
+      d = d.sub(%r{^https?://}, '').sub(%r{/.*$}, '').sub(/^www\./, '')
+      d
+    end
+  end
+end