gsc-cli 2.1.0 → 2.2.0

data/lib/gsc/indexing_queue.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'fileutils'
+require 'date'
+require 'time'
+
+module GSC
+  class IndexingQueue
+    QUEUE_FILE = File.join(Config::CONFIG_DIR, 'indexing_queue.json')
+    DAILY_LIMIT = 200
+
+    attr_reader :state
+
+    def initialize(file_path = QUEUE_FILE)
+      @file_path = file_path
+      @state = load_state
+    end
+
+    def add_urls(urls)
+      normalized = Array(urls).map(&:to_s).map(&:strip).reject(&:empty?).uniq
+      # Only keep valid http(s) URLs
+      valid_urls = normalized.select { |u| u.start_with?('http://', 'https://') }
+
+      existing_pending = @state['pending'] || []
+      new_urls = valid_urls - existing_pending
+
+      @state['pending'] = existing_pending + new_urls
+      save_state
+      new_urls.size
+    end
+
+    def status
+      check_quota_reset!
+      used = @state['daily_quota_used'] || 0
+      remaining = [DAILY_LIMIT - used, 0].max
+
+      {
+        pending_count: (@state['pending'] || []).size,
+        submitted_count: (@state['submitted'] || []).size,
+        failed_count: (@state['failed'] || []).size,
+        daily_quota_limit: DAILY_LIMIT,
+        daily_quota_used: used,
+        daily_quota_remaining: remaining,
+        last_reset_date: @state['last_reset_date']
+      }
+    end
+
+    def clear(scope = :all)
+      if scope == :pending
+        @state['pending'] = []
+      else
+        @state['pending'] = []
+        @state['submitted'] = []
+        @state['failed'] = []
+      end
+      save_state
+    end
+
+    def process_batch(api, batch_size: 50, dry_run: false, delay_sec: 0.15)
+      check_quota_reset!
+      used = @state['daily_quota_used'] || 0
+      remaining_quota = [DAILY_LIMIT - used, 0].max
+
+      if remaining_quota <= 0 && !dry_run
+        return {
+          status: :quota_exhausted,
+          message: "Daily quota of #{DAILY_LIMIT} requests reached for today (#{@state['last_reset_date']}). Next reset at 00:00 UTC.",
+          processed: 0,
+          remaining_in_queue: (@state['pending'] || []).size
+        }
+      end
+
+      to_process_count = [batch_size.to_i, remaining_quota].min
+      urls = (@state['pending'] || []).shift(to_process_count)
+
+      if urls.empty?
+        return {
+          status: :queue_empty,
+          message: 'Indexing queue is empty. Use `gsc index-batch add <url|sitemap>` to enqueue URLs.',
+          processed: 0,
+          remaining_in_queue: 0
+        }
+      end
+
+      results = []
+      successful = 0
+      failed = 0
+
+      urls.each_with_index do |url, idx|
+        if dry_run
+          results << { url: url, status: 'DRY_RUN', ok: true }
+          successful += 1
+          next
+        end
+
+        res = api.publish_url(url, 'URL_UPDATED')
+        if res[:ok]
+          successful += 1
+          @state['submitted'] ||= []
+          @state['submitted'] << {
+            url: url,
+            status: res[:status],
+            submitted_at: Time.now.utc.iso8601
+          }
+          @state['daily_quota_used'] = (@state['daily_quota_used'] || 0) + 1
+          results << { url: url, status: 'SUCCESS', http_code: res[:status], ok: true }
+        else
+          failed += 1
+          @state['failed'] ||= []
+          @state['failed'] << {
+            url: url,
+            error: res[:data],
+            failed_at: Time.now.utc.iso8601
+          }
+          results << { url: url, status: 'FAILED', error: res[:data], ok: false }
+        end
+
+        sleep(delay_sec) if delay_sec > 0 && idx < urls.size - 1
+      end
+
+      # In dry_run, restore pending list so URLs aren't lost
+      if dry_run
+        @state['pending'] = urls + (@state['pending'] || [])
+      else
+        save_state
+      end
+
+      {
+        status: :success,
+        processed: urls.size,
+        successful: successful,
+        failed: failed,
+        dry_run: dry_run,
+        remaining_in_queue: (@state['pending'] || []).size,
+        daily_quota_remaining: dry_run ? remaining_quota : [DAILY_LIMIT - @state['daily_quota_used'], 0].max,
+        results: results
+      }
+    end
+
+    private
+
+    def check_quota_reset!
+      today = Date.today.to_s
+      if @state['last_reset_date'] != today
+        @state['last_reset_date'] = today
+        @state['daily_quota_used'] = 0
+        save_state
+      end
+    end
+
+    def load_state
+      return default_state unless File.exist?(@file_path)
+
+      data = JSON.parse(File.read(@file_path))
+      data.is_a?(Hash) ? data : default_state
+    rescue StandardError
+      default_state
+    end
+
+    def save_state
+      FileUtils.mkdir_p(File.dirname(@file_path))
+      File.write(@file_path, JSON.pretty_generate(@state))
+    rescue StandardError => e
+      # Silently handle disk write errors
+    end
+
+    def default_state
+      {
+        'daily_quota_limit' => DAILY_LIMIT,
+        'daily_quota_used' => 0,
+        'last_reset_date' => Date.today.to_s,
+        'pending' => [],
+        'submitted' => [],
+        'failed' => []
+      }
+    end
+  end
+end

data/lib/gsc/indexnow.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'openssl'
+
+module GSC
+  class IndexNow
+    ENDPOINT = 'https://api.indexnow.org/indexnow'
+
+    def self.generate_key
+      OpenSSL::Random.random_bytes(16).unpack1('H*')
+    end
+
+    def self.get_or_create_key
+      existing = Config.get('indexnow_key') || ENV['INDEXNOW_KEY']
+      return existing if existing && !existing.strip.empty?
+
+      new_key = generate_key
+      Config.set('indexnow_key', new_key)
+      new_key
+    end
+
+    def self.set_key(key)
+      clean = key.to_s.strip
+      Config.set('indexnow_key', clean)
+      clean
+    end
+
+    def self.submit(urls, key: nil, host: nil)
+      urls = Array(urls).map(&:to_s).map(&:strip).reject(&:empty?)
+      raise 'No URLs provided for IndexNow submission' if urls.empty?
+
+      first_uri = URI.parse(urls.first) rescue nil
+      detected_host = host || (first_uri ? first_uri.host : Config.default_domain)
+      raise 'Could not determine host for IndexNow submission. Please provide full URLs (e.g. https://example.com/page)' unless detected_host
+
+      detected_host = detected_host.sub(%r{^https?://}, '').sub(/^sc-domain:/, '').chomp('/')
+
+      active_key = key || get_or_create_key
+
+      payload = {
+        host: detected_host,
+        key: active_key,
+        keyLocation: "https://#{detected_host}/#{active_key}.txt",
+        urlList: urls
+      }
+
+      uri = URI(ENDPOINT)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true
+      http.open_timeout = 10
+      http.read_timeout = 20
+
+      req = Net::HTTP::Post.new(uri.request_uri)
+      req['Content-Type'] = 'application/json; charset=utf-8'
+      req['User-Agent'] = 'gsc-cli IndexNow/1.0'
+      req.body = JSON.generate(payload)
+
+      res = http.request(req)
+
+      status_msg = case res.code.to_i
+                   when 200 then 'OK (URLs submitted successfully)'
+                   when 202 then 'Accepted (Key pending verification)'
+                   when 400 then 'Bad Request (Invalid JSON or URL format)'
+                   when 403 then "Forbidden (Key invalid or https://#{detected_host}/#{active_key}.txt missing)"
+                   when 422 then 'Unprocessable Entity (URLs do not match host)'
+                   when 429 then 'Too Many Requests'
+                   else "HTTP #{res.code}"
+                   end
+
+      {
+        success: [200, 202].include?(res.code.to_i),
+        http_code: res.code.to_i,
+        message: status_msg,
+        host: detected_host,
+        key: active_key,
+        key_location: "https://#{detected_host}/#{active_key}.txt",
+        submitted_urls: urls.size,
+        urls: urls
+      }
+    end
+
+    def self.submit_sitemap(sitemap_path_or_url, key: nil, limit: nil)
+      urls = SitemapLoader.resolve_urls(sitemap_path_or_url, quiet: true)
+      raise "No URLs found in sitemap: #{sitemap_path_or_url}" if urls.empty?
+
+      urls = urls.first(limit) if limit && limit.positive?
+      submit(urls, key: key)
+    end
+  end
+end

data/lib/gsc/intent_shift.rb

@@ -0,0 +1,188 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'json'
+require 'date'
+
+module GSC
+  class IntentShift
+    TRANSACTIONAL_MODIFIERS = %w[
+      buy order purchase discount coupon price pricing cost cheap deal shop store subscription checkout hire
+    ].freeze
+
+    COMMERCIAL_MODIFIERS = %w[
+      best top review reviews vs versus compare comparison alternative alternatives recommended software tool platform
+    ].freeze
+
+    INFORMATIONAL_MODIFIERS = %w[
+      how what why when where who guide tutorial tips steps learn ideas strategy example examples template explain
+    ].freeze
+
+    NAVIGATIONAL_MODIFIERS = %w[
+      login log-in signin sign-in portal account dashboard support helpdesk download
+    ].freeze
+
+    attr_reader :options, :api, :domain
+
+    def initialize(options = {}, api = nil, domain = nil)
+      @options = options
+      @api = api
+      @domain = domain.to_s.strip
+    end
+
+    def self.analyze(options = {}, api = nil, domain = nil)
+      new(options, api, domain).analyze
+    end
+
+    def analyze
+      rows = collect_rows
+      shifts = detect_intent_shifts(rows)
+      summarize_portfolio(shifts)
+    end
+
+    def self.classify_query(query, brand = nil)
+      q = query.to_s.downcase.strip
+
+      if brand && !brand.empty? && q.include?(brand.downcase)
+        return :navigational if NAVIGATIONAL_MODIFIERS.any? { |m| q.include?(m) }
+      end
+
+      return :navigational if NAVIGATIONAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+      return :transactional if TRANSACTIONAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+      return :commercial if COMMERCIAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+      return :informational if INFORMATIONAL_MODIFIERS.any? { |m| q.match?(/\b#{Regexp.escape(m)}\b/) }
+
+      :informational
+    end
+
+    def self.classify_page(url)
+      u = url.to_s.downcase
+      if u =~ %r{/(?:cart|checkout|pricing|buy|products?|shop|store|orders?)(?:/|$|\?|#)}
+        :transactional
+      elsif u =~ %r{/(?:blog|guides?|tutorials?|learn|how-to|articles?|docs|knowledge-base)(?:/|$|\?|#)}
+        :informational
+      elsif u =~ %r{/(?:comparison|compare|vs|alternatives?|best|reviews?)(?:/|$|\?|#)}
+        :commercial
+      elsif u =~ %r{/(?:login|portal|accounts?|dashboard|signin)(?:/|$|\?|#)}
+        :navigational
+      else
+        :hybrid
+      end
+    end
+
+    private
+
+    def collect_rows
+      if @api
+        begin
+          days = (@options[:days] || 28).to_i
+          end_date = (Date.today - 2).strftime('%Y-%m-%d')
+          start_date = (Date.today - 2 - days).strftime('%Y-%m-%d')
+          res = @api.search_analytics(@domain, start_date: start_date, end_date: end_date, dimensions: %w[query page])
+          raw_rows = res['rows'] || []
+          if raw_rows.any?
+            return raw_rows.map do |r|
+              {
+                query: r['keys'][0],
+                page: r['keys'][1],
+                clicks: r['clicks'] || 0,
+                impressions: r['impressions'] || 0,
+                ctr: ((r['ctr'] || 0) * 100.0).round(2),
+                position: (r['position'] || 0).round(1)
+              }
+            end
+          end
+        rescue StandardError
+          # GSC query failed
+        end
+      end
+
+      []
+    end
+
+    def detect_intent_shifts(rows)
+      brand = @options[:brand] || @domain.split('.').first
+
+      shifts = []
+
+      rows.each do |row|
+        q_intent = self.class.classify_query(row[:query], brand)
+        p_intent = self.class.classify_page(row[:page])
+
+        mismatch = false
+        risk_level = :none
+        diagnosis = nil
+        prescription = nil
+
+        # Scenario 1: Informational/Commercial query ranking on a purely Transactional page
+        if (q_intent == :informational || q_intent == :commercial) && p_intent == :transactional
+          mismatch = true
+          risk_level = row[:position] > 10.0 ? :high : :medium
+          diagnosis = "Google expects #{q_intent.to_s.upcase} research content, but ranking URL is a TRANSACTIONAL #{File.basename(row[:page])} page."
+          prescription = "Publish a dedicated #{q_intent} guide/comparison landing page to capture Top 3 SERP intent instead of sending users to checkout/product page."
+
+        # Scenario 2: Transactional query ranking on an Informational blog post
+        elsif q_intent == :transactional && p_intent == :informational
+          mismatch = true
+          risk_level = :medium
+          diagnosis = "Users have HIGH BUYING INTENT (#{row[:query]}), but are landing on an INFORMATIONAL blog article."
+          prescription = "Embed prominent 1-click checkout widgets, pricing tables, and product CTAs directly above the fold in this blog post."
+
+        # Scenario 3: Commercial comparison query landing on generic pricing page
+        elsif q_intent == :commercial && p_intent == :hybrid
+          mismatch = true
+          risk_level = :low
+          diagnosis = "Comparison query landing on generic page."
+          prescription = "Deploy a structured vs/comparison matrix table."
+        end
+
+        shifts << {
+          query: row[:query],
+          page: row[:page],
+          clicks: row[:clicks],
+          impressions: row[:impressions],
+          position: row[:position],
+          ctr: row[:ctr],
+          query_intent: q_intent,
+          page_intent: p_intent,
+          has_mismatch: mismatch,
+          risk_level: risk_level,
+          diagnosis: diagnosis,
+          prescription: prescription
+        }
+      end
+
+      shifts
+    end
+
+    def summarize_portfolio(shifts)
+      total = shifts.size
+      mismatched = shifts.select { |s| s[:has_mismatch] }
+      high_risk  = shifts.select { |s| s[:risk_level] == :high }
+
+      intent_counts = {
+        informational: shifts.count { |s| s[:query_intent] == :informational },
+        transactional: shifts.count { |s| s[:query_intent] == :transactional },
+        commercial: shifts.count { |s| s[:query_intent] == :commercial },
+        navigational: shifts.count { |s| s[:query_intent] == :navigational }
+      }
+
+      volatility_score = ((mismatched.size.to_f / [total, 1].max) * 100.0).round(1)
+
+      prescriptions = mismatched.map { |m| m[:prescription] }.compact.uniq
+      grade = total.zero? ? 'N/A' : (volatility_score > 40.0 ? 'HIGH RISK' : (volatility_score > 20.0 ? 'MODERATE' : 'OPTIMAL'))
+
+      {
+        domain: @domain,
+        total_queries_analyzed: total,
+        intent_distribution: intent_counts,
+        mismatched_queries_count: mismatched.size,
+        high_risk_shifts_count: high_risk.size,
+        portfolio_volatility_pct: volatility_score,
+        risk_grade: grade,
+        prescriptions: prescriptions,
+        shifts: shifts
+      }
+    end
+  end
+end