gsc-cli 2.0.2 → 2.2.0

data/lib/gsc/heading_validator.rb

@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module GSC
+  class HeadingValidator
+    attr_reader :headings, :counts, :violations, :score, :grade, :depth
+
+    def self.analyze(html, target_keywords: nil)
+      new(html, target_keywords: target_keywords).audit
+    end
+
+    def initialize(html, target_keywords: nil)
+      @html = html.to_s.dup.force_encoding('UTF-8').scrub
+      @target_keywords = extract_keyword_tokens(target_keywords)
+      @headings = []
+      @counts = { 'h1' => 0, 'h2' => 0, 'h3' => 0, 'h4' => 0, 'h5' => 0, 'h6' => 0 }
+      @violations = []
+      @score = 100
+      @grade = 'A'
+      @depth = 0
+    end
+
+    def audit
+      parse_headings!
+      validate_hierarchy!
+      calculate_score!
+      audit_keywords! if @target_keywords && !@target_keywords.empty?
+
+      {
+        score: @score,
+        grade: @grade,
+        depth: @depth,
+        count: @headings.size,
+        counts: @counts,
+        h1_count: @counts['h1'],
+        empty_count: @headings.count { |h| h[:empty] },
+        headings: @headings,
+        violations: @violations,
+        keyword_analysis: @keyword_analysis,
+        summary: summary_text,
+        ascii_tree: render_ascii_tree(colorize: false),
+        color_tree: render_ascii_tree(colorize: true)
+      }
+    end
+
+    def render_ascii_tree(colorize: false)
+      return '   (No headings found on page)' if @headings.empty?
+
+      lines = []
+      @headings.each_with_index do |h, idx|
+        level = h[:level]
+        tag_str = "[#{h[:tag].upcase}]"
+        prefix = '   ' + ('  ' * (level - 1)) + (level == 1 ? '■ ' : '├── ')
+
+        text_str = h[:empty] ? '(Empty Heading Tag)' : h[:text]
+        # Truncate very long heading in display
+        display_text = text_str.length > 75 ? "#{text_str[0..72]}..." : text_str
+
+        # Check if this heading has an attached violation
+        v_msgs = h[:violations].map { |v| v[:message] }
+        v_suffix = v_msgs.empty? ? '' : " ⚠️  #{v_msgs.join('; ')}"
+
+        if colorize && defined?(Color)
+          tag_color = case level
+                      when 1 then Color::CYAN
+                      when 2 then Color::BLUE
+                      when 3 then Color::MAGENTA
+                      else Color::GRAY
+                      end
+          colored_tag = Color.c(tag_str, tag_color, Color::BOLD)
+          colored_text = h[:empty] ? Color.c(display_text, Color::RED, Color::DIM) : display_text
+          colored_suffix = v_suffix.empty? ? '' : Color.c(v_suffix, Color::YELLOW, Color::BOLD)
+          lines << "#{prefix}#{colored_tag} #{colored_text}#{colored_suffix}"
+        else
+          lines << "#{prefix}#{tag_str} #{display_text}#{v_suffix}"
+        end
+      end
+
+      lines.join("\n")
+    end
+
+    private
+
+    def parse_headings!
+      idx = 0
+      @html.scan(%r{<(h[1-6])(?:\s+[^>]*)?>(.*?)</\1>}im) do |tag, content|
+        idx += 1
+        clean = clean_text(content)
+        level = tag[1].to_i
+        is_empty = clean.empty?
+
+        h_info = {
+          index: idx,
+          tag: tag.downcase,
+          level: level,
+          text: clean,
+          length: clean.length,
+          empty: is_empty,
+          violations: []
+        }
+
+        @headings << h_info
+        @counts[tag.downcase] += 1
+        @depth = level if level > @depth
+      end
+    end
+
+    def validate_hierarchy!
+      return if @headings.empty?
+
+      # 1. Check if first heading is H1
+      first_h = @headings.first
+      if first_h[:level] != 1
+        v = {
+          type: :first_not_h1,
+          severity: :warning,
+          tag: first_h[:tag],
+          index: 1,
+          message: "Page starts with <#{first_h[:tag]}> instead of <h1>"
+        }
+        @violations << v
+        first_h[:violations] << v
+      end
+
+      # 2. Check H1 count
+      if @counts['h1'] == 0
+        @violations << {
+          type: :missing_h1,
+          severity: :critical,
+          message: 'Missing <h1> tag. Document has 0 top-level headings.'
+        }
+      elsif @counts['h1'] > 1
+        extra_h1s = @headings.select { |h| h[:tag] == 'h1' }[1..]
+        extra_h1s.each do |eh|
+          v = {
+            type: :multiple_h1,
+            severity: :warning,
+            tag: 'h1',
+            index: eh[:index],
+            message: "Multiple <h1> tag detected at position ##{eh[:index]} ('#{eh[:text][0..40]}...')"
+          }
+          @violations << v
+          eh[:violations] << v
+        end
+      end
+
+      # 3. Check sequential depth skips (e.g. H1 -> H3 or H2 -> H4)
+      prev_level = nil
+      @headings.each do |h|
+        curr_level = h[:level]
+
+        if prev_level && curr_level > prev_level + 1
+          skipped = (prev_level + 1...curr_level).map { |lvl| "<h#{lvl}>" }.join(', ')
+          v = {
+            type: :skipped_level,
+            severity: :warning,
+            tag: h[:tag],
+            index: h[:index],
+            from_level: prev_level,
+            to_level: curr_level,
+            message: "Skipped heading level: jumped from <h#{prev_level}> to <h#{curr_level}> (skipped #{skipped})"
+          }
+          @violations << v
+          h[:violations] << v
+        end
+
+        # 4. Check empty heading
+        if h[:empty]
+          v = {
+            type: :empty_heading,
+            severity: :error,
+            tag: h[:tag],
+            index: h[:index],
+            message: "Empty <#{h[:tag]}> tag with no text content"
+          }
+          @violations << v
+          h[:violations] << v
+        end
+
+        # 5. Check overlong heading (> 80 chars)
+        if h[:length] > 80
+          v = {
+            type: :overlong_heading,
+            severity: :info,
+            tag: h[:tag],
+            index: h[:index],
+            message: "Heading length (#{h[:length]} chars) exceeds 80 chars; risk of topical dilution"
+          }
+          @violations << v
+          h[:violations] << v
+        end
+
+        prev_level = curr_level
+      end
+    end
+
+    def calculate_score!
+      if @headings.empty?
+        @score = 0
+        @grade = 'F'
+        return
+      end
+
+      deductions = 0
+      @violations.each do |v|
+        deductions += case v[:type]
+                      when :missing_h1 then 30
+                      when :multiple_h1 then 15
+                      when :skipped_level then 10
+                      when :empty_heading then 10
+                      when :first_not_h1 then 10
+                      when :overlong_heading then 5
+                      else 5
+                      end
+      end
+
+      @score = [100 - deductions, 0].max
+      @grade = case @score
+               when 90..100 then 'A'
+               when 80..89  then 'B'
+               when 70..79  then 'C'
+               when 60..69  then 'D'
+               else 'F'
+               end
+    end
+
+    def audit_keywords!
+      h1 = @headings.find { |h| h[:tag] == 'h1' }
+      h1_matches = []
+      if h1
+        h1_text = h1[:text].downcase
+        h1_matches = @target_keywords.select { |kw| h1_text.include?(kw) }
+      end
+
+      h2_matches = {}
+      @headings.select { |h| h[:tag] == 'h2' }.each do |h2|
+        h2_text = h2[:text].downcase
+        matched = @target_keywords.select { |kw| h2_text.include?(kw) }
+        h2_matches[h2[:text]] = matched unless matched.empty?
+      end
+
+      @keyword_analysis = {
+        target_keywords: @target_keywords,
+        h1_has_keyword: !h1_matches.empty?,
+        h1_matched_keywords: h1_matches,
+        h2_matched_count: h2_matches.size,
+        h2_matches: h2_matches
+      }
+    end
+
+    def extract_keyword_tokens(keywords)
+      return [] if keywords.nil? || keywords.to_s.strip.empty?
+
+      if keywords.is_a?(Array)
+        keywords.map { |k| k.to_s.downcase.strip }.reject(&:empty?)
+      else
+        keywords.to_s.downcase.split(/[,|\s]+/).map(&:strip).reject { |w| w.length < 3 }
+      end
+    end
+
+    def clean_text(str)
+      str.to_s
+         .dup
+         .force_encoding('UTF-8')
+         .scrub
+         .gsub(/<[^>]+>/, ' ')
+         .gsub(/&amp;/, '&')
+         .gsub(/&lt;/, '<')
+         .gsub(/&gt;/, '>')
+         .gsub(/&quot;/, '"')
+         .gsub(/&#39;/, "'")
+         .gsub(/&nbsp;/, ' ')
+         .gsub(/\s+/, ' ')
+         .strip
+    end
+
+    def summary_text
+      counts_str = @counts.select { |_k, v| v > 0 }.map { |k, v| "#{v}x #{k.upcase}" }.join(', ')
+      "Heading Health Score: #{@score}/100 (Grade #{@grade}) | Depth: H#{@depth} | Total: #{@headings.size} (#{counts_str}) | #{@violations.size} violations"
+    end
+  end
+end

data/lib/gsc/hreflang_validator.rb

@@ -0,0 +1,412 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'set'
+
+module GSC
+  class HreflangValidator
+    # Common ISO 639-1 two-letter language codes
+    ISO_639_1_LANGUAGES = Set.new(%w[
+      aa ab ae af ak am an ar as av ay az ba be bg bh bi bm bn bo br bs ca ce ch co cr cs cu cv cy da de dv dz ee el
+      en eo es et eu fa ff fi fj fo fr fy ga gd gl gn gu gv ha he hi ho hr ht hu hy hz ia id ie ig ii ik io is it iu
+      ja jv ka kg ki kj kk kl km kn ko kr ks ku kv kw ky la lb lg li ln lo lt lu lv mg mh mi mk ml mn mr ms mt my na
+      nb nd ne ng nl nn no nr nv ny oc oj om or os pa pi pl ps pt qu rm rn ro ru rw sa sc sd se sg si sk sl sm sn so
+      sq sr ss st su sv sw ta te tg th ti tk tl tn to tr ts tt tw ty ug uk ur uz ve vi vo wa wo xh yi yo za zh zu
+    ]).freeze
+
+    # Common script codes (4 letters, Titlecase)
+    SCRIPT_CODES = Set.new(%w[
+      Hans Hant Cyrl Latn Arab Deva
+    ]).freeze
+
+    # ISO 3166-1 alpha-2 two-letter uppercase country/region codes
+    ISO_3166_1_REGIONS = Set.new(%w[
+      AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW BY BZ
+      CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ FK FM FO
+      FR GA GB GD GE GF GG GH GI GL GM GN GP GQ GR GS GT GU GW GY HK HM HN HR HT HU ID IE IL IM IN IO IQ IR IS IT JE
+      JM JO JP KE KG KH KI KM KN KP KR KW KY KZ LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH MK ML MM MN MO
+      MP MQ MR MS MT MU MV MW MX MY MZ NA NC NE NF NG NI NL NO NP NR NU NZ OM PA PE PF PG PH PK PL PM PN PR PS PT PW
+      PY QA RE RO RS RU RW SA SB SC SD SE SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD TF TG TH TJ TK TL TM
+      TN TO TR TT TV TW TZ UA UG UM US UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW
+    ]).freeze
+
+    # Known common mistakes & corrections
+    CODE_CORRECTIONS = {
+      'en-uk' => 'en-GB (The ISO 3166-1 region code for the UK is "GB", not "UK")',
+      'uk'    => 'uk is Ukrainian; for United Kingdom English, use "en-GB"',
+      'jp'    => 'jp is country code; for Japanese language, use "ja" or "ja-JP"',
+      'kr'    => 'kr is country code; for Korean language, use "ko" or "ko-KR"',
+      'sp'    => 'sp is invalid; for Spanish, use "es"',
+      'ge'    => 'ge is Georgia; for German, use "de"',
+      'cz'    => 'cz is Czech Republic country code; for Czech language, use "cs"',
+      'se'    => 'se is Northern Sami; for Swedish, use "sv" or "sv-SE"',
+      'dk'    => 'dk is Denmark country code; for Danish language, use "da" or "da-DK"',
+      'gr'    => 'gr is Greece country code; for Greek language, use "el" or "el-GR"',
+      'es-la' => 'es-LA is invalid ("LA" is Laos, not Latin America; use "es-419" or country-specific codes)',
+      'es-sa' => 'es-SA is invalid ("SA" is Saudi Arabia; use country-specific codes like es-AR, es-CL)',
+      'cn'    => 'cn is country code; for Chinese, use "zh", "zh-Hans", or "zh-Hant"'
+    }.freeze
+
+    attr_reader :options, :url, :html, :headers
+
+    def initialize(options = {})
+      @options = options
+    end
+
+    def self.audit(target, options = {})
+      new(options).audit(target)
+    end
+
+    def audit(target)
+      @url, @html, @headers = load_target(target)
+      tags = extract_hreflang_tags(@html, @headers, @url)
+
+      # Validate language/region codes
+      tags.each { |tag| validate_tag_code(tag) }
+
+      # Self-referencing check
+      self_ref_found = check_self_reference(tags, @url)
+
+      # x-default check
+      x_default_tag = tags.find { |t| t[:hreflang].downcase == 'x-default' }
+
+      # Graph Reciprocity Check (for remote URLs)
+      if @options[:check_reciprocity] != false && @url.match?(%r{^https?://})
+        verify_reciprocity(tags, @url)
+      end
+
+      evaluate_health(tags, self_ref_found, x_default_tag)
+    end
+
+    private
+
+    def load_target(target)
+      target_str = target.to_s.strip
+      if target_str.match?(%r{^https?://})
+        uri = URI.parse(target_str)
+        req = Net::HTTP::Get.new(uri)
+        req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) gsc-cli/2.1'
+        req['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
+
+        res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 8, read_timeout: 10) do |http|
+          http.request(req)
+        end
+        [target_str, res.body.to_s.dup.force_encoding('UTF-8').scrub, res.to_hash]
+      elsif File.exist?(target_str)
+        [target_str, File.read(target_str, encoding: 'UTF-8'), {}]
+      else
+        # In-memory HTML string (e.g. test fixtures)
+        base = @options[:url] || 'https://example.com/'
+        [base, target_str, {}]
+      end
+    rescue StandardError => e
+      [target_str.to_s, "<html><head><!-- Error: #{e.message} --></head><body></body></html>", {}]
+    end
+
+    def extract_hreflang_tags(html_content, resp_headers, base_url)
+      tags = []
+
+      # 1. Extract from HTML <link ...>
+      html_content.scan(/<link\b([^>]*?)>/im) do |match|
+        attrs_str = match.first
+        rel = extract_attr(attrs_str, 'rel')&.downcase
+        next unless rel == 'alternate'
+
+        hreflang = extract_attr(attrs_str, 'hreflang')
+        href = extract_attr(attrs_str, 'href')
+        next if hreflang.nil? || href.nil? || href.empty?
+
+        resolved_href = resolve_url(href, base_url)
+        tags << {
+          source: :html,
+          hreflang: hreflang.strip,
+          href: resolved_href,
+          raw_href: href,
+          issues: [],
+          status: :pending
+        }
+      end
+
+      # 2. Extract from HTTP response headers (Link: <...>; rel="alternate"; hreflang="...")
+      link_headers = Array(resp_headers['link']) + Array(resp_headers['Link'])
+      link_headers.each do |lh|
+        lh.split(',').each do |part|
+          if part =~ /<([^>]+)>\s*;\s*rel\s*=\s*(?:["']?alternate["']?)/i
+            href = $1
+            hreflang = nil
+            if part =~ /hreflang\s*=\s*["']?([^;"'\s]+)["']?/i
+              hreflang = $1
+            end
+            if hreflang && href
+              resolved_href = resolve_url(href, base_url)
+              tags << {
+                source: :http_header,
+                hreflang: hreflang.strip,
+                href: resolved_href,
+                raw_href: href,
+                issues: [],
+                status: :pending
+              }
+            end
+          end
+        end
+      end
+
+      tags
+    end
+
+    def validate_tag_code(tag)
+      code = tag[:hreflang].strip
+      return if code.downcase == 'x-default'
+
+      # Check for uppercase language code error (e.g., "EN", "EN-us")
+      if code =~ /^[A-Z]{2}(?:-|$)/
+        tag[:issues] << :uppercase_language
+      end
+
+      # Check for known common misconceptions
+      lowered = code.downcase
+      if CODE_CORRECTIONS.key?(lowered)
+        tag[:issues] << :common_mistake
+        tag[:correction] = CODE_CORRECTIONS[lowered]
+      end
+
+      # Split by hyphen
+      parts = code.split('-')
+      lang = parts[0].downcase
+
+      unless ISO_639_1_LANGUAGES.include?(lang)
+        tag[:issues] << :invalid_language unless tag[:issues].include?(:common_mistake)
+      end
+
+      if parts.size == 2
+        subtag = parts[1]
+        if subtag.length == 4 # Script (e.g. Hans, Hant)
+          unless SCRIPT_CODES.include?(subtag.capitalize)
+            tag[:issues] << :invalid_script
+          end
+        elsif subtag.length == 2 # Region (e.g. US, GB)
+          region_upper = subtag.upcase
+          unless ISO_3166_1_REGIONS.include?(region_upper)
+            tag[:issues] << :invalid_region unless tag[:issues].include?(:common_mistake)
+          end
+          if subtag =~ /^[a-z]{2}$/
+            tag[:issues] << :lowercase_region
+          end
+        elsif subtag == '419' # UN M.49 code for Latin America & Caribbean (valid in Google)
+          # valid
+        else
+          tag[:issues] << :invalid_format
+        end
+      elsif parts.size > 2
+        # e.g., zh-Hans-CN
+        script = parts[1]
+        region = parts[2]
+        unless SCRIPT_CODES.include?(script.capitalize)
+          tag[:issues] << :invalid_script
+        end
+        unless ISO_3166_1_REGIONS.include?(region.upcase)
+          tag[:issues] << :invalid_region
+        end
+      end
+    end
+
+    def check_self_reference(tags, origin_url)
+      norm_origin = normalize_url(origin_url)
+      match = tags.find do |t|
+        normalize_url(t[:href]) == norm_origin && t[:hreflang].downcase != 'x-default'
+      end
+
+      !match.nil?
+    end
+
+    def verify_reciprocity(tags, origin_url)
+      norm_origin = normalize_url(origin_url)
+
+      tags.each do |tag|
+        norm_target = normalize_url(tag[:href])
+        if norm_target == norm_origin
+          tag[:status] = :self_referential
+          next
+        end
+
+        # Fetch the alternate page and check for reciprocal link
+        begin
+          uri = URI.parse(tag[:href])
+          req = Net::HTTP::Get.new(uri)
+          req['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) gsc-cli/2.1'
+
+          res = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == 'https', open_timeout: 4, read_timeout: 5) do |http|
+            http.request(req)
+          end
+
+          if res.code.to_i >= 400
+            tag[:status] = :http_error
+            tag[:issues] << "http_#{res.code}".to_sym
+            next
+          end
+
+          dest_html = res.body.to_s.dup.force_encoding('UTF-8').scrub
+          dest_tags = extract_hreflang_tags(dest_html, res.to_hash, tag[:href])
+
+          # Check if dest_tags contains a link back to origin_url
+          return_tag = dest_tags.find { |dt| normalize_url(dt[:href]) == norm_origin }
+
+          if return_tag
+            tag[:status] = :confirmed
+            tag[:return_hreflang] = return_tag[:hreflang]
+          else
+            tag[:status] = :missing_return
+            tag[:issues] << :missing_reciprocal_tag
+          end
+        rescue StandardError => e
+          tag[:status] = :unreachable
+          tag[:issues] << :network_error
+        end
+      end
+    end
+
+    def evaluate_health(tags, self_ref_found, x_default_tag)
+      total = tags.size
+
+      if total.zero?
+        return {
+          url: @url,
+          total_tags: 0,
+          health_score: 50.0,
+          grade: 'C',
+          self_reference: false,
+          has_x_default: false,
+          reciprocal_count: 0,
+          unreciprocated_count: 0,
+          invalid_code_count: 0,
+          issues_summary: { missing_hreflang: 1 },
+          prescriptions: ['Add bidirectional hreflang alternate tags to properly localize international audience.'],
+          cluster_snippets: { html: '', sitemap_xml: '' },
+          tags: []
+        }
+      end
+
+      invalid_codes = tags.count { |t| (t[:issues] & %i[invalid_language invalid_region invalid_script common_mistake uppercase_language]).any? }
+      unreciprocated = tags.count { |t| t[:status] == :missing_return }
+      confirmed = tags.count { |t| t[:status] == :confirmed || t[:status] == :self_referential }
+
+      score = 100.0
+      score -= 25.0 unless self_ref_found
+      score -= 15.0 unless x_default_tag
+      score -= [invalid_codes * 15.0, 30.0].min
+      score -= [unreciprocated * 20.0, 40.0].min
+
+      score = [[score.round(1), 100.0].min, 0.0].max
+      grade = compute_grade(score)
+
+      summary = {
+        total_tags: total,
+        self_referencing: self_ref_found,
+        x_default: !x_default_tag.nil?,
+        confirmed_reciprocal: confirmed,
+        unreciprocated: unreciprocated,
+        invalid_codes: invalid_codes
+      }
+
+      prescriptions = generate_prescriptions(self_ref_found, x_default_tag, invalid_codes, unreciprocated, tags)
+      snippets = generate_cluster_snippets(tags, @url)
+
+      {
+        url: @url,
+        total_tags: total,
+        health_score: score,
+        grade: grade,
+        self_reference: self_ref_found,
+        has_x_default: !x_default_tag.nil?,
+        reciprocal_count: confirmed,
+        unreciprocated_count: unreciprocated,
+        invalid_code_count: invalid_codes,
+        issues_summary: summary,
+        prescriptions: prescriptions,
+        cluster_snippets: snippets,
+        tags: tags
+      }
+    end
+
+    def generate_prescriptions(self_ref, x_def, invalid_count, unrecip_count, tags)
+      recs = []
+
+      unless self_ref
+        recs << "Add a self-referencing hreflang tag pointing back to #{@url}. Per Google guidelines, every language variant must include its own URL."
+      end
+
+      unless x_def
+        recs << "Include an 'x-default' hreflang tag to handle international users whose language/region is not explicitly matched."
+      end
+
+      if invalid_count > 0
+        mistakes = tags.select { |t| t[:correction] }
+        if mistakes.any?
+          mistakes.each do |m|
+            recs << "Fix invalid hreflang code '#{m[:hreflang]}': #{m[:correction]}."
+          end
+        else
+          recs << "Correct #{invalid_count} invalid ISO 639-1 language or ISO 3166-1 region codes."
+        end
+      end
+
+      if unrecip_count > 0
+        recs << "Resolve #{unrecip_count} missing return tags. Hreflang links must be bidirectional; alternate pages must point back to origin."
+      end
+
+      recs
+    end
+
+    def generate_cluster_snippets(tags, base_url)
+      html_lines = []
+      xml_lines = []
+
+      tags.each do |t|
+        html_lines << "<link rel=\"alternate\" hreflang=\"#{t[:hreflang]}\" href=\"#{t[:href]}\" />"
+        xml_lines << "  <xhtml:link rel=\"alternate\" hreflang=\"#{t[:hreflang]}\" href=\"#{t[:href]}\" />"
+      end
+
+      {
+        html: html_lines.join("\n"),
+        sitemap_xml: xml_lines.join("\n")
+      }
+    end
+
+    def compute_grade(score)
+      case score
+      when 90.0..100.0 then 'A+'
+      when 80.0...90.0 then 'A'
+      when 70.0...80.0 then 'B'
+      when 55.0...70.0 then 'C'
+      when 40.0...55.0 then 'D'
+      else 'F'
+      end
+    end
+
+    def extract_attr(tag_str, attr_name)
+      if tag_str =~ /\b#{attr_name}\s*=\s*(['"])(.*?)\1/i
+        $2.strip
+      elsif tag_str =~ /\b#{attr_name}\s*=\s*([^\s>]+)/i
+        $1.strip
+      else
+        nil
+      end
+    end
+
+    def resolve_url(href, base)
+      return href if href.match?(%r{^https?://})
+      URI.join(base, href).to_s
+    rescue StandardError
+      href
+    end
+
+    def normalize_url(url)
+      u = url.to_s.strip.downcase.sub(%r{/$}, '')
+      u.split('#').first
+    end
+  end
+end