fetch_util 0.3.0

data/lib/fetch_util/raw_docs_fallback.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "net/http"
+require "nokogiri"
+require "uri"
+
+module FetchUtil
+  class RawDocsFallback
+    DEFAULT_HEADERS = {
+      "User-Agent" => Browser::DEFAULT_USER_AGENT,
+      "Accept-Language" => Browser::DEFAULT_ACCEPT_LANGUAGE
+    }.freeze
+
+    BLOCK_ELEMENTS = %w[h1 h2 h3 h4 h5 h6 p pre ul ol li table tr].freeze
+    BLOCK_SELECTOR = BLOCK_ELEMENTS.join(", ").freeze
+    DROP_SELECTORS = [
+      "script",
+      "style",
+      "nav",
+      "aside",
+      "footer",
+      ".toc",
+      ".sidebar",
+      ".breadcrumbs",
+      "[aria-label*='breadcrumb']",
+      ".headerlink",
+      ".copybutton",
+      "button"
+    ].freeze
+    DOCS_ROOT_SELECTORS = [
+      "main article",
+      "main",
+      "article",
+      "[role='main']",
+      ".content",
+      ".resource-container"
+    ].freeze
+    PRUNED_TEXT_PATTERN = /\A(?:on this page|table of contents|edit this page|copy page|copy item path|search|settings|help|expand description)\z/i
+
+    def initialize(timeout: 20)
+      @timeout = timeout.to_i
+    end
+
+    def fetch(url)
+      final_url, html = fetch_html(url)
+      payload = payload_from_html(html, requested_url: url, final_url: final_url)
+      return nil unless payload
+
+      [final_url, payload]
+    rescue Error, SocketError, SystemCallError, Timeout::Error, URI::InvalidURIError
+      nil
+    end
+
+    def payload_from_html(html, requested_url:, final_url: requested_url)
+      document = Nokogiri::HTML(html)
+      root = fragment_root(document, final_url) || docs_root(document)
+      return nil unless root
+
+      prune!(root)
+      title = clean_text(fragment_title(document, final_url) || first_heading(root) || meta_title(document) || document.title)
+      markdown = markdown_from_root(root, title)
+      return nil if clean_text(markdown).length < 40
+
+      {
+        "title" => title,
+        "byline" => meta_value(document, "author"),
+        "excerpt" => first_paragraph(root),
+        "siteName" => meta_value(document, "og:site_name", attr: "property") || safe_host(final_url),
+        "publishedTime" => meta_value(document, "article:published_time", attr: "property") || meta_value(document, "publish-date"),
+        "canonicalUrl" => canonical_url(document, final_url),
+        "language" => document.at_css("html")&.[]("lang") || "en",
+        "html" => root.to_html,
+        "markdown" => markdown,
+        "readerMode" => false,
+        "contentType" => "article",
+        "suspect" => false,
+        "warnings" => []
+      }
+    end
+
+    private
+
+    def meta_value(document, name, attr: "name")
+      document.at_css(%(meta[#{attr}="#{name}"]))&.[]("content")
+    end
+
+    def canonical_url(document, final_url)
+      href = document.at_css('link[rel="canonical"]')&.[]("href")
+      return strip_fragment(final_url) unless href && !href.empty?
+
+      URI.join(final_url, href).to_s
+    rescue URI::InvalidURIError
+      strip_fragment(final_url)
+    end
+
+    def fragment_id(url)
+      fragment = URI.parse(url).fragment.to_s
+      CGI.unescape(fragment)
+    rescue URI::InvalidURIError
+      ""
+    end
+
+    def fragment_root(document, url)
+      id = fragment_id(url)
+      return nil if id.empty?
+
+      node = fragment_node(document, id)
+      return nil unless node
+
+      if node.name == "a" && node["name"] == id
+        container = Nokogiri::XML::Node.new("div", document)
+        sibling = node.next_sibling
+        while sibling
+          break if sibling.element? && sibling.name == "a" && sibling["name"]
+
+          container.add_child(sibling.dup)
+          sibling = sibling.next_sibling
+        end
+        return container if clean_text(container.text).length >= 40
+      end
+
+      candidate = node
+      while candidate&.element?
+        text = clean_text(candidate.text)
+        return candidate.dup if text.length >= 80 || candidate["id"] == id
+        candidate = candidate.parent
+      end
+
+      node.dup
+    end
+
+    def fragment_title(document, url)
+      id = fragment_id(url)
+      return nil if id.empty?
+
+      node = fragment_node(document, id)
+      return nil unless node
+
+      heading = if node.name.match?(/h[1-6]/)
+                  node
+                elsif node.name == "a" && node["name"] == id
+                  node.at_xpath("following-sibling::*[1][self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]") ||
+                    node.at_xpath("following-sibling::*[1]//strong[1]")
+                else
+                  node.at_css("h1, h2, h3, h4, h5, h6")
+                end
+      clean_text(heading&.text)
+    end
+
+    def docs_root(document)
+      DOCS_ROOT_SELECTORS.each do |selector|
+        node = document.at_css(selector)
+        return node.dup if node && clean_text(node.text).length >= 120
+      end
+
+      body = document.at_css("body")
+      body&.dup
+    end
+
+    def prune!(root)
+      DROP_SELECTORS.each { |selector| root.css(selector).remove }
+      root.css("*").each do |node|
+        text = clean_text(node.text)
+        node.remove if text.match?(PRUNED_TEXT_PATTERN)
+      end
+    end
+
+    def meta_title(document)
+      meta_value(document, "og:title", attr: "property") || document.title
+    end
+
+    def first_heading(root)
+      clean_text(root.at_css("h1, h2, h3")&.text)
+    end
+
+    def first_paragraph(root)
+      root.css("p").map { |node| clean_text(node.text) }.find { |text| text.length >= 30 }
+    end
+
+    def markdown_from_root(root, title)
+      sections = []
+      root.css(BLOCK_SELECTOR).each do |node|
+        text = clean_text(node.text)
+        next if text.empty?
+
+        case node.name
+        when /h([1-6])/
+          level = Regexp.last_match(1).to_i
+          sections << "#{"#" * level} #{text}"
+        when "p"
+          sections << text
+        when "pre"
+          sections << ["```", node.text.rstrip, "```"].join("\n")
+        when "li"
+          sections << "- #{text}"
+        when "tr"
+          cells = node.css("th, td").map { |cell| clean_text(cell.text) }.reject(&:empty?)
+          sections << "- #{cells.join(": ")}" unless cells.empty?
+        end
+      end
+
+      markdown = sections.join("\n\n").gsub(/\n{3,}/, "\n\n").strip
+      markdown = "# #{title}\n\n#{markdown}" if title && !markdown.start_with?("# #{title}")
+      markdown
+    end
+
+    def clean_text(text)
+      FetchUtil.normalize_whitespace(text)
+    end
+
+    def strip_fragment(url)
+      uri = URI.parse(url)
+      uri.fragment = nil
+      uri.to_s
+    rescue URI::InvalidURIError
+      url
+    end
+
+    def safe_host(url)
+      URI.parse(url).host
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def fragment_node(document, id)
+      document.at_xpath(%(//*[@id=#{xpath_literal(id)}])) || document.at_xpath(%(//a[@name=#{xpath_literal(id)}]))
+    end
+
+    def xpath_literal(value)
+      return %('#{value}') unless value.include?("'")
+      return %("#{value}") unless value.include?('"')
+
+      parts = value.split("'").map { |part| %('#{part}') }
+      %(concat(#{parts.join(%q(, "'", ))}))
+    end
+
+    def fetch_html(url, limit: 5)
+      raise URI::InvalidURIError, "too many redirects" if limit <= 0
+
+      uri = URI.parse(url)
+      request = Net::HTTP::Get.new(uri)
+      DEFAULT_HEADERS.each { |key, value| request[key] = value }
+
+      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: @timeout, read_timeout: @timeout) do |http|
+        http.request(request)
+      end
+
+      case response
+      when Net::HTTPSuccess
+        [uri.to_s, response.body]
+      when Net::HTTPRedirection
+        location = URI.join(uri, response["location"]).to_s
+        fetch_html(location, limit: limit - 1)
+      else
+        raise Error, "HTTP #{response.code}"
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/cache_store.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module CacheStore
+      private
+
+      def cache_fetch(key)
+        path = cache_file_path(key)
+        cached = read_cache(path)
+        return cached if cached
+
+        payload = yield
+        write_cache(path, payload)
+        payload
+      end
+
+      def fetch_record(key, uri, fallback: nil, require_success: true)
+        cache_fetch(key) do
+          response = record_response(uri, require_success: require_success)
+          response ? yield(response.body, response) : fallback
+        end
+      end
+
+      def record_response(uri, require_success:)
+        Array(uri).each do |candidate|
+          response = safe_get(candidate)
+          next unless response
+
+          return response if !require_success || response.status&.between?(200, 299)
+        end
+
+        nil
+      end
+
+      def cache_file_path(key)
+        digest = Digest::SHA256.hexdigest("v#{CACHE_VERSION}:#{key}")
+        File.join(cache_path, "#{digest}.json")
+      end
+
+      def read_cache(path)
+        return nil unless File.exist?(path)
+
+        parsed = JSON.parse(File.read(path))
+        cached_at = Time.parse(parsed.fetch("cached_at"))
+        return nil if Time.now.utc - cached_at > CACHE_TTL
+
+        parsed["payload"]
+      rescue Errno::ENOENT, JSON::ParserError, KeyError, TypeError, ArgumentError
+        nil
+      end
+
+      def write_cache(path, payload)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, JSON.generate({ "cached_at" => Time.now.utc.iso8601, "payload" => json_safe(payload) }))
+      end
+
+      def safe_get(url)
+        client.get(url)
+      rescue ArgumentError, IOError, SocketError, Timeout::Error
+        nil
+      rescue FetchUtil::Error, SystemCallError, OpenSSL::SSL::SSLError
+        nil
+      end
+
+      def deep_copy(value)
+        JSON.parse(JSON.generate(json_safe(value)))
+      end
+
+      def response_chain(response)
+        Array(response&.redirects) + [response].compact
+      end
+
+      def json_safe(value)
+        case value
+        when String
+          value.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+        when Array
+          value.map { |item| json_safe(item) }
+        when Hash
+          safe = {}
+          value.each do |key, item|
+            safe[json_safe(key)] = json_safe(item)
+          end
+          safe
+        else
+          value
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/directives.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module Directives
+      private
+
+      def extract_robot_directive_signals(value, path:, meta_name: nil)
+        current_user_agent = nil
+        tokenize_robot_directives(value).flat_map do |token|
+          prefix, directive = robot_directive_prefix(token)
+          if prefix
+            current_user_agent = prefix
+            directive_signals(directive, path: path, user_agent: prefix)
+          else
+            directive_signals(token, path: path, user_agent: meta_name && !meta_name.casecmp?("robots") ? meta_name : current_user_agent)
+          end
+        end
+      end
+
+      def tokenize_robot_directives(value)
+        protected = value.to_s.gsub(
+          /(unavailable_after\s*:\s*[A-Za-z]{3}),\s*(\d{1,2}\s+[A-Za-z]{3}\s+\d{4}\s+\d{2}:\d{2}:\d{2}\s+[A-Za-z+-]+)/i,
+          '\\1__FETCH_UTIL_COMMA__\\2'
+        )
+
+        protected.split(",").map { |token| token.gsub("__FETCH_UTIL_COMMA__", ",").strip }.reject(&:empty?)
+      end
+
+      def robot_directive_prefix(token)
+        match = token.to_s.match(/\A([A-Za-z0-9*_.-]+)\s*:\s*(.+)\z/)
+        return [nil, nil] unless match
+
+        prefix = match[1]
+        directive = match[2]
+        return [nil, nil] if FetchUtil::Regulatory::Robots::ROBOT_DIRECTIVES.include?(prefix.downcase)
+
+        [prefix, directive]
+      end
+
+      def directive_signals(directive, path:, user_agent: nil)
+        name, raw_value = directive.to_s.split(":", 2)
+        name = name.to_s.strip.downcase
+        value = raw_value.to_s.strip
+        conditions = directive_conditions(user_agent)
+
+        case name
+        when "all"
+          [
+            build_signal("allow", "index", path: path, conditions: conditions),
+            build_signal("allow", "follow", path: path, conditions: conditions)
+          ]
+        when "follow"
+          [build_signal("allow", "follow", path: path, conditions: conditions)]
+        when "index"
+          [build_signal("allow", "index", path: path, conditions: conditions)]
+        when "indexifembedded"
+          [build_signal("allow", "index", path: path, conditions: conditions.merge("if-embedded" => true))]
+        when "max-image-preview"
+          max_image_preview_signal(path: path, conditions: conditions, value: value)
+        when "max-snippet"
+          [build_signal("allow", "snippet", path: path, conditions: conditions.merge("max-chars" => integer_or_value(value)))]
+        when "max-video-preview"
+          [build_signal("allow", "video-preview", path: path, conditions: conditions.merge("max-seconds" => integer_or_value(value)))]
+        when "noai"
+          [build_signal("disallow", "ai-training", path: path, conditions: conditions)]
+        when "noarchive", "nocache"
+          [build_signal("disallow", "archive", path: path, conditions: conditions)]
+        when "nofollow"
+          [build_signal("disallow", "follow", path: path, conditions: conditions)]
+        when "noimageai"
+          [build_signal("disallow", "image-ai-training", path: path, conditions: conditions)]
+        when "noimageindex"
+          [build_signal("disallow", "image-index", path: path, conditions: conditions)]
+        when "noindex"
+          [build_signal("disallow", "index", path: path, conditions: conditions)]
+        when "none"
+          [
+            build_signal("disallow", "index", path: path, conditions: conditions),
+            build_signal("disallow", "follow", path: path, conditions: conditions)
+          ]
+        when "nosnippet"
+          [build_signal("disallow", "snippet", path: path, conditions: conditions)]
+        when "notranslate"
+          [build_signal("disallow", "translate", path: path, conditions: conditions)]
+        when "unavailable_after"
+          [build_signal("disallow", "index", path: path, conditions: conditions.merge("after" => value))]
+        else
+          []
+        end
+      end
+
+      def max_image_preview_signal(path:, conditions:, value:)
+        return [build_signal("disallow", "image-preview", path: path, conditions: conditions)] if value.casecmp?("none")
+
+        [build_signal("allow", "image-preview", path: path, conditions: conditions.merge("max" => value))]
+      end
+
+      def directive_conditions(user_agent)
+        return {} if user_agent.to_s.strip.empty?
+
+        { "user-agent" => robot_user_agent_glob(user_agent) }
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/fetch_records.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module FetchRecords
+      private
+
+      def normalize_output_path(value)
+        text = value.to_s.strip
+        return "/*" if text.empty? || text == "/"
+
+        text
+      end
+
+      def request_target(uri)
+        path = uri.path.to_s.empty? ? "/" : uri.path.to_s
+        query = uri.query.to_s
+        return path if query.empty?
+
+        "#{path}?#{query}"
+      end
+
+      def origin_query?(uri)
+        uri.path.to_s.empty? && uri.query.to_s.empty?
+      end
+
+      def page_query_target(record, fallback:)
+        final_url = record["final_url"].to_s
+        return fallback if final_url.empty?
+
+        request_target(parse_http_uri(final_url))
+      end
+
+      def robots_uri(uri)
+        "#{base_origin(uri)}/robots.txt"
+      end
+
+      def tdmrep_uri(uri)
+        "#{base_origin(uri)}/.well-known/tdmrep.json"
+      end
+
+      def trusttxt_uri(uri)
+        "#{base_origin(uri)}/trust.txt"
+      end
+
+      def trusttxt_well_known_uri(uri)
+        "#{base_origin(uri)}/.well-known/trust.txt"
+      end
+
+      def base_origin(uri)
+        port = if (uri.scheme == "https" && uri.port == 443) || (uri.scheme == "http" && uri.port == 80)
+                 nil
+               else
+                 ":#{uri.port}"
+               end
+        "#{uri.scheme}://#{uri.host}#{port}"
+      end
+
+      def origin_key(uri)
+        base_origin(uri)
+      end
+
+      def parse_http_uri(value)
+        uri = URI.parse(value.to_s.strip)
+        unless uri.is_a?(URI::HTTP) && uri.host
+          raise ArgumentError, "unsupported url: #{value}"
+        end
+
+        uri
+      rescue URI::InvalidURIError
+        raise ArgumentError, "unsupported url: #{value}"
+      end
+
+      def first_header_value(headers, name)
+        Array(headers[name]).first.to_s.strip
+      end
+
+      def header_values(headers, name)
+        headers.fetch(name, [])
+      end
+
+      def html_content?(headers, body)
+        content_type = first_header_value(headers, "content-type")
+        return true if content_type.include?("text/html") || content_type.include?("application/xhtml+xml")
+
+        body.to_s.lstrip.start_with?("<!DOCTYPE html", "<html", "<HTML")
+      end
+
+      def parse_meta_tags(body)
+        body.to_s.scan(/<meta\b[^>]*>/im).map do |tag|
+          attributes = {}
+          tag.scan(/([A-Za-z_:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/).each do |name, quoted, single, bare|
+            attributes[name.downcase] = CGI.unescapeHTML(quoted || single || bare || "")
+          end
+          attributes
+        end
+      end
+
+      def json_like_response?(headers, body)
+        content_type = first_header_value(headers, "content-type")
+        return true if content_type.include?("application/json") || content_type.include?("application/ld+json")
+
+        stripped = body.to_s.lstrip
+        stripped.start_with?("{", "[")
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/headers.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module Headers
+      def extract_x_robot_signals(headers, path:)
+        header_values(headers, "x-robots-tag").flat_map do |value|
+          extract_robot_directive_signals(value, path: path)
+        end
+      end
+
+      def extract_content_usage_header_signals(headers, path:)
+        header_values(headers, "content-usage").flat_map do |value|
+          extract_content_usage_statement_signals(value, path: path)
+        end
+      end
+
+      def extract_meta_robot_signals(meta_tags, path:)
+        signals = []
+
+        meta_tags.each do |attributes|
+          if attributes["http-equiv"].to_s.casecmp?("x-robots-tag")
+            signals.concat(extract_robot_directive_signals(attributes["content"], path: path))
+            next
+          end
+
+          name = attributes["name"].to_s.strip
+          next if name.empty?
+          next if name.casecmp?("tdm-reservation") || name.casecmp?("tdm-policy")
+          next unless name.casecmp?("robots") || name.match?(/bot/i)
+
+          signals.concat(extract_robot_directive_signals(attributes["content"], path: path, meta_name: name))
+        end
+
+        signals
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/http_client.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "net/http"
+
+module FetchUtil
+  class Regulatory
+    class HttpClient
+      REDIRECT_LIMIT = 5
+
+      def initialize(timeout:, user_agent:)
+        @timeout = timeout.to_f
+        @user_agent = user_agent.to_s.strip
+      end
+
+      def get(url, limit: REDIRECT_LIMIT)
+        uri = parse_http_uri(url)
+        fetch(uri, limit, [])
+      end
+
+      private
+
+      attr_reader :timeout, :user_agent
+
+      def fetch(uri, limit, redirects)
+        response = request(uri)
+        return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
+
+        raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
+
+        location = response["location"].to_s.strip
+        return build_response(uri, response, redirects: redirects) if location.empty?
+
+        redirect_response = build_response(uri, response)
+        fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
+      end
+
+      def request(uri)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = timeout
+        http.read_timeout = timeout
+        request = Net::HTTP::Get.new(uri.request_uri.empty? ? "/" : uri.request_uri)
+        request["Accept"] = "text/html,application/json,text/plain,*/*"
+        request["User-Agent"] = user_agent unless user_agent.empty?
+        http.request(request)
+      end
+
+      def build_response(uri, response, redirects: [])
+        FetchUtil::Regulatory::Response.new(
+          url: uri.to_s,
+          status: response.code.to_i,
+          headers: response.to_hash.transform_keys(&:downcase),
+          body: response.body.to_s,
+          redirects: redirects
+        )
+      end
+
+      def parse_http_uri(url)
+        uri = URI.parse(url.to_s)
+        unless uri.is_a?(URI::HTTP) && uri.host
+          raise ArgumentError, "unsupported url: #{url}"
+        end
+
+        uri
+      rescue URI::InvalidURIError
+        raise ArgumentError, "unsupported url: #{url}"
+      end
+    end
+  end
+end