fetch_util 0.5.2 → 0.6.0

data/lib/fetch_util/parallel_fetcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "uri"
+
 module FetchUtil
   class ParallelFetcher
     Failure = Struct.new(:index, :url, :error, keyword_init: true)
@@ -8,7 +10,7 @@ module FetchUtil
       attr_reader :failures, :results
 
       def initialize(failures, results = nil)
-        @failures = failures.freeze
+        @failures = ordered_failures(failures).freeze
         @results = results&.freeze
         super(self.class.build_message(@failures))
       end
@@ -19,60 +21,107 @@ module FetchUtil
 
       def self.build_message(failures)
         preview = failures.first(3).map do |failure|
-          label = failure.url || "<initialization>"
+          label = if failure.index.nil?
+                    "<initialization>"
+                  elsif failure.url.empty?
+                    "<blank>"
+                  else
+                    failure.url
+                  end
           "#{label} (#{failure.error.class}: #{failure.error.message})"
         end.join(", ")
         suffix = failures.length > 3 ? ", +#{failures.length - 3} more" : ""
         "parallel fetch failed for #{failures.length} URLs: #{preview}#{suffix}"
       end
+
+      private
+
+      def ordered_failures(failures)
+        initialization_failures, url_failures = failures.partition { |failure| failure.index.nil? }
+        initialization_failures.sort_by! { |failure| [failure.error.class.name.to_s, failure.error.message] }
+        url_failures.sort_by!(&:index)
+        initialization_failures + url_failures
+      end
     end
 
     DEFAULT_CONCURRENCY = 4
+    FETCHER_COLLABORATOR_OPTIONS = %i[browser extractor raw_docs_fallback request_log pdf_header_probe].freeze
+    private_constant :FETCHER_COLLABORATOR_OPTIONS
 
     def initialize(fetcher_factory: nil, concurrency: DEFAULT_CONCURRENCY, **fetch_options)
-      @fetcher_factory = fetcher_factory || -> { Fetcher.new(**fetch_options) }
-      @concurrency = [concurrency.to_i, 1].max
+      unless concurrency.is_a?(Integer) && concurrency.positive?
+        raise InputError, "concurrency must be a positive Integer"
+      end
+
+      @shared_browser = fetcher_factory.nil? && fetch_options[:browser]
+      @fetcher_factory = fetcher_factory || default_fetcher_factory(fetch_options)
+      @concurrency = concurrency
     end
 
     def fetch(urls)
-      work = Array(urls).compact.map(&:to_s).reject(&:empty?)
+      work = Array(urls).map { |url| url.to_s.dup.freeze }
       return [] if work.empty?
 
       results = Array.new(work.length)
-      worker_count = [@concurrency, work.length].min
       failures = []
+      pending_indices = []
+      work.each_with_index do |url, index|
+        if url.empty?
+          error = URI::InvalidURIError.new("unsupported url: #{url}")
+          failures << Failure.new(index: index, url: url, error: error)
+        else
+          pending_indices << index
+        end
+      end
+
+      worker_count = [@concurrency, pending_indices.length].min
+      if worker_count > 1 && @shared_browser
+        raise InputError, "browser cannot be shared across parallel workers; use fetcher_factory"
+      end
       next_index = 0
       mutex = Mutex.new
-
-      threads = Array.new(worker_count) do
-        Thread.new do
-          fetcher = @fetcher_factory.call
-
-          begin
-            loop do
-              index = mutex.synchronize do
-                if next_index < work.length
-                  current = next_index
-                  next_index += 1
-                  current
+      stopping = false
+      threads = []
+
+      begin
+        worker_count.times do
+          threads << Thread.new do
+            fetcher = @fetcher_factory.call
+
+            begin
+              loop do
+                index = mutex.synchronize do
+                  if !stopping && next_index < pending_indices.length
+                    current = pending_indices[next_index]
+                    next_index += 1
+                    current
+                  end
                 end
-              end
-              break if index.nil?
+                break if index.nil?
 
-              url = work[index]
+                url = work[index]
 
+                begin
+                  results[index] = fetcher.fetch(url)
+                rescue StandardError => e
+                  mutex.synchronize { failures << Failure.new(index: index, url: url, error: e) }
+                end
+              end
+            ensure
               begin
-                results[index] = fetcher.fetch(url)
-              rescue StandardError => e
-                mutex.synchronize { failures << Failure.new(index: index, url: url, error: e) }
+                fetcher.quit if fetcher.respond_to?(:quit)
+              rescue Ferrum::Error
+                nil
               end
             end
-          ensure
-            fetcher.quit if fetcher.respond_to?(:quit)
+          rescue StandardError => e
+            mutex.synchronize { failures << Failure.new(index: nil, url: nil, error: e) }
           end
-        rescue StandardError => e
-          mutex.synchronize { failures << Failure.new(index: nil, url: nil, error: e) }
         end
+      rescue ThreadError
+        mutex.synchronize { stopping = true }
+        threads.each(&:join)
+        raise
       end
 
       threads.each(&:join)
@@ -83,6 +132,54 @@ module FetchUtil
 
     private
 
+    def default_fetcher_factory(fetch_options)
+      owned_options = immutable_fetch_options(fetch_options)
+      -> { Fetcher.new(**immutable_fetch_options(owned_options)) }
+    end
+
+    def immutable_fetch_options(fetch_options)
+      memo = {}.compare_by_identity
+      owned = {}
+      memo[fetch_options] = owned
+
+      fetch_options.each do |key, value|
+        owned_key = immutable_fetch_option(key, memo)
+        owned[owned_key] = if FETCHER_COLLABORATOR_OPTIONS.include?(key)
+                             value
+                           else
+                             immutable_fetch_option(value, memo)
+                           end
+      end
+      owned.freeze
+    end
+
+    def immutable_fetch_option(value, memo)
+      return memo[value] if memo.key?(value)
+
+      case value
+      when String
+        memo[value] = value.dup.freeze
+      when Array
+        owned = value.dup
+        memo[value] = owned
+        owned.map! { |item| immutable_fetch_option(item, memo) }
+        owned.freeze
+      when Hash
+        owned = value.dup
+        memo[value] = owned
+        owned.clear
+        owned.default = immutable_fetch_option(value.default, memo) unless value.default_proc
+        value.each do |key, item|
+          owned_key = value.compare_by_identity? ? key : immutable_fetch_option(key, memo)
+          owned[owned_key] = immutable_fetch_option(item, memo)
+        end
+        owned.freeze unless owned.default_proc
+        owned
+      else
+        value
+      end
+    end
+
     def raise_for_failures(failures, results)
       return if failures.empty?
 

data/lib/fetch_util/raw_docs_fallback.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require "cgi"
 require "nokogiri"
+require "openssl"
 require "uri"
 
-require_relative "regulatory/http_client"
+require_relative "http_redirect_client"
 
 module FetchUtil
   class RawDocsFallback
@@ -15,6 +15,10 @@ module FetchUtil
 
     BLOCK_ELEMENTS = %w[h1 h2 h3 h4 h5 h6 p pre ul ol li table tr].freeze
     BLOCK_SELECTOR = BLOCK_ELEMENTS.join(", ").freeze
+    URL_ATTRIBUTES = %w[action background cite classid codebase data data-lazy data-lazy-src data-original data-src formaction
+                        href itemid longdesc manifest poster profile src usemap xlink:href].freeze
+    SRCSET_ATTRIBUTES = %w[imagesrcset srcset].freeze
+    DROP_ATTRIBUTES = %w[archive ping srcdoc style].freeze
     DROP_SELECTORS = [
       "script",
       "style",
@@ -49,7 +53,7 @@ module FetchUtil
     PRUNED_TEXT_PATTERN = /\A(?:on this page|table of contents|edit this page|copy page|copy item path|search|settings|help|expand description)\z/i
 
     def initialize(timeout: 20, http_client: nil)
-      @timeout = timeout.to_i
+      @timeout = timeout.to_f
       @http_client = http_client || FetchUtil::HttpRedirectClient.new(timeout: @timeout, headers: DEFAULT_HEADERS)
     end
 
@@ -59,7 +63,7 @@ module FetchUtil
       return nil unless payload
 
       [final_url, payload]
-    rescue Error, SocketError, SystemCallError, Timeout::Error, URI::InvalidURIError
+    rescue Error, IOError, SocketError, SystemCallError, Timeout::Error, OpenSSL::SSL::SSLError, URI::InvalidURIError
       nil
     end
 
@@ -69,20 +73,24 @@ module FetchUtil
       return nil unless root
 
       prune!(root)
+      prune_credential_urls!(root, final_url)
       title = [fragment_title(document, final_url), first_heading(root), meta_title(document), document.title]
               .map { |candidate| clean_text(candidate) }
               .find { |candidate| !candidate.empty? }
       markdown = markdown_from_root(root, title)
       return nil if clean_text(markdown).length < 40
 
+      language = clean_optional_text(document.at_css("html")&.[]("lang"))
+
       {
         "title" => title,
-        "byline" => meta_value(document, "author"),
+        "byline" => clean_optional_text(meta_value(document, "author")),
         "excerpt" => first_paragraph(root),
-        "siteName" => meta_value(document, "og:site_name", attr: "property") || safe_host(final_url),
-        "publishedTime" => meta_value(document, "article:published_time", attr: "property") || meta_value(document, "publish-date"),
+        "siteName" => clean_optional_text(meta_value(document, "og:site_name", attr: "property")) || safe_host(final_url),
+        "publishedTime" => clean_optional_text(meta_value(document, "article:published_time", attr: "property") ||
+                                                meta_value(document, "publish-date")),
         "canonicalUrl" => canonical_url(document, final_url),
-        "language" => document.at_css("html")&.[]("lang") || "en",
+        "language" => language,
         "html" => root.to_html,
         "markdown" => markdown,
         "readerMode" => false,
@@ -99,17 +107,64 @@ module FetchUtil
     end
 
     def canonical_url(document, final_url)
-      href = document.at_css('link[rel="canonical"]')&.[]("href")
-      return strip_fragment(final_url) unless href && !href.empty?
+      fallback_url = http_url(strip_fragment(final_url))
+      document.css('link[rel="canonical"]').each do |node|
+        href = node["href"]
+        next unless href && !href.empty?
+
+        url = http_url(URI.join(final_url, href).to_s)
+        return url if url
+      rescue URI::InvalidURIError
+        next
+      end
+      fallback_url
+    end
+
+    def http_url(value)
+      uri = URI.parse(value.to_s)
+      uri.to_s if uri.is_a?(URI::HTTP) && !uri.host.to_s.empty? && uri.userinfo.nil?
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def prune_credential_urls!(root, base_url)
+      ([root] + root.css("*").to_a).each do |node|
+        DROP_ATTRIBUTES.each { |attribute| node.remove_attribute(attribute) }
+        URL_ATTRIBUTES.each do |attribute|
+          value = node[attribute]
+          next unless value
+
+          node.remove_attribute(attribute) if credential_url?(value, base_url)
+        end
+        SRCSET_ATTRIBUTES.each do |attribute|
+          value = node[attribute]
+          next unless value
+
+          entries = value.split(",").map(&:strip).reject do |entry|
+            reference = entry.split(/\s+/, 2).first
+            credential_url?(reference, base_url)
+          end
+          entries.empty? ? node.remove_attribute(attribute) : node[attribute] = entries.join(", ")
+        end
+      end
+      root.xpath(".//text() | self::text()").each do |node|
+        sanitized = credential_free_text(node.text)
+        node.content = sanitized unless sanitized == node.text
+      end
+    end
 
-      URI.join(final_url, href).to_s
+    def credential_url?(value, base_url)
+      return true if value.to_s.match?(%r{\A\s*(?:https?:)?[\\/]{2}[^\\/\s?#@]+@}i)
+
+      uri = URI.join(base_url, value.to_s)
+      uri.is_a?(URI::HTTP) && !uri.userinfo.nil?
     rescue URI::InvalidURIError
-      strip_fragment(final_url)
+      false
     end
 
     def fragment_id(url)
       fragment = URI.parse(url).fragment.to_s
-      CGI.unescape(fragment)
+      URI::DEFAULT_PARSER.unescape(fragment)
     rescue URI::InvalidURIError
       ""
     end
@@ -121,11 +176,13 @@ module FetchUtil
       node = fragment_node(document, id)
       return nil unless node
 
-      if node.name == "a" && node["name"] == id
+      return fragment_heading_root(document, node) if fragment_heading_level(node)
+
+      if fragment_anchor?(node, id)
         container = Nokogiri::XML::Node.new("div", document)
         sibling = node.next_sibling
         while sibling
-          break if sibling.element? && sibling.name == "a" && sibling["name"]
+          break if sibling.element? && fragment_anchor?(sibling)
 
           container.add_child(sibling.dup)
           sibling = sibling.next_sibling
@@ -143,6 +200,29 @@ module FetchUtil
       node.dup
     end
 
+    def fragment_heading_root(document, heading)
+      level = fragment_heading_level(heading)
+      container = Nokogiri::XML::Node.new("div", document)
+      sibling = heading
+
+      while sibling
+        if sibling != heading && sibling.element?
+          sibling_level = fragment_heading_level(sibling)
+          break if empty_fragment_anchor?(sibling) || (sibling_level && sibling_level <= level)
+        end
+
+        container.add_child(sibling.dup)
+        sibling = sibling.next_sibling
+      end
+
+      container
+    end
+
+    def fragment_heading_level(node)
+      match = node&.name&.match(/\Ah([1-6])\z/)
+      match && Integer(match[1], 10)
+    end
+
     def fragment_title(document, url)
       id = fragment_id(url)
       return nil if id.empty?
@@ -152,7 +232,7 @@ module FetchUtil
 
       heading = if node.name.match?(/h[1-6]/)
                   node
-                elsif node.name == "a" && node["name"] == id
+                elsif fragment_anchor?(node, id)
                   node.at_xpath("following-sibling::*[1][self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]") ||
                     node.at_xpath("following-sibling::*[1]//strong[1]")
                 else
@@ -219,7 +299,11 @@ module FetchUtil
 
     def markdown_from_root(root, title)
       sections = []
-      root.css(BLOCK_SELECTOR).each do |node|
+      blocks = root.css(BLOCK_SELECTOR).to_a
+      blocks.unshift(root) if %w[ul ol].include?(root.name)
+      blocks.each do |node|
+        next if node.ancestors.any? { |ancestor| %w[ul ol].include?(ancestor.name) }
+
         text = clean_text(node.text)
         next if text.empty?
 
@@ -230,9 +314,12 @@ module FetchUtil
         when "p"
           sections << text
         when "pre"
-          sections << ["```", node.text.rstrip, "```"].join("\n")
-        when "li"
-          sections << "- #{text}"
+          code = node.text.rstrip
+          fence_length = [3, code.scan(/`+/).map(&:length).max.to_i + 1].max
+          fence = "`" * fence_length
+          sections << [fence, code, fence].join("\n")
+        when "ul", "ol"
+          sections << markdown_list(node)
         when "tr"
           cells = node.css("th, td").map { |cell| clean_text(cell.text) }.reject(&:empty?)
           sections << "- #{cells.join(": ")}" unless cells.empty?
@@ -244,8 +331,45 @@ module FetchUtil
       markdown
     end
 
+    def markdown_list(list, depth = 0)
+      ordinal = integer_attribute(list, "start") || 1
+      items = list.css("li").select do |item|
+        item.ancestors.find { |ancestor| %w[ul ol].include?(ancestor.name) } == list
+      end
+      lines = items.flat_map do |item|
+        marker_value = integer_attribute(item, "value")
+        ordinal = marker_value if marker_value
+        marker = list.name == "ol" ? "#{ordinal}." : "-"
+        content = item.dup
+        content.css("ul, ol").remove
+        line = "#{"  " * depth}#{marker} #{clean_text(content.text)}"
+        ordinal += 1 if list.name == "ol"
+
+        nested = item.css("ul, ol").select do |child|
+          child.ancestors.find { |ancestor| ancestor.name == "li" } == item
+        end
+        [line] + nested.flat_map { |child| markdown_list(child, depth + 1).lines(chomp: true) }
+      end
+      lines.join("\n")
+    end
+
+    def integer_attribute(node, name)
+      Integer(node[name], 10)
+    rescue ArgumentError, TypeError
+      nil
+    end
+
     def clean_text(text)
-      FetchUtil.normalize_whitespace(text)
+      FetchUtil.normalize_whitespace(credential_free_text(text))
+    end
+
+    def clean_optional_text(text)
+      value = clean_text(text)
+      value unless value.empty?
+    end
+
+    def credential_free_text(text)
+      text.to_s.gsub(%r{(https?:[\\/]{2})[^\\/\s?#@]+@}i, "\\1")
     end
 
     def strip_fragment(url)
@@ -266,6 +390,19 @@ module FetchUtil
       document.at_xpath(%(//*[@id=#{xpath_literal(id)}])) || document.at_xpath(%(//a[@name=#{xpath_literal(id)}]))
     end
 
+    def fragment_anchor?(node, id = nil)
+      return false unless node.name == "a"
+
+      anchor_id = node["name"]
+      anchor_id ||= node["id"] if node.element_children.empty? && clean_text(node.text).empty?
+      id ? anchor_id == id : !anchor_id.to_s.empty?
+    end
+
+    def empty_fragment_anchor?(node)
+      node.name == "a" && node.element_children.empty? && clean_text(node.text).empty? &&
+        ![node["name"], node["id"]].compact.empty?
+    end
+
     def xpath_literal(value)
       return %('#{value}') unless value.include?("'")
       return %("#{value}") unless value.include?('"')

data/lib/fetch_util/regulatory/cache_store.rb

@@ -10,27 +10,34 @@ module FetchUtil
         cached = read_cache(path)
         return cached if cached
 
-        payload = yield
-        write_cache(path, payload)
+        payload, cacheable = yield
+        begin
+          write_cache(path, payload) if cacheable
+        rescue SystemCallError, IOError
+          nil
+        end
         payload
       end
 
       def fetch_record(key, uri, fallback: nil, require_success: true)
         cache_fetch(key) do
-          response = record_response(uri, require_success: require_success)
-          response ? yield(response.body, response) : fallback
+          response, cacheable = record_response(uri, require_success: require_success)
+          payload = response ? yield(response.body, response) : fallback
+          [payload, cacheable]
         end
       end
 
       def record_response(uri, require_success:)
+        cacheable = true
         Array(uri).each do |candidate|
-          response = safe_get(candidate)
+          response, completed = safe_get(candidate)
+          cacheable &&= completed
           next unless response
 
-          return response if !require_success || response.status&.between?(200, 299)
+          return [response, cacheable] if !require_success || response.status&.between?(200, 299)
         end
 
-        nil
+        [nil, cacheable]
       end
 
       def cache_file_path(key)
@@ -42,25 +49,51 @@ module FetchUtil
         return nil unless File.exist?(path)
 
         parsed = JSON.parse(File.read(path))
+        return nil unless parsed.is_a?(Hash)
+
         cached_at = Time.parse(parsed.fetch("cached_at"))
-        return nil if Time.now.utc - cached_at > CACHE_TTL
+        now = Time.now.utc
+        return nil if cached_at > now || now - cached_at > CACHE_TTL
 
         parsed["payload"]
-      rescue Errno::ENOENT, JSON::ParserError, KeyError, TypeError, ArgumentError
+      rescue SystemCallError, IOError, JSON::ParserError, KeyError, TypeError, ArgumentError
         nil
       end
 
       def write_cache(path, payload)
-        FileUtils.mkdir_p(File.dirname(path))
-        File.write(path, JSON.generate({ "cached_at" => Time.now.utc.iso8601, "payload" => json_safe(payload) }))
+        directory = File.dirname(path)
+        serialized = JSON.generate({ "cached_at" => Time.now.utc.iso8601, "payload" => json_safe(payload) })
+        FileUtils.mkdir_p(directory)
+        existing_mode = cache_file_mode(path)
+        temp_path = File.join(directory, ".#{File.basename(path)}.#{Process.pid}.#{SecureRandom.hex(6)}.tmp")
+
+        File.open(temp_path, File::WRONLY | File::CREAT | File::EXCL, 0o666) do |file|
+          file.write(serialized)
+        end
+        File.chmod(existing_mode, temp_path) if existing_mode
+        File.rename(temp_path, path)
+      ensure
+        FileUtils.rm_f(temp_path) if temp_path
+      end
+
+      def cache_file_mode(path)
+        File.stat(path).mode & 0o777
+      rescue Errno::ENOENT
+        nil
       end
 
       def safe_get(url)
-        client.get(url)
+        response = client.get(url)
+        [response, cacheable_response?(response)]
       rescue ArgumentError, IOError, SocketError, Timeout::Error
-        nil
+        [nil, false]
       rescue FetchUtil::Error, SystemCallError, OpenSSL::SSL::SSLError
-        nil
+        [nil, false]
+      end
+
+      def cacheable_response?(response)
+        status = response.status
+        ![408, 429].include?(status) && !status&.between?(500, 599)
       end
 
       def deep_copy(value)

data/lib/fetch_util/regulatory/fetch_records.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "nokogiri"
+
 module FetchUtil
   class Regulatory
     module FetchRecords
@@ -63,12 +65,12 @@ module FetchUtil
       def parse_http_uri(value)
         uri = URI.parse(value.to_s.strip)
         unless uri.is_a?(URI::HTTP) && uri.host
-          raise ArgumentError, "unsupported url: #{value}"
+          raise InputError, "unsupported url: #{value}"
         end
 
         uri
       rescue URI::InvalidURIError
-        raise ArgumentError, "unsupported url: #{value}"
+        raise InputError, "unsupported url: #{value}"
       end
 
       def first_header_value(headers, name)
@@ -87,12 +89,13 @@ module FetchUtil
       end
 
       def parse_meta_tags(body)
-        body.to_s.scan(/<meta\b[^>]*>/im).map do |tag|
-          attributes = {}
-          tag.scan(/([A-Za-z_:.-]+)\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+))/).each do |name, quoted, single, bare|
-            attributes[name.downcase] = CGI.unescapeHTML(quoted || single || bare || "")
+        document = Nokogiri::HTML(body.to_s)
+        document.css("meta").filter_map do |tag|
+          next if tag.ancestors.any? { |ancestor| %w[template script].include?(ancestor.name) }
+
+          tag.attribute_nodes.each_with_object({}) do |attribute, attributes|
+            attributes[attribute.name.downcase] = attribute.value
           end
-          attributes
         end
       end
 

data/lib/fetch_util/regulatory/headers.rb

@@ -27,13 +27,20 @@ module FetchUtil
           name = attributes["name"].to_s.strip
           next if name.empty?
           next if name.casecmp?("tdm-reservation") || name.casecmp?("tdm-policy")
-          next unless name.casecmp?("robots") || name.match?(/bot/i)
+          next unless robot_meta_name?(name)
 
           signals.concat(extract_robot_directive_signals(attributes["content"], path: path, meta_name: name))
         end
 
         signals
       end
+
+      def robot_meta_name?(name)
+        return true if name.casecmp?("robots")
+        return false if name.casecmp?("robot")
+
+        name.match?(/\A[a-z0-9][a-z0-9_.-]*bot(?:[-_.][a-z0-9]+)*\z/i)
+      end
     end
   end
 end