fetch_util 0.3.0 → 0.3.2

data/lib/fetch_util/parallel_fetcher.rb

@@ -38,11 +38,11 @@ module FetchUtil
       work = Array(urls).compact.map(&:to_s).reject(&:empty?)
       return [] if work.empty?
 
-      jobs = Queue.new
-      failures = Queue.new
-      work.each_with_index { |url, index| jobs << [index, url] }
       results = Array.new(work.length)
       worker_count = [@concurrency, work.length].min
+      failures = []
+      next_index = 0
+      mutex = Mutex.new
 
       threads = Array.new(worker_count) do
         Thread.new do
@@ -50,44 +50,39 @@ module FetchUtil
 
           begin
             loop do
-              begin
-                index, url = jobs.pop(true)
-              rescue ThreadError
-                break
+              index = mutex.synchronize do
+                if next_index < work.length
+                  current = next_index
+                  next_index += 1
+                  current
+                end
               end
+              break if index.nil?
+
+              url = work[index]
 
               begin
                 results[index] = fetcher.fetch(url)
               rescue StandardError => e
-                failures << Failure.new(index: index, url: url, error: e)
+                mutex.synchronize { failures << Failure.new(index: index, url: url, error: e) }
               end
             end
           ensure
             fetcher.quit if fetcher.respond_to?(:quit)
           end
         rescue StandardError => e
-          failures << Failure.new(index: nil, url: nil, error: e)
+          mutex.synchronize { failures << Failure.new(index: nil, url: nil, error: e) }
         end
       end
 
       threads.each(&:join)
-      raise_for_failures(drain_queue(failures), results)
+      raise_for_failures(failures, results)
 
       results
     end
 
     private
 
-    def drain_queue(queue)
-      items = []
-      loop do
-        items << queue.pop(true)
-      rescue ThreadError
-        break
-      end
-      items
-    end
-
     def raise_for_failures(failures, results)
       return if failures.empty?
 

data/lib/fetch_util/raw_docs_fallback.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 require "cgi"
-require "net/http"
 require "nokogiri"
 require "uri"
 
+require_relative "regulatory/http_client"
+
 module FetchUtil
   class RawDocsFallback
     DEFAULT_HEADERS = {
@@ -20,6 +21,7 @@ module FetchUtil
       "nav",
       "aside",
       "footer",
+      ".discord",
       ".toc",
       ".sidebar",
       ".breadcrumbs",
@@ -29,17 +31,26 @@ module FetchUtil
       "button"
     ].freeze
     DOCS_ROOT_SELECTORS = [
+      "main .sl-markdown-content",
+      "main [data-pagefind-body]",
+      ".sl-markdown-content",
+      "[data-pagefind-body]",
       "main article",
       "main",
       "article",
       "[role='main']",
+      "[class*='content']",
+      "[id*='content']",
+      "[class*='article']",
+      "[id*='article']",
       ".content",
       ".resource-container"
     ].freeze
     PRUNED_TEXT_PATTERN = /\A(?:on this page|table of contents|edit this page|copy page|copy item path|search|settings|help|expand description)\z/i
 
-    def initialize(timeout: 20)
+    def initialize(timeout: 20, http_client: nil)
       @timeout = timeout.to_i
+      @http_client = http_client || FetchUtil::HttpRedirectClient.new(timeout: @timeout, headers: DEFAULT_HEADERS)
     end
 
     def fetch(url)
@@ -58,7 +69,9 @@ module FetchUtil
       return nil unless root
 
       prune!(root)
-      title = clean_text(fragment_title(document, final_url) || first_heading(root) || meta_title(document) || document.title)
+      title = [fragment_title(document, final_url), first_heading(root), meta_title(document), document.title]
+              .map { |candidate| clean_text(candidate) }
+              .find { |candidate| !candidate.empty? }
       markdown = markdown_from_root(root, title)
       return nil if clean_text(markdown).length < 40
 
@@ -149,23 +162,49 @@ module FetchUtil
     end
 
     def docs_root(document)
+      candidates = []
       DOCS_ROOT_SELECTORS.each do |selector|
-        node = document.at_css(selector)
-        return node.dup if node && clean_text(node.text).length >= 120
+        document.css(selector).each do |node|
+          text = clean_text(node.text)
+          next if text.length < 120
+
+          candidates << [node, text.length + (node.css("p").length * 200)]
+        end
       end
 
+      best = candidates.max_by { |_node, score| score }
+      return best.first.dup if best
+
       body = document.at_css("body")
       body&.dup
     end
 
     def prune!(root)
       DROP_SELECTORS.each { |selector| root.css(selector).remove }
+      prune_leading_promo_cards!(root)
       root.css("*").each do |node|
         text = clean_text(node.text)
         node.remove if text.match?(PRUNED_TEXT_PATTERN)
       end
     end
 
+    def prune_leading_promo_cards!(root)
+      return unless root.at_css("h1, h2")
+
+      root.css("article.card, .card").each do |node|
+        text = clean_text(node.text)
+        next if node.ancestors.any? { |ancestor| class_list(ancestor).any? { |klass| %w[landing-card card--fullwidth].include?(klass) } }
+        next if node.at_css(".title, .body")
+        next if text.empty? || text.length > 600 || node.at_css("h1, h2, h3, h4, h5, h6")
+
+        node.remove
+      end
+    end
+
+    def class_list(node)
+      node["class"].to_s.split
+    end
+
     def meta_title(document)
       meta_value(document, "og:title", attr: "property") || document.title
     end
@@ -235,26 +274,11 @@ module FetchUtil
       %(concat(#{parts.join(%q(, "'", ))}))
     end
 
-    def fetch_html(url, limit: 5)
-      raise URI::InvalidURIError, "too many redirects" if limit <= 0
-
-      uri = URI.parse(url)
-      request = Net::HTTP::Get.new(uri)
-      DEFAULT_HEADERS.each { |key, value| request[key] = value }
-
-      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: @timeout, read_timeout: @timeout) do |http|
-        http.request(request)
-      end
+    def fetch_html(url)
+      response = @http_client.get(url)
+      raise Error, "HTTP #{response.status}" unless response.status.between?(200, 299)
 
-      case response
-      when Net::HTTPSuccess
-        [uri.to_s, response.body]
-      when Net::HTTPRedirection
-        location = URI.join(uri, response["location"]).to_s
-        fetch_html(location, limit: limit - 1)
-      else
-        raise Error, "HTTP #{response.code}"
-      end
+      [response.url, response.body]
     end
   end
 end

data/lib/fetch_util/regulatory/http_client.rb

@@ -1,70 +1,138 @@
 # frozen_string_literal: true
 
 require "net/http"
+require "uri"
 
 module FetchUtil
-  class Regulatory
-    class HttpClient
-      REDIRECT_LIMIT = 5
+  class HttpRedirectClient
+    REDIRECT_LIMIT = 5
+    TRANSIENT_ERRORS = [EOFError, IOError, SocketError, SystemCallError, Timeout::Error].freeze
+    Response = Struct.new(:url, :status, :headers, :body, :redirects, keyword_init: true)
 
-      def initialize(timeout:, user_agent:)
-        @timeout = timeout.to_f
-        @user_agent = user_agent.to_s.strip
-      end
+    def initialize(timeout:, headers: {})
+      @timeout = timeout.to_f
+      @headers = headers.reject { |_key, value| value.to_s.empty? }
+    end
 
-      def get(url, limit: REDIRECT_LIMIT)
-        uri = parse_http_uri(url)
-        fetch(uri, limit, [])
-      end
+    def get(url, limit: REDIRECT_LIMIT)
+      @connections = {}
+      fetch(parse_http_uri(url), limit, [])
+    ensure
+      close_connections
+    end
 
-      private
+    private
 
-      attr_reader :timeout, :user_agent
+    attr_reader :timeout, :headers
 
-      def fetch(uri, limit, redirects)
-        response = request(uri)
-        return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
+    def fetch(uri, limit, redirects)
+      response = request(uri)
+      return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
 
-        raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
+      raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
 
-        location = response["location"].to_s.strip
-        return build_response(uri, response, redirects: redirects) if location.empty?
+      location = response["location"].to_s.strip
+      return build_response(uri, response, redirects: redirects) if location.empty?
 
-        redirect_response = build_response(uri, response)
-        fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
-      end
+      redirect_response = build_response(uri, response)
+      fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
+    end
 
-      def request(uri)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = uri.scheme == "https"
-        http.open_timeout = timeout
-        http.read_timeout = timeout
+    def request(uri)
+      attempts = 0
+      begin
+        http = connection_for(uri)
         request = Net::HTTP::Get.new(uri.request_uri.empty? ? "/" : uri.request_uri)
-        request["Accept"] = "text/html,application/json,text/plain,*/*"
-        request["User-Agent"] = user_agent unless user_agent.empty?
+        headers.each { |key, value| request[key] = value }
         http.request(request)
+      rescue *TRANSIENT_ERRORS
+        close_connection(uri)
+        attempts += 1
+        retry if attempts <= 1
+        raise
       end
+    end
 
-      def build_response(uri, response, redirects: [])
-        FetchUtil::Regulatory::Response.new(
-          url: uri.to_s,
-          status: response.code.to_i,
-          headers: response.to_hash.transform_keys(&:downcase),
-          body: response.body.to_s,
-          redirects: redirects
-        )
+    def connection_for(uri)
+      key = [uri.scheme, uri.host, uri.port]
+      @connections[key] ||= Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: uri.scheme == "https",
+        open_timeout: timeout,
+        read_timeout: timeout
+      )
+    end
+
+    def close_connection(uri)
+      key = [uri.scheme, uri.host, uri.port]
+      @connections.delete(key)&.finish
+    rescue IOError
+      nil
+    end
+
+    def close_connections
+      @connections&.each_value do |http|
+        http.finish if http.started?
+      rescue IOError
+        nil
       end
+      @connections = nil
+    end
+
+    def build_response(uri, response, redirects: [])
+      Response.new(
+        url: uri.to_s,
+        status: response.code.to_i,
+        headers: response.to_hash.transform_keys(&:downcase),
+        body: response.body.to_s,
+        redirects: redirects
+      )
+    end
+
+    def parse_http_uri(url)
+      uri = URI.parse(url.to_s)
+      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && uri.host
+
+      uri
+    end
+  end
 
-      def parse_http_uri(url)
-        uri = URI.parse(url.to_s)
-        unless uri.is_a?(URI::HTTP) && uri.host
-          raise ArgumentError, "unsupported url: #{url}"
-        end
+  class Regulatory
+    class HttpClient
+      DEFAULT_ACCEPT = "text/html,application/json,text/plain,*/*"
+
+      def initialize(timeout:, user_agent:, redirect_client: nil)
+        @user_agent = user_agent.to_s.strip
+        @redirect_client = redirect_client || FetchUtil::HttpRedirectClient.new(timeout: timeout, headers: request_headers)
+      end
 
-        uri
+      def get(url, limit: FetchUtil::HttpRedirectClient::REDIRECT_LIMIT)
+        build_response(redirect_client.get(url, limit: limit))
       rescue URI::InvalidURIError
         raise ArgumentError, "unsupported url: #{url}"
       end
+
+      private
+
+      attr_reader :redirect_client, :user_agent
+
+      def build_response(response)
+        FetchUtil::Regulatory::Response.new(
+          url: response.url,
+          status: response.status,
+          headers: response.headers,
+          body: response.body,
+          redirects: response.redirects.map { |redirect| build_response(redirect) }
+        )
+      end
+
+      def request_headers
+        {
+          "Accept" => DEFAULT_ACCEPT,
+          "User-Agent" => user_agent
+        }
+      end
     end
   end
 end

data/lib/fetch_util/result.rb

@@ -5,11 +5,106 @@ module FetchUtil
     attr_reader :url, :final_url, :title, :byline, :excerpt, :site_name,
                 :published_time, :canonical_url, :language, :html, :markdown,
                 :metadata, :reader_mode, :content_type, :suspect, :warnings,
-                :content_completeness_ratio, :content_format, :paywall_state
+                :content_completeness_ratio, :content_format, :paywall_state,
+                :error_message
+
+    class << self
+      def from_payload(url:, final_url:, payload:, canonical_url:, content_type:, warnings:, suspect:)
+        content_completeness_ratio = payload["contentCompletenessRatio"]&.to_f || 1.0
+        content_format = payload["contentFormat"]
+        paywall_state = payload["paywallState"]
+        metadata = payload_metadata(
+          payload,
+          canonical_url: canonical_url,
+          final_url: final_url,
+          content_type: content_type,
+          suspect: suspect,
+          warnings: warnings,
+          content_completeness_ratio: content_completeness_ratio,
+          content_format: content_format,
+          paywall_state: paywall_state
+        )
+
+        new(
+          url: url,
+          final_url: final_url,
+          title: payload["title"],
+          byline: payload["byline"],
+          excerpt: payload["excerpt"],
+          site_name: payload["siteName"],
+          published_time: payload["publishedTime"],
+          canonical_url: canonical_url,
+          language: payload["language"],
+          html: payload["html"],
+          markdown: payload["markdown"],
+          metadata: metadata,
+          reader_mode: payload["readerMode"],
+          content_type: content_type,
+          suspect: suspect,
+          warnings: warnings,
+          content_completeness_ratio: content_completeness_ratio,
+          content_format: content_format,
+          paywall_state: paywall_state
+        )
+      end
+
+      def error(url:, warning:, message:)
+        metadata = {
+          content_url: url,
+          content_type: "error",
+          suspect: true,
+          warnings: [warning],
+          error_message: message
+        }.freeze
+
+        new(
+          url: url,
+          final_url: url,
+          title: nil,
+          byline: nil,
+          excerpt: nil,
+          site_name: nil,
+          published_time: nil,
+          canonical_url: nil,
+          language: nil,
+          html: nil,
+          markdown: "",
+          metadata: metadata,
+          reader_mode: nil,
+          content_type: "error",
+          suspect: true,
+          warnings: [warning],
+          error_message: message
+        )
+      end
+
+      private
+
+      def payload_metadata(payload, canonical_url:, final_url:, content_type:, suspect:, warnings:,
+                           content_completeness_ratio:, content_format:, paywall_state:)
+        {
+          title: payload["title"],
+          byline: payload["byline"],
+          excerpt: payload["excerpt"],
+          site_name: payload["siteName"],
+          published_time: payload["publishedTime"],
+          canonical_url: canonical_url,
+          language: payload["language"],
+          content_url: final_url,
+          reader_mode: payload["readerMode"],
+          content_type: content_type,
+          suspect: suspect,
+          warnings: warnings,
+          content_completeness_ratio: content_completeness_ratio,
+          content_format: content_format,
+          paywall_state: paywall_state
+        }.freeze
+      end
+    end
 
     def initialize(url:, final_url:, title:, byline:, excerpt:, site_name:, published_time:,
                    canonical_url:, language:, html:, markdown:, metadata:, reader_mode:, content_type:, suspect:, warnings:,
-                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil)
+                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil, error_message: nil)
       @url = url
       @final_url = final_url
       @title = title
@@ -29,6 +124,7 @@ module FetchUtil
       @content_completeness_ratio = content_completeness_ratio
       @content_format = content_format&.freeze
       @paywall_state = paywall_state&.freeze
+      @error_message = error_message
     end
 
     def to_h
@@ -51,7 +147,8 @@ module FetchUtil
         warnings: warnings,
         content_completeness_ratio: content_completeness_ratio,
         content_format: content_format,
-        paywall_state: paywall_state
+        paywall_state: paywall_state,
+        error_message: error_message
       }
     end
   end

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.3.0"
+  VERSION = "0.3.2"
 end

data/lib/fetch_util.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "addressable/uri"
 require "uri"
 
 require_relative "fetch_util/version"
@@ -88,12 +89,21 @@ module FetchUtil
     text.gsub(/\u00A0/, " ").gsub(/\s+/, " ").strip
   end
 
+  def normalize_url(url)
+    return url if url.nil? || url.to_s.empty?
+
+    Addressable::URI.parse(url.to_s).normalize.to_s
+  rescue Addressable::URI::InvalidURIError, URI::InvalidURIError
+    url.to_s
+  end
+
   def strip_www_host(url)
-    URI.parse(url.to_s).host.to_s.downcase.sub(/\Awww\./, "")
+    uri = url.is_a?(URI::Generic) ? url : URI.parse(url.to_s)
+    uri.host.to_s.downcase.sub(/\Awww\./, "")
   end
 
   def docs_like_url?(value)
-    uri = value.is_a?(URI::Generic) ? value : URI.parse(value.to_s.strip)
+    uri = value.is_a?(URI::Generic) ? value : URI.parse(normalize_url(value.to_s.strip))
     return false unless uri.is_a?(URI::HTTP) && uri.host
 
     host = strip_www_host(uri)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.2
 platform: ruby
 authors:
 - hmdne
@@ -82,6 +82,7 @@ files:
 - lib/fetch_util/browser/navigation/navigator_patch.rb
 - lib/fetch_util/browser/site_stabilization.rb
 - lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
+- lib/fetch_util/browser/site_stabilization/gitlab_repo.rb
 - lib/fetch_util/browser/site_stabilization/social_platforms.rb
 - lib/fetch_util/browser/stabilization.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb
@@ -139,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.15
 specification_version: 4
 summary: AI for fetching in Ruby
 test_files: []