fetch_util 0.3.1 → 0.4.0

data/lib/fetch_util/parallel_fetcher.rb

@@ -38,11 +38,11 @@ module FetchUtil
       work = Array(urls).compact.map(&:to_s).reject(&:empty?)
       return [] if work.empty?
 
-      jobs = Queue.new
-      failures = Queue.new
-      work.each_with_index { |url, index| jobs << [index, url] }
       results = Array.new(work.length)
       worker_count = [@concurrency, work.length].min
+      failures = []
+      next_index = 0
+      mutex = Mutex.new
 
       threads = Array.new(worker_count) do
         Thread.new do
@@ -50,44 +50,39 @@ module FetchUtil
 
           begin
             loop do
-              begin
-                index, url = jobs.pop(true)
-              rescue ThreadError
-                break
+              index = mutex.synchronize do
+                if next_index < work.length
+                  current = next_index
+                  next_index += 1
+                  current
+                end
               end
+              break if index.nil?
+
+              url = work[index]
 
               begin
                 results[index] = fetcher.fetch(url)
               rescue StandardError => e
-                failures << Failure.new(index: index, url: url, error: e)
+                mutex.synchronize { failures << Failure.new(index: index, url: url, error: e) }
               end
             end
           ensure
             fetcher.quit if fetcher.respond_to?(:quit)
           end
         rescue StandardError => e
-          failures << Failure.new(index: nil, url: nil, error: e)
+          mutex.synchronize { failures << Failure.new(index: nil, url: nil, error: e) }
         end
       end
 
       threads.each(&:join)
-      raise_for_failures(drain_queue(failures), results)
+      raise_for_failures(failures, results)
 
       results
     end
 
     private
 
-    def drain_queue(queue)
-      items = []
-      loop do
-        items << queue.pop(true)
-      rescue ThreadError
-        break
-      end
-      items
-    end
-
     def raise_for_failures(failures, results)
       return if failures.empty?
 

data/lib/fetch_util/raw_docs_fallback.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 require "cgi"
-require "net/http"
 require "nokogiri"
 require "uri"
 
+require_relative "regulatory/http_client"
+
 module FetchUtil
   class RawDocsFallback
     DEFAULT_HEADERS = {
@@ -47,8 +48,9 @@ module FetchUtil
     ].freeze
     PRUNED_TEXT_PATTERN = /\A(?:on this page|table of contents|edit this page|copy page|copy item path|search|settings|help|expand description)\z/i
 
-    def initialize(timeout: 20)
+    def initialize(timeout: 20, http_client: nil)
       @timeout = timeout.to_i
+      @http_client = http_client || FetchUtil::HttpRedirectClient.new(timeout: @timeout, headers: DEFAULT_HEADERS)
     end
 
     def fetch(url)
@@ -272,26 +274,11 @@ module FetchUtil
       %(concat(#{parts.join(%q(, "'", ))}))
     end
 
-    def fetch_html(url, limit: 5)
-      raise URI::InvalidURIError, "too many redirects" if limit <= 0
-
-      uri = URI.parse(url)
-      request = Net::HTTP::Get.new(uri)
-      DEFAULT_HEADERS.each { |key, value| request[key] = value }
-
-      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: @timeout, read_timeout: @timeout) do |http|
-        http.request(request)
-      end
+    def fetch_html(url)
+      response = @http_client.get(url)
+      raise Error, "HTTP #{response.status}" unless response.status.between?(200, 299)
 
-      case response
-      when Net::HTTPSuccess
-        [uri.to_s, response.body]
-      when Net::HTTPRedirection
-        location = URI.join(uri, response["location"]).to_s
-        fetch_html(location, limit: limit - 1)
-      else
-        raise Error, "HTTP #{response.code}"
-      end
+      [response.url, response.body]
     end
   end
 end

data/lib/fetch_util/regulatory/http_client.rb

@@ -1,70 +1,138 @@
 # frozen_string_literal: true
 
 require "net/http"
+require "uri"
 
 module FetchUtil
-  class Regulatory
-    class HttpClient
-      REDIRECT_LIMIT = 5
+  class HttpRedirectClient
+    REDIRECT_LIMIT = 5
+    TRANSIENT_ERRORS = [EOFError, IOError, SocketError, SystemCallError, Timeout::Error].freeze
+    Response = Struct.new(:url, :status, :headers, :body, :redirects, keyword_init: true)
 
-      def initialize(timeout:, user_agent:)
-        @timeout = timeout.to_f
-        @user_agent = user_agent.to_s.strip
-      end
+    def initialize(timeout:, headers: {})
+      @timeout = timeout.to_f
+      @headers = headers.reject { |_key, value| value.to_s.empty? }
+    end
 
-      def get(url, limit: REDIRECT_LIMIT)
-        uri = parse_http_uri(url)
-        fetch(uri, limit, [])
-      end
+    def get(url, limit: REDIRECT_LIMIT)
+      @connections = {}
+      fetch(parse_http_uri(url), limit, [])
+    ensure
+      close_connections
+    end
 
-      private
+    private
 
-      attr_reader :timeout, :user_agent
+    attr_reader :timeout, :headers
 
-      def fetch(uri, limit, redirects)
-        response = request(uri)
-        return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
+    def fetch(uri, limit, redirects)
+      response = request(uri)
+      return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
 
-        raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
+      raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
 
-        location = response["location"].to_s.strip
-        return build_response(uri, response, redirects: redirects) if location.empty?
+      location = response["location"].to_s.strip
+      return build_response(uri, response, redirects: redirects) if location.empty?
 
-        redirect_response = build_response(uri, response)
-        fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
-      end
+      redirect_response = build_response(uri, response)
+      fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
+    end
 
-      def request(uri)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = uri.scheme == "https"
-        http.open_timeout = timeout
-        http.read_timeout = timeout
+    def request(uri)
+      attempts = 0
+      begin
+        http = connection_for(uri)
         request = Net::HTTP::Get.new(uri.request_uri.empty? ? "/" : uri.request_uri)
-        request["Accept"] = "text/html,application/json,text/plain,*/*"
-        request["User-Agent"] = user_agent unless user_agent.empty?
+        headers.each { |key, value| request[key] = value }
         http.request(request)
+      rescue *TRANSIENT_ERRORS
+        close_connection(uri)
+        attempts += 1
+        retry if attempts <= 1
+        raise
       end
+    end
 
-      def build_response(uri, response, redirects: [])
-        FetchUtil::Regulatory::Response.new(
-          url: uri.to_s,
-          status: response.code.to_i,
-          headers: response.to_hash.transform_keys(&:downcase),
-          body: response.body.to_s,
-          redirects: redirects
-        )
+    def connection_for(uri)
+      key = [uri.scheme, uri.host, uri.port]
+      @connections[key] ||= Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: uri.scheme == "https",
+        open_timeout: timeout,
+        read_timeout: timeout
+      )
+    end
+
+    def close_connection(uri)
+      key = [uri.scheme, uri.host, uri.port]
+      @connections.delete(key)&.finish
+    rescue IOError
+      nil
+    end
+
+    def close_connections
+      @connections&.each_value do |http|
+        http.finish if http.started?
+      rescue IOError
+        nil
       end
+      @connections = nil
+    end
+
+    def build_response(uri, response, redirects: [])
+      Response.new(
+        url: uri.to_s,
+        status: response.code.to_i,
+        headers: response.to_hash.transform_keys(&:downcase),
+        body: response.body.to_s,
+        redirects: redirects
+      )
+    end
+
+    def parse_http_uri(url)
+      uri = URI.parse(url.to_s)
+      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && uri.host
+
+      uri
+    end
+  end
 
-      def parse_http_uri(url)
-        uri = URI.parse(url.to_s)
-        unless uri.is_a?(URI::HTTP) && uri.host
-          raise ArgumentError, "unsupported url: #{url}"
-        end
+  class Regulatory
+    class HttpClient
+      DEFAULT_ACCEPT = "text/html,application/json,text/plain,*/*"
+
+      def initialize(timeout:, user_agent:, redirect_client: nil)
+        @user_agent = user_agent.to_s.strip
+        @redirect_client = redirect_client || FetchUtil::HttpRedirectClient.new(timeout: timeout, headers: request_headers)
+      end
 
-        uri
+      def get(url, limit: FetchUtil::HttpRedirectClient::REDIRECT_LIMIT)
+        build_response(redirect_client.get(url, limit: limit))
       rescue URI::InvalidURIError
         raise ArgumentError, "unsupported url: #{url}"
       end
+
+      private
+
+      attr_reader :redirect_client, :user_agent
+
+      def build_response(response)
+        FetchUtil::Regulatory::Response.new(
+          url: response.url,
+          status: response.status,
+          headers: response.headers,
+          body: response.body,
+          redirects: response.redirects.map { |redirect| build_response(redirect) }
+        )
+      end
+
+      def request_headers
+        {
+          "Accept" => DEFAULT_ACCEPT,
+          "User-Agent" => user_agent
+        }
+      end
     end
   end
 end

data/lib/fetch_util/result.rb

@@ -3,10 +3,13 @@
 module FetchUtil
   class Result
     attr_reader :url, :final_url, :title, :byline, :excerpt, :site_name,
-                :published_time, :canonical_url, :language, :html, :markdown,
+                :published_time, :canonical_url, :language, :name, :company, :location,
+                :description, :ingredients, :instructions, :bedrooms, :bathrooms,
+                :area_sqft, :html, :markdown,
                 :metadata, :reader_mode, :content_type, :suspect, :warnings,
                 :content_completeness_ratio, :content_format, :paywall_state,
-                :error_message
+                :price, :rating, :address, :social_kind, :platform, :handle,
+                :reply_count, :community, :score, :error_message
 
     class << self
       def from_payload(url:, final_url:, payload:, canonical_url:, content_type:, warnings:, suspect:)
@@ -22,7 +25,25 @@ module FetchUtil
           warnings: warnings,
           content_completeness_ratio: content_completeness_ratio,
           content_format: content_format,
-          paywall_state: paywall_state
+          paywall_state: paywall_state,
+          name: payload["name"],
+          company: payload["company"],
+          location: payload["location"],
+          description: payload["description"],
+          ingredients: payload["ingredients"],
+          instructions: payload["instructions"],
+          bedrooms: payload["bedrooms"],
+          bathrooms: payload["bathrooms"],
+          area_sqft: payload["areaSqft"],
+          price: payload["price"],
+          rating: payload["rating"],
+          address: payload["address"],
+          social_kind: payload["socialKind"],
+          platform: payload["platform"],
+          handle: payload["handle"],
+          reply_count: payload["replyCount"],
+          community: payload["community"],
+          score: payload["score"]
         )
 
         new(
@@ -35,6 +56,15 @@ module FetchUtil
           published_time: payload["publishedTime"],
           canonical_url: canonical_url,
           language: payload["language"],
+          name: payload["name"],
+          company: payload["company"],
+          location: payload["location"],
+          description: payload["description"],
+          ingredients: payload["ingredients"],
+          instructions: payload["instructions"],
+          bedrooms: payload["bedrooms"],
+          bathrooms: payload["bathrooms"],
+          area_sqft: payload["areaSqft"],
           html: payload["html"],
           markdown: payload["markdown"],
           metadata: metadata,
@@ -44,7 +74,16 @@ module FetchUtil
           warnings: warnings,
           content_completeness_ratio: content_completeness_ratio,
           content_format: content_format,
-          paywall_state: paywall_state
+          paywall_state: paywall_state,
+          price: payload["price"],
+          rating: payload["rating"],
+          address: payload["address"],
+          social_kind: payload["socialKind"],
+          platform: payload["platform"],
+          handle: payload["handle"],
+          reply_count: payload["replyCount"],
+          community: payload["community"],
+          score: payload["score"]
         )
       end
 
@@ -54,7 +93,13 @@ module FetchUtil
           content_type: "error",
           suspect: true,
           warnings: [warning],
-          error_message: message
+          error_message: message,
+          social_kind: nil,
+          platform: nil,
+          handle: nil,
+          reply_count: nil,
+          community: nil,
+          score: nil
         }.freeze
 
         new(
@@ -67,6 +112,15 @@ module FetchUtil
           published_time: nil,
           canonical_url: nil,
           language: nil,
+          name: nil,
+          company: nil,
+          location: nil,
+          description: nil,
+          ingredients: nil,
+          instructions: nil,
+          bedrooms: nil,
+          bathrooms: nil,
+          area_sqft: nil,
           html: nil,
           markdown: "",
           metadata: metadata,
@@ -81,7 +135,10 @@ module FetchUtil
       private
 
       def payload_metadata(payload, canonical_url:, final_url:, content_type:, suspect:, warnings:,
-                           content_completeness_ratio:, content_format:, paywall_state:)
+                           content_completeness_ratio:, content_format:, paywall_state:,
+                           name:, company:, location:, description:, ingredients:, instructions:,
+                           bedrooms:, bathrooms:, area_sqft:, price:, rating:, address:,
+                           social_kind:, platform:, handle:, reply_count:, community:, score:)
         {
           title: payload["title"],
           byline: payload["byline"],
@@ -90,6 +147,15 @@ module FetchUtil
           published_time: payload["publishedTime"],
           canonical_url: canonical_url,
           language: payload["language"],
+          name: name,
+          company: company,
+          location: location,
+          description: description,
+          ingredients: ingredients,
+          instructions: instructions,
+          bedrooms: bedrooms,
+          bathrooms: bathrooms,
+          area_sqft: area_sqft,
           content_url: final_url,
           reader_mode: payload["readerMode"],
           content_type: content_type,
@@ -97,14 +163,27 @@ module FetchUtil
           warnings: warnings,
           content_completeness_ratio: content_completeness_ratio,
           content_format: content_format,
-          paywall_state: paywall_state
+          paywall_state: paywall_state,
+          price: price,
+          rating: rating,
+          address: address,
+          social_kind: social_kind,
+          platform: platform,
+          handle: handle,
+          reply_count: reply_count,
+          community: community,
+          score: score
         }.freeze
       end
     end
 
     def initialize(url:, final_url:, title:, byline:, excerpt:, site_name:, published_time:,
-                   canonical_url:, language:, html:, markdown:, metadata:, reader_mode:, content_type:, suspect:, warnings:,
-                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil, error_message: nil)
+                   canonical_url:, language:, name:, company:, location:, description:, ingredients:,
+                   instructions:, bedrooms:, bathrooms:, area_sqft:, html:, markdown:, metadata:,
+                   reader_mode:, content_type:, suspect:, warnings:,
+                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil,
+                   price: nil, rating: nil, address: nil, social_kind: nil, platform: nil,
+                   handle: nil, reply_count: nil, community: nil, score: nil, error_message: nil)
       @url = url
       @final_url = final_url
       @title = title
@@ -114,6 +193,15 @@ module FetchUtil
       @published_time = published_time
       @canonical_url = canonical_url
       @language = language
+      @name = name
+      @company = company
+      @location = location
+      @description = description
+      @ingredients = ingredients
+      @instructions = instructions
+      @bedrooms = bedrooms
+      @bathrooms = bathrooms
+      @area_sqft = area_sqft
       @html = html
       @markdown = markdown
       @metadata = metadata.freeze
@@ -124,6 +212,15 @@ module FetchUtil
       @content_completeness_ratio = content_completeness_ratio
       @content_format = content_format&.freeze
       @paywall_state = paywall_state&.freeze
+      @price = price
+      @rating = rating
+      @address = address
+      @social_kind = social_kind
+      @platform = platform
+      @handle = handle
+      @reply_count = reply_count
+      @community = community
+      @score = score
       @error_message = error_message
     end
 
@@ -138,6 +235,15 @@ module FetchUtil
         published_time: published_time,
         canonical_url: canonical_url,
         language: language,
+        name: name,
+        company: company,
+        location: location,
+        description: description,
+        ingredients: ingredients,
+        instructions: instructions,
+        bedrooms: bedrooms,
+        bathrooms: bathrooms,
+        area_sqft: area_sqft,
         html: html,
         markdown: markdown,
         metadata: metadata,
@@ -148,6 +254,15 @@ module FetchUtil
         content_completeness_ratio: content_completeness_ratio,
         content_format: content_format,
         paywall_state: paywall_state,
+        price: price,
+        rating: rating,
+        address: address,
+        social_kind: social_kind,
+        platform: platform,
+        handle: handle,
+        reply_count: reply_count,
+        community: community,
+        score: score,
         error_message: error_message
       }
     end

data/lib/fetch_util/searcher.rb

@@ -5,8 +5,6 @@ require "uri"
 
 module FetchUtil
   class Searcher
-    MAX_SNIPPET_LENGTH = 180
-
     SOURCES = {
       "duckduckgo" => "https://duckduckgo.com/?q=%<query>s&ia=web&kl=us-en",
       "google" => "https://www.google.com/search?hl=en&q=%<query>s",
@@ -21,10 +19,10 @@ module FetchUtil
     include ResultFiltering
     private_constant :ResultFiltering
 
-    def initialize(fetcher: nil, request_log: RequestLog.new, sources: nil, limit: 10, concurrency: 2, verbose: false, **fetch_options)
+    def initialize(fetcher: nil, request_log: RequestLog.new, sources: nil, limit: nil, concurrency: 2, verbose: false, **fetch_options)
       @request_log = request_log
       @sources = Array(sources || DEFAULT_SOURCES).map(&:to_s)
-      @limit = limit.to_i
+      @limit = limit
       @verbose = verbose
       @fetcher = fetcher || ParallelFetcher.new(concurrency: concurrency, request_log: request_log, **fetch_options)
     end
@@ -45,7 +43,7 @@ module FetchUtil
 
       {
         query: encoded_query,
-        results: formatted_results(aggregate(urls.keys, fetched).first(limit))
+        results: formatted_results(apply_limit(aggregate(urls.keys, fetched)))
       }
     end
 
@@ -53,6 +51,10 @@ module FetchUtil
 
     attr_reader :limit
 
+    def apply_limit(results)
+      limit.nil? ? results : results.first(limit)
+    end
+
     def search_urls(query)
       urls = {}
 
@@ -131,7 +133,7 @@ module FetchUtil
           url: result[:url]
         }
         item[:snippet] = result[:snippet] if result[:snippet]
-        if verbose?
+        if @verbose
           item[:sources] = result[:sources]
           item[:ranks] = result[:ranks]
         end
@@ -204,10 +206,6 @@ module FetchUtil
       item
     end
 
-    def verbose?
-      @verbose
-    end
-
     def compact_text(value)
       FetchUtil.normalize_whitespace(value)
     end
@@ -296,13 +294,7 @@ module FetchUtil
       return nil if metadata_only_snippet?(text)
       return nil if jammed_navigation_text?(text)
 
-      truncate(text, MAX_SNIPPET_LENGTH)
-    end
-
-    def truncate(text, max_length)
-      return text if text.length <= max_length
-
-      "#{text[0, max_length - 3].rstrip}..."
+      text
     end
 
     def domain_only?(text, host)

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.3.1"
+  VERSION = "0.4.0"
 end

data/lib/fetch_util.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "addressable/uri"
 require "uri"
 
 require_relative "fetch_util/version"
@@ -88,12 +89,21 @@ module FetchUtil
     text.gsub(/\u00A0/, " ").gsub(/\s+/, " ").strip
   end
 
+  def normalize_url(url)
+    return url if url.nil? || url.to_s.empty?
+
+    Addressable::URI.parse(url.to_s).normalize.to_s
+  rescue Addressable::URI::InvalidURIError, URI::InvalidURIError
+    url.to_s
+  end
+
   def strip_www_host(url)
-    URI.parse(url.to_s).host.to_s.downcase.sub(/\Awww\./, "")
+    uri = url.is_a?(URI::Generic) ? url : URI.parse(url.to_s)
+    uri.host.to_s.downcase.sub(/\Awww\./, "")
   end
 
   def docs_like_url?(value)
-    uri = value.is_a?(URI::Generic) ? value : URI.parse(value.to_s.strip)
+    uri = value.is_a?(URI::Generic) ? value : URI.parse(normalize_url(value.to_s.strip))
     return false unless uri.is_a?(URI::HTTP) && uri.host
 
     host = strip_www_host(uri)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - hmdne
@@ -84,6 +84,7 @@ files:
 - lib/fetch_util/browser/site_stabilization/community_and_marketplace.rb
 - lib/fetch_util/browser/site_stabilization/gitlab_repo.rb
 - lib/fetch_util/browser/site_stabilization/social_platforms.rb
+- lib/fetch_util/browser/site_stabilization/travel_and_lodging.rb
 - lib/fetch_util/browser/stabilization.rb
 - lib/fetch_util/browser/stabilization/page_flow.rb
 - lib/fetch_util/browser/stabilization/spa_hydration.rb
@@ -140,7 +141,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.15
 specification_version: 4
 summary: AI for fetching in Ruby
 test_files: []