fetch_util 0.5.2 → 0.6.0

data/lib/fetch_util/extractor.rb

@@ -20,7 +20,7 @@ module FetchUtil
 
     def initialize(reader_mode: true, asset_root: nil)
       @reader_mode = reader_mode
-      @asset_root = asset_root || File.join(__dir__, "assets")
+      @asset_root = (asset_root || File.join(__dir__, "assets")).dup.freeze
       @extraction_call = nil
     end
 
@@ -62,7 +62,13 @@ module FetchUtil
       inject_assets_inline(page)
       page.evaluate(extraction_call)
     ensure
-      page.timeout = original_timeout if timeout_supported && original_timeout
+      restore_page_timeout(page, original_timeout) if timeout_supported
+    end
+
+    def restore_page_timeout(page, timeout)
+      page.timeout = timeout
+    rescue Ferrum::Error
+      nil
     end
 
     def extraction_call

data/lib/fetch_util/fetcher.rb

@@ -2,6 +2,8 @@
 
 require "uri"
 require "net/http"
+require "timeout"
+require "English"
 
 module FetchUtil
   class Fetcher
@@ -33,6 +35,12 @@ module FetchUtil
       /(?:20\d{2}|\d{4}/\d{2}/\d{2}|article|articles|blog|blogs|column|columns|
       entry|entries|post|posts|news/[\w-]+|wiki|dictionary|definition|definitions|thesaurus|\d{5,}[\w-]*\.html?)\b
     }ix
+    REDIRECT_QUERY_NOISE_PATTERN = /\A(?:_gl|click_?id|dclid|fbclid|gclid|jsessionid|mc_cid|mc_eid|msclkid|phpsessid|ref|session_?id|sid|utm_[a-z0-9_]+|yclid)\z/i
+    MARKDOWN_HTTP_RESOURCE_PATTERN = %r{
+      \]\(\s*(https?://[^\s)]+)|
+      <\s*(https?://[^>\s]+)\s*>|
+      (?:href|src)\s*=\s*["'](https?://[^"']+)["']
+    }ix
     LINKED_MARKDOWN_HEADING_PATTERN = /(?:^|\s)(?:(?:\d+\.|[-*])\s+)?\#{1,4}\s+\[[^\]]{8,220}\]\(/
     LINKED_MARKDOWN_ITEM_PATTERN = /(?:^|\s)(?:\d+\.|[-*])\s+\[[^\]]{8,220}\]\(/
     INDEX_QUERY_PATTERN = /(?:^|[&?])(?:q|query|search|searchtext|keyword|k)=/i
@@ -69,7 +77,6 @@ module FetchUtil
       tag tags topic topics wholesale
     ].freeze
     CONTENT_ROUTE_SEGMENTS = %w[article articles content paper papers preprint preprints].freeze
-    SECOND_LEVEL_COUNTRY_TLDS = /\A(co|com|org|net|gov|edu|ac)\z/
     GOOGLE_HOST_PATTERN = /\Agoogle\.[a-z.]+\z/
     NETWORK_ERROR_PATTERN = Regexp.new(
       "\\b(?:net::ERR_|ERR_NAME_NOT_RESOLVED|pending connections|DNS|resolve|resolution|ENOTFOUND|" \
@@ -80,6 +87,17 @@ module FetchUtil
     PENDING_CONNECTIONS_FETCH_RETRIES = 1
     PENDING_CONNECTIONS_FETCH_RETRY_WAIT = 1.0
 
+    def self.fetch_once(url, **options)
+      fetcher = new(**options)
+      fetcher.fetch(url)
+    ensure
+      begin
+        fetcher&.quit
+      rescue Ferrum::Error
+        nil
+      end
+    end
+
     class PayloadSnapshot
       attr_reader :payload, :requested_url, :final_url, :canonical_url, :raw_final_url, :raw_canonical_url,
                   :markdown, :normalized_markdown, :content_downcase, :context, :context_downcase,
@@ -171,28 +189,30 @@ module FetchUtil
     end
 
     def fetch(url)
+      requested_url = url.dup.freeze
       t0 = monotonic_now
       pending_connection_retries = 0
 
       begin
-        if (pdf_result = direct_pdf_result(url))
-          log_request(url, t0)
+        transport_url = FetchUtil.normalize_url(requested_url)
+        parse_http_uri(transport_url)
+
+        if (pdf_result = direct_pdf_result(transport_url, requested_url: requested_url))
           return pdf_result
         end
 
-        result = @browser.with_page(url) do |page|
+        result = @browser.with_page(transport_url) do |page|
           payload = @extractor.extract(page)
-          if telegram_focal_preview?(url) && payload["contentType"] == "article"
+          if telegram_focal_preview?(transport_url) && payload["contentType"] == "article"
             sleep Browser::PRE_EXTRACTION_SETTLE_WAIT
             payload = @extractor.extract(page)
           end
-          build_result(url, page.current_url, payload)
+          build_result(requested_url, page.current_url, payload)
         end
-        fallback = seznam_cmp_redirect_fallback_candidate?(url, result) ? @raw_docs_fallback.fetch(url) : nil
-        fallback ||= docs_fallback_candidate?(url, result) && poor_docs_result?(result) ? @raw_docs_fallback.fetch(url) : nil
+        fallback = seznam_cmp_redirect_fallback_candidate?(transport_url, result) ? @raw_docs_fallback.fetch(transport_url) : nil
+        fallback ||= docs_fallback_candidate?(transport_url, result) && poor_docs_result?(result) ? @raw_docs_fallback.fetch(transport_url) : nil
         fallback ||= article_body_fallback_candidate?(result) ? @raw_docs_fallback.fetch(result.final_url) : nil
-        result = fallback_result(url, fallback) if fallback
-        log_request(url, t0)
+        result = fallback_result(requested_url, fallback) if fallback
         result
       rescue BrowserError, ExtractionError => e
         if e.is_a?(BrowserError) && pending_connections_error?(e) && pending_connection_retries < PENDING_CONNECTIONS_FETCH_RETRIES
@@ -201,17 +221,22 @@ module FetchUtil
           retry
         end
 
-        fallback = docs_fallback_candidate?(url) ? @raw_docs_fallback.fetch(url) : nil
+        fallback = docs_fallback_candidate?(transport_url) ? @raw_docs_fallback.fetch(transport_url) : nil
         if fallback
-          result = fallback_result(url, fallback)
-          log_request(url, t0)
+          result = fallback_result(requested_url, fallback)
           return result
         end
 
-        log_request(url, t0)
-        return network_error_result(url, e) if e.is_a?(BrowserError) && network_error?(e)
+        return network_error_result(requested_url, e) if e.is_a?(BrowserError) && network_error?(e)
 
         raise e
+      ensure
+        active_exception = $ERROR_INFO
+        begin
+          log_request(requested_url, t0)
+        rescue SystemCallError, IOError
+          raise unless active_exception
+        end
       end
     end
 
@@ -229,7 +254,11 @@ module FetchUtil
       raw_final_url = final_url
       raw_canonical_url = payload["canonicalUrl"]
       final_url = normalized_result_url(final_url)
-      canonical_url = normalized_result_url(raw_canonical_url)
+      canonical_url = normalized_canonical_url(raw_canonical_url)
+      if raw_canonical_url && canonical_url.nil?
+        payload = payload.merge("canonicalUrl" => nil)
+        raw_canonical_url = nil
+      end
       snapshot = PayloadSnapshot.new(
         payload: payload, requested_url: url, final_url: final_url, canonical_url: canonical_url,
         raw_final_url: raw_final_url, raw_canonical_url: raw_canonical_url
@@ -574,7 +603,7 @@ module FetchUtil
     def pdf_document?(requested_url, final_url, payload)
       return true if payload["contentType"].to_s == "pdf"
 
-      [requested_url, final_url, payload["canonicalUrl"]].compact.any? do |url|
+      [requested_url, final_url].compact.any? do |url|
         parsed = URI.parse(url)
         [parsed.path, parsed.query].compact.join("?").match?(PDF_PATH_PATTERN)
       end
@@ -685,7 +714,8 @@ module FetchUtil
       content = snapshot.content_downcase
       matches = tokens.select { |token| content.include?(token) }
 
-      matches.any? { |token| code_like_identifier?(token) } || matches.length >= 2
+      matches.any? { |token| code_like_identifier?(token) } || matches.length >= 2 ||
+        matching_redirect_query_pair?(snapshot)
     rescue URI::InvalidURIError
       false
     end
@@ -697,8 +727,11 @@ module FetchUtil
       return false unless cross_domain_redirect?(requested_url, final_url)
       return false unless scholarly_article_markdown?(final_url, snapshot)
 
+      requested_doi = doi_from_url(requested_url)
+      return false if requested_doi.nil?
+
       dois = snapshot.resource_urls.filter_map { |url| doi_from_url(url) }.uniq
-      dois.length == 1
+      dois == [requested_doi]
     end
 
     def doi_from_url(url)
@@ -723,6 +756,48 @@ module FetchUtil
       []
     end
 
+    def matching_redirect_query_pair?(snapshot)
+      requested_pairs = redirect_query_pairs(snapshot.requested_url)
+      return false if requested_pairs.empty?
+
+      final_pairs = redirect_query_pairs(snapshot.final_url)
+      return true if redirect_query_pairs_preserved?(requested_pairs, final_pairs)
+
+      final_host = FetchUtil.strip_www_host(snapshot.final_url)
+      markdown_http_resource_urls(snapshot.markdown).any? do |url|
+        FetchUtil.strip_www_host(url) == final_host &&
+          redirect_query_pairs_preserved?(requested_pairs, redirect_query_pairs(url))
+      end
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def redirect_query_pairs(url)
+      query = URI.parse(url).query.to_s
+      URI.decode_www_form(query).filter_map do |key, value|
+        next if key.empty? || value.empty? || key.match?(REDIRECT_QUERY_NOISE_PATTERN)
+
+        [key, value]
+      end
+    rescue ArgumentError, URI::InvalidURIError
+      []
+    end
+
+    def redirect_query_pairs_preserved?(requested_pairs, candidate_pairs)
+      remaining = candidate_pairs.dup
+      requested_pairs.all? do |pair|
+        index = remaining.index(pair)
+        next false unless index
+
+        remaining.delete_at(index)
+        true
+      end
+    end
+
+    def markdown_http_resource_urls(markdown)
+      markdown.scan(MARKDOWN_HTTP_RESOURCE_PATTERN).flat_map(&:compact)
+    end
+
     def matching_apex_instrument_redirect?(payload, requested_url, final_url)
       requested_instrument = apex_instrument_id(requested_url)
       final_instrument = apex_instrument_id(final_url)
@@ -803,7 +878,7 @@ module FetchUtil
       Result.error(url: url, warning: warning, message: message)
     end
 
-    def direct_pdf_result(url)
+    def direct_pdf_result(url, requested_url: url)
       pdf_info = if pdf_url?(url)
                    { final_url: url, headers: {} }
                  else
@@ -814,7 +889,7 @@ module FetchUtil
       final_url = normalized_result_url(pdf_info.fetch(:final_url, url))
       headers = pdf_info.fetch(:headers, {})
       Result.from_payload(
-        url: url,
+        url: requested_url,
         final_url: final_url,
         canonical_url: nil,
         payload: {
@@ -831,33 +906,43 @@ module FetchUtil
 
     def pdf_header_info(url)
       info = @pdf_header_probe ? @pdf_header_probe.call(url) : probe_pdf_headers(url)
-      return nil unless pdf_content_type?(info[:headers])
+      return nil unless info && pdf_content_type?(info[:headers])
 
       info
-    rescue URI::InvalidURIError, ArgumentError, FetchUtil::Error, IOError, SocketError, SystemCallError, Timeout::Error
+    rescue URI::InvalidURIError, ArgumentError, FetchUtil::Error, IOError, SocketError, SystemCallError, Timeout::Error,
+           OpenSSL::SSL::SSLError
       nil
     end
 
-    def probe_pdf_headers(url, limit = PDF_REDIRECT_LIMIT)
+    def probe_pdf_headers(url, limit = PDF_REDIRECT_LIMIT, deadline: nil)
+      deadline ||= monotonic_now + @timeout
       uri = parse_http_uri(url)
-      response = request_head(uri)
+      remaining = deadline - monotonic_now
+      return nil unless remaining.positive?
+
+      response = request_head(uri, timeout: remaining)
+      return nil unless deadline > monotonic_now
+
       if response.is_a?(Net::HTTPRedirection) && limit.positive? && response["location"].to_s.strip != ""
-        return probe_pdf_headers(uri.merge(response["location"]).to_s, limit - 1)
+        return probe_pdf_headers(uri.merge(response["location"]).to_s, limit - 1, deadline: deadline)
       end
+      return nil unless response.is_a?(Net::HTTPSuccess)
 
       { final_url: uri.to_s, headers: response.to_hash.transform_keys(&:downcase) }
     end
 
-    def request_head(uri)
-      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: @timeout, read_timeout: @timeout) do |http|
-        request = Net::HTTP::Head.new(uri.request_uri.empty? ? "/" : uri.request_uri)
-        http.request(request)
+    def request_head(uri, timeout: @timeout)
+      Timeout.timeout(timeout) do
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: timeout, read_timeout: timeout) do |http|
+          request = Net::HTTP::Head.new(uri.request_uri.empty? ? "/" : uri.request_uri)
+          http.request(request)
+        end
       end
     end
 
     def parse_http_uri(url)
       uri = URI.parse(url.to_s)
-      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && uri.host
+      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
 
       uri
     end
@@ -883,7 +968,7 @@ module FetchUtil
       filename = value[/filename\*=UTF-8''([^;]+)/i, 1] || value[/filename="?([^";]+)"?/i, 1]
       return nil if filename.to_s.strip.empty?
 
-      URI.decode_www_form_component(filename).strip
+      URI::DEFAULT_PARSER.unescape(filename).strip
     rescue ArgumentError
       filename.to_s.strip
     end
@@ -892,7 +977,7 @@ module FetchUtil
       path = URI.parse(url.to_s).path.to_s
       basename = File.basename(path)
       title = basename.empty? || basename == "/" ? URI.parse(url.to_s).host.to_s : basename
-      URI.decode_www_form_component(title.tr("+", " ")).strip
+      URI::DEFAULT_PARSER.unescape(title).strip
     rescue URI::InvalidURIError, ArgumentError
       url.to_s
     end
@@ -925,7 +1010,24 @@ module FetchUtil
     end
 
     def log_request(url, t0)
-      @request_log&.append(url, duration: monotonic_now - t0)
+      @request_log&.append(request_log_url(url), duration: monotonic_now - t0)
+    end
+
+    def request_log_url(url)
+      value = url.to_s
+      scheme = value.match(/\Ahttps?:/i)
+      return value unless scheme
+
+      separators = value[scheme.end(0)..].to_s.match(%r{\A[\\/]+})
+      return value unless separators
+
+      authority_start = scheme.end(0) + separators[0].length
+      authority_end = value.index(%r{[\\/?#\s]}, authority_start) || value.length
+      authority = value[authority_start...authority_end]
+      userinfo_end = authority.rindex("@")
+      return value unless userinfo_end
+
+      value[0...authority_start] + authority[(userinfo_end + 1)..] + value[authority_end..]
     end
 
     def monotonic_now
@@ -947,15 +1049,8 @@ module FetchUtil
 
     def effective_domain(url)
       host = FetchUtil.strip_www_host(url)
-      parts = host.split(".")
-      return host if parts.length <= 2
-
-      if parts.length >= 3 && parts[-2].match?(SECOND_LEVEL_COUNTRY_TLDS) && parts[-1].length == 2
-        parts.last(3).join(".")
-      else
-        parts.last(2).join(".")
-      end
-    rescue URI::InvalidURIError
+      PublicSuffix.domain(host, default_rule: nil) || host
+    rescue URI::InvalidURIError, PublicSuffix::Error
       nil
     end
 
@@ -990,6 +1085,12 @@ module FetchUtil
       strip_tracking_params(url)
     end
 
+    def normalized_canonical_url(url)
+      parse_http_uri(normalized_result_url(url)).to_s
+    rescue URI::InvalidURIError
+      nil
+    end
+
     def strip_tracking_params(url)
       return url if url.nil? || url.empty?
 

data/lib/fetch_util/http_redirect_client.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+require "timeout"
+require "uri"
+
+module FetchUtil
+  class HttpRedirectClient
+    REDIRECT_LIMIT = 5
+    MAX_RESPONSE_BYTES = 10 * 1024 * 1024
+    TRANSIENT_ERRORS = [EOFError, IOError, SocketError, SystemCallError, Timeout::Error].freeze
+    CLEANUP_ERRORS = (TRANSIENT_ERRORS + [OpenSSL::SSL::SSLError]).freeze
+    Response = Struct.new(:url, :status, :headers, :body, :redirects, keyword_init: true)
+
+    def initialize(timeout:, headers: {}, max_response_bytes: MAX_RESPONSE_BYTES, clock: nil)
+      @timeout = positive_timeout(timeout)
+      owned_headers = {}
+      headers.each do |key, value|
+        next if value.to_s.empty?
+
+        owned_headers[key.dup.freeze] = value.dup.freeze
+      end
+      @headers = owned_headers.freeze
+      @max_response_bytes = positive_max_response_bytes(max_response_bytes)
+      @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+    end
+
+    def get(url, limit: REDIRECT_LIMIT)
+      connections = {}
+      deadline = clock.call + timeout
+      fetch(parse_http_uri(url), limit, [], connections, deadline)
+    ensure
+      close_connections(connections) if connections
+    end
+
+    private
+
+    attr_reader :timeout, :headers, :max_response_bytes, :clock
+
+    def positive_timeout(value)
+      timeout = Float(value)
+      return timeout if timeout.positive? && timeout.finite?
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError
+      raise InputError, "timeout must be positive"
+    end
+
+    def positive_max_response_bytes(value)
+      bytes = Integer(value)
+      return bytes if bytes.positive?
+
+      raise ArgumentError
+    rescue ArgumentError, TypeError, FloatDomainError
+      raise InputError, "max_response_bytes must be positive"
+    end
+
+    def fetch(uri, limit, redirects, connections, deadline)
+      response, body = request(uri, connections, deadline)
+      return build_response(uri, response, body: body, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
+
+      raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
+
+      location = response["location"].to_s.strip
+      return build_response(uri, response, body: body, redirects: redirects) if location.empty?
+
+      redirect_response = build_response(uri, response, body: body)
+      redirect_uri = parse_http_uri(uri.merge(location))
+      fetch(redirect_uri, limit - 1, redirects + [redirect_response], connections, deadline)
+    end
+
+    def request(uri, connections, deadline)
+      attempts = 0
+      begin
+        response, body = within_deadline(deadline) do |remaining|
+          http = connection_for(uri, connections, remaining)
+          request = Net::HTTP::Get.new(uri.request_uri.empty? ? "/" : uri.request_uri)
+          headers.each { |key, value| request[key] = value }
+          body = +""
+          response = http.request(request) do |incoming|
+            incoming.read_body do |chunk|
+              ensure_remaining!(deadline)
+              if body.bytesize + chunk.bytesize > max_response_bytes
+                raise FetchUtil::Error, "response body exceeds #{max_response_bytes} bytes for #{uri}"
+              end
+              body << chunk
+            end
+          end
+          [response, body]
+        end
+      rescue *TRANSIENT_ERRORS
+        close_connection(uri, connections)
+        attempts += 1
+        retry if attempts <= 1
+        raise
+      end
+    end
+
+    def connection_for(uri, connections, remaining)
+      key = [uri.scheme, uri.host, uri.port]
+      connections[key] ||= Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: uri.scheme == "https",
+        open_timeout: remaining,
+        read_timeout: remaining
+      )
+    end
+
+    def within_deadline(deadline)
+      remaining = remaining_timeout(deadline)
+      Timeout.timeout(remaining) { yield remaining }
+    end
+
+    def ensure_remaining!(deadline)
+      remaining_timeout(deadline)
+    end
+
+    def remaining_timeout(deadline)
+      remaining = deadline - clock.call
+      raise Timeout::Error, "execution expired" unless remaining.positive?
+
+      remaining
+    end
+
+    def close_connection(uri, connections)
+      key = [uri.scheme, uri.host, uri.port]
+      connections.delete(key)&.finish
+    rescue *CLEANUP_ERRORS
+      nil
+    end
+
+    def close_connections(connections)
+      connections.each_value do |http|
+        http.finish if http.started?
+      rescue *CLEANUP_ERRORS
+        nil
+      end
+    ensure
+      connections.clear
+    end
+
+    def build_response(uri, response, body:, redirects: [])
+      Response.new(
+        url: uri.to_s,
+        status: response.code.to_i,
+        headers: response.to_hash.transform_keys(&:downcase),
+        body: body,
+        redirects: redirects
+      )
+    end
+
+    def parse_http_uri(url)
+      uri = URI.parse(url.to_s)
+      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && !uri.host.to_s.empty?
+
+      uri
+    end
+  end
+end