fetch_util 0.5.2 → 0.6.0

data/lib/fetch_util/regulatory/http_client.rb

@@ -1,103 +1,8 @@
 # frozen_string_literal: true
 
-require "net/http"
-require "uri"
+require_relative "../http_redirect_client"
 
 module FetchUtil
-  class HttpRedirectClient
-    REDIRECT_LIMIT = 5
-    TRANSIENT_ERRORS = [EOFError, IOError, SocketError, SystemCallError, Timeout::Error].freeze
-    Response = Struct.new(:url, :status, :headers, :body, :redirects, keyword_init: true)
-
-    def initialize(timeout:, headers: {})
-      @timeout = timeout.to_f
-      @headers = headers.reject { |_key, value| value.to_s.empty? }
-    end
-
-    def get(url, limit: REDIRECT_LIMIT)
-      @connections = {}
-      fetch(parse_http_uri(url), limit, [])
-    ensure
-      close_connections
-    end
-
-    private
-
-    attr_reader :timeout, :headers
-
-    def fetch(uri, limit, redirects)
-      response = request(uri)
-      return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
-
-      raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
-
-      location = response["location"].to_s.strip
-      return build_response(uri, response, redirects: redirects) if location.empty?
-
-      redirect_response = build_response(uri, response)
-      fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
-    end
-
-    def request(uri)
-      attempts = 0
-      begin
-        http = connection_for(uri)
-        request = Net::HTTP::Get.new(uri.request_uri.empty? ? "/" : uri.request_uri)
-        headers.each { |key, value| request[key] = value }
-        http.request(request)
-      rescue *TRANSIENT_ERRORS
-        close_connection(uri)
-        attempts += 1
-        retry if attempts <= 1
-        raise
-      end
-    end
-
-    def connection_for(uri)
-      key = [uri.scheme, uri.host, uri.port]
-      @connections[key] ||= Net::HTTP.start(
-        uri.host,
-        uri.port,
-        use_ssl: uri.scheme == "https",
-        open_timeout: timeout,
-        read_timeout: timeout
-      )
-    end
-
-    def close_connection(uri)
-      key = [uri.scheme, uri.host, uri.port]
-      @connections.delete(key)&.finish
-    rescue IOError
-      nil
-    end
-
-    def close_connections
-      @connections&.each_value do |http|
-        http.finish if http.started?
-      rescue IOError
-        nil
-      end
-      @connections = nil
-    end
-
-    def build_response(uri, response, redirects: [])
-      Response.new(
-        url: uri.to_s,
-        status: response.code.to_i,
-        headers: response.to_hash.transform_keys(&:downcase),
-        body: response.body.to_s,
-        redirects: redirects
-      )
-    end
-
-    def parse_http_uri(url)
-      uri = URI.parse(url.to_s)
-      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && uri.host
-
-      uri
-    end
-  end
-
   class Regulatory
     class HttpClient
       DEFAULT_ACCEPT = "text/html,application/json,text/plain,*/*"
@@ -110,7 +15,7 @@ module FetchUtil
       def get(url, limit: FetchUtil::HttpRedirectClient::REDIRECT_LIMIT)
         build_response(redirect_client.get(url, limit: limit))
       rescue URI::InvalidURIError
-        raise ArgumentError, "unsupported url: #{url}"
+        raise InputError, "unsupported url: #{url}"
       end
 
       private

data/lib/fetch_util/regulatory/orchestration.rb

@@ -5,8 +5,8 @@ module FetchUtil
     module Orchestration
       def initialize(client: nil, cache_path: DEFAULT_CACHE_PATH, sources: nil, timeout: 20, user_agent: nil)
         @client = client || HttpClient.new(timeout: timeout, user_agent: user_agent || default_user_agent)
-        @cache_path = cache_path || DEFAULT_CACHE_PATH
-        @source_tokens = sources
+        @cache_path = (cache_path || DEFAULT_CACHE_PATH).dup.freeze
+        @source_tokens = normalized_source_selection(sources)
       end
 
       def call(url)
@@ -14,7 +14,7 @@ module FetchUtil
         origin_query = origin_query?(requested_uri)
         query_target = request_target(requested_uri)
         effective_query_target = query_target
-        selected_sources = resolve_sources(@source_tokens)
+        selected_sources = source_selection
         result = {}
         policy_refs = []
 
@@ -33,7 +33,7 @@ module FetchUtil
 
         if needs_robots_fetch?(selected_sources)
           record = robots_record(requested_uri)
-          %w[robotstxt contentsignal contentusagerobots].each do |source|
+          ROBOTS_RECORD_SOURCES.each do |source|
             next unless selected_sources.include?(source)
 
             add_source_payload(
@@ -47,7 +47,7 @@ module FetchUtil
         if needs_page_fetch?(selected_sources)
           record = page_record(requested_uri)
           effective_query_target = page_query_target(record, fallback: query_target)
-          %w[xrobotstag metarobots tdmheaders tdmmeta contentusageheader human].each do |source|
+          PAGE_RECORD_SOURCES.each do |source|
             next unless selected_sources.include?(source)
 
             add_source_payload(
@@ -63,7 +63,11 @@ module FetchUtil
           add_source_payload(
             result,
             "tdmpolicy",
-            scoped_signals(expanded_tdm_policy_signals(policy_refs), origin_query: origin_query, query_target: effective_query_target)
+            scoped_signals(
+              expanded_tdm_policy_signals(policy_refs, target_origin: requested_uri),
+              origin_query: origin_query,
+              query_target: effective_query_target
+            )
           )
         end
 

data/lib/fetch_util/regulatory/page.rb

@@ -13,13 +13,17 @@ module FetchUtil
           header_policies = []
 
           response_chain(response).each do |step|
-            path = request_target(parse_http_uri(step.url))
+            step_uri = parse_http_uri(step.url)
+            next unless origin_key(step_uri).casecmp?(origin_key(final_uri))
+
+            path = request_target(step_uri)
             xrobotstag.concat(extract_x_robot_signals(step.headers, path: path))
             contentusageheader.concat(extract_content_usage_header_signals(step.headers, path: path))
             step_tdmheaders, step_policies = extract_tdm_value_signals(
               reservation: first_header_value(step.headers, "tdm-reservation"),
               policy_url: first_header_value(step.headers, "tdm-policy"),
-              path: path
+              path: path,
+              target_origin: step_uri
             )
             tdmheaders.concat(step_tdmheaders)
             header_policies.concat(step_policies)
@@ -32,7 +36,7 @@ module FetchUtil
           if html_content?(response.headers, response.body)
             meta_tags = parse_meta_tags(response.body)
             metarobots = sort_generic_signals(extract_meta_robot_signals(meta_tags, path: final_path))
-            tdmmeta, meta_policies = extract_tdm_meta_signals(meta_tags, path: final_path)
+            tdmmeta, meta_policies = extract_tdm_meta_signals(meta_tags, path: final_path, target_origin: final_uri)
             human = sort_generic_signals(extract_human_signals(response.body, path: final_path))
           end
 
@@ -54,14 +58,7 @@ module FetchUtil
       def empty_page_record
         {
           "final_url" => nil,
-          "signals" => {
-            "xrobotstag" => [],
-            "metarobots" => [],
-            "tdmheaders" => [],
-            "tdmmeta" => [],
-            "contentusageheader" => [],
-            "human" => []
-          },
+          "signals" => PAGE_RECORD_SOURCES.to_h { |source| [source, []] },
           "policies" => []
         }
       end

data/lib/fetch_util/regulatory/robots.rb

@@ -37,11 +37,7 @@ module FetchUtil
       end
 
       def extract_robots_source_signals(body)
-        signals = {
-          "robotstxt" => [],
-          "contentsignal" => [],
-          "contentusagerobots" => []
-        }
+        signals = ROBOTS_RECORD_SOURCES.to_h { |source| [source, []] }
         user_agents = []
         in_rules = false
 
@@ -91,11 +87,7 @@ module FetchUtil
 
       def empty_robots_record
         {
-          "signals" => {
-            "robotstxt" => [],
-            "contentsignal" => [],
-            "contentusagerobots" => []
-          }
+          "signals" => ROBOTS_RECORD_SOURCES.to_h { |source| [source, []] }
         }
       end
 

data/lib/fetch_util/regulatory/source_selection.rb

@@ -32,14 +32,24 @@ module FetchUtil
         selected
       end
 
+      def source_selection
+        @source_tokens ? @source_tokens.dup : resolve_sources(nil)
+      end
+
+      def normalized_source_selection(sources)
+        return if sources.nil?
+
+        resolve_sources(sources).map { |source| source.dup.freeze }.freeze
+      end
+
       def validate_source!(source)
         return if all_sources.include?(source)
 
-        raise ArgumentError, "unsupported regulatory source: #{source}"
+        raise InputError, "unsupported regulatory source: #{source}"
       end
 
       def needs_page_fetch?(selected_sources)
-        (selected_sources & (HUMAN_SOURCES + %w[xrobotstag metarobots tdmheaders tdmmeta contentusageheader tdmpolicy])).any?
+        (selected_sources & PAGE_RECORD_SOURCES).any? || selected_sources.include?("tdmpolicy")
       end
 
       def needs_tdmrep_fetch?(selected_sources)
@@ -47,7 +57,7 @@ module FetchUtil
       end
 
       def needs_robots_fetch?(selected_sources)
-        (selected_sources & %w[robotstxt contentsignal contentusagerobots]).any?
+        (selected_sources & ROBOTS_RECORD_SOURCES).any?
       end
 
       def add_source_payload(result, source, signals)

data/lib/fetch_util/regulatory/tdm_page.rb

@@ -3,7 +3,7 @@
 module FetchUtil
   class Regulatory
     module TdmPage
-      def extract_tdm_meta_signals(meta_tags, path:)
+      def extract_tdm_meta_signals(meta_tags, path:, target_origin:)
         reservation = nil
         policy_url = nil
 
@@ -13,10 +13,15 @@ module FetchUtil
           policy_url ||= attributes["content"] if name == "tdm-policy"
         end
 
-        extract_tdm_value_signals(reservation: reservation, policy_url: policy_url, path: path)
+        extract_tdm_value_signals(
+          reservation: reservation,
+          policy_url: policy_url,
+          path: path,
+          target_origin: target_origin
+        )
       end
 
-      def extract_tdm_value_signals(reservation:, policy_url:, path:)
+      def extract_tdm_value_signals(reservation:, policy_url:, path:, target_origin:)
         value = reservation.to_s.strip
         return [[], []] unless %w[0 1].include?(value)
 
@@ -31,7 +36,7 @@ module FetchUtil
             conditions: conditions
           )
         ]
-        policies = value == "1" ? [policy_ref(policy, path)] : []
+        policies = value == "1" ? [policy_ref(policy, path, target_origin: target_origin)] : []
         [signals, dedupe_policy_refs(policies)]
       end
     end

data/lib/fetch_util/regulatory/tdm_policy.rb

@@ -3,9 +3,11 @@
 module FetchUtil
   class Regulatory
     module TdmPolicy
-      def expanded_tdm_policy_signals(policy_refs)
+      def expanded_tdm_policy_signals(policy_refs, target_origin:)
         dedupe_policy_refs(policy_refs).flat_map do |policy_ref|
-          record = tdm_policy_record(policy_ref["url"])
+          policy_origin = policy_ref["target_origin"]
+          policy_origin = policy_origin ? parse_http_uri(policy_origin) : target_origin
+          record = tdm_policy_record(policy_ref["url"], target_origin: policy_origin)
           Array(record["signals"]).map do |template|
             signal = deep_copy(template)
             signal["path"] ||= policy_ref["path"]
@@ -17,25 +19,31 @@ module FetchUtil
         end
       end
 
-      def tdm_policy_record(url)
-        cache_fetch("tdmpolicy:#{url}") do
-          response = safe_get(url)
+      def tdm_policy_record(url, target_origin:)
+        cache_fetch("tdmpolicy:#{origin_key(target_origin)}:#{url}") do
+          response, cacheable = safe_get(url)
           signals = []
           if response&.status&.between?(200, 299) && json_like_response?(response.headers, response.body)
-            signals = extract_tdm_policy_signals(response.body)
+            signals = extract_tdm_policy_signals(response.body, target_origin: target_origin)
           end
-          { "signals" => sort_specificity_signals(signals) }
+          [{ "signals" => sort_specificity_signals(signals) }, cacheable]
         end
       end
 
-      def extract_tdm_policy_signals(body)
+      def extract_tdm_policy_signals(body, target_origin: nil)
         payload = JSON.parse(body.to_s)
+        return [] unless payload.is_a?(Hash)
+
         permissions = array_value(payload, "permission")
 
         permissions.filter_map do |permission|
           next unless permission.is_a?(Hash)
           next unless tdm_policy_action?(permission["action"])
 
+          target = permission["target"]
+          target_path = policy_target_path(target, target_origin: target_origin)
+          next if !target.to_s.strip.empty? && target_path.nil?
+
           conditions = {}
           duties = array_value(permission, "duty").filter_map { |item| duty_name(item) }
           conditions["duty"] = duties if duties.any?
@@ -43,7 +51,6 @@ module FetchUtil
           conditions["purpose"] = purpose if purpose
 
           signal = build_signal("allow", "text-and-data-mining", conditions: conditions)
-          target_path = policy_target_path(permission["target"])
           signal["path"] = target_path if target_path
           signal
         end

data/lib/fetch_util/regulatory/tdm_rep.rb

@@ -9,7 +9,7 @@ module FetchUtil
           tdmrep_uri(requested_uri),
           fallback: { "signals" => [], "policies" => [] }
         ) do |body|
-          signals, policies = extract_tdmrep_signals(body)
+          signals, policies = extract_tdmrep_signals(body, target_origin: requested_uri)
           {
             "signals" => sort_specificity_signals(signals),
             "policies" => policies
@@ -17,7 +17,7 @@ module FetchUtil
         end
       end
 
-      def extract_tdmrep_signals(body)
+      def extract_tdmrep_signals(body, target_origin:)
         payload = JSON.parse(body.to_s)
         return [[], []] unless payload.is_a?(Array)
 
@@ -35,7 +35,8 @@ module FetchUtil
           rule_signals, rule_policies = extract_tdm_value_signals(
             reservation: reservation,
             policy_url: policy_url,
-            path: normalize_output_path(location)
+            path: normalize_output_path(location),
+            target_origin: target_origin
           )
           signals.concat(rule_signals)
           policies.concat(rule_policies)

data/lib/fetch_util/regulatory/tdm_support.rb

@@ -38,12 +38,16 @@ module FetchUtil
         nil
       end
 
-      def policy_target_path(value)
+      def policy_target_path(value, target_origin:)
         target = value.to_s.strip
         return nil if target.empty?
 
         uri = URI.parse(target)
-        return normalize_output_path(request_target(uri)) if uri.is_a?(URI::HTTP)
+        if uri.is_a?(URI::HTTP)
+          return nil unless target_origin && origin_key(uri).casecmp?(origin_key(target_origin))
+
+          return normalize_output_path(request_target(uri))
+        end
         return normalize_output_path(target) if target.start_with?("/")
 
         nil
@@ -63,12 +67,12 @@ module FetchUtil
         [value]
       end
 
-      def policy_ref(url, path)
+      def policy_ref(url, path, target_origin:)
         candidate = url.to_s.strip
         return nil if candidate.empty?
 
         parse_http_uri(candidate)
-        { "url" => candidate, "path" => path }
+        { "url" => candidate, "path" => path, "target_origin" => origin_key(target_origin) }
       rescue ArgumentError
         nil
       end
@@ -77,7 +81,7 @@ module FetchUtil
         seen = {}
         list = []
         Array(policy_refs).compact.each do |policy_ref|
-          key = [policy_ref["url"], policy_ref["path"]]
+          key = [policy_ref["url"], policy_ref["path"], policy_ref["target_origin"]]
           next if seen[key]
 
           seen[key] = true
@@ -87,7 +91,7 @@ module FetchUtil
       end
 
       def odrl_token(value)
-        value.to_s.strip.downcase.split(/[#:]/).last.to_s
+        value.to_s.strip.downcase.split(%r{[/#:]}).reject(&:empty?).last.to_s
       end
     end
   end

data/lib/fetch_util/regulatory.rb

@@ -5,6 +5,7 @@ require "digest"
 require "fileutils"
 require "json"
 require "openssl"
+require "securerandom"
 require "time"
 require "timeout"
 require "uri"
@@ -12,8 +13,10 @@ require "uri"
 module FetchUtil
   class Regulatory
     CACHE_TTL = 86_400
-    CACHE_VERSION = 2
+    CACHE_VERSION = 3
     DEFAULT_CACHE_PATH = File.expand_path("~/.local/state/fetch_util/regulatory-cache")
+    ROBOTS_RECORD_SOURCES = %w[robotstxt contentsignal contentusagerobots].freeze
+    PAGE_RECORD_SOURCES = %w[xrobotstag metarobots tdmheaders tdmmeta contentusageheader human].freeze
     MACHINE_SOURCES = %w[
       robotstxt
       contentsignal
@@ -32,6 +35,7 @@ module FetchUtil
       "machine" => MACHINE_SOURCES,
       "human" => HUMAN_SOURCES
     }.freeze
+    private_constant :ROBOTS_RECORD_SOURCES, :PAGE_RECORD_SOURCES
 
     Response = Struct.new(:url, :status, :headers, :body, :redirects, keyword_init: true)
     autoload :HttpClient, "fetch_util/regulatory/http_client"

data/lib/fetch_util/request_log.rb

@@ -6,19 +6,40 @@ require "time"
 module FetchUtil
   class RequestLog
     DEFAULT_PATH = File.expand_path("~/.local/state/fetch_util/requests.log")
+    DELIMITER_ESCAPES = { "\t" => "\\t", "\r" => "\\r", "\n" => "\\n" }.freeze
 
-    def initialize(path: ENV.fetch("FETCH_UTIL_REQUEST_LOG", DEFAULT_PATH))
-      @path = path
+    def initialize(path: nil)
+      configured_path = path || ENV.fetch("FETCH_UTIL_REQUEST_LOG", DEFAULT_PATH)
+      @path = (path.nil? ? configured_path.dup : normalized_path(path)).freeze
     end
 
     attr_reader :path
 
     def append(entry, duration: nil)
       FileUtils.mkdir_p(File.dirname(path))
-      line = "#{Time.now.utc.iso8601}\t#{entry}"
+      escaped_entry = entry.to_s.gsub(/[\t\r\n]/, DELIMITER_ESCAPES)
+      line = "#{Time.now.utc.iso8601}\t#{escaped_entry}"
       line = "#{line}\t#{format("%.2f", duration)}s" if duration
       File.open(path, "a") { |file| file.puts(line) }
       path
     end
+
+    private
+
+    def normalized_path(value)
+      raw_path = value.to_s
+      return raw_path.dup if raw_path.empty? || directory_path?(raw_path)
+
+      expanded_path = File.expand_path(raw_path)
+      basename = File.basename(raw_path)
+      return expanded_path if basename.start_with?(".") || !File.extname(basename).empty?
+
+      "#{expanded_path}.log"
+    end
+
+    def directory_path?(path)
+      separators = [File::SEPARATOR, File::ALT_SEPARATOR].compact
+      separators.any? { |separator| path.end_with?(separator) } || %w[. ..].include?(File.basename(path))
+    end
   end
 end