fetch_util 0.3.0

data/lib/fetch_util/regulatory/human.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module Human
+      HUMAN_PATTERNS = [
+        {
+          "verb" => "disallow",
+          "noun" => "text-and-data-mining",
+          "regex" => /
+            \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
+            \b(?:text\s+and\s+data\s+mining|text-and-data-mining|tdm)\b
+          /ix
+        },
+        {
+          "verb" => "disallow",
+          "noun" => "ai-training",
+          "regex" => /
+            \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
+            \b(?:ai\s+training|training\s+(?:of|for)\s+(?:ai|models?)|machine\s+learning|large\s+language\s+model|large\s+language\s+models|llm|generative\s+ai)\b
+          /ix
+        },
+        {
+          "verb" => "disallow",
+          "noun" => "index",
+          "regex" => /
+            \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
+            \b(?:index(?:ing)?|search\s+engine(?:s)?)\b
+          /ix
+        },
+        {
+          "verb" => "disallow",
+          "noun" => "fetch",
+          "regex" => /
+            \b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b.{0,120}
+            \b(?:crawl(?:ing)?|scrap(?:e|ing)|fetch(?:ing)?|bot(?:s)?)\b
+          /ix
+        },
+        {
+          "verb" => "allow",
+          "noun" => "text-and-data-mining",
+          "regex" => /\b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}\b(?:text\s+and\s+data\s+mining|text-and-data-mining|tdm)\b/i
+        },
+        {
+          "verb" => "allow",
+          "noun" => "ai-training",
+          "regex" => /
+            \b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}
+            \b(?:ai\s+training|training\s+(?:of|for)\s+(?:ai|models?)|machine\s+learning|large\s+language\s+model|large\s+language\s+models|llm|generative\s+ai)\b
+          /ix
+        },
+        {
+          "verb" => "allow",
+          "noun" => "index",
+          "regex" => /\b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}\b(?:index(?:ing)?|search\s+engine(?:s)?)\b/i
+        },
+        {
+          "verb" => "allow",
+          "noun" => "fetch",
+          "regex" => /\b(?:allow(?:ed|s)?|permit(?:ted|s)?|may|can)\b.{0,120}\b(?:crawl(?:ing)?|scrap(?:e|ing)|fetch(?:ing)?|bot(?:s)?)\b/i
+        }
+      ].freeze
+
+      def extract_human_signals(body, path:)
+        chunks = human_text_chunks(body)
+        seen = {}
+
+        HUMAN_PATTERNS.filter_map do |entry|
+          evidence = chunks.find { |chunk| chunk.match?(entry["regex"]) }
+          next unless evidence
+          next if entry["verb"] == "allow" && negative_human_chunk?(evidence)
+
+          key = [entry["verb"], entry["noun"], evidence]
+          next if seen[key]
+
+          seen[key] = true
+          build_signal(
+            entry["verb"],
+            entry["noun"],
+            path: path,
+            conditions: { "evidence" => evidence }
+          )
+        end
+      end
+
+      private
+
+      def human_text_chunks(body)
+        text = body.to_s
+        text = text.gsub(%r{<script\b.*?</script>}mi, " ")
+        text = text.gsub(%r{<style\b.*?</style>}mi, " ")
+        text = text.gsub(%r{<noscript\b.*?</noscript>}mi, " ")
+        text = text.gsub(/<[^>]+>/, " ")
+        text = CGI.unescapeHTML(text)
+        text = FetchUtil.normalize_whitespace(text)
+        text.split(/(?<=[.!?])\s+/).map { |chunk| chunk.strip }.reject(&:empty?)
+      end
+
+      def negative_human_chunk?(chunk)
+        chunk.match?(/\b(?:do\s+not|does\s+not|must\s+not|may\s+not|shall\s+not|not\s+permit(?:ted)?|not\s+allow(?:ed)?|forbid(?:den|s)?|prohibit(?:ed|s)?)\b/i)
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/orchestration.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module Orchestration
+      def initialize(client: nil, cache_path: DEFAULT_CACHE_PATH, sources: nil, timeout: 20, user_agent: nil)
+        @client = client || HttpClient.new(timeout: timeout, user_agent: user_agent || default_user_agent)
+        @cache_path = cache_path || DEFAULT_CACHE_PATH
+        @source_tokens = sources
+      end
+
+      def call(url)
+        requested_uri = parse_http_uri(url)
+        origin_query = origin_query?(requested_uri)
+        query_target = request_target(requested_uri)
+        effective_query_target = query_target
+        selected_sources = resolve_sources(@source_tokens)
+        result = {}
+        policy_refs = []
+
+        if needs_tdmrep_fetch?(selected_sources)
+          record = tdmrep_record(requested_uri)
+          if selected_sources.include?("tdmrep")
+            add_source_payload(result, "tdmrep", scoped_signals(record["signals"], origin_query: origin_query, query_target: query_target))
+          end
+          policy_refs.concat(record["policies"])
+        end
+
+        if selected_sources.include?("trusttxt")
+          record = trusttxt_record(requested_uri)
+          add_source_payload(result, "trusttxt", scoped_signals(record["signals"], origin_query: origin_query, query_target: query_target))
+        end
+
+        if needs_robots_fetch?(selected_sources)
+          record = robots_record(requested_uri)
+          %w[robotstxt contentsignal contentusagerobots].each do |source|
+            next unless selected_sources.include?(source)
+
+            add_source_payload(
+              result,
+              source,
+              scoped_signals(record.dig("signals", source), origin_query: origin_query, query_target: query_target)
+            )
+          end
+        end
+
+        if needs_page_fetch?(selected_sources)
+          record = page_record(requested_uri)
+          effective_query_target = page_query_target(record, fallback: query_target)
+          %w[xrobotstag metarobots tdmheaders tdmmeta contentusageheader human].each do |source|
+            next unless selected_sources.include?(source)
+
+            add_source_payload(
+              result,
+              source,
+              scoped_signals(record.dig("signals", source), origin_query: origin_query, query_target: effective_query_target)
+            )
+          end
+          policy_refs.concat(record["policies"])
+        end
+
+        if selected_sources.include?("tdmpolicy")
+          add_source_payload(
+            result,
+            "tdmpolicy",
+            scoped_signals(expanded_tdm_policy_signals(policy_refs), origin_query: origin_query, query_target: effective_query_target)
+          )
+        end
+
+        result
+      end
+
+      private
+
+      attr_reader :cache_path, :client
+
+      def default_user_agent
+        "fetch_util/#{FetchUtil::VERSION}"
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/page.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module Page
+      def page_record(requested_uri)
+        fetch_record("page:#{requested_uri}", requested_uri.to_s, fallback: empty_page_record, require_success: false) do |_body, response|
+          final_uri = parse_http_uri(response.url)
+          final_path = request_target(final_uri)
+          xrobotstag = []
+          contentusageheader = []
+          tdmheaders = []
+          header_policies = []
+
+          response_chain(response).each do |step|
+            path = request_target(parse_http_uri(step.url))
+            xrobotstag.concat(extract_x_robot_signals(step.headers, path: path))
+            contentusageheader.concat(extract_content_usage_header_signals(step.headers, path: path))
+            step_tdmheaders, step_policies = extract_tdm_value_signals(
+              reservation: first_header_value(step.headers, "tdm-reservation"),
+              policy_url: first_header_value(step.headers, "tdm-policy"),
+              path: path
+            )
+            tdmheaders.concat(step_tdmheaders)
+            header_policies.concat(step_policies)
+          end
+
+          metarobots = []
+          tdmmeta = []
+          meta_policies = []
+          human = []
+          if html_content?(response.headers, response.body)
+            meta_tags = parse_meta_tags(response.body)
+            metarobots = sort_generic_signals(extract_meta_robot_signals(meta_tags, path: final_path))
+            tdmmeta, meta_policies = extract_tdm_meta_signals(meta_tags, path: final_path)
+            human = sort_generic_signals(extract_human_signals(response.body, path: final_path))
+          end
+
+          {
+            "final_url" => final_uri.to_s,
+            "signals" => {
+              "xrobotstag" => sort_generic_signals(xrobotstag),
+              "metarobots" => metarobots,
+              "tdmheaders" => sort_generic_signals(tdmheaders),
+              "tdmmeta" => sort_generic_signals(tdmmeta),
+              "contentusageheader" => sort_usage_preference_signals(contentusageheader),
+              "human" => human
+            },
+            "policies" => dedupe_policy_refs(header_policies + meta_policies)
+          }
+        end
+      end
+
+      def empty_page_record
+        {
+          "final_url" => nil,
+          "signals" => {
+            "xrobotstag" => [],
+            "metarobots" => [],
+            "tdmheaders" => [],
+            "tdmmeta" => [],
+            "contentusageheader" => [],
+            "human" => []
+          },
+          "policies" => []
+        }
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/robot_globs.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module RobotGlobs
+      private
+
+      def robot_user_agent_glob(user_agent)
+        token = user_agent.to_s.strip
+        return "*" if token.empty? || token == "*"
+        return token if token.end_with?("*")
+
+        "#{token}*"
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/robots.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module Robots
+      ROBOT_DIRECTIVES = %w[
+        all
+        follow
+        index
+        indexifembedded
+        max-image-preview
+        max-snippet
+        max-video-preview
+        noai
+        noarchive
+        nocache
+        nofollow
+        noimageai
+        noimageindex
+        noindex
+        none
+        nosnippet
+        notranslate
+        unavailable_after
+      ].freeze
+      def robots_record(requested_uri)
+        fetch_record("robotstxt:#{origin_key(requested_uri)}", robots_uri(requested_uri), fallback: empty_robots_record) do |body|
+          payload = extract_robots_source_signals(body)
+          {
+            "signals" => {
+              "robotstxt" => sort_robot_signals(payload["robotstxt"]),
+              "contentsignal" => sort_usage_preference_signals(payload["contentsignal"]),
+              "contentusagerobots" => sort_usage_preference_signals(payload["contentusagerobots"])
+            }
+          }
+        end
+      end
+
+      def extract_robots_source_signals(body)
+        signals = {
+          "robotstxt" => [],
+          "contentsignal" => [],
+          "contentusagerobots" => []
+        }
+        user_agents = []
+        in_rules = false
+
+        body.to_s.gsub("\r\n", "\n").gsub("\r", "\n").each_line do |line|
+          content = line.sub(/\s*#.*\z/, "").strip
+          next if content.empty?
+
+          field, value = content.split(":", 2)
+          next unless value
+
+          field = field.strip.downcase
+          value = value.strip
+
+          case field
+          when "user-agent"
+            user_agents = [] if in_rules
+            in_rules = false
+            user_agents << value unless value.empty?
+          when "allow", "disallow"
+            next if user_agents.empty?
+
+            in_rules = true
+            user_agents.each do |user_agent|
+              signals["robotstxt"] << robot_signal(field, user_agent, value)
+            end
+          when "content-signal"
+            next if user_agents.empty?
+
+            in_rules = true
+            user_agents.each do |user_agent|
+              signals["contentsignal"].concat(extract_content_signal_signals(value, user_agent: user_agent))
+            end
+          when "content-usage"
+            next if user_agents.empty?
+
+            in_rules = true
+            user_agents.each do |user_agent|
+              signals["contentusagerobots"].concat(extract_content_usage_robot_signals(value, user_agent: user_agent))
+            end
+          end
+        end
+
+        signals
+      end
+
+      private
+
+      def empty_robots_record
+        {
+          "signals" => {
+            "robotstxt" => [],
+            "contentsignal" => [],
+            "contentusagerobots" => []
+          }
+        }
+      end
+
+      def robot_signal(field, user_agent, value)
+        verb = value.to_s.strip.empty? ? "allow" : field
+        conditions = {}
+        user_agent_glob = robot_user_agent_glob(user_agent)
+        conditions["user-agent"] = user_agent_glob unless user_agent_glob == "*"
+
+        build_signal(
+          verb,
+          "*",
+          path: normalize_output_path(value),
+          conditions: conditions
+        )
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/signals.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module Signals
+      private
+
+      def scoped_signals(signals, origin_query:, query_target:)
+        list = Array(signals).map { |signal| deep_copy(signal) }
+        return list if origin_query
+
+        list.filter_map do |signal|
+          next unless signal_matches_target?(signal["path"], query_target)
+
+          signal.reject { |key, _value| key == "path" }
+        end
+      end
+
+      def signal_matches_target?(path, query_target)
+        return true if path.nil? || path.empty?
+
+        Regexp.new("\\A#{signal_path_pattern(path)}").match?(query_target)
+      end
+
+      def signal_path_pattern(path)
+        escaped = Regexp.escape(path.to_s)
+        escaped = escaped.gsub("\\*", ".*")
+        if escaped.end_with?("\\$")
+          "#{escaped[0...-2]}$"
+        else
+          "#{escaped}.*"
+        end
+      end
+
+      def sort_specificity_signals(signals)
+        Array(signals).sort_by do |signal|
+          [
+            *signal_sort_prefix(signal),
+            signal.dig("conditions", "policy").to_s
+          ]
+        end
+      end
+
+      def signal_sort_prefix(signal)
+        [
+          -path_specificity(signal["path"]),
+          allow_signal?(signal) ? 0 : 1
+        ]
+      end
+
+      def sort_generic_signals(signals)
+        Array(signals).sort_by do |signal|
+          [
+            allow_signal?(signal) ? 1 : 0,
+            wildcard_signal?(signal) ? 1 : 0,
+            signal_verb(signal),
+            signal_noun(signal)
+          ]
+        end
+      end
+
+      def signal_verb(signal)
+        signal.keys.find { |key| %w[allow disallow].include?(key) }.to_s
+      end
+
+      def signal_noun(signal)
+        signal.values_at("allow", "disallow").compact.first.to_s
+      end
+
+      def allow_signal?(signal)
+        signal.key?("allow")
+      end
+
+      def wildcard_signal?(signal)
+        signal.dig("conditions", "user-agent").to_s == "*" || signal.dig("conditions", "user-agent").to_s.empty?
+      end
+
+      def path_specificity(path)
+        path.to_s.delete("*$").length
+      end
+
+      def integer_or_value(value)
+        Integer(value, exception: false) || value
+      end
+
+      def build_signal(verb, noun, path: nil, conditions: nil)
+        signal = { verb => noun }
+        signal["path"] = path if path
+        signal["conditions"] = conditions if conditions && !conditions.empty?
+        signal
+      end
+
+      def sort_robot_signals(signals)
+        signals.sort_by do |signal|
+          prefix = signal_sort_prefix(signal)
+          [
+            prefix.first,
+            wildcard_signal?(signal) ? 1 : 0,
+            prefix.last,
+            signal.dig("conditions", "user-agent").to_s
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/source_selection.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module SourceSelection
+      private
+
+      def all_sources
+        @all_sources ||= (MACHINE_SOURCES + HUMAN_SOURCES).freeze
+      end
+
+      def resolve_sources(selection)
+        tokens = Array(selection || "machine").flat_map { |value| value.to_s.split(",") }
+        tokens = tokens.map(&:strip).reject(&:empty?)
+        selected = []
+
+        tokens.each do |token|
+          remove = token.start_with?("-")
+          name = remove ? token[1..] : token
+          expansions = SOURCE_CLASSES.fetch(name, [name])
+
+          expansions.each do |source|
+            validate_source!(source)
+            if remove
+              selected.delete(source)
+            else
+              selected << source unless selected.include?(source)
+            end
+          end
+        end
+
+        selected
+      end
+
+      def validate_source!(source)
+        return if all_sources.include?(source)
+
+        raise ArgumentError, "unsupported regulatory source: #{source}"
+      end
+
+      def needs_page_fetch?(selected_sources)
+        (selected_sources & (HUMAN_SOURCES + %w[xrobotstag metarobots tdmheaders tdmmeta contentusageheader tdmpolicy])).any?
+      end
+
+      def needs_tdmrep_fetch?(selected_sources)
+        (selected_sources & %w[tdmrep tdmpolicy]).any?
+      end
+
+      def needs_robots_fetch?(selected_sources)
+        (selected_sources & %w[robotstxt contentsignal contentusagerobots]).any?
+      end
+
+      def add_source_payload(result, source, signals)
+        return if signals.nil? || signals.empty?
+
+        result[source] = signals
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/tdm_page.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module TdmPage
+      def extract_tdm_meta_signals(meta_tags, path:)
+        reservation = nil
+        policy_url = nil
+
+        meta_tags.each do |attributes|
+          name = attributes["name"].to_s.strip.downcase
+          reservation ||= attributes["content"] if name == "tdm-reservation"
+          policy_url ||= attributes["content"] if name == "tdm-policy"
+        end
+
+        extract_tdm_value_signals(reservation: reservation, policy_url: policy_url, path: path)
+      end
+
+      def extract_tdm_value_signals(reservation:, policy_url:, path:)
+        value = reservation.to_s.strip
+        return [[], []] unless %w[0 1].include?(value)
+
+        conditions = {}
+        policy = policy_url.to_s.strip
+        conditions["policy"] = policy if value == "1" && !policy.empty?
+        signals = [
+          build_signal(
+            value == "1" ? "disallow" : "allow",
+            "text-and-data-mining",
+            path: path,
+            conditions: conditions
+          )
+        ]
+        policies = value == "1" ? [policy_ref(policy, path)] : []
+        [signals, dedupe_policy_refs(policies)]
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/tdm_policy.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module TdmPolicy
+      def expanded_tdm_policy_signals(policy_refs)
+        dedupe_policy_refs(policy_refs).flat_map do |policy_ref|
+          record = tdm_policy_record(policy_ref["url"])
+          Array(record["signals"]).map do |template|
+            signal = deep_copy(template)
+            signal["path"] ||= policy_ref["path"]
+            conditions = signal["conditions"] || {}
+            conditions["policy"] = policy_ref["url"]
+            signal["conditions"] = conditions
+            signal
+          end
+        end
+      end
+
+      def tdm_policy_record(url)
+        cache_fetch("tdmpolicy:#{url}") do
+          response = safe_get(url)
+          signals = []
+          if response&.status&.between?(200, 299) && json_like_response?(response.headers, response.body)
+            signals = extract_tdm_policy_signals(response.body)
+          end
+          { "signals" => sort_specificity_signals(signals) }
+        end
+      end
+
+      def extract_tdm_policy_signals(body)
+        payload = JSON.parse(body.to_s)
+        permissions = array_value(payload, "permission")
+
+        permissions.filter_map do |permission|
+          next unless permission.is_a?(Hash)
+          next unless tdm_policy_action?(permission["action"])
+
+          conditions = {}
+          duties = array_value(permission, "duty").filter_map { |item| duty_name(item) }
+          conditions["duty"] = duties if duties.any?
+          purpose = permission_purpose(permission)
+          conditions["purpose"] = purpose if purpose
+
+          signal = build_signal("allow", "text-and-data-mining", conditions: conditions)
+          target_path = policy_target_path(permission["target"])
+          signal["path"] = target_path if target_path
+          signal
+        end
+      rescue JSON::ParserError
+        []
+      end
+    end
+  end
+end