fetch_util 0.3.0

data/lib/fetch_util/regulatory/tdm_rep.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module TdmRep
+      def tdmrep_record(requested_uri)
+        fetch_record(
+          "tdmrep:#{origin_key(requested_uri)}",
+          tdmrep_uri(requested_uri),
+          fallback: { "signals" => [], "policies" => [] }
+        ) do |body|
+          signals, policies = extract_tdmrep_signals(body)
+          {
+            "signals" => sort_specificity_signals(signals),
+            "policies" => policies
+          }
+        end
+      end
+
+      def extract_tdmrep_signals(body)
+        payload = JSON.parse(body.to_s)
+        return [[], []] unless payload.is_a?(Array)
+
+        signals = []
+        policies = []
+        payload.each do |rule|
+          next unless rule.is_a?(Hash)
+
+          location = string_value(rule, "location")
+          reservation = string_value(rule, "tdm-reservation")
+          policy_url = string_value(rule, "tdm-policy")
+          next if location.empty?
+          next unless %w[0 1].include?(reservation)
+
+          rule_signals, rule_policies = extract_tdm_value_signals(
+            reservation: reservation,
+            policy_url: policy_url,
+            path: normalize_output_path(location)
+          )
+          signals.concat(rule_signals)
+          policies.concat(rule_policies)
+        end
+
+        [signals, dedupe_policy_refs(policies)]
+      rescue JSON::ParserError
+        [[], []]
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/tdm_support.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module TdmSupport
+      private
+
+      def tdm_policy_action?(value)
+        odrl_token(value) == "mine"
+      end
+
+      def duty_name(value)
+        action = value.is_a?(Hash) ? value["action"].to_s : value.to_s
+        return nil if action.empty?
+
+        normalized = odrl_token(action)
+        case normalized.downcase
+        when "obtainconsent"
+          "obtain-consent"
+        when "compensate"
+          "compensate"
+        else
+          normalized.downcase.gsub(/[^a-z0-9]+/, "-").sub(/\A-+|-+\z/, "")
+        end
+      end
+
+      def permission_purpose(permission)
+        array_value(permission, "constraint").each do |constraint|
+          next unless constraint.is_a?(Hash)
+          next unless odrl_token(constraint["leftOperand"]) == "purpose"
+          next unless odrl_token(constraint["operator"]) == "eq"
+
+          right_operand = odrl_token(constraint["rightOperand"])
+          return "research" if right_operand == "research"
+          return "non-research" if right_operand == "non-research"
+        end
+
+        nil
+      end
+
+      def policy_target_path(value)
+        target = value.to_s.strip
+        return nil if target.empty?
+
+        uri = URI.parse(target)
+        return normalize_output_path(request_target(uri)) if uri.is_a?(URI::HTTP)
+        return normalize_output_path(target) if target.start_with?("/")
+
+        nil
+      rescue URI::InvalidURIError
+        nil
+      end
+
+      def string_value(hash, key)
+        hash[key].to_s.strip
+      end
+
+      def array_value(hash, key)
+        value = hash[key]
+        return value if value.is_a?(Array)
+        return [] if value.nil?
+
+        [value]
+      end
+
+      def policy_ref(url, path)
+        candidate = url.to_s.strip
+        return nil if candidate.empty?
+
+        parse_http_uri(candidate)
+        { "url" => candidate, "path" => path }
+      rescue ArgumentError
+        nil
+      end
+
+      def dedupe_policy_refs(policy_refs)
+        seen = {}
+        list = []
+        Array(policy_refs).compact.each do |policy_ref|
+          key = [policy_ref["url"], policy_ref["path"]]
+          next if seen[key]
+
+          seen[key] = true
+          list << policy_ref
+        end
+        list
+      end
+
+      def odrl_token(value)
+        value.to_s.strip.downcase.split(/[#:]/).last.to_s
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/trust_txt.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module TrustTxt
+      def trusttxt_record(requested_uri)
+        fetch_record(
+          "trusttxt:#{origin_key(requested_uri)}",
+          [trusttxt_uri(requested_uri), trusttxt_well_known_uri(requested_uri)],
+          fallback: { "signals" => [] }
+        ) do |body|
+          signals = extract_trusttxt_signals(body)
+          { "signals" => sort_usage_preference_signals(signals) }
+        end
+      end
+
+      def extract_trusttxt_signals(body)
+        preference = nil
+
+        body.to_s.gsub("\r\n", "\n").gsub("\r", "\n").each_line do |line|
+          content = line.sub(/\s*#.*\z/, "").strip
+          next if content.empty?
+
+          key, raw_value = content.split("=", 2)
+          next unless raw_value
+          next unless key.to_s.strip.casecmp?("datatrainingallowed")
+
+          preference = case raw_value.to_s.strip.downcase
+                       when "yes"
+                         "allow"
+                       when "no"
+                         "disallow"
+                       end
+        end
+
+        return [] unless preference
+
+        [
+          build_signal(
+            preference,
+            "ai-training",
+            path: "/*",
+            conditions: { "label" => "datatrainingallowed" }
+          )
+        ]
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory/usage_preferences.rb

@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Regulatory
+    module UsagePreferences
+      def extract_content_signal_signals(value, user_agent: nil)
+        parse_yes_no_preferences(value).map do |label, verb|
+          usage_preference_signal(label, verb, path: "/*", user_agent: user_agent)
+        end
+      end
+
+      def extract_content_usage_robot_signals(value, user_agent: nil)
+        path, statement = parse_content_usage_rule(value)
+        return [] if statement.nil? || statement.empty?
+
+        extract_content_usage_statement_signals(statement, path: normalize_output_path(path || "/*"), user_agent: user_agent)
+      end
+
+      def extract_content_usage_statement_signals(value, path:, user_agent: nil)
+        parse_structured_usage_preferences(value).map do |label, verb|
+          usage_preference_signal(label, verb, path: path, user_agent: user_agent)
+        end
+      end
+
+      def sort_usage_preference_signals(signals)
+        Array(signals).sort_by do |signal|
+          [
+            -path_specificity(signal["path"]),
+            wildcard_signal?(signal) ? 1 : 0,
+            signal_noun(signal),
+            allow_signal?(signal) ? 1 : 0,
+            signal.dig("conditions", "user-agent").to_s
+          ]
+        end
+      end
+
+      private
+
+      def usage_preference_signal(label, verb, path:, user_agent: nil)
+        conditions = {}
+        user_agent_glob = robot_user_agent_glob(user_agent)
+        conditions["user-agent"] = user_agent_glob unless user_agent_glob == "*"
+        normalized_noun = usage_preference_noun(label)
+        conditions["label"] = label if normalized_noun != label
+
+        build_signal(verb, normalized_noun, path: path, conditions: conditions)
+      end
+
+      def usage_preference_noun(label)
+        case label
+        when "ai-train", "train-ai"
+          "ai-training"
+        else
+          label
+        end
+      end
+
+      def parse_content_usage_rule(value)
+        text = value.to_s.strip
+        return [nil, text] unless text.start_with?("/")
+
+        path, statement = text.split(/[ \t]+/, 2)
+        [path, statement.to_s.strip]
+      end
+
+      def parse_yes_no_preferences(value)
+        preferences = {}
+        value.to_s.split(",").each do |entry|
+          label, raw_value = entry.split("=", 2)
+          next unless raw_value
+
+          verb = case raw_value.to_s.strip.downcase
+                 when "yes"
+                   "allow"
+                 when "no"
+                   "disallow"
+                 end
+          next unless verb
+
+          preferences[label.to_s.strip.downcase] = verb
+        end
+        preferences.to_a
+      end
+
+      def parse_structured_usage_preferences(value)
+        preferences = {}
+        value.to_s.split(",").each do |entry|
+          label_part, raw_value = entry.split("=", 2)
+          next unless raw_value
+
+          label = label_part.to_s.split(";", 2).first.to_s.strip.downcase
+          token = raw_value.to_s.split(";", 2).first.to_s.strip.delete_prefix('"').delete_suffix('"').downcase
+          verb = if token == "y"
+                   "allow"
+                 else
+                   (token == "n" ? "disallow" : nil)
+                 end
+          next unless verb
+
+          preferences[label] = verb
+        end
+        preferences.to_a
+      end
+    end
+  end
+end

data/lib/fetch_util/regulatory.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "cgi"
+require "digest"
+require "fileutils"
+require "json"
+require "openssl"
+require "time"
+require "timeout"
+require "uri"
+
+module FetchUtil
+  class Regulatory
+    CACHE_TTL = 86_400
+    CACHE_VERSION = 2
+    DEFAULT_CACHE_PATH = File.expand_path("~/.local/state/fetch_util/regulatory-cache")
+    MACHINE_SOURCES = %w[
+      robotstxt
+      contentsignal
+      contentusagerobots
+      contentusageheader
+      trusttxt
+      xrobotstag
+      metarobots
+      tdmrep
+      tdmheaders
+      tdmmeta
+      tdmpolicy
+    ].freeze
+    HUMAN_SOURCES = %w[human].freeze
+    SOURCE_CLASSES = {
+      "machine" => MACHINE_SOURCES,
+      "human" => HUMAN_SOURCES
+    }.freeze
+
+    Response = Struct.new(:url, :status, :headers, :body, :redirects, keyword_init: true)
+    autoload :HttpClient, "fetch_util/regulatory/http_client"
+    autoload :Orchestration, "fetch_util/regulatory/orchestration"
+    autoload :SourceSelection, "fetch_util/regulatory/source_selection"
+    autoload :Signals, "fetch_util/regulatory/signals"
+    autoload :FetchRecords, "fetch_util/regulatory/fetch_records"
+    autoload :CacheStore, "fetch_util/regulatory/cache_store"
+    autoload :Robots, "fetch_util/regulatory/robots"
+    autoload :RobotGlobs, "fetch_util/regulatory/robot_globs"
+    autoload :Headers, "fetch_util/regulatory/headers"
+    autoload :Directives, "fetch_util/regulatory/directives"
+    autoload :TdmSupport, "fetch_util/regulatory/tdm_support"
+    autoload :TdmPage, "fetch_util/regulatory/tdm_page"
+    autoload :TrustTxt, "fetch_util/regulatory/trust_txt"
+    autoload :UsagePreferences, "fetch_util/regulatory/usage_preferences"
+    autoload :Page, "fetch_util/regulatory/page"
+    autoload :TdmRep, "fetch_util/regulatory/tdm_rep"
+    autoload :TdmPolicy, "fetch_util/regulatory/tdm_policy"
+    autoload :Human, "fetch_util/regulatory/human"
+
+    include Orchestration
+    include SourceSelection
+    include Signals
+    include FetchRecords
+    include CacheStore
+    include Robots
+    include RobotGlobs
+    include Headers
+    include Directives
+    include TdmSupport
+    include TdmPage
+    include TrustTxt
+    include UsagePreferences
+    include Page
+    include TdmRep
+    include TdmPolicy
+    include Human
+  end
+end

data/lib/fetch_util/request_log.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "time"
+
+module FetchUtil
+  class RequestLog
+    DEFAULT_PATH = File.expand_path("~/.local/state/fetch_util/requests.log")
+
+    def initialize(path: ENV.fetch("FETCH_UTIL_REQUEST_LOG", DEFAULT_PATH))
+      @path = path
+    end
+
+    attr_reader :path
+
+    def append(entry, duration: nil)
+      FileUtils.mkdir_p(File.dirname(path))
+      line = "#{Time.now.utc.iso8601}\t#{entry}"
+      line = "#{line}\t#{format("%.2f", duration)}s" if duration
+      File.open(path, "a") { |file| file.puts(line) }
+      path
+    end
+  end
+end

data/lib/fetch_util/result.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Result
+    attr_reader :url, :final_url, :title, :byline, :excerpt, :site_name,
+                :published_time, :canonical_url, :language, :html, :markdown,
+                :metadata, :reader_mode, :content_type, :suspect, :warnings,
+                :content_completeness_ratio, :content_format, :paywall_state
+
+    def initialize(url:, final_url:, title:, byline:, excerpt:, site_name:, published_time:,
+                   canonical_url:, language:, html:, markdown:, metadata:, reader_mode:, content_type:, suspect:, warnings:,
+                   content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil)
+      @url = url
+      @final_url = final_url
+      @title = title
+      @byline = byline
+      @excerpt = excerpt
+      @site_name = site_name
+      @published_time = published_time
+      @canonical_url = canonical_url
+      @language = language
+      @html = html
+      @markdown = markdown
+      @metadata = metadata.freeze
+      @reader_mode = reader_mode
+      @content_type = content_type
+      @suspect = suspect
+      @warnings = warnings.freeze
+      @content_completeness_ratio = content_completeness_ratio
+      @content_format = content_format&.freeze
+      @paywall_state = paywall_state&.freeze
+    end
+
+    def to_h
+      {
+        url: url,
+        final_url: final_url,
+        title: title,
+        byline: byline,
+        excerpt: excerpt,
+        site_name: site_name,
+        published_time: published_time,
+        canonical_url: canonical_url,
+        language: language,
+        html: html,
+        markdown: markdown,
+        metadata: metadata,
+        reader_mode: reader_mode,
+        content_type: content_type,
+        suspect: suspect,
+        warnings: warnings,
+        content_completeness_ratio: content_completeness_ratio,
+        content_format: content_format,
+        paywall_state: paywall_state
+      }
+    end
+  end
+end

data/lib/fetch_util/searcher/result_filtering.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Searcher
+    module ResultFiltering
+      private
+
+      def search_engine_self_link?(title, url, snippet)
+        host, path = result_location(url)
+        text = compact_text([title, snippet].compact.join(" ")).downcase
+        return false if host.empty?
+
+        return true if duckduckgo_self_link?(host, path, title, text)
+        return true if google_self_link?(host, path, title, text)
+        return true if search_shell_result?(host, path, title)
+
+        false
+      end
+
+      def low_value_result?(title, url, snippet)
+        host, path = result_location(url)
+        return false if host.empty?
+        return true if non_html_document_url?(url)
+        return true if host == "duckduckgo.com" && path == "/y.js"
+        return true if host.start_with?("translate.google.")
+        return true if facebook_noise_result?(host, path, title, snippet)
+        return true if pinterest_noise_result?(host, path, title)
+        return true if host.end_with?("threads.net") || host.end_with?("threads.com")
+        return true if tiktok_noise_result?(host, path, snippet)
+        return true if host.end_with?("walmart.com") && path.match?(%r{\A/(search|browse|c|cp|b)\b})
+
+        false
+      end
+
+      def result_location(url)
+        [host_for(url).to_s, path_for(url)]
+      end
+
+      def search_action_text?(text)
+        /\b(redo search without this site|block this site from all results|go to google home|duckduckgo)\b/.match?(text)
+      end
+
+      def duckduckgo_self_link?(host, path, title, text)
+        return false unless host.end_with?("duckduckgo.com")
+
+        (path == "/" && (title.casecmp?("DuckDuckGo") || search_action_text?(text))) ||
+          (path.start_with?("/html") && search_action_text?(text))
+      end
+
+      def google_self_link?(host, path, title, text)
+        if host.end_with?("google.com")
+          return true if path == "/" && search_action_text?(text)
+          return true if %w[/search /preferences /advanced_search /setprefs].include?(path)
+        end
+
+        return false unless host.match?(/\Agoogle\.[a-z.]+\z/)
+
+        google_home_shell = /before you continue to google|go to google home/.match?(text) || title.casecmp?("Before you continue to Google")
+        (%w[/ /webhp].include?(path) && google_home_shell) ||
+          (path.start_with?("/intl/") && /\bgoogle apps|about google|products\b/.match?(text))
+      end
+
+      def search_shell_result?(host, path, title)
+        return true if host.end_with?("search.brave.com") && path == "/search" && title.casecmp?("Brave Search")
+        return true if host.end_with?("bing.com") && path == "/search" && title.casecmp?("Bing")
+        return true if host.end_with?("ecosia.org") && path == "/search" && title.casecmp?("Ecosia")
+
+        false
+      end
+
+      def facebook_noise_result?(host, path, title, snippet)
+        return false unless host.end_with?("facebook.com")
+
+        path.match?(%r{\A/(groups|events|watch|share|reel|photo)\b}) ||
+          title.end_with?(" - Facebook") ||
+          title.match?(/\(@[^)]+\)/) ||
+          snippet.to_s.match?(/\b\d+[,\dKMB+.]*\s*(followers?|likes?|members?)\b/i)
+      end
+
+      def pinterest_noise_result?(host, path, title)
+        return false unless host.include?("pinterest.")
+
+        !path.match?(%r{\A/search/}) || title.end_with?(" - Pinterest")
+      end
+
+      def tiktok_noise_result?(host, path, snippet)
+        return false unless host.end_with?("tiktok.com")
+
+        host.start_with?("shop.") ||
+          path.match?(%r{\A/@[^/]+/video/}) ||
+          snippet.to_s.match?(/\bAll Categories\b/i)
+      end
+
+      def non_html_document_url?(url)
+        normalized = url.to_s.downcase
+        path = path_for(normalized).downcase
+
+        path.end_with?(".pdf") || path.match?(%r{/pdf(?:/|\z)}) || normalized.match?(/[?&](?:format|download)=pdf\b/)
+      end
+    end
+  end
+end