html2rss 0.32.4 → 0.34.0

data/lib/html2rss/request_service/compressed_body.rb

@@ -7,56 +7,58 @@ require 'zlib'
 module Html2rss
   class RequestService
     # Decodes gzip/brotli bodies when Content-Encoding is missing or the type is octet-stream.
-    module CompressedBody
+    class CompressedBody
       # Gzip stream magic (RFC 1952).
       GZIP_MAGIC = "\x1F\x8B".b.freeze
       # HTML prefix for unlabeled brotli trial decode. Same object as {Response::HTML_BODY_SNIFF}.
       HTML_BODY_SNIFF = Response::HTML_BODY_SNIFF
 
-      module_function
-
       # @param body [String, nil] raw response body
       # @param headers [Hash] response headers (Content-Encoding / Content-Type)
       # @return [String, nil] decoded body when compressed; otherwise the original body
-      def decode(body, headers: {})
-        return body if body.nil? || body.empty?
-
+      def self.decode(body, headers: {})
         raw = body.to_s.b
-        decode_labeled(raw, headers) || decode_unlabeled(raw, headers) || body
+        new(raw, headers).decoded_body || body
+      end
+
+      # @param raw [String] binary response body
+      # @param headers [Hash] response headers (Content-Encoding / Content-Type)
+      def initialize(raw, headers)
+        @raw = raw
+        @headers = headers
       end
+      private_class_method :new
 
-      def decode_labeled(raw, headers)
-        encodings = content_encodings(headers)
+      # @return [String, nil] inflated body, or nil when the bytes are not a recognized stream
+      def decoded_body
+        decode_labeled || decode_unlabeled
+      end
+
+      private
+
+      def decode_labeled
+        encodings = content_encodings
         return if encodings.empty?
 
-        encodings.reverse.reduce(raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
+        encodings.reverse.reduce(@raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
       end
-      module_function :decode_labeled
-      private_class_method :decode_labeled
 
-      def decode_unlabeled(raw, headers)
-        return uncompress_gzip(raw) if raw.start_with?(GZIP_MAGIC)
-        return unless octet_stream?(headers)
+      def decode_unlabeled
+        return uncompress_gzip(@raw) if @raw.start_with?(GZIP_MAGIC)
+        return unless octet_stream?
 
-        try_brotli(raw)
+        try_brotli(@raw)
       end
-      module_function :decode_unlabeled
-      private_class_method :decode_unlabeled
 
       def inflate(raw, encoding)
         case encoding
-        when 'gzip'
-          return raw unless raw.start_with?(GZIP_MAGIC)
-
-          uncompress_gzip(raw)
+        when 'gzip' then uncompress_gzip(raw)
         when 'br' then try_brotli(raw, require_html: false)
         when 'deflate' then inflate_deflate(raw)
         end
       rescue Zlib::Error, Brotli::Error, ArgumentError
         nil
       end
-      module_function :inflate
-      private_class_method :inflate
 
       def inflate_deflate(raw)
         inflater = nil
@@ -67,51 +69,37 @@ module Html2rss
       ensure
         inflater&.close
       end
-      module_function :inflate_deflate
-      private_class_method :inflate_deflate
 
       def uncompress_gzip(raw)
-        Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
+        Zlib::GzipReader.wrap(StringIO.new(raw), &:read).b
       rescue Zlib::Error
         nil
       end
-      module_function :uncompress_gzip
-      private_class_method :uncompress_gzip
 
       def try_brotli(raw, require_html: true)
         inflated = Brotli.inflate(raw)
         return inflated unless require_html
-        return inflated if inflated.b.match?(HTML_BODY_SNIFF)
 
-        nil
+        inflated.b.match?(HTML_BODY_SNIFF) ? inflated : nil
       rescue Brotli::Error, ArgumentError
         nil
       end
-      module_function :try_brotli
-      private_class_method :try_brotli
 
-      def content_encodings(headers)
-        header(headers, 'content-encoding')
+      def content_encodings
+        header('content-encoding')
           .to_s
           .split(',')
-          .filter_map { |val| (s = val.strip.downcase).empty? ? nil : s }
+          .map { |val| val.strip.downcase }
+          .reject(&:empty?)
       end
-      module_function :content_encodings
-      private_class_method :content_encodings
 
-      def octet_stream?(headers)
-        header(headers, 'content-type').to_s.downcase.include?('application/octet-stream')
+      def octet_stream?
+        header('content-type').to_s.downcase.include?('application/octet-stream')
       end
-      module_function :octet_stream?
-      private_class_method :octet_stream?
 
-      def header(headers, name)
-        headers.fetch(name) do
-          headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
-        end
+      def header(name)
+        @headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
       end
-      module_function :header
-      private_class_method :header
     end
   end
 end

data/lib/html2rss/request_service/context.rb

@@ -6,6 +6,9 @@ module Html2rss
     # Holds information needed to send requests to websites.
     # To be passed down to the RequestService's strategies.
     class Context
+      # Raised when a context is built without its runtime policy or request budget.
+      class InvalidError < Html2rss::ContractError; end
+
       ##
       # @param url [String, Html2rss::Url] the URL to request
       # @param headers [Hash] HTTP request headers
@@ -15,7 +18,8 @@ module Html2rss
       # @option request_options [String, Html2rss::Url, nil] :origin_url originating URL for same-origin checks
       # @option request_options [Policy] :policy runtime request policy
       # @option request_options [Budget] :budget shared request budget for the feed build
-      # @raise [ArgumentError] if policy or budget is explicitly nil
+      # @option request_options [FeedAdvertisement, nil] :advertisement page proof for an +:advertised_feed+ follow-up
+      # @raise [InvalidError] if policy or budget is nil or budget is missing
       def initialize(url:, headers: {}, request: {}, **request_options)
         @url = Html2rss::Url.from_absolute(url)
         @headers = normalize_headers(headers).freeze
@@ -44,23 +48,19 @@ module Html2rss
       # @return [Budget] the shared request budget
       attr_reader :budget
 
+      # @return [FeedAdvertisement, nil] the page's advertised feeds, for an +:advertised_feed+ follow-up
+      attr_reader :advertisement
+
       ##
       # Builds a follow-up request context sharing headers, budget, and policy.
       #
       # @param url [String, Html2rss::Url] the follow-up URL
       # @param relation [Symbol] why the follow-up is being made
       # @param origin_url [String, Html2rss::Url] effective origin for same-origin checks
+      # @param advertisement [FeedAdvertisement, nil] page proof for an +:advertised_feed+ follow-up
       # @return [Context] derived request context
-      def follow_up(url:, relation:, origin_url: self.origin_url)
-        self.class.new(
-          url:,
-          headers:,
-          request:,
-          relation:,
-          origin_url:,
-          policy:,
-          budget:
-        )
+      def follow_up(url:, relation:, origin_url: self.origin_url, advertisement: nil)
+        self.class.new(url:, headers:, request:, relation:, origin_url:, policy:, budget:, advertisement:)
       end
 
       private
@@ -68,11 +68,12 @@ module Html2rss
       def assign_request_options(request_options)
         @relation = request_options.fetch(:relation, :initial)
         @policy = request_options.fetch(:policy, Policy.default)
-        raise ArgumentError, 'policy must not be nil' if @policy.nil?
+        raise InvalidError, 'policy must not be nil' if @policy.nil?
 
         @origin_url = normalized_origin_url(request_options[:origin_url])
-        @budget = request_options.fetch(:budget) { raise ArgumentError, 'budget is required' }
-        raise ArgumentError, 'budget must not be nil' if @budget.nil?
+        @advertisement = request_options[:advertisement]
+        @budget = request_options.fetch(:budget) { raise InvalidError, 'budget is required' }
+        raise InvalidError, 'budget must not be nil' if @budget.nil?
       end
 
       def normalized_origin_url(origin_url)
@@ -87,9 +88,7 @@ module Html2rss
       end
 
       def normalize_request(request)
-        normalized = HashUtil.deep_symbolize_keys(request, context: 'request')
-        HashUtil.assert_symbol_keys!(normalized, context: 'request')
-        normalized
+        HashUtil.deep_symbolize_keys(request, context: 'request')
       end
     end
   end

data/lib/html2rss/request_service/cookie_jar.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # One host/egress-scoped view onto the {CookieStore}: replay, refresh and evict with count-only telemetry.
+    # Cookie names and values are session credentials and never reach logs.
+    class CookieJar
+      # Share of replayable requests that deliberately skip replay, so the pass-rate effect stays measurable.
+      HOLDOUT_RATE = Float(ENV.fetch('BOTASAURUS_COOKIE_HOLDOUT_RATE', 0.1))
+      # Shared across jars so blocks accumulate per key within the process.
+      BREAKER = ReplayBreaker.new
+
+      ##
+      # @param host [String] target host
+      # @param scraper_base_url [String] scrape-api base URL
+      # @param proxy [String, nil] egress proxy
+      # @param user_agent [String, nil] browser user agent
+      # @param store [#fetch, #write, #evict] the configured cookie store
+      def initialize(host:, scraper_base_url:, proxy:, user_agent:, store: RequestService.cookie_store)
+        @store = store
+        @host = host
+        @key = CookieStore.key(host:, scraper_base_url:, proxy:, user_agent:)
+      end
+
+      # @return [Hash{String => String}] live stored cookies ({} when none)
+      def stored = store.fetch(key)
+
+      ##
+      # @param stored [Hash{String => String}] cookies in the jar
+      # @return [Boolean] whether to replay them (not when empty, the breaker is open, or in the holdout)
+      def replayable?(stored)
+        return false if stored.empty?
+
+        reason = skip_reason
+        log(reason, 0) if reason
+        reason.nil?
+      end
+
+      ##
+      # Records what a replay led to: a pass resets the breaker; a block evicts the jar and counts toward it.
+      #
+      # @param outcome [Symbol] +:pass+ or +:block+
+      # @param stored [Hash{String => String}] cookies that were replayed
+      # @return [void]
+      def replayed(outcome, stored)
+        log("replay_#{outcome}", stored.size)
+        return BREAKER.record_pass(key) if outcome == :pass
+
+        evict(stored)
+        log('breaker_open', 0) if BREAKER.record_block(key, ttl: CookieStore::DEFAULT_TTL_SECONDS)
+      end
+
+      ##
+      # @param stored [Hash{String => String}] cookies that were replayed
+      # @param earned [Hash{String => String}] cookies returned by the scrape
+      # @return [void]
+      def refresh(stored, earned)
+        ttl = CookieStore::DEFAULT_TTL_SECONDS
+        return if earned.empty? || ttl <= 0
+
+        store.write(key, stored.merge(earned), ttl:)
+        log('refresh', earned.size)
+      end
+
+      ##
+      # @param stored [Hash{String => String}] cookies that were rejected
+      # @return [void]
+      def evict(stored)
+        store.evict(key)
+        log('evict', stored.size)
+      end
+
+      private
+
+      attr_reader :store, :key, :host
+
+      def skip_reason
+        return 'breaker_open' if BREAKER.open?(key)
+
+        'holdout' if rand < HOLDOUT_RATE
+      end
+
+      def log(action, count)
+        Log.info({ event_name: 'botasaurus.cookie_jar', action:, host:, cookie_count: count }.to_json)
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/cookie_store.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Html2rss
+  class RequestService
+    ##
+    # In-process cookie jar for anti-bot clearance cookies returned by botasaurus-scrape-api.
+    #
+    # The scrape API is stateless, so html2rss owns reuse. Any object responding to
+    # +fetch(key)+, +write(key, cookies, ttl:)+ and +evict(key)+ can replace this store via
+    # {RequestService.cookie_store=} (e.g. Redis or a database in html2rss-web).
+    #
+    # Values are session credentials: never log, dump or persist them in feed config.
+    class CookieStore
+      # Seconds stored cookies are replayed (the API returns no expiry; 0 disables reuse).
+      DEFAULT_TTL_SECONDS = Integer(ENV.fetch('BOTASAURUS_COOKIE_TTL_SECONDS', 1800))
+
+      ##
+      # Builds a jar key from everything a clearance cookie is bound to.
+      # The digest keeps proxy credentials out of keys, logs and external stores.
+      #
+      # @param host [String] target host
+      # @param scraper_base_url [String] scrape-api base URL
+      # @param proxy [String, nil] egress proxy
+      # @param user_agent [String, nil] browser user agent
+      # @return [String] opaque jar key
+      def self.key(host:, scraper_base_url:, proxy:, user_agent:)
+        Digest::SHA256.hexdigest([host, scraper_base_url, proxy, user_agent].join("\0"))
+      end
+
+      def initialize
+        @entries = {}
+        @mutex = Mutex.new
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [Hash{String => String}] live cookies, {} when absent or expired
+      def fetch(key)
+        @mutex.synchronize do
+          cookies, expires_at = @entries[key]
+          next {} if cookies.nil?
+          next cookies.dup if now < expires_at
+
+          @entries.delete(key)
+          {}
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @param cookies [Hash{String => String}] cookies to keep
+      # @param ttl [Numeric] seconds until expiry
+      # @return [void]
+      def write(key, cookies, ttl: DEFAULT_TTL_SECONDS)
+        @mutex.synchronize do
+          purge_expired
+          @entries[key] = [cookies.dup.freeze, now + ttl]
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [void]
+      def evict(key)
+        @mutex.synchronize { @entries.delete(key) }
+      end
+
+      private
+
+      def purge_expired
+        @entries.delete_if { |_key, (_cookies, expires_at)| expires_at <= now }
+      end
+
+      def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/html2rss/request_service/feed_advertisement.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Proof that a page advertises feed URLs in its head (+link[rel=alternate]+ with an
+    # RSS/Atom/RDF type and an +http(s)+ href, see {Html::FeedLink}).
+    #
+    # Built only from a parsed document ({.from_document}), so a caller cannot mint permission
+    # for an arbitrary URL: +new+ and +allocate+ are private, an instance and its +feeds+ are
+    # frozen, and +dup+/+clone+ return the instance itself (there is no +with+ copy).
+    # Reflection (+send(:new)+, +instance_variable_set+ on an allocated object, +Marshal.load+)
+    # still can, as nothing in-process can stop it; {Policy} also rejects an unfrozen proof.
+    #
+    # {Policy} lets an +:advertised_feed+ follow-up leave the page origin only for a URL listed
+    # here. A feed off the page origin is recorded as +https+ (an advertised +http:+ feed is
+    # fetched as +https:+ or not at all).
+    #
+    # +feeds+ maps each advertised feed URL to its link title, in document order.
+    class FeedAdvertisement
+      private_class_method :new, :allocate
+
+      ##
+      # @param doc [Nokogiri::HTML::Document] parsed page
+      # @param page_url [String, Html2rss::Url] URL the page was fetched from
+      # @return [FeedAdvertisement] the page's advertised feeds (possibly none)
+      def self.from_document(doc, page_url:)
+        page = Url.from_absolute(page_url)
+        feeds = Html::FeedLink.from_document(doc).each_with_object({}) do |link, acc|
+          url = feed_url(link.href, page)
+          acc[url] ||= link.title if url
+        end
+        new(page_url: page, feeds: feeds.freeze)
+      end
+
+      def self.feed_url(href, page)
+        return unless (url = Url.resolve_web(href, page))
+
+        Policy.same_origin?(url, page) ? url : url.with_scheme('https')
+      end
+      private_class_method :feed_url
+
+      ##
+      # @return [Html2rss::Url] the page the feeds were read from
+      attr_reader :page_url
+
+      # @return [Hash{Html2rss::Url => String, nil}] advertised feed URL => link title, in document order
+      attr_reader :feeds
+
+      # @param page_url [Html2rss::Url]
+      # @param feeds [Hash{Html2rss::Url => String, nil}] frozen
+      def initialize(page_url:, feeds:)
+        @page_url = page_url
+        @feeds = feeds
+        freeze
+      end
+
+      ##
+      # @param url [Html2rss::Url]
+      # @return [Boolean] whether the page advertises +url+
+      def advertises?(url) = feeds.key?(url)
+
+      # @return [FeedAdvertisement] itself: a proof is immutable, so no copy is ever minted
+      def dup = self
+
+      # The proof stays frozen whatever +freeze:+ asks.
+      #
+      # @return [FeedAdvertisement] itself
+      def clone(**) = self
+    end
+  end
+end

data/lib/html2rss/request_service/httpx_strategy.rb

@@ -154,7 +154,7 @@ module Html2rss
       end
 
       def begin_terminal_url_request!(terminal_url)
-        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
+        validate_redirect!(terminal_url)
         @request_url_override = terminal_url
         @last_redirect_to = nil
         @current_url = terminal_url
@@ -213,14 +213,18 @@ module Html2rss
       def redirect_callback
         lambda do |redirect_uri|
           to_url = normalize_url(redirect_uri)
-          from_url = @current_url
           @last_redirect_to = to_url
-          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+          validate_redirect!(to_url)
           @current_url = to_url
           true
         end
       end
 
+      def validate_redirect!(to_url)
+        ctx.policy.validate_redirect!(from_url: @current_url, to_url:, origin_url: ctx.origin_url,
+                                      relation: ctx.relation, advertisement: ctx.advertisement)
+      end
+
       def build_response(response)
         headers = response.headers.to_h
         Response.new(

data/lib/html2rss/request_service/local_file_strategy.rb

@@ -20,12 +20,11 @@ module Html2rss
 
       ##
       # @return [Response] the mock response wrapped around the file contents
-      # @raise [ArgumentError] if the local file path is missing
+      # @raise [InvalidOption] if the local file path is missing
       # @raise [Errno::ENOENT] if the file does not exist
       def fetch
         file_path = ctx.request[:local_file_path]
-        raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
-        raise Errno::ENOENT, "File not found: #{file_path}" unless File.exist?(file_path)
+        raise InvalidOption, 'Local file path is required for local_file strategy' unless file_path
 
         body = File.read(file_path)
         Response.new(

data/lib/html2rss/request_service/network_guard.rb

@@ -68,7 +68,7 @@ module Html2rss
       def validate_remote_ip!(ip:, url:)
         return if allow_private_networks?
 
-        parsed_ip = ip && !ip.to_s.empty? ? parse_ip(ip) : nil
+        parsed_ip = parse_ip(ip.to_s)
         raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
         return unless blocked_ip?(parsed_ip)
 
@@ -122,7 +122,8 @@ module Html2rss
       end
 
       def blocked_ip?(address)
-        BLOCKED_IP_RANGES.any? { |range| range.include?(address) }
+        native = address.native
+        BLOCKED_IP_RANGES.any? { |range| range.include?(native) }
       end
     end
   end