html2rss 0.33.0 → 0.34.0

data/lib/html2rss/request_service/cookie_store.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Html2rss
+  class RequestService
+    ##
+    # In-process cookie jar for anti-bot clearance cookies returned by botasaurus-scrape-api.
+    #
+    # The scrape API is stateless, so html2rss owns reuse. Any object responding to
+    # +fetch(key)+, +write(key, cookies, ttl:)+ and +evict(key)+ can replace this store via
+    # {RequestService.cookie_store=} (e.g. Redis or a database in html2rss-web).
+    #
+    # Values are session credentials: never log, dump or persist them in feed config.
+    class CookieStore
+      # Seconds stored cookies are replayed (the API returns no expiry; 0 disables reuse).
+      DEFAULT_TTL_SECONDS = Integer(ENV.fetch('BOTASAURUS_COOKIE_TTL_SECONDS', 1800))
+
+      ##
+      # Builds a jar key from everything a clearance cookie is bound to.
+      # The digest keeps proxy credentials out of keys, logs and external stores.
+      #
+      # @param host [String] target host
+      # @param scraper_base_url [String] scrape-api base URL
+      # @param proxy [String, nil] egress proxy
+      # @param user_agent [String, nil] browser user agent
+      # @return [String] opaque jar key
+      def self.key(host:, scraper_base_url:, proxy:, user_agent:)
+        Digest::SHA256.hexdigest([host, scraper_base_url, proxy, user_agent].join("\0"))
+      end
+
+      def initialize
+        @entries = {}
+        @mutex = Mutex.new
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [Hash{String => String}] live cookies, {} when absent or expired
+      def fetch(key)
+        @mutex.synchronize do
+          cookies, expires_at = @entries[key]
+          next {} if cookies.nil?
+          next cookies.dup if now < expires_at
+
+          @entries.delete(key)
+          {}
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @param cookies [Hash{String => String}] cookies to keep
+      # @param ttl [Numeric] seconds until expiry
+      # @return [void]
+      def write(key, cookies, ttl: DEFAULT_TTL_SECONDS)
+        @mutex.synchronize do
+          purge_expired
+          @entries[key] = [cookies.dup.freeze, now + ttl]
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [void]
+      def evict(key)
+        @mutex.synchronize { @entries.delete(key) }
+      end
+
+      private
+
+      def purge_expired
+        @entries.delete_if { |_key, (_cookies, expires_at)| expires_at <= now }
+      end
+
+      def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/html2rss/request_service/feed_advertisement.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Proof that a page advertises feed URLs in its head (+link[rel=alternate]+ with an
+    # RSS/Atom/RDF type and an +http(s)+ href, see {Html::FeedLink}).
+    #
+    # Built only from a parsed document ({.from_document}), so a caller cannot mint permission
+    # for an arbitrary URL: +new+ and +allocate+ are private, an instance and its +feeds+ are
+    # frozen, and +dup+/+clone+ return the instance itself (there is no +with+ copy).
+    # Reflection (+send(:new)+, +instance_variable_set+ on an allocated object, +Marshal.load+)
+    # still can, as nothing in-process can stop it; {Policy} also rejects an unfrozen proof.
+    #
+    # {Policy} lets an +:advertised_feed+ follow-up leave the page origin only for a URL listed
+    # here. A feed off the page origin is recorded as +https+ (an advertised +http:+ feed is
+    # fetched as +https:+ or not at all).
+    #
+    # +feeds+ maps each advertised feed URL to its link title, in document order.
+    class FeedAdvertisement
+      private_class_method :new, :allocate
+
+      ##
+      # @param doc [Nokogiri::HTML::Document] parsed page
+      # @param page_url [String, Html2rss::Url] URL the page was fetched from
+      # @return [FeedAdvertisement] the page's advertised feeds (possibly none)
+      def self.from_document(doc, page_url:)
+        page = Url.from_absolute(page_url)
+        feeds = Html::FeedLink.from_document(doc).each_with_object({}) do |link, acc|
+          url = feed_url(link.href, page)
+          acc[url] ||= link.title if url
+        end
+        new(page_url: page, feeds: feeds.freeze)
+      end
+
+      def self.feed_url(href, page)
+        return unless (url = Url.resolve_web(href, page))
+
+        Policy.same_origin?(url, page) ? url : url.with_scheme('https')
+      end
+      private_class_method :feed_url
+
+      ##
+      # @return [Html2rss::Url] the page the feeds were read from
+      attr_reader :page_url
+
+      # @return [Hash{Html2rss::Url => String, nil}] advertised feed URL => link title, in document order
+      attr_reader :feeds
+
+      # @param page_url [Html2rss::Url]
+      # @param feeds [Hash{Html2rss::Url => String, nil}] frozen
+      def initialize(page_url:, feeds:)
+        @page_url = page_url
+        @feeds = feeds
+        freeze
+      end
+
+      ##
+      # @param url [Html2rss::Url]
+      # @return [Boolean] whether the page advertises +url+
+      def advertises?(url) = feeds.key?(url)
+
+      # @return [FeedAdvertisement] itself: a proof is immutable, so no copy is ever minted
+      def dup = self
+
+      # The proof stays frozen whatever +freeze:+ asks.
+      #
+      # @return [FeedAdvertisement] itself
+      def clone(**) = self
+    end
+  end
+end

data/lib/html2rss/request_service/httpx_strategy.rb

@@ -154,7 +154,7 @@ module Html2rss
       end
 
       def begin_terminal_url_request!(terminal_url)
-        ctx.policy.validate_request!(url: terminal_url, origin_url: ctx.origin_url, relation: ctx.relation)
+        validate_redirect!(terminal_url)
         @request_url_override = terminal_url
         @last_redirect_to = nil
         @current_url = terminal_url
@@ -213,14 +213,18 @@ module Html2rss
       def redirect_callback
         lambda do |redirect_uri|
           to_url = normalize_url(redirect_uri)
-          from_url = @current_url
           @last_redirect_to = to_url
-          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+          validate_redirect!(to_url)
           @current_url = to_url
           true
         end
       end
 
+      def validate_redirect!(to_url)
+        ctx.policy.validate_redirect!(from_url: @current_url, to_url:, origin_url: ctx.origin_url,
+                                      relation: ctx.relation, advertisement: ctx.advertisement)
+      end
+
       def build_response(response)
         headers = response.headers.to_h
         Response.new(

data/lib/html2rss/request_service/local_file_strategy.rb

@@ -20,11 +20,11 @@ module Html2rss
 
       ##
       # @return [Response] the mock response wrapped around the file contents
-      # @raise [ArgumentError] if the local file path is missing
+      # @raise [InvalidOption] if the local file path is missing
       # @raise [Errno::ENOENT] if the file does not exist
       def fetch
         file_path = ctx.request[:local_file_path]
-        raise ArgumentError, 'Local file path is required for local_file strategy' unless file_path
+        raise InvalidOption, 'Local file path is required for local_file strategy' unless file_path
 
         body = File.read(file_path)
         Response.new(

data/lib/html2rss/request_service/policy.rb

@@ -9,18 +9,26 @@ module Html2rss
     #
     # Public façade for request limits, same-origin/follow-up rules, and
     # network reachability. DNS/host/IP enforcement lives on {NetworkGuard}.
+    #
+    # Follow-ups stay on the origin (scheme, host, port) unless the relation is
+    # +:advertised_feed+ and the URL is listed in the page's {FeedAdvertisement}; redirects
+    # of such a request may change host over +https+. Budget, scheme, public-network
+    # and response-size guards apply to every request and hop either way.
     class Policy
       # Hard ceiling for configured max_requests during a feed build.
       MAX_REQUESTS_CEILING = 10
 
+      # Default response-size ceiling (10 MiB); owns both streamed and decompressed defaults.
+      MAX_RESPONSE_BYTES = 10 * 1024 * 1024
+
       # Default policy values used when request controls are not explicitly set.
       DEFAULTS = {
         connect_timeout_seconds: Integer(ENV.fetch('HTML2RSS_CONNECT_TIMEOUT_SECONDS', 5)),
         read_timeout_seconds: Integer(ENV.fetch('HTML2RSS_READ_TIMEOUT_SECONDS', 10)),
         total_timeout_seconds: Integer(ENV.fetch('HTML2RSS_TOTAL_TIMEOUT_SECONDS', 30)),
         max_redirects: 5,
-        max_response_bytes: 5_242_880,
-        max_decompressed_bytes: 10_485_760,
+        max_response_bytes: MAX_RESPONSE_BYTES,
+        max_decompressed_bytes: MAX_RESPONSE_BYTES,
         max_requests: 1,
         allow_private_networks: false,
         allow_cross_origin_followups: false
@@ -80,6 +88,12 @@ module Html2rss
         @allow_cross_origin_followups
       end
 
+      ##
+      # @return [Array] the policy fields that shape a fetched response (for response caching)
+      def cache_key
+        [allow_private_networks?, max_redirects, max_response_bytes, max_decompressed_bytes]
+      end
+
       ##
       # Returns the default request policy.
       #
@@ -89,17 +103,30 @@ module Html2rss
         new
       end
 
+      ##
+      # Whether two URLs share an origin (scheme, host, and port).
+      #
+      # @param url [Html2rss::Url]
+      # @param other [Html2rss::Url]
+      # @return [Boolean]
+      def self.same_origin?(url, other)
+        [url.scheme, url.host, url.port] == [other.scheme, other.host, other.port]
+      end
+
       ##
       # Validates whether a request target is permitted for the given context.
       #
       # @param url [Html2rss::Url] destination URL
       # @param origin_url [Html2rss::Url] initial URL of the feed build
       # @param relation [Symbol] logical reason for the request
+      # @param advertisement [FeedAdvertisement, nil] the page's advertised feeds (+:advertised_feed+ only)
       # @return [void]
       # @raise [CrossOriginFollowUpDenied] if a follow-up leaves the origin host
       # @raise [PrivateNetworkDenied] if the target resolves to a private address
-      def validate_request!(url:, origin_url:, relation:)
-        enforce_same_origin!(url, origin_url, relation)
+      def validate_request!(url:, origin_url:, relation:, advertisement: nil)
+        enforce_same_origin!(url, origin_url, relation) do
+          advertised_feed?(origin_url, relation, advertisement) && advertisement.advertises?(url)
+        end
         network_guard.enforce_public_network!(url)
       end
 
@@ -110,14 +137,20 @@ module Html2rss
       # @param to_url [Html2rss::Url] redirect destination
       # @param origin_url [Html2rss::Url] initial URL of the feed build
       # @param relation [Symbol] logical reason for the request
+      # @param advertisement [FeedAdvertisement, nil] the page's advertised feeds (+:advertised_feed+ only)
       # @return [void]
       # @raise [UnsupportedUrlScheme] if the redirect downgrades from HTTPS to HTTP
-      def validate_redirect!(from_url:, to_url:, origin_url:, relation:)
+      # @raise [CrossOriginFollowUpDenied] if a follow-up hop leaves the origin host
+      # @raise [PrivateNetworkDenied] if the hop resolves to a private address
+      def validate_redirect!(from_url:, to_url:, origin_url:, relation:, advertisement: nil)
         if from_url.scheme == 'https' && to_url.scheme == 'http'
           raise UnsupportedUrlScheme, "Redirect downgraded from #{from_url} to #{to_url}"
         end
 
-        validate_request!(url: to_url, origin_url:, relation:)
+        enforce_same_origin!(to_url, origin_url, relation) do
+          advertised_feed?(origin_url, relation, advertisement) && to_url.scheme == 'https'
+        end
+        network_guard.enforce_public_network!(to_url)
       end
 
       ##
@@ -135,24 +168,28 @@ module Html2rss
 
       attr_reader :network_guard
 
+      # Yields only for a cross-origin follow-up: a truthy block admits it.
       def enforce_same_origin!(url, origin_url, relation)
         return if relation == :initial || allow_cross_origin_followups?
 
         enforce_follow_up_scheme!(url, origin_url)
-        return if comparable_origin(url) == comparable_origin(origin_url)
+        return if self.class.same_origin?(url, origin_url) || yield
 
         raise CrossOriginFollowUpDenied, "Cross-origin follow-up denied for #{url}"
       end
 
+      # The proof must come from the page this follow-up originates from; an unfrozen one was not built by
+      # {FeedAdvertisement.from_document}.
+      def advertised_feed?(origin_url, relation, advertisement)
+        relation == :advertised_feed && advertisement.instance_of?(FeedAdvertisement) && advertisement.frozen? &&
+          self.class.same_origin?(advertisement.page_url, origin_url)
+      end
+
       def enforce_follow_up_scheme!(url, origin_url)
         return unless origin_url.scheme == 'https' && url.scheme == 'http'
 
         raise UnsupportedUrlScheme, "Follow-up downgraded from https to http for #{url}"
       end
-
-      def comparable_origin(url)
-        [url.scheme, url.host, url.port]
-      end
     end
 
     # Shared immutable policy instance used for default request execution.

data/lib/html2rss/request_service/replay_breaker.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Process-local circuit breaker for cookie replay: after {MAX_BLOCKS} consecutive replay-then-block
+    # outcomes for a jar key, replay stays off until the cookie TTL elapses.
+    class ReplayBreaker
+      # Consecutive blocked replays before the breaker opens.
+      MAX_BLOCKS = Integer(ENV.fetch('BOTASAURUS_COOKIE_REPLAY_MAX_BLOCKS', 3))
+
+      def initialize
+        @state = {}
+        @mutex = Mutex.new
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [Boolean] true while replay is suspended for the key
+      def open?(key)
+        @mutex.synchronize do
+          open_until = @state[key]&.last
+          next false unless open_until
+          next true if now < open_until
+
+          @state.delete(key)
+          false
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @param ttl [Numeric] seconds replay stays off once the breaker trips
+      # @return [Boolean] true when this block tripped the breaker
+      def record_block(key, ttl:)
+        @mutex.synchronize do
+          blocks = (@state[key]&.first || 0) + 1
+          tripped = blocks >= MAX_BLOCKS
+          @state[key] = [blocks, (now + ttl if tripped)]
+          tripped
+        end
+      end
+
+      ##
+      # @param key [String] jar key
+      # @return [void]
+      def record_pass(key)
+        @mutex.synchronize { @state.delete(key) }
+      end
+
+      private
+
+      def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/html2rss/request_service/response.rb

@@ -6,18 +6,20 @@ module Html2rss
   class RequestService
     ##
     # To be used by strategies to provide their response.
-    class Response
+    class Response # rubocop:disable Metrics/ClassLength -- transport + capture + cache fields stay co-located
       # Default when a strategy does not attach transport telemetry.
       EMPTY_TRANSPORT_META = {}.freeze
       # Default when a strategy does not capture sub-resource responses.
       EMPTY_CAPTURED_RESPONSES = [].freeze
+      # Default when a strategy returns no cookies.
+      EMPTY_COOKIES = {}.freeze
 
       # Bodies that look like HTML even when Content-Type is missing, empty, or wrong.
       HTML_BODY_SNIFF = /\A\s*(?:<!DOCTYPE\s+html|<html)/i
-      # Content-Type markers for RSS/Atom syndication responses.
-      FEED_CT_MARKERS = %w[rss+xml atom+xml rss atom].freeze
+      # Content-Type markers for RSS/Atom/RDF syndication responses.
+      FEED_CT_MARKERS = %w[rss+xml atom+xml rdf+xml rss atom].freeze
       # Body sniff for unlabeled syndication (first 800 bytes, downcased).
-      FEED_BODY_MARKERS = ['<rss', '<feed xmlns', '<feed '].freeze
+      FEED_BODY_MARKERS = ['<rss', '<feed xmlns', '<feed ', '<rdf:rdf'].freeze
       # Charset from Content-Type or a leading <meta charset>.
       CHARSET_PARAMETER = /charset\s*=\s*["']?([\w.:-]+)/i
       # Bytes scanned for a meta charset hint (HTML spec looks at the first 1024).
@@ -31,20 +33,20 @@ module Html2rss
       # @param status [Integer, nil] the HTTP status code when available
       # @param transport_meta [Hash] allowlisted upstream telemetry (frozen when present)
       # @param captured_responses [Array<Hash>] JSON XHR/fetch bodies captured during browser scrapes
+      # @param cookies [Hash{String => String}] cookies earned during the scrape (session credentials; never log)
+      # @param cache_age [Integer, nil] seconds since fetch when served from {ResponseCache}; nil when fetched live
       # rubocop:disable-next Metrics/ParameterLists -- transport + capture fields stay co-located
       def initialize(body:, url:, headers: {}, status: nil, transport_meta: EMPTY_TRANSPORT_META,
-                     captured_responses: EMPTY_CAPTURED_RESPONSES)
+                     captured_responses: EMPTY_CAPTURED_RESPONSES, cookies: EMPTY_COOKIES, cache_age: nil)
         @body = body
+        @cache_age = cache_age
 
         @headers = (headers || {}).transform_keys(&:to_s)
         @status = status
         @url = url
-        @transport_meta = transport_meta.nil? || transport_meta.empty? ? EMPTY_TRANSPORT_META : transport_meta.freeze
-        @captured_responses = if captured_responses.nil? || captured_responses.empty?
-                                EMPTY_CAPTURED_RESPONSES
-                              else
-                                captured_responses.freeze
-                              end
+        @transport_meta = freeze_or(transport_meta, EMPTY_TRANSPORT_META)
+        @cookies = freeze_or(cookies, EMPTY_COOKIES)
+        @captured_responses = freeze_or(captured_responses, EMPTY_CAPTURED_RESPONSES)
       end
 
       # @return [String] the raw body of the response
@@ -65,6 +67,19 @@ module Html2rss
       # @return [Array<Hash>] captured JSON XHR/fetch responses (empty when unsupported)
       attr_reader :captured_responses
 
+      # @return [Hash{String => String}] cookies earned during the scrape (empty when unsupported)
+      attr_reader :cookies
+
+      # @return [Integer, nil] seconds since fetch when served from {ResponseCache}; nil when fetched live
+      attr_reader :cache_age
+
+      # @param cache_age [Integer, nil] cache age of the copy (defaults to this response's)
+      # @return [Response] an independent copy that shares no parsed document with this one
+      def replica(cache_age: @cache_age)
+        self.class.new(body: body.dup, url:, headers:, status:, transport_meta:, captured_responses:, cookies:,
+                       cache_age:)
+      end
+
       # @return [String] normalized content type header value
       def content_type = header('content-type').to_s
 
@@ -79,7 +94,7 @@ module Html2rss
         content_type.include?('text/html') || (!json_response? && !feed_response? && html_looking_body?)
       end
 
-      # @return [Boolean] whether response content is RSS/Atom (header or sniffed body)
+      # @return [Boolean] whether response content is RSS/Atom/RDF (header or sniffed body)
       def feed_response?
         return @feed_response if defined?(@feed_response)
 
@@ -101,6 +116,8 @@ module Html2rss
 
       private
 
+      def freeze_or(value, default) = value.nil? || value.empty? ? default : value.freeze
+
       # @param name [String] canonical header name
       # @return [Object, nil] header value when present
       def header(name)
@@ -155,7 +172,7 @@ module Html2rss
 
       def feed_content_type?
         ct = content_type.downcase
-        return false if ct.empty? || ct.include?('html') || ct.include?('json')
+        return false if ct.include?('html') || ct.include?('json')
 
         # Substring match against Content-Type (not Array#intersect? on a String).
         FEED_CT_MARKERS.any? { |marker| ct.include?(marker) }

data/lib/html2rss/request_service/response_cache.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Bounded, thread-safe TTL cache for initial page fetches; install with
+    # {RequestService.response_cache=}. Only successful (2xx) responses are stored, so a
+    # block page or rate limit is never replayed. Hits are returned tagged with their
+    # {Response#cache_age}; memory is capped in body bytes as well as entries.
+    class ResponseCache
+      # Default entry ceiling; oldest entries are evicted first.
+      DEFAULT_MAX_ENTRIES = 32
+      # Default body-byte budget across all entries.
+      DEFAULT_MAX_BYTES = 32 * 1024 * 1024
+      # Monotonic seconds source, immune to wall-clock jumps.
+      MONOTONIC_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      Entry = Data.define(:stored_at, :response, :bytes)
+      private_constant :Entry
+
+      # @return [Integer] seconds an entry stays fresh
+      attr_reader :ttl
+
+      ##
+      # @param ttl [Integer] seconds an entry stays fresh
+      # @param max_entries [Integer] maximum stored entries
+      # @param max_bytes [Integer] maximum summed body bytes
+      # @param clock [#call] monotonic seconds source (injectable for specs)
+      def initialize(ttl:, max_entries: DEFAULT_MAX_ENTRIES, max_bytes: DEFAULT_MAX_BYTES, clock: MONOTONIC_CLOCK)
+        @ttl = Option.integer(:ttl, ttl, min: 1)
+        @max_entries = Option.integer(:max_entries, max_entries, min: 1)
+        @max_bytes = Option.integer(:max_bytes, max_bytes, min: 1)
+        @clock = clock
+        @entries = {}
+        @bytes = 0
+        @mutex = Mutex.new
+        @flight_done = ConditionVariable.new
+        @inflight = Set.new
+      end
+
+      ##
+      # Concurrent callers missing the same key share one fetch: the first builds the response while
+      # the others wait, then each is served its own replica. A raising or uncacheable fetch is not
+      # stored; waiters then retry in turn so none is left blocked.
+      # @param key [Object] hashable cache key
+      # @yieldreturn [Response] the freshly fetched response, built only on a miss
+      # @return [Response] a cache-aged replica on a hit, otherwise the freshly built response
+      def fetch(key, &)
+        cached = claim(key)
+        return cached if cached
+
+        lead(key, &)
+      end
+
+      private
+
+      # Returns a fresh replica, or nil after registering the caller as the key's fetcher.
+      def claim(key)
+        @mutex.synchronize do
+          loop do
+            hit = lookup(key)
+            break hit if hit
+            break @inflight.add(key) && nil unless @inflight.include?(key)
+
+            @flight_done.wait(@mutex)
+          end
+        end
+      end
+
+      def lead(key)
+        yield.tap { |response| store(key, response) if cacheable?(response) }
+      ensure
+        @mutex.synchronize do
+          @inflight.delete(key)
+          @flight_done.broadcast
+        end
+      end
+
+      # Callers must hold @mutex.
+      def lookup(key)
+        entry = @entries[key]
+        age = entry && (@clock.call - entry.stored_at)
+        entry.response.replica(cache_age: age.floor) if age && age < @ttl
+      end
+
+      def store(key, response)
+        @mutex.synchronize do
+          evict(key)
+          bytes = response.body.bytesize
+          @entries[key] = Entry.new(stored_at: @clock.call, response:, bytes:)
+          @bytes += bytes
+          evict_oldest while @entries.size > @max_entries || @bytes > @max_bytes
+        end
+      end
+
+      def evict(key)
+        @bytes -= @entries.delete(key)&.bytes.to_i
+      end
+
+      def evict_oldest
+        _key, entry = @entries.shift
+        @bytes -= entry.bytes
+      end
+
+      def cacheable?(response)
+        (200..299).cover?(response.status) && response.body.bytesize <= @max_bytes
+      end
+    end
+  end
+end