html2rss 0.22.2 → 0.24.0

data/lib/html2rss/html/rendering/description_builder.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module Rendering
+      # Builds a sanitized article description from the base text, title, and optional media.
+      #
+      # Combines media elements (images, audio, video, PDFs) with sanitized text content
+      # to create rich RSS descriptions that reveal more scraped information.
+      #
+      # @example Basic usage
+      #   builder = DescriptionBuilder.new(
+      #     base: "Article content",
+      #     title: "Article Title",
+      #     url: "https://example.com",
+      #     enclosures: [enclosure_object],
+      #     image: "https://example.com/image.jpg"
+      #   )
+      #   description = builder.call
+      class DescriptionBuilder
+        # Removes the specified pattern from the beginning of the text
+        # within a given range if the pattern occurs before the range's end.
+        #
+        # @param text [String]
+        # @param pattern [String]
+        # @param end_of_range [Integer] Optional, defaults to half the text length
+        # @return [String]
+        def self.remove_pattern_from_start(text, pattern, end_of_range: (text.size * 0.5).to_i)
+          return text unless text.is_a?(String) && pattern.is_a?(String) && !pattern.empty?
+
+          index = text.index(pattern)
+          return text if index.nil? || index > end_of_range
+
+          "#{text[0, index]}#{text[(index + pattern.size)..]}"
+        end
+
+        # @param base [String] The base text content for the description
+        # @param title [String] The article title (used for alt text and title removal)
+        # @param url [String, Html2rss::Url] The article URL (used for sanitization)
+        # @param enclosures [Array<Html2rss::Article::Enclosure>, nil] Media enclosures
+        # @param image [String, Html2rss::Url, nil] Fallback image URL
+        def initialize(base:, title:, url:, enclosures:, image:)
+          @base = base.to_s
+          @title = title
+          @url = url
+          @enclosures = Array(enclosures)
+          @image = image
+        end
+
+        # Generates the complete description with media and sanitized text.
+        #
+        # @return [String, nil] The complete description or nil if empty
+        def call
+          fragments = []
+          fragments.concat(Array(rendered_media))
+          fragments << processed_base_description
+
+          result = fragments.compact.join("\n\n").strip
+          result.empty? ? nil : result
+        end
+
+        private
+
+        def rendered_media
+          rendered = render_enclosures
+          return rendered if rendered.any?
+          return render_fallback_image if @image
+
+          []
+        end
+
+        def render_enclosures
+          @enclosures.filter_map do |enclosure|
+            MediaRenderer.for(enclosure:, image: @image, title: @title)&.to_html
+          end
+        end
+
+        def render_fallback_image
+          [MediaRenderer.for(enclosure: nil, image: @image, title: @title)&.to_html]
+        end
+
+        def processed_base_description
+          text = self.class.remove_pattern_from_start(@base, @title)
+          Html2rss::Selectors::PostProcessors::SanitizeHtml.get(text, @url)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/image_renderer.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <img> tag from a URL and title.
+      class ImageRenderer
+        # @param url [String, Html2rss::Url] image URL for the src attribute
+        # @param title [String, nil] title/alt text for the image
+        def initialize(url:, title:)
+          @url = url
+          @title = title
+        end
+
+        # @return [String] HTML image snippet for article rendering
+        def to_html
+          attributes = [
+            %(src="#{escaped_url}"),
+            %(alt="#{escaped_title}"),
+            %(title="#{escaped_title}"),
+            'loading="lazy"',
+            'referrerpolicy="no-referrer"',
+            'decoding="async"',
+            'crossorigin="anonymous"'
+          ]
+          "<img #{attributes.join(' ')}>"
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+
+        def escaped_title
+          CGI.escapeHTML(@title.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/media_renderer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    module Rendering
+      # Factory: picks the appropriate renderer for a given enclosure or fallback image.
+      class MediaRenderer
+        # @param enclosure [Html2rss::Article::Enclosure, nil]
+        # @param image [String, Html2rss::Url, nil] Fallback image URL
+        # @param title [String]
+        # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
+        def self.for(enclosure:, image:, title:)
+          return ImageRenderer.new(url: image, title:) if enclosure.nil? && image
+          return nil unless enclosure
+
+          create_renderer_for_type(enclosure.type, url: enclosure.url, title:)
+        end
+
+        # @private
+        # @param type [String, nil] enclosure MIME type
+        # @param url [String, Html2rss::Url] enclosure URL
+        # @param title [String, nil] title used by image renderer
+        # @return [ImageRenderer, VideoRenderer, AudioRenderer, PdfRenderer, nil]
+        def self.create_renderer_for_type(type, url:, title:)
+          case type
+          when %r{^image/}
+            ImageRenderer.new(url:, title:)
+          when %r{^video/}
+            VideoRenderer.new(url:, type:)
+          when %r{^audio/}
+            AudioRenderer.new(url:, type:)
+          when 'application/pdf'
+            PdfRenderer.new(url:)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/pdf_renderer.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <iframe> for PDF documents.
+      class PdfRenderer
+        # @param url [String, Html2rss::Url] PDF URL rendered in the iframe
+        def initialize(url:)
+          @url = url
+        end
+
+        # @return [String] HTML iframe snippet for PDF rendering
+        def to_html
+          attributes = [
+            %(src="#{escaped_url}"),
+            'width="100%"',
+            'height="75vh"',
+            'sandbox=""',
+            'referrerpolicy="no-referrer"',
+            'loading="lazy"'
+          ]
+          "<iframe #{attributes.join(' ')}></iframe>"
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering/video_renderer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Html
+    module Rendering
+      # Renders an HTML <video> tag from a URL and type.
+      class VideoRenderer
+        # @param url [String, Html2rss::Url] media URL for the video source
+        # @param type [String] MIME type for the video source
+        def initialize(url:, type:)
+          @url = url
+          @type = type
+        end
+
+        # @return [String] HTML video snippet for article rendering
+        def to_html
+          [
+            '<video controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous" playsinline>',
+            %(<source src="#{escaped_url}" type="#{escaped_type}">),
+            '</video>'
+          ].join
+        end
+
+        private
+
+        def escaped_url
+          CGI.escapeHTML(@url.to_s)
+        end
+
+        def escaped_type
+          CGI.escapeHTML(@type.to_s)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/html/rendering.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    # Namespace for HTML rendering logic, used to generate rich content such as
+    # images, audio, video, or embedded documents for feed descriptions.
+    #
+    # @see Html2rss::Html::Rendering::DescriptionBuilder
+    #
+    # @example
+    #   Html2rss::Html::Rendering::ImageRenderer.new(
+    #     url: "https://example.com/image.jpg",
+    #     title: "Example"
+    #   ).to_html
+    #
+    # @example
+    #   Html2rss::Html::Rendering::MediaRenderer.for(
+    #     enclosure: nil,
+    #     image: "https://example.com/image.jpg",
+    #     title: "Example"
+    #   )
+    module Rendering
+    end
+  end
+end

data/lib/html2rss/html.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # DOM helpers for navigating and extracting from HTML trees.
+  module Html
+  end
+end

data/lib/html2rss/request_service/blocked_surface.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Shared anti-bot/interstitial signatures used by request and auto-source flows.
+    #
+    # This module centralizes signature matching so request-time guards and
+    # auto-source surface classification stay consistent.
+    module BlockedSurface
+      # Known interstitial fingerprints used to detect blocked or anti-bot surfaces.
+      INTERSTITIAL_SIGNATURES = [
+        {
+          key: :cloudflare_interstitial,
+          min_matches: 2,
+          patterns: [
+            %r{<title>\s*just a moment\.\.\.\s*</title>}i,
+            /checking your browser before accessing/i,
+            /please (?:enable|turn on) javascript and cookies/i,
+            %r{cdn-cgi/challenge-platform}i,
+            /cloudflare ray id/i
+          ],
+          message: 'Blocked surface detected: Cloudflare anti-bot interstitial page. ' \
+                   'Retry with --strategy browserless, try a more specific public listing URL, ' \
+                   'or run from an environment that can complete anti-bot checks.'
+        }
+      ].freeze
+
+      ##
+      # Returns the first matching interstitial signature for the provided body.
+      #
+      # @param body [String, nil] response body candidate
+      # @return [Hash, nil] signature hash when matched, otherwise nil
+      def self.interstitial_signature_for(body)
+        normalized_body = normalize_body(body)
+        INTERSTITIAL_SIGNATURES.find { |signature| interstitial_signature_match?(normalized_body, signature) }
+      end
+
+      ##
+      # @param body [String, nil] response body candidate
+      # @return [Boolean] true when body matches a known interstitial signature
+      def self.interstitial?(body)
+        !interstitial_signature_for(body).nil?
+      end
+
+      def self.interstitial_signature_match?(body, signature)
+        min_matches = signature.fetch(:min_matches, 1)
+        matches = 0
+
+        signature.fetch(:patterns).each do |pattern|
+          matches += 1 if pattern.match?(body)
+          return true if matches >= min_matches
+        end
+
+        false
+      end
+      private_class_method :interstitial_signature_match?
+
+      def self.normalize_body(body)
+        body.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: '')
+      rescue Encoding::CompatibilityError, Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+        body.to_s.force_encoding(Encoding::UTF_8).scrub
+      end
+      private_class_method :normalize_body
+    end
+  end
+end

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -2,11 +2,7 @@
 
 require 'json'
 
-##
-# Main html2rss namespace.
 module Html2rss
-  ##
-  # Request transport orchestration and strategies.
   class RequestService
     ##
     # Maps html2rss request/response handling to the botasaurus-scrape-api contract.
@@ -14,7 +10,7 @@ module Html2rss
       # Default Botasaurus scrape options when no explicit config is provided.
       DEFAULT_OPTIONS = {
         navigation_mode: 'auto',
-        max_retries: 2,
+        max_retries: 1,
         headless: false
       }.freeze
 
@@ -34,6 +30,17 @@ module Html2rss
         lang
       ].freeze
 
+      # Allowlisted upstream response keys exposed as Response#transport_meta.
+      META_KEYS = %w[
+        request_id strategy_used render_ms challenge_detected blocked_detected attempts error_category
+      ].freeze
+
+      # Remaining seconds at or below which Botasaurus retries are disabled.
+      TIGHT_BUDGET_SECONDS = 12
+
+      # Seconds reserved from remaining budget before setting wait_timeout_seconds.
+      BUDGET_WAIT_RESERVE_SECONDS = 2
+
       # Parsed Botasaurus response wrapper.
       class ParsedResponse
         # Fallback headers when upstream omits response headers.
@@ -94,6 +101,9 @@ module Html2rss
         # @return [String, nil] final URL reported by upstream
         def final_url = payload['final_url']
 
+        # @return [Hash{String => Object}] allowlisted upstream telemetry (frozen)
+        def transport_meta = payload.slice(*META_KEYS).compact.freeze
+
         private
 
         attr_reader :payload, :transport_status
@@ -113,6 +123,7 @@ module Html2rss
       ##
       # @param url [Html2rss::Url] canonical URL to scrape
       # @param options [Hash] validated request.botasaurus options
+      # @param remaining_timeout_seconds [Numeric, nil] shared request budget remainder for clamps
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
       # @option options [String] :wait_for_selector
@@ -125,14 +136,15 @@ module Html2rss
       # @option options [String] :user_agent
       # @option options [Array<Integer>] :window_size
       # @option options [String] :lang
-      def initialize(url:, options: {})
+      def initialize(url:, options: {}, remaining_timeout_seconds: nil)
         @url = url
         @options = options
+        @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
-      # @return [Hash] payload for POST /scrape
+      # @return [Hash] payload for POST /scrape (budget-clamped when remaining is known)
       def request_payload
-        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s)
+        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s).then { clamp_for_budget(_1) }
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
@@ -149,13 +161,25 @@ module Html2rss
 
       private
 
-      attr_reader :url, :options
+      attr_reader :url, :options, :remaining_timeout_seconds
 
       def filtered_options
         OPTION_KEYS.each_with_object({}) do |key, normalized|
           normalized[key] = options[key] if options.key?(key)
         end
       end
+
+      def clamp_for_budget(payload)
+        remaining = remaining_timeout_seconds
+        return payload if remaining.nil?
+
+        clamped = payload.dup
+        clamped[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
+        budget_wait = [1, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
+        configured = clamped[:wait_timeout_seconds]
+        clamped[:wait_timeout_seconds] = configured ? [configured, budget_wait].min : budget_wait
+        clamped
+      end
     end
   end
 end

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -8,47 +8,27 @@ module Html2rss
     ##
     # Strategy to delegate fetching to a Botasaurus scrape API.
     class BotasaurusStrategy < Strategy
-      ##
-      # Executes a Botasaurus-backed request with shared request policy guards.
-      #
-      # @return [Response] normalized request response
-      # @raise [BotasaurusConfigurationError] when BOTASAURUS_SCRAPER_URL is missing or invalid
-      # @raise [BotasaurusConnectionFailed] when Botasaurus cannot be reached or returns an invalid payload
-      # @raise [RequestTimedOut] when the Botasaurus request exceeds configured timeout
-      def execute
-        check_timeout!
-        validate_request!
+      private
+
+      def fetch
         parsed_response = post_scrape_request
         raise_if_challenge_blocked!(parsed_response)
         raise_if_upstream_failed!(parsed_response)
         build_response(parsed_response)
-      rescue Faraday::TimeoutError, Timeout::Error => error
-        raise RequestTimedOut, error.message
-      rescue Faraday::ConnectionFailed, Faraday::SSLError => error
-        raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
       end
 
-      private
-
       def post_scrape_request
         transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
         contract.parse_response(transport_response)
       end
 
-      def validate_request!
-        ctx.budget.consume!
-        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
       def build_response(parsed_response)
-        body = parsed_response.html
-        ResponseGuard.new(policy: ctx.policy).inspect_body!(body)
-
         Response.new(
-          body:,
+          body: parsed_response.html,
           headers: parsed_response.headers,
           url: response_url(parsed_response.final_url),
-          status: parsed_response.status
+          status: parsed_response.status,
+          transport_meta: parsed_response.transport_meta
         )
       end
 
@@ -73,7 +53,11 @@ module Html2rss
       end
 
       def contract
-        @contract ||= BotasaurusContract.new(url: ctx.url, options: ctx.request.fetch(:botasaurus, {}))
+        @contract ||= BotasaurusContract.new(
+          url: ctx.url,
+          options: ctx.request.fetch(:botasaurus, {}),
+          remaining_timeout_seconds: attempt_timeout_seconds
+        )
       end
 
       def client
@@ -81,10 +65,13 @@ module Html2rss
       end
 
       def request_options
-        timeout = ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds
-        raise RequestTimedOut, 'Request timed out' if timeout <= 0
+        { timeout: attempt_timeout_seconds.to_i }
+      end
 
-        { timeout: timeout.to_i }
+      def attempt_timeout_seconds
+        @attempt_timeout_seconds ||= ctx.budget.effective_timeout_seconds(
+          fallback: ctx.policy.total_timeout_seconds
+        )
       end
 
       def content_type_header
@@ -101,6 +88,10 @@ module Html2rss
           raise BotasaurusConfigurationError, "BOTASAURUS_SCRAPER_URL is invalid: #{error.message}"
         end
       end
+
+      def translate_connection_error(error)
+        raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
+      end
     end
   end
 end

data/lib/html2rss/request_service/browserless_strategy.rb

@@ -31,19 +31,6 @@ module Html2rss
     # are aligned with the default values.
     # @see https://github.com/browserless/browserless/pkgs/container/chromium
     class BrowserlessStrategy < Strategy
-      ##
-      # Executes a Browserless-backed request with the shared request policy.
-      #
-      # @return [Response] normalized request response
-      # @raise [RequestTimedOut] if the browser session exceeds the configured timeout
-      def execute
-        check_timeout!
-        validate_request!
-        execute_browserless_request
-      rescue Puppeteer::TimeoutError => error
-        raise RequestTimedOut, error.message
-      end
-
       ##
       # @return [String] the Browserless websocket endpoint with token query param
       # @raise [ArgumentError] if a custom endpoint is configured without an API token
@@ -58,12 +45,7 @@ module Html2rss
 
       private
 
-      def validate_request!
-        ctx.budget.consume!
-        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
-      end
-
-      def execute_browserless_request
+      def fetch
         connect_with_timeout_support do |browser|
           PuppetCommander.new(ctx, browser).call
         ensure
@@ -72,10 +54,7 @@ module Html2rss
       end
 
       def protocol_timeout_ms
-        timeout = ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds
-        raise RequestTimedOut, 'Request timed out' if timeout <= 0
-
-        (timeout * 1000).to_i
+        ctx.budget.effective_timeout_ms(fallback: ctx.policy.total_timeout_seconds)
       end
 
       def connect_with_timeout_support(&)