html2rss 0.17.0 → 0.19.0

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'json'
+
+##
+# Main html2rss namespace.
+module Html2rss
+  ##
+  # Request transport orchestration and strategies.
+  class RequestService
+    ##
+    # Maps html2rss request/response handling to the botasaurus-scrape-api contract.
+    class BotasaurusContract
+      # Default Botasaurus scrape options when no explicit config is provided.
+      DEFAULT_OPTIONS = {
+        navigation_mode: 'auto',
+        max_retries: 2,
+        headless: false
+      }.freeze
+
+      # Allowlisted request.botasaurus keys forwarded to upstream.
+      OPTION_KEYS = %i[
+        navigation_mode
+        max_retries
+        wait_for_selector
+        wait_timeout_seconds
+        block_images
+        block_images_and_css
+        wait_for_complete_page_load
+        headless
+        proxy
+        user_agent
+        window_size
+        lang
+      ].freeze
+
+      # Parsed Botasaurus response wrapper.
+      class ParsedResponse
+        # Fallback headers when upstream omits response headers.
+        DEFAULT_HEADERS = { 'content-type' => 'text/html' }.freeze
+
+        # @param payload [Hash{String => Object}] parsed Botasaurus response payload
+        # @param transport_status [Integer] HTTP status returned by Botasaurus
+        def initialize(payload:, transport_status:)
+          @payload = payload
+          @transport_status = transport_status
+        end
+
+        # @return [Boolean] true when upstream classified request as challenge blocked
+        def challenge_block? = error_category == 'challenge_block'
+
+        # @return [Boolean] true when upstream returned non-200 or an error payload
+        def upstream_failure?
+          status != 200 || error_message?
+        end
+
+        # @return [String] normalized challenge error message
+        def challenge_message
+          error || 'Botasaurus challenge block detected.'
+        end
+
+        # @return [String] actionable upstream failure summary
+        def upstream_failure_message
+          details = ["status=#{status}"]
+          details << "error_category=#{error_category}" if error_category
+          details << "error=#{error}" if error
+          details << "request_id=#{request_id}" if request_id
+          "Botasaurus scrape failed (#{details.join(', ')})."
+        end
+
+        # @return [String] rendered HTML body from Botasaurus
+        # @raise [BotasaurusConnectionFailed] when html is missing
+        def html
+          value = payload['html']
+          raise BotasaurusConnectionFailed, "Botasaurus response missing required 'html' field" if value.nil?
+
+          value.to_s
+        end
+
+        # @return [Hash{String => String}] normalized response headers
+        def headers
+          raw_headers = payload['headers']
+          return DEFAULT_HEADERS.dup unless raw_headers.is_a?(Hash) && raw_headers.any?
+
+          raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
+        end
+
+        # @return [Integer] resolved status code (payload status_code or transport status)
+        def status
+          status_code = payload['status_code']
+          status_code.is_a?(Integer) ? status_code : transport_status
+        end
+
+        # @return [String, nil] final URL reported by upstream
+        def final_url = payload['final_url']
+
+        private
+
+        attr_reader :payload, :transport_status
+
+        def error = payload['error']
+
+        def request_id = payload['request_id']
+
+        def error_category = payload['error_category']
+
+        def error_message?
+          value = error
+          value.is_a?(String) ? !value.empty? : !value.nil?
+        end
+      end
+
+      ##
+      # @param url [Html2rss::Url] canonical URL to scrape
+      # @param options [Hash] validated request.botasaurus options
+      # @option options [String] :navigation_mode
+      # @option options [Integer] :max_retries
+      # @option options [String] :wait_for_selector
+      # @option options [Integer] :wait_timeout_seconds
+      # @option options [Boolean] :block_images
+      # @option options [Boolean] :block_images_and_css
+      # @option options [Boolean] :wait_for_complete_page_load
+      # @option options [Boolean] :headless
+      # @option options [String] :proxy
+      # @option options [String] :user_agent
+      # @option options [Array<Integer>] :window_size
+      # @option options [String] :lang
+      def initialize(url:, options: {})
+        @url = url
+        @options = options
+      end
+
+      # @return [Hash] payload for POST /scrape
+      def request_payload
+        DEFAULT_OPTIONS.merge(filtered_options).merge(url: url.to_s)
+      end
+
+      # @param transport_response [Faraday::Response] upstream HTTP response
+      # @return [ParsedResponse]
+      # @raise [BotasaurusConnectionFailed] when payload is not valid JSON object
+      def parse_response(transport_response)
+        payload = JSON.parse(transport_response.body.to_s)
+        raise BotasaurusConnectionFailed, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
+
+        ParsedResponse.new(payload:, transport_status: transport_response.status)
+      rescue JSON::ParserError => error
+        raise BotasaurusConnectionFailed, "Botasaurus response JSON parse failed: #{error.message}"
+      end
+
+      private
+
+      attr_reader :url, :options
+
+      def filtered_options
+        OPTION_KEYS.each_with_object({}) do |key, normalized|
+          normalized[key] = options[key] if options.key?(key)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'faraday'
+require 'json'
+
+module Html2rss
+  class RequestService
+    ##
+    # Strategy to delegate fetching to a Botasaurus scrape API.
+    class BotasaurusStrategy < Strategy
+      ##
+      # Executes a Botasaurus-backed request with shared request policy guards.
+      #
+      # @return [Response] normalized request response
+      # @raise [BotasaurusConfigurationError] when BOTASAURUS_SCRAPER_URL is missing or invalid
+      # @raise [BotasaurusConnectionFailed] when Botasaurus cannot be reached or returns an invalid payload
+      # @raise [RequestTimedOut] when the Botasaurus request exceeds configured timeout
+      def execute
+        validate_request!
+        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
+        parsed_response = contract.parse_response(transport_response)
+        raise_if_challenge_blocked!(parsed_response)
+        raise_if_upstream_failed!(parsed_response)
+        build_response(parsed_response)
+      rescue Faraday::TimeoutError, Timeout::Error => error
+        raise RequestTimedOut, error.message
+      rescue Faraday::ConnectionFailed, Faraday::SSLError => error
+        raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
+      end
+
+      private
+
+      def validate_request!
+        ctx.budget.consume!
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      def build_response(parsed_response)
+        body = parsed_response.html
+        ResponseGuard.new(policy: ctx.policy).inspect_body!(body)
+
+        Response.new(
+          body:,
+          headers: parsed_response.headers,
+          url: response_url(parsed_response.final_url),
+          status: parsed_response.status
+        )
+      end
+
+      def raise_if_challenge_blocked!(parsed_response)
+        return unless parsed_response.challenge_block?
+
+        raise BlockedSurfaceDetected, "Blocked surface detected: #{parsed_response.challenge_message}"
+      end
+
+      def raise_if_upstream_failed!(parsed_response)
+        return unless parsed_response.upstream_failure?
+
+        raise BotasaurusConnectionFailed, parsed_response.upstream_failure_message
+      end
+
+      def response_url(final_url)
+        return ctx.url if final_url.nil?
+
+        Html2rss::Url.from_absolute(final_url)
+      rescue ArgumentError
+        ctx.url
+      end
+
+      def contract
+        @contract ||= BotasaurusContract.new(url: ctx.url, options: ctx.request.fetch(:botasaurus, {}))
+      end
+
+      def client
+        @client ||= Faraday.new(url: scraper_base_url.to_s, request: request_options)
+      end
+
+      def request_options
+        { timeout: ctx.policy.total_timeout_seconds }
+      end
+
+      def content_type_header
+        { 'Content-Type' => 'application/json' }
+      end
+
+      def scraper_base_url
+        @scraper_base_url ||= begin
+          configured = ENV.fetch('BOTASAURUS_SCRAPER_URL') do
+            raise BotasaurusConfigurationError, 'BOTASAURUS_SCRAPER_URL is required for strategy=botasaurus.'
+          end
+          Html2rss::Url.for_channel(configured)
+        rescue ArgumentError => error
+          raise BotasaurusConfigurationError, "BOTASAURUS_SCRAPER_URL is invalid: #{error.message}"
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/browserless_strategy.rb

@@ -31,23 +31,119 @@ module Html2rss
     # are aligned with the default values.
     # @see https://github.com/browserless/browserless/pkgs/container/chromium
     class BrowserlessStrategy < Strategy
-      # return [Response]
+      ##
+      # Executes a Browserless-backed request with the shared request policy.
+      #
+      # @return [Response] normalized request response
+      # @raise [RequestTimedOut] if the browser session exceeds the configured timeout
       def execute
-        Puppeteer.connect(browser_ws_endpoint:) do |browser|
-          PuppetCommander.new(ctx, browser).call
-        ensure
-          browser.disconnect
-        end
+        validate_request!
+        execute_browserless_request
+      rescue Puppeteer::TimeoutError => error
+        raise RequestTimedOut, error.message
       end
 
+      ##
+      # @return [String] the Browserless websocket endpoint with token query param
+      # @raise [ArgumentError] if a custom endpoint is configured without an API token
       def browser_ws_endpoint
         @browser_ws_endpoint ||= begin
-          api_token = ENV.fetch('BROWSERLESS_IO_API_TOKEN', '6R0W53R135510')
           ws_url = ENV.fetch('BROWSERLESS_IO_WEBSOCKET_URL', 'ws://127.0.0.1:3000')
+          api_token = browserless_api_token(ws_url)
 
           "#{ws_url}?token=#{api_token}"
         end
       end
+
+      private
+
+      def validate_request!
+        ctx.budget.consume!
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      def execute_browserless_request
+        connect_with_timeout_support do |browser|
+          PuppetCommander.new(ctx, browser).call
+        ensure
+          browser.disconnect
+        end
+      end
+
+      def protocol_timeout_ms
+        ctx.policy.total_timeout_seconds * 1000
+      end
+
+      def connect_with_timeout_support(&)
+        connect_browserless(protocol_timeout: protocol_timeout_ms, &)
+      rescue ArgumentError => error
+        raise unless unsupported_protocol_timeout?(error)
+
+        connect_browserless(&)
+      end
+
+      def unsupported_protocol_timeout?(error)
+        error.message.include?('unknown keyword: :protocol_timeout')
+      end
+
+      def connect_browserless(protocol_timeout: nil, &)
+        connected = false
+
+        Puppeteer.connect(**browserless_connect_options(protocol_timeout)) do |browser|
+          connected = true
+          yield browser
+        end
+      rescue ArgumentError => error
+        handle_connection_error(error, connected:, protocol_timeout:)
+      rescue StandardError => error
+        handle_connection_error(error, connected:)
+      end
+
+      def browserless_connect_options(protocol_timeout)
+        { browser_ws_endpoint:, protocol_timeout: }.compact
+      end
+
+      def handle_connection_error(error, connected:, protocol_timeout: nil)
+        raise if connected || compatibility_timeout_error?(error, protocol_timeout:)
+
+        raise BrowserlessConnectionFailed, browserless_connection_message(error), cause: error
+      end
+
+      def compatibility_timeout_error?(error, protocol_timeout:)
+        protocol_timeout && unsupported_protocol_timeout?(error)
+      end
+
+      def browserless_connection_message(error)
+        base = "Browserless connection failed (#{error.class}: #{error.message})."
+        endpoint_hint = "Check BROWSERLESS_IO_WEBSOCKET_URL (currently #{browserless_websocket_url})."
+        token_hint = 'Check BROWSERLESS_IO_API_TOKEN and ensure it matches your Browserless TOKEN.'
+        local_hint = 'For local Browserless, confirm the service is running and reachable.'
+
+        if likely_authentication_error?(error)
+          "#{base} #{token_hint} #{endpoint_hint}"
+        else
+          "#{base} #{endpoint_hint} #{token_hint} #{local_hint}"
+        end
+      end
+
+      def likely_authentication_error?(error)
+        message = error.message.downcase
+        message.include?('unauthorized') || message.include?('forbidden') || message.include?('401')
+      end
+
+      def browserless_websocket_url
+        ENV.fetch('BROWSERLESS_IO_WEBSOCKET_URL', 'ws://127.0.0.1:3000')
+      end
+
+      def browserless_api_token(ws_url)
+        ENV.fetch('BROWSERLESS_IO_API_TOKEN') do
+          return '6R0W53R135510' if ws_url == 'ws://127.0.0.1:3000'
+
+          raise BrowserlessConfigurationError,
+                'BROWSERLESS_IO_API_TOKEN is required for custom Browserless endpoints. ' \
+                'Set BROWSERLESS_IO_API_TOKEN or use ws://127.0.0.1:3000 for local defaults.'
+        end
+      end
     end
   end
 end

data/lib/html2rss/request_service/budget.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Tracks how many outbound requests a single feed build may still perform.
+    class Budget
+      ##
+      # @param max_requests [Integer] the maximum number of requests allowed
+      def initialize(max_requests:)
+        unless max_requests.is_a?(Integer) && max_requests.positive?
+          raise ArgumentError, 'max_requests must be positive'
+        end
+
+        @remaining = max_requests
+        @mutex = Mutex.new
+      end
+
+      ##
+      # Consumes one request from the budget.
+      #
+      # @return [Integer] remaining request count after consumption
+      # @raise [RequestBudgetExceeded] if no requests remain
+      def consume!
+        @mutex.synchronize do
+          raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining.zero?
+
+          @remaining -= 1
+        end
+      end
+
+      ##
+      # @return [Integer] requests still available
+      def remaining
+        @mutex.synchronize { @remaining }
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/context.rb

@@ -1,45 +1,101 @@
 # frozen_string_literal: true
 
-require 'addressable/uri'
-
 module Html2rss
   class RequestService
     ##
     # Holds information needed to send requests to websites.
     # To be passed down to the RequestService's strategies.
     class Context
-      SUPPORTED_URL_SCHEMES = %w[http https].to_set.freeze
-
       ##
-      # @param url [String, Addressable::URI] the URL to request
+      # @param url [String, Html2rss::Url] the URL to request
       # @param headers [Hash] HTTP request headers
-      def initialize(url:, headers: {})
-        @url = Addressable::URI.parse(url)
-        assert_valid_url!
-
-        @headers = headers
+      # @param request [Hash] request specific options passed to strategies
+      # @param request_options [Hash] runtime request options
+      # @option request_options [Symbol] :relation why this request is being made
+      # @option request_options [String, Html2rss::Url, nil] :origin_url originating URL for same-origin checks
+      # @option request_options [Policy] :policy runtime request policy
+      # @option request_options [Budget] :budget shared request budget for the feed build
+      # @raise [ArgumentError] if policy or budget is explicitly nil
+      def initialize(url:, headers: {}, request: {}, **request_options)
+        @url = Html2rss::Url.from_absolute(url)
+        @headers = normalize_headers(headers).freeze
+        @request = normalize_request(request).freeze
+        assign_request_options(request_options)
       end
 
-      # @return [Addressable::URI] the parsed URL
+      # @return [Html2rss::Url] the parsed and normalized URL
       attr_reader :url
 
       # @return [Hash] the HTTP request headers
       attr_reader :headers
 
-      private
+      # @return [Hash] the request specific options
+      attr_reader :request
+
+      # @return [Hash] browserless specific options
+      def browserless = request.fetch(:browserless, {})
+
+      # @return [Hash, nil] preload options for browserless requests
+      def browserless_preload = browserless[:preload]
+
+      # @return [Symbol] the request relation
+      attr_reader :relation
+
+      # @return [Html2rss::Url] the initial URL for the feed build
+      attr_reader :origin_url
+
+      # @return [Policy] the runtime request policy
+      attr_reader :policy
+
+      # @return [Budget] the shared request budget
+      attr_reader :budget
 
       ##
-      # Validates the URL.
-      # @raise [InvalidUrl] if the URL is not valid
-      # @raise [UnsupportedUrlScheme] if the URL scheme is not supported
-      def assert_valid_url!
-        raise InvalidUrl, 'URL must be absolute' unless url.absolute?
-        raise InvalidUrl, 'URL must not contain an @ character' if url.to_s.include?('@')
+      # Builds a follow-up request context sharing headers, budget, and policy.
+      #
+      # @param url [String, Html2rss::Url] the follow-up URL
+      # @param relation [Symbol] why the follow-up is being made
+      # @param origin_url [String, Html2rss::Url] effective origin for same-origin checks
+      # @return [Context] derived request context
+      def follow_up(url:, relation:, origin_url: self.origin_url)
+        self.class.new(
+          url:,
+          headers:,
+          request:,
+          relation:,
+          origin_url:,
+          policy:,
+          budget:
+        )
+      end
+
+      private
 
-        return if SUPPORTED_URL_SCHEMES.include?(url.scheme)
+      def assign_request_options(request_options)
+        @relation = request_options.fetch(:relation, :initial)
+        @policy = request_options.fetch(:policy, Policy.default)
+        raise ArgumentError, 'policy must not be nil' if @policy.nil?
+
+        @origin_url = normalized_origin_url(request_options[:origin_url])
+        @budget = request_options.fetch(:budget) { Budget.new(max_requests: policy.max_requests) }
+        raise ArgumentError, 'budget must not be nil' if @budget.nil?
+      end
+
+      def normalized_origin_url(origin_url)
+        source = origin_url || @url
+        Html2rss::Url.from_absolute(source)
+      end
+
+      def normalize_headers(headers)
+        headers.to_h do |key, value|
+          [key.to_s, value]
+        end
+      end
 
-        raise UnsupportedUrlScheme,
-              "URL scheme '#{url.scheme}' is not supported"
+      def normalize_request(request)
+        normalized = HashUtil.deep_symbolize_keys(request, context: 'request')
+        HashUtil.assert_symbol_keys!(normalized, context: 'request')
+        normalized
       end
     end
   end