html2rss 0.16.0 → 0.18.0

data/lib/html2rss/rendering/video_renderer.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'cgi'
+
+module Html2rss
+  module Rendering
+    # Renders an HTML <video> tag from a URL and type.
+    class VideoRenderer
+      def initialize(url:, type:)
+        @url = url
+        @type = type
+      end
+
+      def to_html
+        %(<video controls preload="none" referrerpolicy="no-referrer" crossorigin="anonymous" playsinline>
+            <source src="#{escaped_url}" type="#{escaped_type}">
+          </video>)
+      end
+
+      private
+
+      def escaped_url
+        CGI.escapeHTML(@url.to_s)
+      end
+
+      def escaped_type
+        CGI.escapeHTML(@type.to_s)
+      end
+    end
+  end
+end

data/lib/html2rss/rendering.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Html2rss
+  # Namespace for HTML rendering logic, used to generate rich content such as
+  # images, audio, video, or embedded documents for feed descriptions.
+  #
+  # @example
+  #   Html2rss::Rendering::ImageRenderer.new(...).to_html
+  #   Html2rss::Rendering::MediaRenderer.for(...)
+  #
+  # @see Html2rss::Rendering::DescriptionBuilder
+  module Rendering
+  end
+end

data/lib/html2rss/request_controls.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Tracks runtime request controls together with whether each value was explicitly set.
+  class RequestControls
+    TOP_LEVEL_KEYS = %i[strategy].freeze
+    REQUEST_KEYS = %i[max_redirects max_requests].freeze
+
+    ##
+    # @param config [Hash<Symbol, Object>, Hash<String, Object>] raw config input
+    # @return [RequestControls] request controls extracted from the config hash
+    def self.from_config(config)
+      new(
+        strategy: value_for(config, :strategy),
+        max_redirects: request_value_for(config, :max_redirects),
+        max_requests: request_value_for(config, :max_requests),
+        explicit_keys: explicit_keys_for(config)
+      )
+    end
+
+    def self.explicit_keys_for(config)
+      TOP_LEVEL_KEYS.filter { top_level_key?(config, _1) } +
+        REQUEST_KEYS.filter { request_key?(config, _1) }
+    end
+
+    def self.value_for(config, key)
+      return config[key] if config.key?(key)
+      return config[key.to_s] if config.key?(key.to_s)
+
+      nil
+    end
+
+    def self.request_value_for(config, key)
+      request_config = value_for(config, :request)
+      return nil unless request_config.is_a?(Hash)
+
+      value_for(request_config, key)
+    end
+
+    def self.top_level_key?(config, key)
+      config.key?(key) || config.key?(key.to_s)
+    end
+
+    def self.request_key?(config, key)
+      request_config = value_for(config, :request)
+      request_config.is_a?(Hash) && top_level_key?(request_config, key)
+    end
+    private_class_method :explicit_keys_for, :request_value_for, :top_level_key?, :request_key?, :value_for
+
+    ##
+    # @param strategy [Symbol, nil] effective request strategy
+    # @param max_redirects [Integer, nil] effective redirect limit
+    # @param max_requests [Integer, nil] effective request budget
+    # @param explicit_keys [Array<Symbol>] controls explicitly supplied by the caller
+    def initialize(strategy: nil, max_redirects: nil, max_requests: nil, explicit_keys: [])
+      @strategy = strategy
+      @max_redirects = max_redirects
+      @max_requests = max_requests
+      @explicit_keys = explicit_keys.map(&:to_sym).uniq.freeze
+      freeze
+    end
+
+    ##
+    # @return [Symbol, nil] effective request strategy
+    attr_reader :strategy
+
+    ##
+    # @return [Integer, nil] effective redirect limit
+    attr_reader :max_redirects
+
+    ##
+    # @return [Integer, nil] effective request budget
+    attr_reader :max_requests
+
+    ##
+    # @param name [Symbol, String] request control name
+    # @return [Boolean] whether the control was explicitly supplied
+    def explicit?(name)
+      explicit_keys.include?(name.to_sym)
+    end
+
+    ##
+    # @param strategy [Symbol, nil] validated request strategy
+    # @param max_redirects [Integer, nil] validated redirect limit
+    # @param max_requests [Integer, nil] validated request budget
+    # @return [RequestControls] controls updated with validated effective values
+    def with_effective_values(strategy:, max_redirects:, max_requests:)
+      self.class.new(
+        strategy:,
+        max_redirects:,
+        max_requests:,
+        explicit_keys:
+      )
+    end
+
+    ##
+    # Applies only explicitly set controls to the provided config hash.
+    #
+    # @param config [Hash<Symbol, Object>] mutable config hash
+    # @return [Hash<Symbol, Object>] the same hash with explicit controls written
+    def apply_to(config)
+      config[:strategy] = strategy if explicit?(:strategy)
+      apply_request_value(config, :max_redirects, max_redirects)
+      apply_request_value(config, :max_requests, max_requests)
+      config
+    end
+
+    private
+
+    attr_reader :explicit_keys
+
+    def apply_request_value(config, key, value)
+      return unless explicit?(key)
+
+      ensure_request_config!(config)
+      config[:request][key] = value
+    end
+
+    def ensure_request_config!(config)
+      request_config = config[:request]
+      return config[:request] = {} if request_config.nil?
+      return if request_config.is_a?(Hash)
+
+      raise ArgumentError, 'request config must be a hash'
+    end
+  end
+end

data/lib/html2rss/request_service/browserless_strategy.rb

@@ -31,23 +31,119 @@ module Html2rss
     # are aligned with the default values.
     # @see https://github.com/browserless/browserless/pkgs/container/chromium
     class BrowserlessStrategy < Strategy
-      # return [Response]
+      ##
+      # Executes a Browserless-backed request with the shared request policy.
+      #
+      # @return [Response] normalized request response
+      # @raise [RequestTimedOut] if the browser session exceeds the configured timeout
       def execute
-        Puppeteer.connect(browser_ws_endpoint:) do |browser|
-          PuppetCommander.new(ctx, browser).call
-        ensure
-          browser.disconnect
-        end
+        validate_request!
+        execute_browserless_request
+      rescue Puppeteer::TimeoutError => error
+        raise RequestTimedOut, error.message
       end
 
+      ##
+      # @return [String] the Browserless websocket endpoint with token query param
+      # @raise [ArgumentError] if a custom endpoint is configured without an API token
       def browser_ws_endpoint
         @browser_ws_endpoint ||= begin
-          api_token = ENV.fetch('BROWSERLESS_IO_API_TOKEN', '6R0W53R135510')
           ws_url = ENV.fetch('BROWSERLESS_IO_WEBSOCKET_URL', 'ws://127.0.0.1:3000')
+          api_token = browserless_api_token(ws_url)
 
           "#{ws_url}?token=#{api_token}"
         end
       end
+
+      private
+
+      def validate_request!
+        ctx.budget.consume!
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      def execute_browserless_request
+        connect_with_timeout_support do |browser|
+          PuppetCommander.new(ctx, browser).call
+        ensure
+          browser.disconnect
+        end
+      end
+
+      def protocol_timeout_ms
+        ctx.policy.total_timeout_seconds * 1000
+      end
+
+      def connect_with_timeout_support(&)
+        connect_browserless(protocol_timeout: protocol_timeout_ms, &)
+      rescue ArgumentError => error
+        raise unless unsupported_protocol_timeout?(error)
+
+        connect_browserless(&)
+      end
+
+      def unsupported_protocol_timeout?(error)
+        error.message.include?('unknown keyword: :protocol_timeout')
+      end
+
+      def connect_browserless(protocol_timeout: nil, &)
+        connected = false
+
+        Puppeteer.connect(**browserless_connect_options(protocol_timeout)) do |browser|
+          connected = true
+          yield browser
+        end
+      rescue ArgumentError => error
+        handle_connection_error(error, connected:, protocol_timeout:)
+      rescue StandardError => error
+        handle_connection_error(error, connected:)
+      end
+
+      def browserless_connect_options(protocol_timeout)
+        { browser_ws_endpoint:, protocol_timeout: }.compact
+      end
+
+      def handle_connection_error(error, connected:, protocol_timeout: nil)
+        raise if connected || compatibility_timeout_error?(error, protocol_timeout:)
+
+        raise BrowserlessConnectionFailed, browserless_connection_message(error), cause: error
+      end
+
+      def compatibility_timeout_error?(error, protocol_timeout:)
+        protocol_timeout && unsupported_protocol_timeout?(error)
+      end
+
+      def browserless_connection_message(error)
+        base = "Browserless connection failed (#{error.class}: #{error.message})."
+        endpoint_hint = "Check BROWSERLESS_IO_WEBSOCKET_URL (currently #{browserless_websocket_url})."
+        token_hint = 'Check BROWSERLESS_IO_API_TOKEN and ensure it matches your Browserless TOKEN.'
+        local_hint = 'For local Browserless, confirm the service is running and reachable.'
+
+        if likely_authentication_error?(error)
+          "#{base} #{token_hint} #{endpoint_hint}"
+        else
+          "#{base} #{endpoint_hint} #{token_hint} #{local_hint}"
+        end
+      end
+
+      def likely_authentication_error?(error)
+        message = error.message.downcase
+        message.include?('unauthorized') || message.include?('forbidden') || message.include?('401')
+      end
+
+      def browserless_websocket_url
+        ENV.fetch('BROWSERLESS_IO_WEBSOCKET_URL', 'ws://127.0.0.1:3000')
+      end
+
+      def browserless_api_token(ws_url)
+        ENV.fetch('BROWSERLESS_IO_API_TOKEN') do
+          return '6R0W53R135510' if ws_url == 'ws://127.0.0.1:3000'
+
+          raise BrowserlessConfigurationError,
+                'BROWSERLESS_IO_API_TOKEN is required for custom Browserless endpoints. ' \
+                'Set BROWSERLESS_IO_API_TOKEN or use ws://127.0.0.1:3000 for local defaults.'
+        end
+      end
     end
   end
 end

data/lib/html2rss/request_service/budget.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Html2rss
+  class RequestService
+    ##
+    # Tracks how many outbound requests a single feed build may still perform.
+    class Budget
+      ##
+      # @param max_requests [Integer] the maximum number of requests allowed
+      def initialize(max_requests:)
+        unless max_requests.is_a?(Integer) && max_requests.positive?
+          raise ArgumentError, 'max_requests must be positive'
+        end
+
+        @remaining = max_requests
+        @mutex = Mutex.new
+      end
+
+      ##
+      # Consumes one request from the budget.
+      #
+      # @return [Integer] remaining request count after consumption
+      # @raise [RequestBudgetExceeded] if no requests remain
+      def consume!
+        @mutex.synchronize do
+          raise RequestBudgetExceeded, 'Request budget exhausted' if @remaining.zero?
+
+          @remaining -= 1
+        end
+      end
+
+      ##
+      # @return [Integer] requests still available
+      def remaining
+        @mutex.synchronize { @remaining }
+      end
+    end
+  end
+end

data/lib/html2rss/request_service/context.rb

@@ -1,45 +1,89 @@
 # frozen_string_literal: true
 
-require 'addressable/uri'
-
 module Html2rss
   class RequestService
     ##
     # Holds information needed to send requests to websites.
     # To be passed down to the RequestService's strategies.
     class Context
-      SUPPORTED_URL_SCHEMES = %w[http https].to_set.freeze
-
       ##
-      # @param url [String, Addressable::URI] the URL to request
+      # @param url [String, Html2rss::Url] the URL to request
       # @param headers [Hash] HTTP request headers
-      def initialize(url:, headers: {})
-        @url = Addressable::URI.parse(url)
-        assert_valid_url!
-
+      # @param request [Hash] request specific options passed to strategies
+      # @param request_options [Hash] runtime request options
+      # @option request_options [Symbol] :relation why this request is being made
+      # @option request_options [String, Html2rss::Url, nil] :origin_url originating URL for same-origin checks
+      # @option request_options [Policy] :policy runtime request policy
+      # @option request_options [Budget] :budget shared request budget for the feed build
+      # @raise [ArgumentError] if policy or budget is explicitly nil
+      def initialize(url:, headers: {}, request: {}, **request_options)
+        @url = Html2rss::Url.from_absolute(url)
         @headers = headers
+        @request = request.freeze
+        assign_request_options(request_options)
       end
 
-      # @return [Addressable::URI] the parsed URL
+      # @return [Html2rss::Url] the parsed and normalized URL
       attr_reader :url
 
       # @return [Hash] the HTTP request headers
       attr_reader :headers
 
-      private
+      # @return [Hash] the request specific options
+      attr_reader :request
+
+      # @return [Hash] browserless specific options
+      def browserless = request.fetch(:browserless, {})
+
+      # @return [Hash, nil] preload options for browserless requests
+      def browserless_preload = browserless[:preload]
+
+      # @return [Symbol] the request relation
+      attr_reader :relation
+
+      # @return [Html2rss::Url] the initial URL for the feed build
+      attr_reader :origin_url
+
+      # @return [Policy] the runtime request policy
+      attr_reader :policy
+
+      # @return [Budget] the shared request budget
+      attr_reader :budget
 
       ##
-      # Validates the URL.
-      # @raise [InvalidUrl] if the URL is not valid
-      # @raise [UnsupportedUrlScheme] if the URL scheme is not supported
-      def assert_valid_url!
-        raise InvalidUrl, 'URL must be absolute' unless url.absolute?
-        raise InvalidUrl, 'URL must not contain an @ character' if url.to_s.include?('@')
+      # Builds a follow-up request context sharing headers, budget, and policy.
+      #
+      # @param url [String, Html2rss::Url] the follow-up URL
+      # @param relation [Symbol] why the follow-up is being made
+      # @param origin_url [String, Html2rss::Url] effective origin for same-origin checks
+      # @return [Context] derived request context
+      def follow_up(url:, relation:, origin_url: self.origin_url)
+        self.class.new(
+          url:,
+          headers:,
+          request:,
+          relation:,
+          origin_url:,
+          policy:,
+          budget:
+        )
+      end
 
-        return if SUPPORTED_URL_SCHEMES.include?(url.scheme)
+      private
+
+      def assign_request_options(request_options)
+        @relation = request_options.fetch(:relation, :initial)
+        @policy = request_options.fetch(:policy, Policy.default)
+        raise ArgumentError, 'policy must not be nil' if @policy.nil?
+
+        @origin_url = normalized_origin_url(request_options[:origin_url])
+        @budget = request_options.fetch(:budget) { Budget.new(max_requests: policy.max_requests) }
+        raise ArgumentError, 'budget must not be nil' if @budget.nil?
+      end
 
-        raise UnsupportedUrlScheme,
-              "URL scheme '#{url.scheme}' is not supported"
+      def normalized_origin_url(origin_url)
+        source = origin_url || @url
+        Html2rss::Url.from_absolute(source)
       end
     end
   end

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -2,6 +2,7 @@
 
 require 'faraday'
 require 'faraday/follow_redirects'
+require 'faraday/gzip'
 
 module Html2rss
   class RequestService
@@ -9,15 +10,144 @@ module Html2rss
     # Strategy to use Faraday for the request.
     # @see https://rubygems.org/gems/faraday
     class FaradayStrategy < Strategy
-      # return [Response]
+      ##
+      # Restores buffered streamed bytes so response middleware can process them.
+      class StreamingBodyMiddleware < Faraday::Middleware
+        STREAM_BUFFER_KEY = :html2rss_stream_buffer
+
+        def on_complete(env)
+          buffer = env.request.context&.delete(STREAM_BUFFER_KEY)
+          return if buffer.nil? || buffer.empty?
+
+          env.body = buffer
+        end
+      end
+
+      ##
+      # NOTE: Unlike BrowserlessStrategy, Faraday does not expose the remote IP after connect.
+      # SSRF protection here is pre-connection only (DNS resolution via Policy).
+      # A DNS rebinding attack between resolution and connect cannot be caught at this layer.
+      #
+      # Executes a request with runtime policy enforcement.
+      #
+      # @return [Response] normalized request response
       def execute
-        request = Faraday.new(url: ctx.url, headers: ctx.headers) do |faraday|
-          faraday.use Faraday::FollowRedirects::Middleware
+        deadline = request_deadline
+        response_guard, response = perform_request(deadline:)
+        response_guard.inspect_body!(response.body)
+        build_response(response)
+      rescue Faraday::TimeoutError, Timeout::Error => error
+        raise RequestTimedOut, error.message
+      end
+
+      private
+
+      def request_deadline
+        monotonic_now + ctx.policy.total_timeout_seconds
+      end
+
+      def perform_request(deadline:)
+        response_guard = ResponseGuard.new(policy: ctx.policy)
+        response = faraday_request(response_guard, deadline:, streaming_buffer: true)
+        response = retry_without_streaming(response_guard, deadline:) if retry_without_streaming?(response)
+        [response_guard, response]
+      end
+
+      def build_response(response)
+        Response.new(body: response.body, headers: response.headers, url: response_url(response),
+                     status: response.status)
+      end
+
+      def validate_request!(consume_budget: true)
+        ctx.budget.consume! if consume_budget
+        ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
+      end
+
+      def faraday_request(response_guard, deadline:, streaming_buffer:, consume_budget: true)
+        validate_request!(consume_budget:)
+
+        client.get do |req|
+          apply_timeouts(req, deadline:)
+          buffer = prepare_stream_buffer(req) if streaming_buffer
+          req.options.on_data = on_data_callback(response_guard, buffer)
+        end
+      end
+
+      def retry_without_streaming(response_guard, deadline:)
+        faraday_request(response_guard, deadline:, streaming_buffer: false, consume_budget: false)
+      end
+
+      def client
+        @client ||= Faraday.new(url: ctx.url.to_s, headers: ctx.headers) do |faraday|
+          faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
+          faraday.request :gzip
+          faraday.use StreamingBodyMiddleware
           faraday.adapter Faraday.default_adapter
         end
-        response = request.get
+      end
+
+      def apply_timeouts(request, deadline:)
+        remaining_timeout = remaining_timeout_seconds(deadline)
+        request.options.timeout = remaining_timeout
+        request.options.open_timeout = [ctx.policy.connect_timeout_seconds, remaining_timeout].min
+        request.options.read_timeout = [ctx.policy.read_timeout_seconds, remaining_timeout].min
+      end
+
+      def prepare_stream_buffer(request)
+        request.options.context ||= {}
+        request.options.context[StreamingBodyMiddleware::STREAM_BUFFER_KEY] = +''
+      end
+
+      def on_data_callback(response_guard, buffer)
+        proc do |chunk, total_bytes, env|
+          response_guard.inspect_chunk!(total_bytes:, headers: env&.response_headers)
+          buffer&.<< chunk
+        end
+      end
+
+      def remaining_timeout_seconds(deadline)
+        remaining = deadline - monotonic_now
+        raise RequestTimedOut, 'Request timed out' if remaining <= 0
+
+        remaining
+      end
+
+      def retry_without_streaming?(response)
+        return false if response.body.to_s.empty? == false
+        return false unless response_success?(response)
+
+        final_url = response.env&.url
+        return false unless final_url
+
+        final_url.to_s != ctx.url.to_s
+      end
+
+      def response_success?(response)
+        return true if response.status.nil?
+
+        response.status >= 200 && response.status < 300
+      end
+
+      def response_url(response)
+        return ctx.url unless (url = response.env&.url)
+
+        Html2rss::Url.from_absolute(url.to_s)
+      end
+
+      def redirect_callback
+        lambda do |old_env, new_env|
+          from_url = normalize_url(old_env[:url])
+          to_url = normalize_url(new_env[:url])
+          ctx.policy.validate_redirect!(from_url:, to_url:, origin_url: ctx.origin_url, relation: ctx.relation)
+        end
+      end
+
+      def normalize_url(url)
+        Html2rss::Url.from_absolute(url.to_s)
+      end
 
-        Response.new(body: response.body, headers: response.headers)
+      def monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
     end
   end