html2rss 0.22.1 → 0.22.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 750b7fb967b328cef2238b66729cafe122d1bae23bee05fd8504bb31e760b8a7
-  data.tar.gz: 327406de9c7c97ea13e90c89bec1c2653c962bbcaccfd29ddb78b282477f7578
+  metadata.gz: eb40d85981c1bc11576b0c9d7a3eae3948bd49239ad8b9a65bb6a4ea103b54f9
+  data.tar.gz: 6c4636413e1e995e98adec8d8cc742a5833208c2197a024abe90f4b81fce1774
 SHA512:
-  metadata.gz: 5f41e00edfdd19ceb012900db7518f28236b417662dc4f11f45d9c498ede0720bdbbc0fb31443d80495eaf6076df646062fdf7f972b27bd71c50cc4e198b4540
-  data.tar.gz: 7a7eff85bd7f98cd872131041aa58faf9d3fba1aff47893a6d286378b6f52904ee11120f053e3ca8beca3060d86d1438cf2037e3e4e7e1586d3d3c4679b026f0
+  metadata.gz: 9134076ad05417b8b7176da26033e4d66137f862781378b44080d4ea4fe67e8932156c8eb7d22b58ad97f0d7f5e32102c96ef84741d681c2c7571b3add09afb0
+  data.tar.gz: 61958c85cb791adc78d6d18debcdbfb7a90948332696dac0a4ab4aee39f6b098c3366f948d8905d3d36e2052e03e75037006bd2e6cfa3420b9a11e46e3ebce18

data/lib/html2rss/feed_pipeline.rb

@@ -3,7 +3,7 @@
 module Html2rss
   ##
   # Builds feeds from validated config through request, extraction, and rendering stages.
-  class FeedPipeline
+  class FeedPipeline # rubocop:disable Metrics/ClassLength
     ##
     # @param raw_config [Hash{Symbol => Object}] user-provided feed config
     def initialize(raw_config)
@@ -77,11 +77,16 @@ module Html2rss
       auto_fallback_for(config).call
     end
 
+    # rubocop:disable Metrics/MethodLength
     def auto_fallback_for(config)
       AutoFallback.new(
         strategies: AutoFallback::CHAIN,
         budget: auto_pipeline_budget(config),
         session_for: lambda do |strategy:, budget:|
+          if budget.remaining_timeout_seconds && budget.remaining_timeout_seconds <= 0
+            raise RequestService::RequestTimedOut, 'Request timed out'
+          end
+
           request_session_for(config, strategy:, budget:)
         end,
         articles_for: lambda do |response:, request_session:|
@@ -89,10 +94,14 @@ module Html2rss
         end
       )
     end
+    # rubocop:enable Metrics/MethodLength
 
     def auto_pipeline_budget(config)
-      max_requests = RequestSession::RuntimePolicy.from_config(config).max_requests
-      RequestService::Budget.new(max_requests:)
+      policy = RequestSession::RuntimePolicy.from_config(config)
+      RequestService::Budget.new(
+        max_requests: policy.max_requests,
+        total_timeout_seconds: policy.total_timeout_seconds
+      )
     end
 
     def collect_articles(response:, config:, request_session:)

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -16,9 +16,9 @@ module Html2rss
       # @raise [BotasaurusConnectionFailed] when Botasaurus cannot be reached or returns an invalid payload
       # @raise [RequestTimedOut] when the Botasaurus request exceeds configured timeout
       def execute
+        check_timeout!
         validate_request!
-        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
-        parsed_response = contract.parse_response(transport_response)
+        parsed_response = post_scrape_request
         raise_if_challenge_blocked!(parsed_response)
         raise_if_upstream_failed!(parsed_response)
         build_response(parsed_response)
@@ -30,6 +30,11 @@ module Html2rss
 
       private
 
+      def post_scrape_request
+        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
+        contract.parse_response(transport_response)
+      end
+
       def validate_request!
         ctx.budget.consume!
         ctx.policy.validate_request!(url: ctx.url, origin_url: ctx.origin_url, relation: ctx.relation)
@@ -76,7 +81,10 @@ module Html2rss
       end
 
       def request_options
-        { timeout: ctx.policy.total_timeout_seconds }
+        timeout = ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds
+        raise RequestTimedOut, 'Request timed out' if timeout <= 0
+
+        { timeout: timeout.to_i }
       end
 
       def content_type_header

data/lib/html2rss/request_service/browserless_strategy.rb

@@ -37,6 +37,7 @@ module Html2rss
       # @return [Response] normalized request response
       # @raise [RequestTimedOut] if the browser session exceeds the configured timeout
       def execute
+        check_timeout!
         validate_request!
         execute_browserless_request
       rescue Puppeteer::TimeoutError => error
@@ -71,7 +72,10 @@ module Html2rss
       end
 
       def protocol_timeout_ms
-        ctx.policy.total_timeout_seconds * 1000
+        timeout = ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds
+        raise RequestTimedOut, 'Request timed out' if timeout <= 0
+
+        (timeout * 1000).to_i
       end
 
       def connect_with_timeout_support(&)

data/lib/html2rss/request_service/budget.rb

@@ -5,14 +5,18 @@ module Html2rss
     ##
     # Tracks how many outbound requests a single feed build may still perform.
     class Budget
+      ##
       ##
       # @param max_requests [Integer] the maximum number of requests allowed
-      def initialize(max_requests:)
+      # @param total_timeout_seconds [Integer, nil] the total timeout for the feed build
+      def initialize(max_requests:, total_timeout_seconds: nil)
         unless max_requests.is_a?(Integer) && max_requests.positive?
           raise ArgumentError, 'max_requests must be positive'
         end
 
         @remaining = max_requests
+        @start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        @total_timeout_seconds = total_timeout_seconds
         @mutex = Mutex.new
       end
 
@@ -34,6 +38,16 @@ module Html2rss
       def remaining
         @mutex.synchronize { @remaining }
       end
+
+      ##
+      # @return [Float, nil] the remaining timeout in seconds, or nil if not tracked
+      def remaining_timeout_seconds
+        return unless @total_timeout_seconds
+
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_time
+        remaining = @total_timeout_seconds - elapsed
+        [remaining, 0.0].max
+      end
     end
   end
 end

data/lib/html2rss/request_service/context.rb

@@ -77,7 +77,9 @@ module Html2rss
         raise ArgumentError, 'policy must not be nil' if @policy.nil?
 
         @origin_url = normalized_origin_url(request_options[:origin_url])
-        @budget = request_options.fetch(:budget) { Budget.new(max_requests: policy.max_requests) }
+        @budget = request_options.fetch(:budget) do
+          Budget.new(max_requests: policy.max_requests, total_timeout_seconds: policy.total_timeout_seconds)
+        end
         raise ArgumentError, 'budget must not be nil' if @budget.nil?
       end
 

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -34,6 +34,7 @@ module Html2rss
       #   SSRF protection here is pre-connection only (DNS resolution via Policy).
       #   A DNS rebinding attack between resolution and connect cannot be caught at this layer.
       def execute
+        check_timeout!
         deadline = request_deadline
         response_guard, response = perform_request(deadline:)
         response_guard.inspect_body!(response.body)
@@ -45,7 +46,7 @@ module Html2rss
       private
 
       def request_deadline
-        monotonic_now + ctx.policy.total_timeout_seconds
+        monotonic_now + (ctx.budget.remaining_timeout_seconds || ctx.policy.total_timeout_seconds)
       end
 
       def perform_request(deadline:)

data/lib/html2rss/request_service/strategy.rb

@@ -23,6 +23,11 @@ module Html2rss
 
       # @return [Context] the context for the request
       attr_reader :ctx
+
+      def check_timeout!
+        remaining = ctx.budget.remaining_timeout_seconds
+        raise RequestTimedOut, 'Request timed out' if remaining && remaining <= 0
+      end
     end
   end
 end

data/lib/html2rss/version.rb

@@ -4,6 +4,6 @@
 # The Html2rss namespace.
 module Html2rss
   # Current application version.
-  VERSION = '0.22.1'
+  VERSION = '0.22.2'
   public_constant :VERSION
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.22.1
+  version: 0.22.2
 platform: ruby
 authors:
 - Gil Desmarais
@@ -381,7 +381,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.22.1
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.22.2
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: