html2rss 0.27.1 → 0.27.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 898e609f0c0ae81285356d4c1793d89c8bd76cd3615f51aa28ed5bc352d1bce6
-  data.tar.gz: e384fccef00524f53c733959298f8f363a4400b8142fcb7b5484e06423e478ca
+  metadata.gz: '09b4a391aa87101ab6612c07c0c434b9ebc6198fc3bed8c14c2f8b90bd61bc6a'
+  data.tar.gz: d7fac305311719a6b131be4e89fe78919aef1b45c763a22c471b24583b41d7fa
 SHA512:
-  metadata.gz: 70e479939781e18fa0206659fade0bc12f1aa272932e2e479ff07c5ea46c55e825c2d428bc5b51a62095a01bcc638615e346f895b179062b8f90347227e572c4
-  data.tar.gz: ada830ba830f0f0637122fb4f8d0f0803f292fed2afa1024cdaf29eee46c89c638a7d958459380e9b3d1009a9f1ed46571d553e1fe164d58be13156cff74e6b7
+  metadata.gz: d5fb2cdfc2b95e64de5ac42ffbe1479c15fe63b01f4b922d721d3099ffdd418c9293a1672217838612d5cf4f7bbc2e153cf83ffbcc85e8f30bf8a1a17bb20a79
+  data.tar.gz: a9f51aa30dfdf8e3e222ed1a49216e857cd056f473408273ad3781e862d743f657a67473fe691423e1fb1252fbe2935ee6eb1cef009d3b2662d03b66f7bdb90c

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -7,7 +7,7 @@ module Html2rss
     ##
     # Maps html2rss request/response handling to botasaurus-scrape-api OpenAPI 2.0
     # +ScrapeRequest+ / +ScrapeSuccess+ / +ScrapeError+ (sibling +openapi.yaml+).
-    class BotasaurusContract # rubocop:disable Metrics/ClassLength -- Success/Error envelopes stay with the owner
+    class BotasaurusContract
       # Closed set from OpenAPI ExecutionMode.
       EXECUTION_MODES = %w[auto request browser].freeze
       # Closed set from OpenAPI NavigationMode.
@@ -25,12 +25,19 @@ module Html2rss
       # OpenAPI WindowSize required keys (positive integers).
       WINDOW_SIZE_PROPERTIES = %i[width height].freeze
 
-      # Published wait clamp floor (`[1, 20]` in OpenAPI description).
+      # Faraday POST /scrape cap mirroring botasaurus-scrape-api total scrape wall (`SCRAPE_TIMEOUT_SECONDS`).
+      SCRAPE_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_TIMEOUT_SECONDS', 45))
+      # Post-boot navigate/wait budget mirrored from scrape-api (`SCRAPE_WORK_TIMEOUT_SECONDS`).
+      SCRAPE_WORK_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS', 30))
+      # Seconds added to the scrape total for client transport overhead.
+      TRANSPORT_BUFFER_SECONDS = 2
+
+      # Published wait clamp floor (`[1, SCRAPE_WORK_TIMEOUT_SECONDS]` in OpenAPI description).
       MIN_WAIT_TIMEOUT_SECONDS = 1
       # Published wait clamp ceiling; YAML values above this are rejected.
-      MAX_WAIT_TIMEOUT_SECONDS = 20
+      MAX_WAIT_TIMEOUT_SECONDS = SCRAPE_WORK_TIMEOUT_SECONDS
       # OpenAPI wait_timeout_seconds default (omitted from the client payload).
-      DEFAULT_WAIT_TIMEOUT_SECONDS = 15
+      DEFAULT_WAIT_TIMEOUT_SECONDS = [15, SCRAPE_WORK_TIMEOUT_SECONDS].min
 
       # OpenAPI max_retries maximum (default 2 is applied upstream when omitted).
       MAX_RETRIES = 3
@@ -40,12 +47,6 @@ module Html2rss
       # Allowlisted ChallengeSignal keys nested under diagnostics.challenge.
       CHALLENGE_KEYS = %w[blocked detected marker].freeze
 
-      # Remaining seconds at or below which Botasaurus retries are disabled.
-      TIGHT_BUDGET_SECONDS = 12
-
-      # Seconds reserved from remaining budget before setting wait_timeout_seconds.
-      BUDGET_WAIT_RESERVE_SECONDS = 2
-
       ##
       # Parsed OpenAPI ScrapeSuccess envelope (HTTP 200).
       class Success
@@ -246,7 +247,6 @@ module Html2rss
       # @param url [Html2rss::Url] canonical URL to scrape
       # @param headers [Hash] request headers from context
       # @param options [Hash] validated request.botasaurus options
-      # @param remaining_timeout_seconds [Numeric, nil] shared request budget remainder for clamps
       # @option options [String] :execution_mode
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
@@ -264,19 +264,19 @@ module Html2rss
       # @option options [String] :lang
       # @option options [Hash] :cookies
       # @option options [Hash] :headers
-      def initialize(url:, headers: {}, options: {}, remaining_timeout_seconds: nil)
+      def initialize(url:, headers: {}, options: {})
         @url = url
         @headers = headers
         @options = options
-        @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
-      # @return [Hash] payload for POST /scrape (explicit options plus budget clamp-downs)
+      # @return [Hash] payload for POST /scrape (explicit options plus wait cap)
       def request_payload
         payload = { url: url.to_s }.merge(filtered_options)
         forwarded_headers = merged_headers
         payload[:headers] = forwarded_headers if forwarded_headers&.any?
-        clamp_for_budget(payload)
+        cap_wait_timeout!(payload)
+        payload
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
@@ -291,7 +291,7 @@ module Html2rss
 
       private
 
-      attr_reader :url, :headers, :options, :remaining_timeout_seconds
+      attr_reader :url, :headers, :options
 
       def json_object(body)
         payload = JSON.parse(body)
@@ -315,36 +315,6 @@ module Html2rss
         end
       end
 
-      def clamp_for_budget(payload)
-        remaining = remaining_timeout_seconds
-        clamped = payload.dup
-        unless remaining.nil?
-          apply_tight_retries!(clamped, remaining)
-          apply_budget_wait!(clamped, remaining)
-        end
-        cap_wait_timeout!(clamped)
-        clamped
-      end
-
-      def apply_tight_retries!(payload, remaining)
-        payload[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
-      end
-
-      def apply_budget_wait!(payload, remaining)
-        budget_wait = budget_wait_seconds(remaining)
-        configured = payload[:wait_timeout_seconds]
-        if configured
-          payload[:wait_timeout_seconds] = [configured, budget_wait].min
-        elsif budget_wait < DEFAULT_WAIT_TIMEOUT_SECONDS
-          payload[:wait_timeout_seconds] = budget_wait
-        end
-      end
-
-      def budget_wait_seconds(remaining)
-        reserved = [MIN_WAIT_TIMEOUT_SECONDS, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
-        [reserved, MAX_WAIT_TIMEOUT_SECONDS].min
-      end
-
       def cap_wait_timeout!(payload)
         wait = payload[:wait_timeout_seconds]
         return unless wait

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -2,12 +2,20 @@
 
 require 'faraday'
 require 'json'
+require 'securerandom'
 
 module Html2rss
   class RequestService
     ##
     # Strategy to delegate fetching to a Botasaurus scrape API.
     class BotasaurusStrategy < Strategy
+      # Content-Type negotiation for the scrape API transport hop.
+      TRANSPORT_ACCEPT = 'application/json'
+      # Disable compressed bodies so Faraday returns raw JSON without implicit decoding surprises.
+      TRANSPORT_ENCODING = 'identity'
+      # Correlates each POST /scrape with botasaurus-scrape-api request logs.
+      REQUEST_ID_HEADER = 'X-Request-Id'
+
       private
 
       def fetch
@@ -18,7 +26,9 @@ module Html2rss
       end
 
       def post_scrape_request
-        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
+        request_id = SecureRandom.uuid
+        Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
+        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), post_headers(request_id))
         contract.parse_response(transport_response)
       end
 
@@ -64,13 +74,28 @@ module Html2rss
         @contract ||= BotasaurusContract.new(
           url: ctx.url,
           headers: ctx.headers,
-          options: ctx.request.fetch(:botasaurus, {}),
-          remaining_timeout_seconds: attempt_timeout_seconds
+          options: ctx.request.fetch(:botasaurus, {})
         )
       end
 
       def client
-        @client ||= Faraday.new(url: scraper_base_url.to_s, request: request_options)
+        # No :gzip middleware on this client — compression is for remote target fetches only.
+        @client ||= Faraday.new(url: scraper_base_url.to_s, headers: client_headers, request: request_options)
+      end
+
+      def client_headers
+        {
+          'User-Agent' => Config::RequestHeaders::DEFAULT_USER_AGENT,
+          'Accept' => TRANSPORT_ACCEPT,
+          'Accept-Encoding' => TRANSPORT_ENCODING
+        }
+      end
+
+      def post_headers(request_id)
+        {
+          'Content-Type' => 'application/json',
+          REQUEST_ID_HEADER => request_id
+        }
       end
 
       def request_options
@@ -78,13 +103,10 @@ module Html2rss
       end
 
       def attempt_timeout_seconds
-        @attempt_timeout_seconds ||= ctx.budget.effective_timeout_seconds(
-          fallback: ctx.policy.total_timeout_seconds
-        )
-      end
-
-      def content_type_header
-        { 'Content-Type' => 'application/json' }
+        @attempt_timeout_seconds ||= begin
+          budget = ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+          [budget, BotasaurusContract::SCRAPE_TIMEOUT_SECONDS + BotasaurusContract::TRANSPORT_BUFFER_SECONDS].min
+        end
       end
 
       def scraper_base_url

data/lib/html2rss/version.rb

@@ -2,6 +2,6 @@
 
 module Html2rss
   # Current application version.
-  VERSION = '0.27.1'
+  VERSION = '0.27.2'
   public_constant :VERSION
 end

data/schema/html2rss-config.schema.json

@@ -698,7 +698,7 @@
                 "type": "null"
               },
               "minimum": 1,
-              "maximum": 20
+              "maximum": 30
             },
             "scroll": {
               "type": "boolean",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: html2rss
 version: !ruby/object:Gem::Version
-  version: 0.27.1
+  version: 0.27.2
 platform: ruby
 authors:
 - Gil Desmarais
@@ -486,7 +486,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.27.1
+  changelog_uri: https://github.com/html2rss/html2rss/releases/tag/v0.27.2
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths: