ask-web-fetch 0.5.4 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9126d245115b48e3be3d0f08166ae1bdae5d917d9bc84603c90b99a81ada84d
-  data.tar.gz: 5a6b70c34ab86157f850a5158838efc94361c46032eb184b8f611558150a6590
+  metadata.gz: db21a4a35495ff89b8b91b769d4a15743da85beb961a5ddbbc430bdde37988f5
+  data.tar.gz: 61fb64631bbfe1d478b3b6ed6587de8e5b9b3890dd5348924ca574929f85acb7
 SHA512:
-  metadata.gz: 31f43065058dc40f29aeeced52b8d11c900c2e7a49b7bbf41cf9dd090a3b0b8ccd09507c2df4596ff73ef60a134828410005d4c19c2c4ae6e326a3f4abcb4d62
-  data.tar.gz: 94d089feccbc51350c7c0f7ebf06636ca53246c3e960dcb2c9aa8ca820bc96a2a1c3e2ebbc2b29a1faacbcea906388f2c56ad9faae4f7cb4a148967b5eb5b7a8
+  metadata.gz: f53d538cf613c02e1411c9f09901c3b93db98c51c33d0d3dfca2d7ea58420f9e463f074c1f2e9a6c97da59023037abcfce59d757ef3ab689eada87fc6155939d
+  data.tar.gz: fde5f3000a2ec77294d1bceec3484d9d4ec68b3b3d0b91d8e29083b3bb711f31b391092a1cd544b329858ffb510e116d39c39ed4a23323c110af133db6cef928

data/lib/ask/web_fetch/backends/browser.rb

@@ -3,6 +3,7 @@
 require 'ferrum'
 require 'json'
 require 'net/http'
+require 'set'
 require 'uri'
 require_relative '../backend'
 require_relative '../content_filter'
@@ -80,6 +81,13 @@ module Ask
 
           attr_writer :challenge_timeout, :poll_interval
 
+          # Domains already given a warm pass this process — a failed warm
+          # (DataDome-class wall) is not re-paid at CHALLENGE_TIMEOUT on
+          # every fetch of that domain.
+          def warmed_domains
+            @warmed_domains ||= Set.new
+          end
+
           # A shared browser, built once and reused. Ferrum is loaded lazily
           # so consumers who never hit this backend pay nothing for it.
           def browser
@@ -153,6 +161,30 @@ module Ask
           raise FetchError, 'ferrum gem unavailable' unless self.class.browser
 
           page = self.class.browser.create_page
+          fetch_attempt(page, url)
+        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
+          raise TimeoutError, "#{e.class}: #{e.message}"
+        rescue Ferrum::StatusError => e
+          raise FetchError, "browser could not load #{url}: #{e.message}"
+        rescue Ferrum::Error => e
+          raise ServerError, "browser #{e.class}: #{e.message}"
+        rescue Errno::ECONNREFUSED, SocketError => e
+          raise TimeoutError, "browser connection #{e.class}: #{e.message}"
+        ensure
+          page&.close
+        end
+
+        # One fetch of +url+ on the page, with the warm-and-retry pass: a
+        # challenge page means this domain hasn't issued the profile a
+        # clearance cookie yet. Visiting the DOMAIN ROOT first (where a
+        # managed challenge auto-solves for a trusted browser) earns the
+        # cookie — pinned to this browser + IP and persisted in the
+        # profile — then the URL is retried once. Subsequent fetches for
+        # the same domain find the cookie and never warm again. Bounded:
+        # one warm per domain per process (warmed_domains), one retry per
+        # fetch — a still-challenged URL fails as before, never wedging
+        # the queue on a DataDome-class wall.
+        def fetch_attempt(page, url, warmed: false)
           page.go_to(url)
           wait_for_challenge(page)
           wait_for_idle(page)
@@ -161,23 +193,42 @@ module Ask
           raise FetchError, "got #{status} at #{url}" if status && status >= 400
 
           body = page.body
-          raise FetchError, "challenge page at #{url}" if challenge_page?(body)
+          if challenge_page?(body)
+            raise FetchError, "challenge page at #{url}" if warmed
+            raise FetchError, "challenge page at #{url}" unless warm_domain(page, url)
+
+            return fetch_attempt(page, url, warmed: true)
+          end
 
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
           result[:outlinks] = outlink_urls(body, url)
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(result[:content])
 
           result
-        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
-          raise TimeoutError, "#{e.class}: #{e.message}"
-        rescue Ferrum::StatusError => e
-          raise FetchError, "browser could not load #{url}: #{e.message}"
-        rescue Ferrum::Error => e
-          raise ServerError, "browser #{e.class}: #{e.message}"
-        rescue Errno::ECONNREFUSED, SocketError => e
-          raise TimeoutError, "browser connection #{e.class}: #{e.message}"
-        ensure
-          page&.close
+        end
+
+        # Earns the domain's clearance cookie: navigates to the domain
+        # root (the challenge JS runs there, not on the deep URL), waits
+        # for the managed challenge to auto-solve, and lets the cookie
+        # land in the profile. Returns true when the domain is now warm.
+        # Domains already attempted this process are skipped — a failed
+        # warm is not re-paid at CHALLENGE_TIMEOUT per fetch.
+        def warm_domain(page, url)
+          domain = URI(url).host
+          return false if self.class.warmed_domains.include?(domain)
+
+          self.class.warmed_domains << domain
+          page.go_to(domain_root(url))
+          wait_for_challenge(page)
+          wait_for_idle(page)
+          true
+        rescue Ferrum::Error, URI::InvalidURIError
+          false
+        end
+
+        def domain_root(url)
+          uri = URI(url)
+          "#{uri.scheme}://#{uri.host}"
         end
 
         private

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.4'
+    VERSION = '0.5.5'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.5
 platform: ruby
 authors:
 - Kaka Ruto