ask-web-fetch 0.5.4 → 0.5.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9126d245115b48e3be3d0f08166ae1bdae5d917d9bc84603c90b99a81ada84d
-  data.tar.gz: 5a6b70c34ab86157f850a5158838efc94361c46032eb184b8f611558150a6590
+  metadata.gz: a4c0b6735b967e38e8b837e7128bb501875ea1ce239f2df1549518396740ba8e
+  data.tar.gz: 0c6b95aba29f4914d062b35b3e21a43d1616a95ab01111a8d81d73326bd7f985
 SHA512:
-  metadata.gz: 31f43065058dc40f29aeeced52b8d11c900c2e7a49b7bbf41cf9dd090a3b0b8ccd09507c2df4596ff73ef60a134828410005d4c19c2c4ae6e326a3f4abcb4d62
-  data.tar.gz: 94d089feccbc51350c7c0f7ebf06636ca53246c3e960dcb2c9aa8ca820bc96a2a1c3e2ebbc2b29a1faacbcea906388f2c56ad9faae4f7cb4a148967b5eb5b7a8
+  metadata.gz: 6f7f82bc5b93d368395989049757d86675e89d7645f7b351f3d24edff50a8303ee86cdc04dec9240a9e71ef2f6f7dedd0d6641da93ddc3b9d5494b116fa5ccd0
+  data.tar.gz: c160aac9363740829eaceeb53b18e91b009dcdc278b194d7774ba692b3b6d14d3404cb5b4f38e0d50fb73a62390fad7b2e0bbc1ca60a47d8dce8de8c1e745072

data/lib/ask/web_fetch/backends/browser.rb

@@ -3,6 +3,7 @@
 require 'ferrum'
 require 'json'
 require 'net/http'
+require 'set'
 require 'uri'
 require_relative '../backend'
 require_relative '../content_filter'
@@ -80,6 +81,13 @@ module Ask
 
           attr_writer :challenge_timeout, :poll_interval
 
+          # Domains already given a warm pass this process — a failed warm
+          # (DataDome-class wall) is not re-paid at CHALLENGE_TIMEOUT on
+          # every fetch of that domain.
+          def warmed_domains
+            @warmed_domains ||= Set.new
+          end
+
           # A shared browser, built once and reused. Ferrum is loaded lazily
           # so consumers who never hit this backend pay nothing for it.
           def browser
@@ -153,6 +161,30 @@ module Ask
           raise FetchError, 'ferrum gem unavailable' unless self.class.browser
 
           page = self.class.browser.create_page
+          fetch_attempt(page, url)
+        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
+          raise TimeoutError, "#{e.class}: #{e.message}"
+        rescue Ferrum::StatusError => e
+          raise FetchError, "browser could not load #{url}: #{e.message}"
+        rescue Ferrum::Error => e
+          raise ServerError, "browser #{e.class}: #{e.message}"
+        rescue Errno::ECONNREFUSED, SocketError => e
+          raise TimeoutError, "browser connection #{e.class}: #{e.message}"
+        ensure
+          page&.close
+        end
+
+        # One fetch of +url+ on the page, with the warm-and-retry pass: a
+        # challenge page means this domain hasn't issued the profile a
+        # clearance cookie yet. Visiting the DOMAIN ROOT first (where a
+        # managed challenge auto-solves for a trusted browser) earns the
+        # cookie — pinned to this browser + IP and persisted in the
+        # profile — then the URL is retried once. Subsequent fetches for
+        # the same domain find the cookie and never warm again. Bounded:
+        # one warm per domain per process (warmed_domains), one retry per
+        # fetch — a still-challenged URL fails as before, never wedging
+        # the queue on a DataDome-class wall.
+        def fetch_attempt(page, url, warmed: false)
           page.go_to(url)
           wait_for_challenge(page)
           wait_for_idle(page)
@@ -161,23 +193,42 @@ module Ask
           raise FetchError, "got #{status} at #{url}" if status && status >= 400
 
           body = page.body
-          raise FetchError, "challenge page at #{url}" if challenge_page?(body)
+          if challenge_page?(body)
+            raise FetchError, "challenge page at #{url}" if warmed
+            raise FetchError, "challenge page at #{url}" unless warm_domain(page, url)
+
+            return fetch_attempt(page, url, warmed: true)
+          end
 
           result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
           result[:outlinks] = outlink_urls(body, url)
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(result[:content])
 
           result
-        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
-          raise TimeoutError, "#{e.class}: #{e.message}"
-        rescue Ferrum::StatusError => e
-          raise FetchError, "browser could not load #{url}: #{e.message}"
-        rescue Ferrum::Error => e
-          raise ServerError, "browser #{e.class}: #{e.message}"
-        rescue Errno::ECONNREFUSED, SocketError => e
-          raise TimeoutError, "browser connection #{e.class}: #{e.message}"
-        ensure
-          page&.close
+        end
+
+        # Earns the domain's clearance cookie: navigates to the domain
+        # root (the challenge JS runs there, not on the deep URL), waits
+        # for the managed challenge to auto-solve, and lets the cookie
+        # land in the profile. Returns true when the domain is now warm.
+        # Domains already attempted this process are skipped — a failed
+        # warm is not re-paid at CHALLENGE_TIMEOUT per fetch.
+        def warm_domain(page, url)
+          domain = URI(url).host
+          return false if self.class.warmed_domains.include?(domain)
+
+          self.class.warmed_domains << domain
+          page.go_to(domain_root(url))
+          wait_for_challenge(page)
+          wait_for_idle(page)
+          true
+        rescue Ferrum::Error, URI::InvalidURIError
+          false
+        end
+
+        def domain_root(url)
+          uri = URI(url)
+          "#{uri.scheme}://#{uri.host}"
         end
 
         private

data/lib/ask/web_fetch/backends/local.rb

@@ -46,6 +46,19 @@ module Ask
           page = to_markdown(body, url)
           page[:redirected] = redirect
           raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
+          # The completeness signal: a JS-app shell whose server HTML
+          # renders little is a TRUNCATED page, not a complete one — the
+          # real content awaits client-side JS that Local cannot run.
+          # Storing the shell as success would silently under-deliver
+          # (airbnb: 613KB server HTML -> 143 chars of markdown); failing
+          # through lets the chain prefer a rendering backend (Browser),
+          # and keeps a partial page from ever being stored as the real
+          # thing. Two detectors: known framework markers, or a large
+          # HTML page with almost no server-rendered text.
+          if js_app_shell?(body) && page[:content].length < SHELL_CONTENT_THRESHOLD
+            raise EmptyContentError,
+              "JS-app shell at #{url} — server HTML renders only #{page[:content].length} chars; a rendering backend is required"
+          end
 
           page
         rescue Errno::ECONNREFUSED, Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
@@ -55,6 +68,40 @@ module Ask
           raise TimeoutError, "#{e.class}: #{e.message}"
         end
 
+        # Client-rendered framework markers: the server HTML is (at least
+        # partly) a shell awaiting JS. Deliberately narrow — generic terms
+        # like "script" or "root" appear on every page; these are the
+        # specific footprints of React/Vue/Next/Nuxt app shells.
+        JS_APP_SHELL_MARKERS = /id=["'](?:root|app|__next|site-content)["']|__NEXT_DATA__|window\.__NUXT__|ng-app|data-reactroot/
+
+        # Markdown below this from a JS-app shell is "server sent a shell",
+        # not "page is genuinely short" — a real page (even a short one)
+        # is usually server-rendered above this. Tunable; the chain turns
+        # the signal into "prefer Browser for this URL".
+        SHELL_CONTENT_THRESHOLD = 4_000
+
+        # A page whose server HTML is large but yields almost no text is a
+        # shell whatever framework it uses (airbnb: 613KB HTML -> 143 chars
+        # of markdown). Framework markers miss these; the size ratio
+        # catches them. nytimes (1.4MB -> 7.5k text) stays above the text
+        # floor and is correctly left to Local.
+        SHELL_HTML_BYTES = 20_000
+
+        def js_app_shell?(body)
+          body.to_s.match?(JS_APP_SHELL_MARKERS) ||
+            (body.to_s.bytesize > SHELL_HTML_BYTES && markdown_visible_chars(body) < SHELL_CONTENT_THRESHOLD)
+        end
+
+        # A cheap text estimate from the raw HTML (tags stripped) — the
+        # "how much did the server actually render" number. Deliberately
+        # rough: it only feeds a shell-vs-page heuristic.
+        def markdown_visible_chars(body)
+          body.to_s.gsub(/<script[\s\S]*?<\/script>/i, "")
+            .gsub(/<style[\s\S]*?<\/style>/i, "")
+            .gsub(/<[^>]+>/, " ")
+            .gsub(/\s+/, " ").strip.length
+        end
+
         # Parses +html+ and returns { title:, description:, content:,
         # licenses:, outlinks: } where content is clean markdown (pruned by
         # the ContentFilter), licenses are the page's declared license

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.4'
+    VERSION = '0.5.6'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.6
 platform: ruby
 authors:
 - Kaka Ruto