ask-web-fetch 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: efa999aac8d4321c2e8e6d463ace6c3f5ec5e95a62973ad06593c4714e1ffab1
-  data.tar.gz: 9a9e59e8da81bb4238bf1a08b0ba84634553dfd3caac99763a7bb52a54d9ac6e
+  metadata.gz: 3ad48d09aa8869b4193a703ad6a760753c2767c9e26a6919b1405a5fb186c7a4
+  data.tar.gz: 52960016a3d4740f4913d4879cbf80edacf95e4ffc07601e6dde0d80515ffe81
 SHA512:
-  metadata.gz: af1e312f91b9e5ea72679b7e877232cc3e4e9639b3fdf3e98291810377a450a5ce180bb4664ca651c8ccd2084220632890da92fe4eaf8c1ca472e6ba464d60ff
-  data.tar.gz: 3951989986d476f6bc008712e6451ccb148c9fbe33537544061486d15c79443e0368dd668ef40b35287173ad41f569fc519130a9cb5368e78da16eafbcdf2c91
+  metadata.gz: fa6d778abc6918bb7ae839d4f9c96dafe215014a8b2f4efc8027ab1e49d287e3ca57204f54f013531e38dff7393ac23f6da9619b3afd48efae8b6f7e42dfa0d9
+  data.tar.gz: 7d4e5fe0e70b28cc3349ec96d071d4e1cc7a46c2c21ca8f41c35968d8b15221854c2418fc72ae2a8e56e97c07930592b2ee5b79da921c4a4ccb4fe6990c7e938

data/lib/ask/web_fetch/backends/browser.rb

@@ -96,6 +96,12 @@ module Ask
             browser_mutex.synchronize { @browser ||= build_browser }
           end
 
+          # Drops the shared browser so the next use builds a fresh
+          # session — called when the current one died mid-fetch.
+          def reset_browser
+            browser_mutex.synchronize { @browser = nil }
+          end
+
           def content_filter
             @content_filter ||= ContentFilter.default
           end
@@ -162,7 +168,17 @@ module Ask
 
           page = self.class.browser.create_page
           fetch_attempt(page, url)
-        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError, Ferrum::DeadBrowserError => e
+        rescue Ferrum::DeadBrowserError => e
+          # The shared browser died mid-fetch (the browserless server
+          # killed the session's browser). A dead browser is transient —
+          # the next session starts a fresh one — so reconnect and retry
+          # ONCE instead of failing the page (observed 2026-08-14: the
+          # server's session limits killed browsers and pages were
+          # silently dropped).
+          self.class.reset_browser
+          page = self.class.browser.create_page
+          fetch_attempt(page, url)
+        rescue Ferrum::TimeoutError, Ferrum::ProcessTimeoutError => e
           raise TimeoutError, "#{e.class}: #{e.message}"
         rescue Ferrum::StatusError => e
           raise FetchError, "browser could not load #{url}: #{e.message}"

data/lib/ask/web_fetch/backends/local.rb

@@ -63,7 +63,7 @@ module Ask
           # and keeps a partial page from ever being stored as the real
           # thing. Two detectors: known framework markers, or a large
           # HTML page with almost no server-rendered text.
-          if js_app_shell?(body) && page[:content].length < SHELL_CONTENT_THRESHOLD
+          if js_app_shell?(body) && page[:content].length < SHELL_DEFER_THRESHOLD
             raise EmptyContentError,
               "JS-app shell at #{url} — server HTML renders only #{page[:content].length} chars; a rendering backend is required"
           end
@@ -82,10 +82,19 @@ module Ask
         # specific footprints of React/Vue/Next/Nuxt app shells.
         JS_APP_SHELL_MARKERS = /id=["'](?:root|app|__next|site-content)["']|__NEXT_DATA__|window\.__NUXT__|ng-app|data-reactroot/
 
-        # Markdown below this from a JS-app shell is "server sent a shell",
-        # not "page is genuinely short" — a real page (even a short one)
-        # is usually server-rendered above this. Tunable; the chain turns
-        # the signal into "prefer Browser for this URL".
+        # A framework marker (React/Vue/Next) alone is NOT emptiness: a
+        # marked page can server-render real content (careers.abb job
+        # pages: 2,162 chars of job description) and must be stored as-is
+        # when the rendering backends cannot do better — dropping it lost
+        # real pages (2026-08-14). The shell deferral fires only when the
+        # extraction is genuinely little: below this, the page is a true
+        # shell (airbnb: 613KB HTML -> 143 chars). Tunable; the chain
+        # turns the signal into "prefer Browser for this URL".
+        SHELL_DEFER_THRESHOLD = 500
+
+        # The ratio detector's own bar, kept for compatibility with the
+        # comment below (large HTML + near-empty text is a shell whatever
+        # the framework).
         SHELL_CONTENT_THRESHOLD = 4_000
 
         # A page whose server HTML is large but yields almost no text is a

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.1'
+    VERSION = '0.7.2'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Kaka Ruto
@@ -171,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Web fetch library for the ask-rb ecosystem
 test_files: []