ask-web-fetch 0.5.1 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f49a52ecb57c57ee29ce475ad791f70d30fe52946c94c6bd6842dcccc477392
-  data.tar.gz: b67e97adb54db5b6a775d1ece43df51c0e5443a6ef9e2cd3a17d547af5f7d1dc
+  metadata.gz: ed798fddd5578a39814ea41425937d4e539827dae6c66a8e9bc1621007f5dcda
+  data.tar.gz: 698add682f0ce4f056acfee23130748bb6d359acb8004a07efc6b862dcc462da
 SHA512:
-  metadata.gz: 28dd03ea3e7aae930925ea2d582ff074beba7d3030d762830024e41506a23fd671bb166fa5809417a824e8d2c1729679d00f3d710fa82de05b2ca41038dd39de
-  data.tar.gz: 84815a2d6504c81df9d81aab2fd679d5bb334f64893b05264cc4185be0e7d7439f357461494c65811cbdb16e0e4548b0b2db290f52b747412678eb953c5f40b3
+  metadata.gz: 0aa82e97aef13e08d736024d4927a96a72a554ec1cda21292d501884787ad82e756c51522a179ebfd81bdbcd5baadef6965254957613044e088e511accdfb4f1
+  data.tar.gz: e5d46eef13701008e7c4557675d372b6c4776f38f19b75be4559943b675def5d9370006cb9ef10416a7b70f6a3897f54b7c14f4afff409afd590dee0fa068ac7

data/lib/ask/web_fetch/backends/attached_browser.rb

@@ -50,6 +50,9 @@ module Ask
             @session = @client.session(
               @client.command('Target.attachToTarget', targetId: @target_id, flatten: true)['sessionId']
             )
+            # Subscribed BEFORE any navigation so the idle wait sees the
+            # page's real load traffic (lazy SPAs render in waves).
+            @network = Network.new(self)
           end
 
           # Navigates and does not return until the document has actually
@@ -65,7 +68,6 @@ module Ask
 
             wait_for_load
           end
-
           def title
             evaluate('document.title')
           end
@@ -78,11 +80,11 @@ module Ask
             evaluate('document.documentElement.outerHTML') || ''
           end
 
-          # Minimal network facade for the Browser backend. The status comes
-          # from the Navigation Timing API (the main document's HTTP status);
-          # attached pages need no idle wait — they only exist once loaded.
+          # Minimal network facade for the Browser backend: status from the
+          # Navigation Timing API, idle from CDP Network events (subscribed
+          # in the constructor, before navigation).
           def network
-            @network ||= Network.new(self)
+            @network
           end
 
           def close
@@ -122,10 +124,22 @@ module Ask
             nil
           end
 
-          # Status + no-op idle for the Browser backend's fetch flow.
+          # Real network-idle detection for the attached browser, via CDP's
+          # Network domain events. The launched-browser path gets idle
+          # waiting from Ferrum's network; the attached path previously
+          # skipped it entirely (a no-op), which read lazy-loading SPAs too
+          # early — reddit's post list renders in waves, and the fetch saw
+          # only the first 2.6k of an 8k+ body. Tracks requestStarted /
+          # requestFinished events on the page session and waits until the
+          # network has been quiet for +duration+, capped at +timeout+.
           class Network
             def initialize(page)
               @page = page
+              @session = page.instance_variable_get(:@session)
+              @in_flight = 0
+              @quiet_since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              @mutex = Mutex.new
+              subscribe
             end
 
             def status
@@ -137,7 +151,39 @@ module Ask
               0
             end
 
-            def wait_for_idle(*)
+            def wait_for_idle(duration:, timeout:)
+              deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+              loop do
+                quiet = @mutex.synchronize { @in_flight.zero? }
+                if quiet && Process.clock_gettime(Process::CLOCK_MONOTONIC) - @quiet_since >= duration
+                  return
+                end
+                raise Ferrum::TimeoutError, "network did not go idle within #{timeout}s" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+                sleep 0.1
+              end
+            end
+
+            private
+
+            # CDP events arrive on the session's message loop; Ferrum's
+            # session dispatches them to subscribed handlers. Track the
+            # request lifecycle so "quiet" means no request has been in
+            # flight for the duration — the signal lazy SPAs settle on.
+            # Defensive: a minimal/mock session without event support
+            # (some test doubles) just skips subscription — the load-wait
+            # in go_to is still the floor.
+            def subscribe
+              return unless @session.respond_to?(:on)
+
+              @session.on('Network.requestWillBeSent') { @mutex.synchronize { @in_flight += 1 } }
+              @session.on('Network.responseReceived') { @mutex.synchronize { @quiet_since = Process.clock_gettime(Process::CLOCK_MONOTONIC) } }
+              @session.on('Network.loadingFinished') { @mutex.synchronize { @in_flight -= 1 if @in_flight.positive? } }
+              @session.on('Network.loadingFailed') { @mutex.synchronize { @in_flight -= 1 if @in_flight.positive? } }
+              @session.command('Network.enable')
+            rescue Ferrum::Error
+              # Some attached browsers reject Network.enable (unlikely);
+              # fall back to the load-wait already done in go_to.
               nil
             end
           end

data/lib/ask/web_fetch/http.rb

@@ -48,17 +48,24 @@ module Ask
 
       # One pooled session per thread — httpx sessions are not thread-safe,
       # and a crawl worker thread reusing its session across every page it
-      # fetches is what makes the pooling pay off. Sessions idle-close
-      # themselves after keep-alive timeout, so nothing to reap.
+      # fetches is what keeps timeouts and retries configured once.
+      # Sessions idle-close themselves after keep-alive timeout, so nothing
+      # to reap.
       def self.session
         Thread.current[SESSION_KEY] ||= build_session
       end
 
       def self.build_session
-        # :persistent is what makes the pooling real — without it httpx
-        # opens a fresh connection per request. It loads fiber_concurrency
-        # and the retries plugin internally.
-        HTTPX.plugin(:persistent).with(
+        # Deliberately NOT :persistent: in httpx 1.8.1 that plugin wedges
+        # inside the selector loop when a session that already holds a
+        # pooled connection opens one to a NEW host — the operation
+        # timeout never fires and the fetch hangs forever (reproduced in
+        # plain Ruby: example.com then nytimes.com on one session).
+        # Without it httpx opens a fresh connection per host, which for a
+        # crawler hitting mostly-distinct hosts costs one TLS handshake
+        # per page and never hangs. Retries are explicit so the transient
+        # backoff that :persistent loaded internally is kept.
+        HTTPX.plugin(:retries).with(
           timeout: {
             connect_timeout: CONNECT_TIMEOUT,
             read_timeout: READ_TIMEOUT,

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.1'
+    VERSION = '0.5.3'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: ruby
 authors:
 - Kaka Ruto