ask-web-fetch 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 18e8a7b0274def2380d5b75df069e428aeca9e027352f2fc09ee64b65d946a68
-  data.tar.gz: d69e005edde7afcf8b05a8d89da9655be939ab7e21b039d6d61c617b4a1d74de
+  metadata.gz: ed798fddd5578a39814ea41425937d4e539827dae6c66a8e9bc1621007f5dcda
+  data.tar.gz: 698add682f0ce4f056acfee23130748bb6d359acb8004a07efc6b862dcc462da
 SHA512:
-  metadata.gz: 0e7c44e68630b3b7c2e7a5c04581b2e9992a888c927ecb5de47b4782f92538fa0e5b71eb53ea4fea325c6f354595338698abfac5540260e09d162d30673db347
-  data.tar.gz: 6f67717fbe1aeec17b0cfbc4939c3b9195ebd229976ca744ebe8ef67de08e9bae6630ca8fbd2fa7b50979487ce331d8a2edaf96e12818185d2ea855ed295c477
+  metadata.gz: 0aa82e97aef13e08d736024d4927a96a72a554ec1cda21292d501884787ad82e756c51522a179ebfd81bdbcd5baadef6965254957613044e088e511accdfb4f1
+  data.tar.gz: e5d46eef13701008e7c4557675d372b6c4776f38f19b75be4559943b675def5d9370006cb9ef10416a7b70f6a3897f54b7c14f4afff409afd590dee0fa068ac7

data/lib/ask/web_fetch/backends/attached_browser.rb

@@ -50,6 +50,9 @@ module Ask
             @session = @client.session(
               @client.command('Target.attachToTarget', targetId: @target_id, flatten: true)['sessionId']
             )
+            # Subscribed BEFORE any navigation so the idle wait sees the
+            # page's real load traffic (lazy SPAs render in waves).
+            @network = Network.new(self)
           end
 
           # Navigates and does not return until the document has actually
@@ -65,7 +68,6 @@ module Ask
 
             wait_for_load
           end
-
           def title
             evaluate('document.title')
           end
@@ -78,11 +80,11 @@ module Ask
             evaluate('document.documentElement.outerHTML') || ''
           end
 
-          # Minimal network facade for the Browser backend. The status comes
-          # from the Navigation Timing API (the main document's HTTP status);
-          # attached pages need no idle wait — they only exist once loaded.
+          # Minimal network facade for the Browser backend: status from the
+          # Navigation Timing API, idle from CDP Network events (subscribed
+          # in the constructor, before navigation).
           def network
-            @network ||= Network.new(self)
+            @network
           end
 
           def close
@@ -122,10 +124,22 @@ module Ask
             nil
           end
 
-          # Status + no-op idle for the Browser backend's fetch flow.
+          # Real network-idle detection for the attached browser, via CDP's
+          # Network domain events. The launched-browser path gets idle
+          # waiting from Ferrum's network; the attached path previously
+          # skipped it entirely (a no-op), which read lazy-loading SPAs too
+          # early — reddit's post list renders in waves, and the fetch saw
+          # only the first 2.6k of an 8k+ body. Tracks requestStarted /
+          # requestFinished events on the page session and waits until the
+          # network has been quiet for +duration+, capped at +timeout+.
           class Network
             def initialize(page)
               @page = page
+              @session = page.instance_variable_get(:@session)
+              @in_flight = 0
+              @quiet_since = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              @mutex = Mutex.new
+              subscribe
             end
 
             def status
@@ -137,7 +151,39 @@ module Ask
               0
             end
 
-            def wait_for_idle(*)
+            def wait_for_idle(duration:, timeout:)
+              deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+              loop do
+                quiet = @mutex.synchronize { @in_flight.zero? }
+                if quiet && Process.clock_gettime(Process::CLOCK_MONOTONIC) - @quiet_since >= duration
+                  return
+                end
+                raise Ferrum::TimeoutError, "network did not go idle within #{timeout}s" if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+                sleep 0.1
+              end
+            end
+
+            private
+
+            # CDP events arrive on the session's message loop; Ferrum's
+            # session dispatches them to subscribed handlers. Track the
+            # request lifecycle so "quiet" means no request has been in
+            # flight for the duration — the signal lazy SPAs settle on.
+            # Defensive: a minimal/mock session without event support
+            # (some test doubles) just skips subscription — the load-wait
+            # in go_to is still the floor.
+            def subscribe
+              return unless @session.respond_to?(:on)
+
+              @session.on('Network.requestWillBeSent') { @mutex.synchronize { @in_flight += 1 } }
+              @session.on('Network.responseReceived') { @mutex.synchronize { @quiet_since = Process.clock_gettime(Process::CLOCK_MONOTONIC) } }
+              @session.on('Network.loadingFinished') { @mutex.synchronize { @in_flight -= 1 if @in_flight.positive? } }
+              @session.on('Network.loadingFailed') { @mutex.synchronize { @in_flight -= 1 if @in_flight.positive? } }
+              @session.command('Network.enable')
+            rescue Ferrum::Error
+              # Some attached browsers reject Network.enable (unlikely);
+              # fall back to the load-wait already done in go_to.
               nil
             end
           end

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.5.2'
+    VERSION = '0.5.3'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Kaka Ruto