ask-web-fetch 0.7.6 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae762437d9eb577d28a7f88d7cc82e5cb872e29a572c8bf3ec977f2602e2d398
-  data.tar.gz: 4c52bfb0d6a69d5184eea6ad81281dbea87f3bdfcd2a72219de3f95bbcd4b524
+  metadata.gz: 3e1fdcc511917b85bfe1ab456733d89e09a24eb4dd4bd085d61ebcf5d52ff4d0
+  data.tar.gz: 59365989bd571bd06454513328a5f27c64116f6ced3669bcebb753c94fddfcee
 SHA512:
-  metadata.gz: 4be64a96e76ff2335e5d4e9ced1b6fc918eff6a8e5520b7f87660b554989e9ac6beb0841fcd12b218ac33c8776f9f9d3703c95da072b7f66e3ea3095916f66a6
-  data.tar.gz: 9741eeb18451170efedf920716dd22f1db61c57b71b43e76556d3e18f9303645671e6ee93867b78faefd8e026c2e272891856fc49e4ca95b91b3e791810dbac0
+  metadata.gz: 5ab5dfc8d25e5c9550a08981382a041e148291b6d83ab40c9f375404c7efd35f08d08b6444b10fabe4a8035773ea49606dc96a6c73211aeb7cd4f89e529ca359
+  data.tar.gz: 742d576e4a4f0a71ab89c47d177107f991c384795eef0775fc8579c96d6cbb41688cb30cabfb13d4d861bf6e665c54ee662998e44a66da64904955406562818f

data/lib/ask/web_fetch/backends/browser.rb

@@ -45,7 +45,7 @@ module Ask
       class Browser < Backend
         # Seconds to let a Cloudflare-style challenge auto-solve before
         # giving up (tunable via Browser.challenge_timeout).
-        CHALLENGE_TIMEOUT = 30
+        CHALLENGE_TIMEOUT = 10
 
         # Seconds to wait for the network to go quiet after the page loads,
         # so lazy-loaded content is in before we read the DOM.

data/lib/ask/web_fetch/backends/jina.rb

@@ -15,8 +15,8 @@ module Ask
       # pages that the Local backend cannot.
       class Jina < Backend
         BASE_URL = 'https://r.jina.ai'
-        OPEN_TIMEOUT = 5
-        READ_TIMEOUT = 30
+        OPEN_TIMEOUT = 3
+        READ_TIMEOUT = 10
 
         def fetch(url)
           uri = URI("#{BASE_URL}/#{url}")

data/lib/ask/web_fetch/http.rb

@@ -18,12 +18,12 @@ module Ask
     # TLS) all surface as Ask::WebFetch::TimeoutError, the transient
     # bucket, whatever their underlying class.
     class Http
-      CONNECT_TIMEOUT = 5
-      READ_TIMEOUT = 15
-      WRITE_TIMEOUT = 15
-      # Whole-request cap. A page that can't be read in a minute is a
+      CONNECT_TIMEOUT = 3
+      READ_TIMEOUT = 8
+      WRITE_TIMEOUT = 8
+      # Whole-request cap. A page that can't be read in 20s is a
       # problem page, not a stall worth a crawl worker.
-      OPERATION_TIMEOUT = 60
+      OPERATION_TIMEOUT = 20
       # Retries per request, on top of the crawl ledger's own auto-heal
       # rounds. GETs are idempotent; a couple of cheap retries beat a full
       # ledger round-trip for transient flakiness.

data/lib/ask/web_fetch/version.rb

@@ -2,6 +2,6 @@
 
 module Ask
   module WebFetch
-    VERSION = '0.7.6'
+    VERSION = '0.7.7'
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ask-web-fetch
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors:
 - Kaka Ruto