ask-web-fetch 0.5.1 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/ask/web_fetch/http.rb +13 -6
- data/lib/ask/web_fetch/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 18e8a7b0274def2380d5b75df069e428aeca9e027352f2fc09ee64b65d946a68
|
|
4
|
+
data.tar.gz: d69e005edde7afcf8b05a8d89da9655be939ab7e21b039d6d61c617b4a1d74de
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0e7c44e68630b3b7c2e7a5c04581b2e9992a888c927ecb5de47b4782f92538fa0e5b71eb53ea4fea325c6f354595338698abfac5540260e09d162d30673db347
|
|
7
|
+
data.tar.gz: 6f67717fbe1aeec17b0cfbc4939c3b9195ebd229976ca744ebe8ef67de08e9bae6630ca8fbd2fa7b50979487ce331d8a2edaf96e12818185d2ea855ed295c477
|
data/lib/ask/web_fetch/http.rb
CHANGED
|
@@ -48,17 +48,24 @@ module Ask
|
|
|
48
48
|
|
|
49
49
|
# One pooled session per thread — httpx sessions are not thread-safe,
|
|
50
50
|
# and a crawl worker thread reusing its session across every page it
|
|
51
|
-
# fetches is what
|
|
52
|
-
# themselves after keep-alive timeout, so nothing
|
|
51
|
+
# fetches is what keeps timeouts and retries configured once.
|
|
52
|
+
# Sessions idle-close themselves after keep-alive timeout, so nothing
|
|
53
|
+
# to reap.
|
|
53
54
|
def self.session
|
|
54
55
|
Thread.current[SESSION_KEY] ||= build_session
|
|
55
56
|
end
|
|
56
57
|
|
|
57
58
|
def self.build_session
|
|
58
|
-
# :persistent
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
|
|
59
|
+
# Deliberately NOT :persistent: in httpx 1.8.1 that plugin wedges
|
|
60
|
+
# inside the selector loop when a session that already holds a
|
|
61
|
+
# pooled connection opens one to a NEW host — the operation
|
|
62
|
+
# timeout never fires and the fetch hangs forever (reproduced in
|
|
63
|
+
# plain Ruby: example.com then nytimes.com on one session).
|
|
64
|
+
# Without it httpx opens a fresh connection per host, which for a
|
|
65
|
+
# crawler hitting mostly-distinct hosts costs one TLS handshake
|
|
66
|
+
# per page and never hangs. Retries are explicit so the transient
|
|
67
|
+
# backoff that :persistent loaded internally is kept.
|
|
68
|
+
HTTPX.plugin(:retries).with(
|
|
62
69
|
timeout: {
|
|
63
70
|
connect_timeout: CONNECT_TIMEOUT,
|
|
64
71
|
read_timeout: READ_TIMEOUT,
|