ask-web-fetch 0.5.7 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d8f3dc79c573c21dc567f86edd621ad84da98437ef512faea0e13b36219670d2
|
|
4
|
+
data.tar.gz: 694519194d63c9f4066f943ed2fe472f029ea10afd6e423a41440ff7d8a04964
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 270301527344d9b49c982e1d7c27dd83117f23c56de3e321f67e15aae7696998d423bd7cec72852a188412ee54b4183736917a5ec6c7030eab888b621af611f1
|
|
7
|
+
data.tar.gz: 66933232dbb0f106362e2a9552f19fc152022c5f9ed9b7ca4bda6fe1d2b3935ce3a3b7288bc2d0130aa8d9bddd6a4e5861ff5ed675e11b763b0e0d171e5749ef
|
|
@@ -16,6 +16,13 @@ module Ask
|
|
|
16
16
|
# A backend that fetched the page but found nothing usable in it.
|
|
17
17
|
class EmptyContentError < Error; end
|
|
18
18
|
|
|
19
|
+
# A backend that fetched a REGISTRAR PARKING PAGE — an ad for a
|
|
20
|
+
# parked (for-sale) domain, not the site's content. Deterministic and
|
|
21
|
+
# terminal: retrying will never turn a parking ad into content, so
|
|
22
|
+
# the pipeline must classify (not retry) it. A subclass of
|
|
23
|
+
# EmptyContentError so existing empty-content handling still applies.
|
|
24
|
+
class ParkedDomainError < EmptyContentError; end
|
|
25
|
+
|
|
19
26
|
# Network-level failure — timeout, connection refused/reset, bad
|
|
20
27
|
# socket. Transient: the same URL may succeed on retry.
|
|
21
28
|
class TimeoutError < Error; end
|
|
@@ -27,7 +27,11 @@ module Ask
|
|
|
27
27
|
|
|
28
28
|
# Creates a page as a fresh tab in the attached browser.
|
|
29
29
|
def create_page
|
|
30
|
-
|
|
30
|
+
# Capture the frontmost app once per browser instance, not per
|
|
31
|
+
# page: a batch of fetches steals focus once and restores to the
|
|
32
|
+
# same app, and consecutive fetches don't fight each other.
|
|
33
|
+
@frontmost_app ||= FocusRestorer.capture_frontmost
|
|
34
|
+
Page.new(client, timeout: @timeout, restore_focus_to: @frontmost_app)
|
|
31
35
|
end
|
|
32
36
|
|
|
33
37
|
private
|
|
@@ -43,9 +47,10 @@ module Ask
|
|
|
43
47
|
# enough CDP for the Browser backend: navigate, read title/URL/HTML,
|
|
44
48
|
# and report the main-document HTTP status.
|
|
45
49
|
class Page
|
|
46
|
-
def initialize(client, timeout:)
|
|
50
|
+
def initialize(client, timeout:, restore_focus_to: nil)
|
|
47
51
|
@client = client
|
|
48
52
|
@timeout = timeout
|
|
53
|
+
@restore_focus_to = restore_focus_to
|
|
49
54
|
@target_id = @client.command('Target.createTarget', url: 'about:blank')['targetId']
|
|
50
55
|
@session = @client.session(
|
|
51
56
|
@client.command('Target.attachToTarget', targetId: @target_id, flatten: true)['sessionId']
|
|
@@ -91,6 +96,12 @@ module Ask
|
|
|
91
96
|
@client.command('Target.closeTarget', targetId: @target_id)
|
|
92
97
|
rescue Ferrum::Error
|
|
93
98
|
nil
|
|
99
|
+
ensure
|
|
100
|
+
# The attached dev Chrome window steals focus when a tab is
|
|
101
|
+
# created; give it back once the tab is gone — but only to an
|
|
102
|
+
# app that ISN'T Chrome, so a fetch while the user is already
|
|
103
|
+
# looking at Chrome doesn't ping-pong.
|
|
104
|
+
FocusRestorer.restore_frontmost(@restore_focus_to)
|
|
94
105
|
end
|
|
95
106
|
|
|
96
107
|
def evaluate(expression)
|
|
@@ -189,6 +200,48 @@ module Ask
|
|
|
189
200
|
end
|
|
190
201
|
end
|
|
191
202
|
end
|
|
203
|
+
|
|
204
|
+
# macOS dev convenience: the attached Chrome window steals focus when
|
|
205
|
+
# a tab is created, and a developer working in another app wants it
|
|
206
|
+
# back once the tab is closed. Captures the frontmost app before the
|
|
207
|
+
# page is created and re-activates it after close. Deliberately
|
|
208
|
+
# cheap and quiet: one osascript each way, rescued to nil everywhere
|
|
209
|
+
# else (prod is headless Linux — this module is a no-op there, and
|
|
210
|
+
# even on macOS a failure must never fail a fetch).
|
|
211
|
+
module FocusRestorer
|
|
212
|
+
# The command runner, injectable for tests (minitest 6 ships no
|
|
213
|
+
# mock support, so tests swap this instead of stubbing).
|
|
214
|
+
class << self
|
|
215
|
+
attr_writer :runner
|
|
216
|
+
|
|
217
|
+
def runner
|
|
218
|
+
@runner ||= method(:system)
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
def self.capture_frontmost
|
|
223
|
+
return nil unless RUBY_PLATFORM.include?('darwin')
|
|
224
|
+
|
|
225
|
+
out = `osascript -e 'tell application "System Events" to get name of first application process whose frontmost is true' 2>/dev/null`.strip
|
|
226
|
+
out.empty? ? nil : out
|
|
227
|
+
rescue StandardError
|
|
228
|
+
nil
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def self.restore_frontmost(app)
|
|
232
|
+
return if app.nil? || app.empty?
|
|
233
|
+
|
|
234
|
+
# Never ping-pong: if the user is already looking at Chrome (or
|
|
235
|
+
# headless/CI has no frontmost app at all), there's nothing to
|
|
236
|
+
# give back.
|
|
237
|
+
return if app == 'Google Chrome'
|
|
238
|
+
|
|
239
|
+
runner.call("osascript -e 'tell application \"#{app}\" to activate' 2>/dev/null")
|
|
240
|
+
nil
|
|
241
|
+
rescue StandardError
|
|
242
|
+
nil
|
|
243
|
+
end
|
|
244
|
+
end
|
|
192
245
|
end
|
|
193
246
|
end
|
|
194
247
|
end
|
|
@@ -201,8 +201,9 @@ module Ask
|
|
|
201
201
|
end
|
|
202
202
|
# Browser renders the parked page a JS redirect lands on (the
|
|
203
203
|
# server shell hands /lander to JS) — Local never sees it. The
|
|
204
|
-
# shared parked-domain detector catches it here
|
|
205
|
-
|
|
204
|
+
# shared parked-domain detector catches it here; the distinct
|
|
205
|
+
# ParkedDomainError lets the pipeline classify (never retry) it.
|
|
206
|
+
raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content" if parked_domain?(body)
|
|
206
207
|
|
|
207
208
|
result = Markdown.generate(body, base_url: url, filter: self.class.content_filter)
|
|
208
209
|
result[:outlinks] = outlink_urls(body, url)
|
|
@@ -55,7 +55,7 @@ module Ask
|
|
|
55
55
|
# parked pages are fully server-rendered, so both Local and
|
|
56
56
|
# Browser see the same ad.
|
|
57
57
|
if parked_domain?(body)
|
|
58
|
-
raise
|
|
58
|
+
raise ParkedDomainError, "parked domain at #{url} — registrar parking page, not site content"
|
|
59
59
|
end
|
|
60
60
|
raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
|
|
61
61
|
# The completeness signal: a JS-app shell whose server HTML
|