ask-web-fetch 0.5.5 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/ask/web_fetch/backends/local.rb +47 -0
- data/lib/ask/web_fetch/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a4c0b6735b967e38e8b837e7128bb501875ea1ce239f2df1549518396740ba8e
|
|
4
|
+
data.tar.gz: 0c6b95aba29f4914d062b35b3e21a43d1616a95ab01111a8d81d73326bd7f985
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6f7f82bc5b93d368395989049757d86675e89d7645f7b351f3d24edff50a8303ee86cdc04dec9240a9e71ef2f6f7dedd0d6641da93ddc3b9d5494b116fa5ccd0
|
|
7
|
+
data.tar.gz: c160aac9363740829eaceeb53b18e91b009dcdc278b194d7774ba692b3b6d14d3404cb5b4f38e0d50fb73a62390fad7b2e0bbc1ca60a47d8dce8de8c1e745072
|
|
@@ -46,6 +46,19 @@ module Ask
|
|
|
46
46
|
page = to_markdown(body, url)
|
|
47
47
|
page[:redirected] = redirect
|
|
48
48
|
raise EmptyContentError, "no readable content at #{url}" unless usable_content?(page[:content])
|
|
49
|
+
# The completeness signal: a JS-app shell whose server HTML
|
|
50
|
+
# renders little is a TRUNCATED page, not a complete one — the
|
|
51
|
+
# real content awaits client-side JS that Local cannot run.
|
|
52
|
+
# Storing the shell as success would silently under-deliver
|
|
53
|
+
# (airbnb: 613KB server HTML -> 143 chars of markdown); failing
|
|
54
|
+
# through lets the chain prefer a rendering backend (Browser),
|
|
55
|
+
# and keeps a partial page from ever being stored as the real
|
|
56
|
+
# thing. Two detectors: known framework markers, or a large
|
|
57
|
+
# HTML page with almost no server-rendered text.
|
|
58
|
+
if js_app_shell?(body) && page[:content].length < SHELL_CONTENT_THRESHOLD
|
|
59
|
+
raise EmptyContentError,
|
|
60
|
+
"JS-app shell at #{url} — server HTML renders only #{page[:content].length} chars; a rendering backend is required"
|
|
61
|
+
end
|
|
49
62
|
|
|
50
63
|
page
|
|
51
64
|
rescue Errno::ECONNREFUSED, Errno::ECONNRESET, SocketError, URI::InvalidURIError => e
|
|
@@ -55,6 +68,40 @@ module Ask
|
|
|
55
68
|
raise TimeoutError, "#{e.class}: #{e.message}"
|
|
56
69
|
end
|
|
57
70
|
|
|
71
|
+
# Client-rendered framework markers: the server HTML is (at least
|
|
72
|
+
# partly) a shell awaiting JS. Deliberately narrow — generic terms
|
|
73
|
+
# like "script" or "root" appear on every page; these are the
|
|
74
|
+
# specific footprints of React/Vue/Next/Nuxt app shells.
|
|
75
|
+
JS_APP_SHELL_MARKERS = /id=["'](?:root|app|__next|site-content)["']|__NEXT_DATA__|window\.__NUXT__|ng-app|data-reactroot/
|
|
76
|
+
|
|
77
|
+
# Markdown below this from a JS-app shell is "server sent a shell",
|
|
78
|
+
# not "page is genuinely short" — a real page (even a short one)
|
|
79
|
+
# is usually server-rendered above this. Tunable; the chain turns
|
|
80
|
+
# the signal into "prefer Browser for this URL".
|
|
81
|
+
SHELL_CONTENT_THRESHOLD = 4_000
|
|
82
|
+
|
|
83
|
+
# A page whose server HTML is large but yields almost no text is a
|
|
84
|
+
# shell whatever framework it uses (airbnb: 613KB HTML -> 143 chars
|
|
85
|
+
# of markdown). Framework markers miss these; the size ratio
|
|
86
|
+
# catches them. nytimes (1.4MB -> 7.5k text) stays above the text
|
|
87
|
+
# floor and is correctly left to Local.
|
|
88
|
+
SHELL_HTML_BYTES = 20_000
|
|
89
|
+
|
|
90
|
+
def js_app_shell?(body)
|
|
91
|
+
body.to_s.match?(JS_APP_SHELL_MARKERS) ||
|
|
92
|
+
(body.to_s.bytesize > SHELL_HTML_BYTES && markdown_visible_chars(body) < SHELL_CONTENT_THRESHOLD)
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# A cheap text estimate from the raw HTML (tags stripped) — the
|
|
96
|
+
# "how much did the server actually render" number. Deliberately
|
|
97
|
+
# rough: it only feeds a shell-vs-page heuristic.
|
|
98
|
+
def markdown_visible_chars(body)
|
|
99
|
+
body.to_s.gsub(/<script[\s\S]*?<\/script>/i, "")
|
|
100
|
+
.gsub(/<style[\s\S]*?<\/style>/i, "")
|
|
101
|
+
.gsub(/<[^>]+>/, " ")
|
|
102
|
+
.gsub(/\s+/, " ").strip.length
|
|
103
|
+
end
|
|
104
|
+
|
|
58
105
|
# Parses +html+ and returns { title:, description:, content:,
|
|
59
106
|
# licenses:, outlinks: } where content is clean markdown (pruned by
|
|
60
107
|
# the ContentFilter), licenses are the page's declared license
|