pikuri-web 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +68 -0
- data/lib/pikuri/web/fetch.rb +153 -0
- data/lib/pikuri/web/read_cached.rb +96 -0
- data/lib/pikuri/web/scrape.rb +107 -0
- data/lib/pikuri/web/scraper.rb +188 -0
- data/lib/pikuri/web/search/brave.rb +298 -0
- data/lib/pikuri/web/search/duckduckgo.rb +181 -0
- data/lib/pikuri/web/search/engines.rb +430 -0
- data/lib/pikuri/web/search/exa.rb +243 -0
- data/lib/pikuri/web/search/page_store.rb +142 -0
- data/lib/pikuri/web/search/rate_limiter.rb +114 -0
- data/lib/pikuri/web/search/result.rb +42 -0
- data/lib/pikuri/web/search.rb +229 -0
- data/lib/pikuri/web/url_cache/in_memory.rb +96 -0
- data/lib/pikuri/web/url_cache/on_disk.rb +80 -0
- data/lib/pikuri/web/url_cache.rb +63 -0
- data/lib/pikuri-web.rb +36 -0
- metadata +138 -0
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'faraday'
|
|
4
|
+
require 'json'
|
|
5
|
+
require 'nokogiri'
|
|
6
|
+
|
|
7
|
+
module Pikuri
|
|
8
|
+
module Web
|
|
9
|
+
module Search
|
|
10
|
+
# Performs a Brave Search via the official Web Search API, returning hits
|
|
11
|
+
# as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
|
|
12
|
+
# parser (.parse) so tests exercise the parser against fixture JSON.
|
|
13
|
+
# {Engines#search} owns the final Markdown rendering.
|
|
14
|
+
#
|
|
15
|
+
# Constructed with its API key (+Brave.new(api_key:)+); {Engines} builds
|
|
16
|
+
# one only when a Brave key was configured. pikuri reads no key from the
|
|
17
|
+
# environment (AGENTS.md "Environment is not a secret store"). Key at
|
|
18
|
+
# https://api-dashboard.search.brave.com — the free "Data for Search" tier
|
|
19
|
+
# allows 1 query/sec, ~2k/month.
|
|
20
|
+
#
|
|
21
|
+
# == Page text
|
|
22
|
+
#
|
|
23
|
+
# Built with +page_text: true+, it calls LLM Context ({LLM_CONTEXT_ENDPOINT})
|
|
24
|
+
# instead of Web Search — the same plan, the same price per call — which
|
|
25
|
+
# returns each page's text as chunks from Brave's index, never fetched
|
|
26
|
+
# live. A row's {Result#text} is its chunks joined with a visible +…+ gap
|
|
27
|
+
# (they are excerpts, not one continuous page), {Result#indexed_at} the
|
|
28
|
+
# index's last fetch, and {Result#body} the page's own meta description,
|
|
29
|
+
# else the start of its first chunk. The two endpoints rank differently,
|
|
30
|
+
# so the mode changes which pages come back, not only their snippets.
|
|
31
|
+
#
|
|
32
|
+
# == Privacy posture
|
|
33
|
+
#
|
|
34
|
+
# Brave's Search API privacy notice (updated 3 Nov 2026, covers the whole
|
|
35
|
+
# API, no per-endpoint terms —
|
|
36
|
+
# https://api-dashboard.search.brave.com/privacy-policy) keeps Search
|
|
37
|
+
# Query Logs at most 90 days for billing, troubleshooting and abuse
|
|
38
|
+
# prevention, and logs the caller's IP address and API token for the same
|
|
39
|
+
# purposes. It collects no identifier linking a query to an *end user* of
|
|
40
|
+
# the customer's app — but under pikuri the account holder *is* the end
|
|
41
|
+
# user, so that log is tied to your key. Zero Data Retention exists only
|
|
42
|
+
# on a custom Enterprise plan. The notice is silent on training; the
|
|
43
|
+
# pricing page (https://brave.com/search/api/) promises "Queries never
|
|
44
|
+
# shared or trained on". Bottom line: the cleanest API-level posture of
|
|
45
|
+
# pikuri's three providers (no training, capped retention), but a 90-day
|
|
46
|
+
# log tied to your key, so no ZDR substitute for genuinely sensitive
|
|
47
|
+
# queries.
|
|
48
|
+
class Brave
|
|
49
|
+
# @return [String] Web Search endpoint
|
|
50
|
+
ENDPOINT = 'https://api.search.brave.com/res/v1/web/search'
|
|
51
|
+
# @return [Integer] default number of results returned.
|
|
52
|
+
DEFAULT_MAX_RESULTS = 10
|
|
53
|
+
# @return [RateLimiter] free-tier Brave caps at 1 req/sec; the 5-minute
|
|
54
|
+
# cooldown protects the monthly quota from doomed retries on a 429.
|
|
55
|
+
LIMITER = RateLimiter.new(min_interval: 1.0, cooldown: 300.0)
|
|
56
|
+
# @return [String] LLM Context endpoint, used with +page_text+
|
|
57
|
+
LLM_CONTEXT_ENDPOINT = 'https://api.search.brave.com/res/v1/llm/context'
|
|
58
|
+
# @return [Integer] LLM Context's ceiling on a response's tokens, asked
|
|
59
|
+
# for whole: pikuri takes all the text Brave gives and trims only when
|
|
60
|
+
# it is read
|
|
61
|
+
MAX_TOKENS = 32_768
|
|
62
|
+
# @return [Integer] LLM Context's ceiling on one URL's tokens
|
|
63
|
+
MAX_TOKENS_PER_URL = 8_192
|
|
64
|
+
# @return [Integer] chars of a first chunk kept as a row's snippet when
|
|
65
|
+
# the page has no meta description
|
|
66
|
+
SNIPPET_CHARS = 300
|
|
67
|
+
# @return [String] between two chunks of one page's {Result#text}
|
|
68
|
+
CHUNK_SEPARATOR = "\n\n…\n\n"
|
|
69
|
+
|
|
70
|
+
# @param api_key [String] Brave Search subscription token. Required
|
|
71
|
+
# and non-blank: pikuri reads no key from the environment — the
|
|
72
|
+
# host supplies it ({Engines} only constructs a Brave when a key
|
|
73
|
+
# was configured).
|
|
74
|
+
# @param page_text [Boolean] call LLM Context for each page's stored
|
|
75
|
+
# text; see the class doc's +== Page text+
|
|
76
|
+
# @raise [ArgumentError] if +api_key+ is blank
|
|
77
|
+
def initialize(api_key:, page_text: false)
|
|
78
|
+
raise ArgumentError, 'Brave Search API key is blank' if api_key.to_s.strip.empty?
|
|
79
|
+
|
|
80
|
+
@api_key = api_key
|
|
81
|
+
@page_text = page_text
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# @return [String] short provider label for {Engines}.
|
|
85
|
+
def label
|
|
86
|
+
'Brave'
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Whether {Engines} may keep this provider's results in its on-disk
|
|
90
|
+
# cache. No: the Search API terms (§3(b)(i),
|
|
91
|
+
# https://api-dashboard.search.brave.com/terms-of-service) forbid to
|
|
92
|
+
# "store, cache, or create a database of Search Results … other than
|
|
93
|
+
# transient storage required for operation", so they stay in memory
|
|
94
|
+
# ({UrlCache::InMemory}).
|
|
95
|
+
#
|
|
96
|
+
# @return [Boolean] +false+
|
|
97
|
+
def disk_cacheable?
|
|
98
|
+
false
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Fetch results for +query+ as an +Array<Result>+. Throttled to 1/sec and
|
|
102
|
+
# circuit-broken 5 min on rate-limit/quota (see {LIMITER}). The caller
|
|
103
|
+
# ({Engines#search}) normalizes the query and wraps this in a cache.
|
|
104
|
+
#
|
|
105
|
+
# @param query [String] search query (already normalized)
|
|
106
|
+
# @param max_results [Integer] max entries; Brave's +count+ (1..20)
|
|
107
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable] makes
|
|
108
|
+
# {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
|
|
109
|
+
# @return [Array<Result>] hits, possibly empty when Brave matched nothing
|
|
110
|
+
# @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
|
|
111
|
+
# back), or if {LIMITER} is in cooldown. Other non-2xx (401/403 bad
|
|
112
|
+
# key) bubble up as +RuntimeError+.
|
|
113
|
+
# @raise [RuntimeError] for non-rate-limit HTTP failures, or a
|
|
114
|
+
# zero-result response that isn't a recognized empty-results payload.
|
|
115
|
+
def search(query, max_results: DEFAULT_MAX_RESULTS,
|
|
116
|
+
cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
117
|
+
LIMITER.call(cancellable: cancellable) do
|
|
118
|
+
endpoint, params = if @page_text
|
|
119
|
+
[LLM_CONTEXT_ENDPOINT, self.class.llm_context_params(query, max_results: max_results)]
|
|
120
|
+
else
|
|
121
|
+
[ENDPOINT, { q: query, count: max_results }]
|
|
122
|
+
end
|
|
123
|
+
response = Faraday.get(
|
|
124
|
+
endpoint, params,
|
|
125
|
+
{ 'X-Subscription-Token' => @api_key, 'Accept' => 'application/json' }
|
|
126
|
+
)
|
|
127
|
+
unless response.success?
|
|
128
|
+
if response.status == 429 || response.status >= 500
|
|
129
|
+
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
raise "Brave Search request failed: #{response.status} #{response.body}"
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
if @page_text
|
|
136
|
+
self.class.parse_llm_context(response.body, max_results: max_results)
|
|
137
|
+
else
|
|
138
|
+
self.class.parse(response.body, max_results: max_results)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
# The query parameters for an {LLM_CONTEXT_ENDPOINT} request: every
|
|
144
|
+
# relevance threshold off and the token budgets at their ceilings, so
|
|
145
|
+
# each row brings all the text Brave holds for it.
|
|
146
|
+
#
|
|
147
|
+
# Brave.llm_context_params('ruby', max_results: 5)
|
|
148
|
+
# # => { q: 'ruby', count: 5, maximum_number_of_urls: 5,
|
|
149
|
+
# # maximum_number_of_tokens: 32768, maximum_number_of_tokens_per_url: 8192,
|
|
150
|
+
# # context_threshold_mode: 'disabled', enable_source_metadata: true }
|
|
151
|
+
#
|
|
152
|
+
# @param query [String] search query (already normalized)
|
|
153
|
+
# @param max_results [Integer] max entries (1..50)
|
|
154
|
+
# @return [Hash{Symbol => Object}]
|
|
155
|
+
def self.llm_context_params(query, max_results: DEFAULT_MAX_RESULTS)
|
|
156
|
+
{
|
|
157
|
+
q: query,
|
|
158
|
+
count: max_results,
|
|
159
|
+
maximum_number_of_urls: max_results,
|
|
160
|
+
maximum_number_of_tokens: MAX_TOKENS,
|
|
161
|
+
maximum_number_of_tokens_per_url: MAX_TOKENS_PER_URL,
|
|
162
|
+
context_threshold_mode: 'disabled',
|
|
163
|
+
enable_source_metadata: true
|
|
164
|
+
}
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
# Parse an LLM Context response into {Result} rows with +text+ — see the
|
|
168
|
+
# class doc's +== Page text+ for what each field holds. A response with
|
|
169
|
+
# an empty +grounding.generic+ is Brave's "matched nothing" and returns
|
|
170
|
+
# +[]+; any other zero-result shape raises.
|
|
171
|
+
#
|
|
172
|
+
# @param json [String] response body from {LLM_CONTEXT_ENDPOINT}
|
|
173
|
+
# @param max_results [Integer] max entries
|
|
174
|
+
# @return [Array<Result>]
|
|
175
|
+
# @raise [RuntimeError] on an unrecognized zero-result response
|
|
176
|
+
def self.parse_llm_context(json, max_results: DEFAULT_MAX_RESULTS)
|
|
177
|
+
data = JSON.parse(json)
|
|
178
|
+
sources = data.is_a?(Hash) && data['sources'].is_a?(Hash) ? data['sources'] : {}
|
|
179
|
+
generic = data.is_a?(Hash) ? data.dig('grounding', 'generic') : nil
|
|
180
|
+
results = Array(generic).take(max_results).filter_map do |g|
|
|
181
|
+
href = g['url'].to_s
|
|
182
|
+
next nil if href.empty?
|
|
183
|
+
|
|
184
|
+
source = sources[href].is_a?(Hash) ? sources[href] : {}
|
|
185
|
+
chunks = Array(g['snippets']).map(&:to_s).reject { |c| c.strip.empty? }
|
|
186
|
+
title = strip_html(g['title'] || source['title'])
|
|
187
|
+
fetched = source['fetched_content_timestamp']
|
|
188
|
+
Result.new(
|
|
189
|
+
url: href,
|
|
190
|
+
title: title.empty? ? href : title,
|
|
191
|
+
body: llm_context_snippet(source['description'], chunks.first),
|
|
192
|
+
text: chunks.empty? ? nil : chunks.join(CHUNK_SEPARATOR),
|
|
193
|
+
indexed_at: fetched.is_a?(Integer) ? Time.at(fetched).utc : nil
|
|
194
|
+
)
|
|
195
|
+
end
|
|
196
|
+
return results unless results.empty?
|
|
197
|
+
return [] if data.is_a?(Hash) && data['grounding'].is_a?(Hash) && generic.is_a?(Array)
|
|
198
|
+
|
|
199
|
+
raise diagnose_empty(data, json)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# A row's snippet: the page's meta description, else the start of its
|
|
203
|
+
# first chunk.
|
|
204
|
+
#
|
|
205
|
+
# @param description [String, nil] +sources[url].description+
|
|
206
|
+
# @param first_chunk [String, nil]
|
|
207
|
+
# @return [String] possibly empty
|
|
208
|
+
def self.llm_context_snippet(description, first_chunk)
|
|
209
|
+
snippet = strip_html(description)
|
|
210
|
+
return snippet unless snippet.empty?
|
|
211
|
+
|
|
212
|
+
chunk = first_chunk.to_s.gsub(/\s+/, ' ').strip
|
|
213
|
+
chunk.length > SNIPPET_CHARS ? "#{chunk[0, SNIPPET_CHARS]}…" : chunk
|
|
214
|
+
end
|
|
215
|
+
private_class_method :llm_context_snippet
|
|
216
|
+
|
|
217
|
+
# Parse a Brave JSON response into {Result} rows (+<strong>+ highlights
|
|
218
|
+
# stripped). A genuine "no results" payload (recognized search shape,
|
|
219
|
+
# empty +mixed.main/top/side+) returns +[]+ so {Engines.search} renders
|
|
220
|
+
# its stub; any other zero-result shape raises.
|
|
221
|
+
#
|
|
222
|
+
# @param json [String] response body from {ENDPOINT}
|
|
223
|
+
# @param max_results [Integer] max entries
|
|
224
|
+
# @return [Array<Result>] hits, possibly empty on a recognized
|
|
225
|
+
# empty-results payload
|
|
226
|
+
# @raise [RuntimeError] on an unrecognized zero-result response
|
|
227
|
+
def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
|
|
228
|
+
data = JSON.parse(json)
|
|
229
|
+
results = Array(data.dig('web', 'results')).take(max_results).filter_map do |r|
|
|
230
|
+
href = r['url'].to_s
|
|
231
|
+
next nil if href.empty?
|
|
232
|
+
|
|
233
|
+
Result.new(
|
|
234
|
+
url: href,
|
|
235
|
+
title: strip_html(r['title']),
|
|
236
|
+
body: strip_html(r['description'])
|
|
237
|
+
)
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
if results.empty?
|
|
241
|
+
return [] if genuine_no_results?(data)
|
|
242
|
+
|
|
243
|
+
raise diagnose_empty(data, json)
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
results
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# Strip HTML markup (notably the +<strong>+ highlights Brave wraps around
|
|
250
|
+
# query terms) and collapse whitespace.
|
|
251
|
+
#
|
|
252
|
+
# @param html [String, nil]
|
|
253
|
+
# @return [String] plain text; empty string for nil
|
|
254
|
+
def self.strip_html(html)
|
|
255
|
+
return '' if html.nil?
|
|
256
|
+
|
|
257
|
+
Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
|
|
258
|
+
end
|
|
259
|
+
private_class_method :strip_html
|
|
260
|
+
|
|
261
|
+
# True when a zero-+web.results+ response is Brave's own "ran, matched
|
|
262
|
+
# nothing" payload, not an error. Markers: the +type: "search"+ envelope
|
|
263
|
+
# and an empty +mixed+ block (Brave fills +mixed.main/top/side+ with
|
|
264
|
+
# matched verticals, so all three empty = succeeded-but-found-nothing).
|
|
265
|
+
#
|
|
266
|
+
# @param data [Hash, Object] parsed response
|
|
267
|
+
# @return [Boolean]
|
|
268
|
+
def self.genuine_no_results?(data)
|
|
269
|
+
return false unless data.is_a?(Hash) && data['type'] == 'search'
|
|
270
|
+
|
|
271
|
+
mixed = data['mixed']
|
|
272
|
+
return false unless mixed.is_a?(Hash)
|
|
273
|
+
|
|
274
|
+
%w[main top side].all? { |k| Array(mixed[k]).empty? }
|
|
275
|
+
end
|
|
276
|
+
private_class_method :genuine_no_results?
|
|
277
|
+
|
|
278
|
+
# Build an error message for a zero-result response: quote Brave's
|
|
279
|
+
# +error.detail+ if present, else a truncated raw body.
|
|
280
|
+
#
|
|
281
|
+
# @param data [Hash, Object] parsed response
|
|
282
|
+
# @param raw [String] raw response body
|
|
283
|
+
# @return [String] diagnostic to feed to +raise+
|
|
284
|
+
def self.diagnose_empty(data, raw)
|
|
285
|
+
if data.is_a?(Hash) && data['error'].is_a?(Hash)
|
|
286
|
+
err = data['error']
|
|
287
|
+
return "Brave Search returned an error: #{[err['code'], err['detail'] || err['meta']].compact.join(' — ')}"
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
snippet = raw.to_s[0, 800]
|
|
291
|
+
snippet += '…' if raw.to_s.length > 800
|
|
292
|
+
"Brave Search returned no results. Body: #{snippet}"
|
|
293
|
+
end
|
|
294
|
+
private_class_method :diagnose_empty
|
|
295
|
+
end
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
end
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'faraday'
|
|
4
|
+
require 'nokogiri'
|
|
5
|
+
require 'uri'
|
|
6
|
+
|
|
7
|
+
module Pikuri
|
|
8
|
+
module Web
|
|
9
|
+
module Search
|
|
10
|
+
# Performs a DuckDuckGo search by scraping +html.duckduckgo.com+, returning
|
|
11
|
+
# hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
|
|
12
|
+
# parser (.parse) so tests exercise the parser against fixture HTML.
|
|
13
|
+
# {Engines#search} owns the final Markdown rendering. Constructed with no
|
|
14
|
+
# arguments, sharing the uniform +#search+/+#label+ provider shape with the
|
|
15
|
+
# keyed {Brave} / {Exa}.
|
|
16
|
+
#
|
|
17
|
+
# == Privacy posture
|
|
18
|
+
#
|
|
19
|
+
# DDG doesn't save your IP or tie a per-user profile to queries, and
|
|
20
|
+
# proxies requests so downstream providers can't build a search history —
|
|
21
|
+
# real, but DDG relays web results over Bing, so *query content* still
|
|
22
|
+
# reaches Microsoft (who has no no-training pledge). Better than Exa for
|
|
23
|
+
# sensitive queries, worse than Brave; for anything genuinely embarrassing,
|
|
24
|
+
# don't search the web at all.
|
|
25
|
+
class DuckDuckGo
|
|
26
|
+
# @return [String] HTML search endpoint
|
|
27
|
+
ENDPOINT = 'https://html.duckduckgo.com/html/'
|
|
28
|
+
# @return [Integer] default number of results returned.
|
|
29
|
+
DEFAULT_MAX_RESULTS = 10
|
|
30
|
+
# @return [RateLimiter] paces calls (DDG bans IPs that hammer the HTML
|
|
31
|
+
# endpoint) and circuit-breaks 5 min on {Engines::Unavailable}.
|
|
32
|
+
LIMITER = RateLimiter.new(min_interval: 2.0, cooldown: 300.0)
|
|
33
|
+
|
|
34
|
+
# @return [String] short provider label for {Engines}.
|
|
35
|
+
def label
|
|
36
|
+
'DuckDuckGo'
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Whether {Engines} may keep this provider's results in its on-disk
|
|
40
|
+
# cache. Yes: pikuri scrapes the keyless HTML endpoint, under no API
|
|
41
|
+
# agreement restricting what it keeps.
|
|
42
|
+
#
|
|
43
|
+
# @return [Boolean] +true+
|
|
44
|
+
def disk_cacheable?
|
|
45
|
+
true
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Fetch results for +query+ as an +Array<Result>+. Throttled to one every
|
|
49
|
+
# 2s and circuit-broken 5 min after a soft-block (see {LIMITER}). The
|
|
50
|
+
# caller ({Engines#search}) normalizes the query and wraps this in a cache.
|
|
51
|
+
#
|
|
52
|
+
# @param query [String] search query (already normalized)
|
|
53
|
+
# @param max_results [Integer] max result entries
|
|
54
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable] makes
|
|
55
|
+
# {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
|
|
56
|
+
# @return [Array<Result>] hits, possibly empty when DDG matched nothing
|
|
57
|
+
# @raise [Engines::Unavailable] on a soft-block (anomaly/CAPTCHA page) or
|
|
58
|
+
# HTTP 429/5xx (the cascade falls back), or if {LIMITER} is in cooldown.
|
|
59
|
+
# @raise [RuntimeError] on other HTTP failures or an unrecognized
|
|
60
|
+
# empty-results layout. A genuine empty-results page is not an error;
|
|
61
|
+
# see {.parse}.
|
|
62
|
+
def search(query, max_results: DEFAULT_MAX_RESULTS,
|
|
63
|
+
cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
64
|
+
LIMITER.call(cancellable: cancellable) do
|
|
65
|
+
# DDG soft-blocks a bot-looking request, so it goes out dressed as a browser.
|
|
66
|
+
response = Scraper.browser_connection(ENDPOINT).get(ENDPOINT, { q: query })
|
|
67
|
+
unless response.success?
|
|
68
|
+
if response.status == 429 || response.status >= 500
|
|
69
|
+
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
raise "DuckDuckGo request failed: #{response.status} #{response.body}"
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
self.class.parse(response.body, max_results: max_results)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Parse a +html.duckduckgo.com+ result page into {Result} rows (+<b>+
|
|
80
|
+
# highlights stripped). On zero result nodes: a genuine "no results" page
|
|
81
|
+
# returns +[]+ (so {Engines.search} renders its stub); the anomaly/CAPTCHA
|
|
82
|
+
# modal raises {Engines::Unavailable}; anything else raises {RuntimeError}
|
|
83
|
+
# (likely a layout change) — so an IP soft-block surfaces rather than
|
|
84
|
+
# masquerading as an empty search.
|
|
85
|
+
#
|
|
86
|
+
# @param html [String] HTML from html.duckduckgo.com
|
|
87
|
+
# @param max_results [Integer] max result entries
|
|
88
|
+
# @return [Array<Result>] hits, possibly empty on a genuine no-results page
|
|
89
|
+
# @raise [Engines::Unavailable] on the anomaly/CAPTCHA modal (IP soft-block)
|
|
90
|
+
# @raise [RuntimeError] on zero results with an unrecognized layout
|
|
91
|
+
def self.parse(html, max_results: DEFAULT_MAX_RESULTS)
|
|
92
|
+
doc = Nokogiri::HTML(html)
|
|
93
|
+
results = doc.css('div.result.web-result').take(max_results).filter_map do |node|
|
|
94
|
+
title_link = node.at_css('a.result__a')
|
|
95
|
+
next nil if title_link.nil?
|
|
96
|
+
|
|
97
|
+
snippet = node.at_css('a.result__snippet')
|
|
98
|
+
Result.new(
|
|
99
|
+
url: extract_url(title_link['href']),
|
|
100
|
+
title: title_link.text.strip,
|
|
101
|
+
body: snippet&.text&.strip.to_s
|
|
102
|
+
)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
if results.empty?
|
|
106
|
+
return [] if genuine_no_results?(doc)
|
|
107
|
+
|
|
108
|
+
message = diagnose_empty(doc)
|
|
109
|
+
raise(anomaly_modal?(doc) ? Engines::Unavailable : RuntimeError, message)
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
results
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# True when the page is DDG's anomaly/CAPTCHA modal (IP soft-blocked).
|
|
116
|
+
# {.parse} uses it to pick {Engines::Unavailable} (recoverable) over
|
|
117
|
+
# {RuntimeError} (unknown layout).
|
|
118
|
+
#
|
|
119
|
+
# @param doc [Nokogiri::HTML::Document]
|
|
120
|
+
# @return [Boolean]
|
|
121
|
+
def self.anomaly_modal?(doc)
|
|
122
|
+
!!(doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description'))
|
|
123
|
+
end
|
|
124
|
+
private_class_method :anomaly_modal?
|
|
125
|
+
|
|
126
|
+
# True when a zero-node page is DDG's own "no results found" page, not an
|
|
127
|
+
# anomaly/CAPTCHA. Anomaly wins: if the modal divs are present it's never
|
|
128
|
+
# a genuine empty result, even if the text mentions "No results".
|
|
129
|
+
#
|
|
130
|
+
# @param doc [Nokogiri::HTML::Document]
|
|
131
|
+
# @return [Boolean]
|
|
132
|
+
def self.genuine_no_results?(doc)
|
|
133
|
+
return false if doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description')
|
|
134
|
+
return true if doc.at_css('div.no-results')
|
|
135
|
+
|
|
136
|
+
doc.text.include?('No results found')
|
|
137
|
+
end
|
|
138
|
+
private_class_method :genuine_no_results?
|
|
139
|
+
|
|
140
|
+
# Build an error message for a zero-match page: quote the DDG anomaly
|
|
141
|
+
# modal's title/description if present, else a truncated copy of the
|
|
142
|
+
# body's visible text (+<script>/<style>/<noscript>+ stripped).
|
|
143
|
+
#
|
|
144
|
+
# @param doc [Nokogiri::HTML::Document]
|
|
145
|
+
# @return [String] diagnostic to feed to +raise+
|
|
146
|
+
def self.diagnose_empty(doc)
|
|
147
|
+
title = doc.at_css('.anomaly-modal__title')&.text&.strip
|
|
148
|
+
desc = doc.at_css('.anomaly-modal__description')&.text&.strip
|
|
149
|
+
if title || desc
|
|
150
|
+
return "DuckDuckGo anomaly check (likely IP soft-block): #{[title, desc].compact.reject(&:empty?).join(' — ')}"
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
doc.css('script, style, noscript').remove
|
|
154
|
+
text = doc.text.gsub(/\s+/, ' ').strip
|
|
155
|
+
snippet = text.empty? ? '<empty body>' : text[0, 1500]
|
|
156
|
+
snippet += '…' if text.length > 1500
|
|
157
|
+
"DuckDuckGo returned no results. Page text: #{snippet}"
|
|
158
|
+
end
|
|
159
|
+
private_class_method :diagnose_empty
|
|
160
|
+
|
|
161
|
+
# Decode DDG's +//duckduckgo.com/l/?uddg=<encoded>+ redirect wrapper to
|
|
162
|
+
# the real target URL.
|
|
163
|
+
#
|
|
164
|
+
# @param href [String, nil] href from the result page
|
|
165
|
+
# @return [String] the decoded target, or +href+ unchanged when not a
|
|
166
|
+
# recognized DDG redirect or unparseable
|
|
167
|
+
def self.extract_url(href)
|
|
168
|
+
return href if href.nil? || href.empty?
|
|
169
|
+
|
|
170
|
+
uri = URI.parse(href.start_with?('//') ? "https:#{href}" : href)
|
|
171
|
+
return href unless uri.host&.end_with?('duckduckgo.com') && uri.path == '/l/'
|
|
172
|
+
|
|
173
|
+
params = URI.decode_www_form(uri.query.to_s).to_h
|
|
174
|
+
params['uddg'] || href
|
|
175
|
+
rescue URI::InvalidURIError
|
|
176
|
+
href
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|