pikuri-web 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,298 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'faraday'
4
+ require 'json'
5
+ require 'nokogiri'
6
+
7
+ module Pikuri
8
+ module Web
9
+ module Search
10
+ # Performs a Brave Search via the official Web Search API, returning hits
11
+ # as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
12
+ # parser (.parse) so tests exercise the parser against fixture JSON.
13
+ # {Engines#search} owns the final Markdown rendering.
14
+ #
15
+ # Constructed with its API key (+Brave.new(api_key:)+); {Engines} builds
16
+ # one only when a Brave key was configured. pikuri reads no key from the
17
+ # environment (AGENTS.md "Environment is not a secret store"). Key at
18
+ # https://api-dashboard.search.brave.com — the free "Data for Search" tier
19
+ # allows 1 query/sec, ~2k/month.
20
+ #
21
+ # == Page text
22
+ #
23
+ # Built with +page_text: true+, it calls LLM Context ({LLM_CONTEXT_ENDPOINT})
24
+ # instead of Web Search — the same plan, the same price per call — which
25
+ # returns each page's text as chunks from Brave's index, never fetched
26
+ # live. A row's {Result#text} is its chunks joined with a visible +…+ gap
27
+ # (they are excerpts, not one continuous page), {Result#indexed_at} the
28
+ # index's last fetch, and {Result#body} the page's own meta description,
29
+ # else the start of its first chunk. The two endpoints rank differently,
30
+ # so the mode changes which pages come back, not only their snippets.
31
+ #
32
+ # == Privacy posture
33
+ #
34
+ # Brave's Search API privacy notice (updated 3 Nov 2026, covers the whole
35
+ # API, no per-endpoint terms —
36
+ # https://api-dashboard.search.brave.com/privacy-policy) keeps Search
37
+ # Query Logs at most 90 days for billing, troubleshooting and abuse
38
+ # prevention, and logs the caller's IP address and API token for the same
39
+ # purposes. It collects no identifier linking a query to an *end user* of
40
+ # the customer's app — but under pikuri the account holder *is* the end
41
+ # user, so that log is tied to your key. Zero Data Retention exists only
42
+ # on a custom Enterprise plan. The notice is silent on training; the
43
+ # pricing page (https://brave.com/search/api/) promises "Queries never
44
+ # shared or trained on". Bottom line: the cleanest API-level posture of
45
+ # pikuri's three providers (no training, capped retention), but a 90-day
46
+ # log tied to your key, so no ZDR substitute for genuinely sensitive
47
+ # queries.
48
+ class Brave
49
+ # @return [String] Web Search endpoint
50
+ ENDPOINT = 'https://api.search.brave.com/res/v1/web/search'
51
+ # @return [Integer] default number of results returned.
52
+ DEFAULT_MAX_RESULTS = 10
53
+ # @return [RateLimiter] free-tier Brave caps at 1 req/sec; the 5-minute
54
+ # cooldown protects the monthly quota from doomed retries on a 429.
55
+ LIMITER = RateLimiter.new(min_interval: 1.0, cooldown: 300.0)
56
+ # @return [String] LLM Context endpoint, used with +page_text+
57
+ LLM_CONTEXT_ENDPOINT = 'https://api.search.brave.com/res/v1/llm/context'
58
+ # @return [Integer] LLM Context's ceiling on a response's tokens, asked
59
+ # for whole: pikuri takes all the text Brave gives and trims only when
60
+ # it is read
61
+ MAX_TOKENS = 32_768
62
+ # @return [Integer] LLM Context's ceiling on one URL's tokens
63
+ MAX_TOKENS_PER_URL = 8_192
64
+ # @return [Integer] chars of a first chunk kept as a row's snippet when
65
+ # the page has no meta description
66
+ SNIPPET_CHARS = 300
67
+ # @return [String] between two chunks of one page's {Result#text}
68
+ CHUNK_SEPARATOR = "\n\n…\n\n"
69
+
70
+ # @param api_key [String] Brave Search subscription token. Required
71
+ # and non-blank: pikuri reads no key from the environment — the
72
+ # host supplies it ({Engines} only constructs a Brave when a key
73
+ # was configured).
74
+ # @param page_text [Boolean] call LLM Context for each page's stored
75
+ # text; see the class doc's +== Page text+
76
+ # @raise [ArgumentError] if +api_key+ is blank
77
+ def initialize(api_key:, page_text: false)
78
+ raise ArgumentError, 'Brave Search API key is blank' if api_key.to_s.strip.empty?
79
+
80
+ @api_key = api_key
81
+ @page_text = page_text
82
+ end
83
+
84
+ # @return [String] short provider label for {Engines}.
85
+ def label
86
+ 'Brave'
87
+ end
88
+
89
+ # Whether {Engines} may keep this provider's results in its on-disk
90
+ # cache. No: the Search API terms (§3(b)(i),
91
+ # https://api-dashboard.search.brave.com/terms-of-service) forbid to
92
+ # "store, cache, or create a database of Search Results … other than
93
+ # transient storage required for operation", so they stay in memory
94
+ # ({UrlCache::InMemory}).
95
+ #
96
+ # @return [Boolean] +false+
97
+ def disk_cacheable?
98
+ false
99
+ end
100
+
101
+ # Fetch results for +query+ as an +Array<Result>+. Throttled to 1/sec and
102
+ # circuit-broken 5 min on rate-limit/quota (see {LIMITER}). The caller
103
+ # ({Engines#search}) normalizes the query and wraps this in a cache.
104
+ #
105
+ # @param query [String] search query (already normalized)
106
+ # @param max_results [Integer] max entries; Brave's +count+ (1..20)
107
+ # @param cancellable [Pikuri::Agent::Control::Cancellable] makes
108
+ # {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
109
+ # @return [Array<Result>] hits, possibly empty when Brave matched nothing
110
+ # @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
111
+ # back), or if {LIMITER} is in cooldown. Other non-2xx (401/403 bad
112
+ # key) bubble up as +RuntimeError+.
113
+ # @raise [RuntimeError] for non-rate-limit HTTP failures, or a
114
+ # zero-result response that isn't a recognized empty-results payload.
115
+ def search(query, max_results: DEFAULT_MAX_RESULTS,
116
+ cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
117
+ LIMITER.call(cancellable: cancellable) do
118
+ endpoint, params = if @page_text
119
+ [LLM_CONTEXT_ENDPOINT, self.class.llm_context_params(query, max_results: max_results)]
120
+ else
121
+ [ENDPOINT, { q: query, count: max_results }]
122
+ end
123
+ response = Faraday.get(
124
+ endpoint, params,
125
+ { 'X-Subscription-Token' => @api_key, 'Accept' => 'application/json' }
126
+ )
127
+ unless response.success?
128
+ if response.status == 429 || response.status >= 500
129
+ raise Engines::Unavailable, "HTTP #{response.status}"
130
+ end
131
+
132
+ raise "Brave Search request failed: #{response.status} #{response.body}"
133
+ end
134
+
135
+ if @page_text
136
+ self.class.parse_llm_context(response.body, max_results: max_results)
137
+ else
138
+ self.class.parse(response.body, max_results: max_results)
139
+ end
140
+ end
141
+ end
142
+
143
+ # The query parameters for an {LLM_CONTEXT_ENDPOINT} request: every
144
+ # relevance threshold off and the token budgets at their ceilings, so
145
+ # each row brings all the text Brave holds for it.
146
+ #
147
+ # Brave.llm_context_params('ruby', max_results: 5)
148
+ # # => { q: 'ruby', count: 5, maximum_number_of_urls: 5,
149
+ # # maximum_number_of_tokens: 32768, maximum_number_of_tokens_per_url: 8192,
150
+ # # context_threshold_mode: 'disabled', enable_source_metadata: true }
151
+ #
152
+ # @param query [String] search query (already normalized)
153
+ # @param max_results [Integer] max entries (1..50)
154
+ # @return [Hash{Symbol => Object}]
155
+ def self.llm_context_params(query, max_results: DEFAULT_MAX_RESULTS)
156
+ {
157
+ q: query,
158
+ count: max_results,
159
+ maximum_number_of_urls: max_results,
160
+ maximum_number_of_tokens: MAX_TOKENS,
161
+ maximum_number_of_tokens_per_url: MAX_TOKENS_PER_URL,
162
+ context_threshold_mode: 'disabled',
163
+ enable_source_metadata: true
164
+ }
165
+ end
166
+
167
+ # Parse an LLM Context response into {Result} rows with +text+ — see the
168
+ # class doc's +== Page text+ for what each field holds. A response with
169
+ # an empty +grounding.generic+ is Brave's "matched nothing" and returns
170
+ # +[]+; any other zero-result shape raises.
171
+ #
172
+ # @param json [String] response body from {LLM_CONTEXT_ENDPOINT}
173
+ # @param max_results [Integer] max entries
174
+ # @return [Array<Result>]
175
+ # @raise [RuntimeError] on an unrecognized zero-result response
176
+ def self.parse_llm_context(json, max_results: DEFAULT_MAX_RESULTS)
177
+ data = JSON.parse(json)
178
+ sources = data.is_a?(Hash) && data['sources'].is_a?(Hash) ? data['sources'] : {}
179
+ generic = data.is_a?(Hash) ? data.dig('grounding', 'generic') : nil
180
+ results = Array(generic).take(max_results).filter_map do |g|
181
+ href = g['url'].to_s
182
+ next nil if href.empty?
183
+
184
+ source = sources[href].is_a?(Hash) ? sources[href] : {}
185
+ chunks = Array(g['snippets']).map(&:to_s).reject { |c| c.strip.empty? }
186
+ title = strip_html(g['title'] || source['title'])
187
+ fetched = source['fetched_content_timestamp']
188
+ Result.new(
189
+ url: href,
190
+ title: title.empty? ? href : title,
191
+ body: llm_context_snippet(source['description'], chunks.first),
192
+ text: chunks.empty? ? nil : chunks.join(CHUNK_SEPARATOR),
193
+ indexed_at: fetched.is_a?(Integer) ? Time.at(fetched).utc : nil
194
+ )
195
+ end
196
+ return results unless results.empty?
197
+ return [] if data.is_a?(Hash) && data['grounding'].is_a?(Hash) && generic.is_a?(Array)
198
+
199
+ raise diagnose_empty(data, json)
200
+ end
201
+
202
+ # A row's snippet: the page's meta description, else the start of its
203
+ # first chunk.
204
+ #
205
+ # @param description [String, nil] +sources[url].description+
206
+ # @param first_chunk [String, nil]
207
+ # @return [String] possibly empty
208
+ def self.llm_context_snippet(description, first_chunk)
209
+ snippet = strip_html(description)
210
+ return snippet unless snippet.empty?
211
+
212
+ chunk = first_chunk.to_s.gsub(/\s+/, ' ').strip
213
+ chunk.length > SNIPPET_CHARS ? "#{chunk[0, SNIPPET_CHARS]}…" : chunk
214
+ end
215
+ private_class_method :llm_context_snippet
216
+
217
+ # Parse a Brave JSON response into {Result} rows (+<strong>+ highlights
218
+ # stripped). A genuine "no results" payload (recognized search shape,
219
+ # empty +mixed.main/top/side+) returns +[]+ so {Engines.search} renders
220
+ # its stub; any other zero-result shape raises.
221
+ #
222
+ # @param json [String] response body from {ENDPOINT}
223
+ # @param max_results [Integer] max entries
224
+ # @return [Array<Result>] hits, possibly empty on a recognized
225
+ # empty-results payload
226
+ # @raise [RuntimeError] on an unrecognized zero-result response
227
+ def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
228
+ data = JSON.parse(json)
229
+ results = Array(data.dig('web', 'results')).take(max_results).filter_map do |r|
230
+ href = r['url'].to_s
231
+ next nil if href.empty?
232
+
233
+ Result.new(
234
+ url: href,
235
+ title: strip_html(r['title']),
236
+ body: strip_html(r['description'])
237
+ )
238
+ end
239
+
240
+ if results.empty?
241
+ return [] if genuine_no_results?(data)
242
+
243
+ raise diagnose_empty(data, json)
244
+ end
245
+
246
+ results
247
+ end
248
+
249
+ # Strip HTML markup (notably the +<strong>+ highlights Brave wraps around
250
+ # query terms) and collapse whitespace.
251
+ #
252
+ # @param html [String, nil]
253
+ # @return [String] plain text; empty string for nil
254
+ def self.strip_html(html)
255
+ return '' if html.nil?
256
+
257
+ Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
258
+ end
259
+ private_class_method :strip_html
260
+
261
+ # True when a zero-+web.results+ response is Brave's own "ran, matched
262
+ # nothing" payload, not an error. Markers: the +type: "search"+ envelope
263
+ # and an empty +mixed+ block (Brave fills +mixed.main/top/side+ with
264
+ # matched verticals, so all three empty = succeeded-but-found-nothing).
265
+ #
266
+ # @param data [Hash, Object] parsed response
267
+ # @return [Boolean]
268
+ def self.genuine_no_results?(data)
269
+ return false unless data.is_a?(Hash) && data['type'] == 'search'
270
+
271
+ mixed = data['mixed']
272
+ return false unless mixed.is_a?(Hash)
273
+
274
+ %w[main top side].all? { |k| Array(mixed[k]).empty? }
275
+ end
276
+ private_class_method :genuine_no_results?
277
+
278
+ # Build an error message for a zero-result response: quote Brave's
279
+ # +error.detail+ if present, else a truncated raw body.
280
+ #
281
+ # @param data [Hash, Object] parsed response
282
+ # @param raw [String] raw response body
283
+ # @return [String] diagnostic to feed to +raise+
284
+ def self.diagnose_empty(data, raw)
285
+ if data.is_a?(Hash) && data['error'].is_a?(Hash)
286
+ err = data['error']
287
+ return "Brave Search returned an error: #{[err['code'], err['detail'] || err['meta']].compact.join(' — ')}"
288
+ end
289
+
290
+ snippet = raw.to_s[0, 800]
291
+ snippet += '…' if raw.to_s.length > 800
292
+ "Brave Search returned no results. Body: #{snippet}"
293
+ end
294
+ private_class_method :diagnose_empty
295
+ end
296
+ end
297
+ end
298
+ end
@@ -0,0 +1,181 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'faraday'
4
+ require 'nokogiri'
5
+ require 'uri'
6
+
7
+ module Pikuri
8
+ module Web
9
+ module Search
10
+ # Performs a DuckDuckGo search by scraping +html.duckduckgo.com+, returning
11
+ # hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
12
+ # parser (.parse) so tests exercise the parser against fixture HTML.
13
+ # {Engines#search} owns the final Markdown rendering. Constructed with no
14
+ # arguments, sharing the uniform +#search+/+#label+ provider shape with the
15
+ # keyed {Brave} / {Exa}.
16
+ #
17
+ # == Privacy posture
18
+ #
19
+ # DDG doesn't save your IP or tie a per-user profile to queries, and
20
+ # proxies requests so downstream providers can't build a search history —
21
+ # real, but DDG relays web results over Bing, so *query content* still
22
+ # reaches Microsoft (who has no no-training pledge). Better than Exa for
23
+ # sensitive queries, worse than Brave; for anything genuinely embarrassing,
24
+ # don't search the web at all.
25
+ class DuckDuckGo
26
+ # @return [String] HTML search endpoint
27
+ ENDPOINT = 'https://html.duckduckgo.com/html/'
28
+ # @return [Integer] default number of results returned.
29
+ DEFAULT_MAX_RESULTS = 10
30
+ # @return [RateLimiter] paces calls (DDG bans IPs that hammer the HTML
31
+ # endpoint) and circuit-breaks 5 min on {Engines::Unavailable}.
32
+ LIMITER = RateLimiter.new(min_interval: 2.0, cooldown: 300.0)
33
+
34
+ # @return [String] short provider label for {Engines}.
35
+ def label
36
+ 'DuckDuckGo'
37
+ end
38
+
39
+ # Whether {Engines} may keep this provider's results in its on-disk
40
+ # cache. Yes: pikuri scrapes the keyless HTML endpoint, under no API
41
+ # agreement restricting what it keeps.
42
+ #
43
+ # @return [Boolean] +true+
44
+ def disk_cacheable?
45
+ true
46
+ end
47
+
48
+ # Fetch results for +query+ as an +Array<Result>+. Throttled to one every
49
+ # 2s and circuit-broken 5 min after a soft-block (see {LIMITER}). The
50
+ # caller ({Engines#search}) normalizes the query and wraps this in a cache.
51
+ #
52
+ # @param query [String] search query (already normalized)
53
+ # @param max_results [Integer] max result entries
54
+ # @param cancellable [Pikuri::Agent::Control::Cancellable] makes
55
+ # {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
56
+ # @return [Array<Result>] hits, possibly empty when DDG matched nothing
57
+ # @raise [Engines::Unavailable] on a soft-block (anomaly/CAPTCHA page) or
58
+ # HTTP 429/5xx (the cascade falls back), or if {LIMITER} is in cooldown.
59
+ # @raise [RuntimeError] on other HTTP failures or an unrecognized
60
+ # empty-results layout. A genuine empty-results page is not an error;
61
+ # see {.parse}.
62
+ def search(query, max_results: DEFAULT_MAX_RESULTS,
63
+ cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
64
+ LIMITER.call(cancellable: cancellable) do
65
+ # DDG soft-blocks a bot-looking request, so it goes out dressed as a browser.
66
+ response = Scraper.browser_connection(ENDPOINT).get(ENDPOINT, { q: query })
67
+ unless response.success?
68
+ if response.status == 429 || response.status >= 500
69
+ raise Engines::Unavailable, "HTTP #{response.status}"
70
+ end
71
+
72
+ raise "DuckDuckGo request failed: #{response.status} #{response.body}"
73
+ end
74
+
75
+ self.class.parse(response.body, max_results: max_results)
76
+ end
77
+ end
78
+
79
+ # Parse a +html.duckduckgo.com+ result page into {Result} rows (+<b>+
80
+ # highlights stripped). On zero result nodes: a genuine "no results" page
81
+ # returns +[]+ (so {Engines.search} renders its stub); the anomaly/CAPTCHA
82
+ # modal raises {Engines::Unavailable}; anything else raises {RuntimeError}
83
+ # (likely a layout change) — so an IP soft-block surfaces rather than
84
+ # masquerading as an empty search.
85
+ #
86
+ # @param html [String] HTML from html.duckduckgo.com
87
+ # @param max_results [Integer] max result entries
88
+ # @return [Array<Result>] hits, possibly empty on a genuine no-results page
89
+ # @raise [Engines::Unavailable] on the anomaly/CAPTCHA modal (IP soft-block)
90
+ # @raise [RuntimeError] on zero results with an unrecognized layout
91
+ def self.parse(html, max_results: DEFAULT_MAX_RESULTS)
92
+ doc = Nokogiri::HTML(html)
93
+ results = doc.css('div.result.web-result').take(max_results).filter_map do |node|
94
+ title_link = node.at_css('a.result__a')
95
+ next nil if title_link.nil?
96
+
97
+ snippet = node.at_css('a.result__snippet')
98
+ Result.new(
99
+ url: extract_url(title_link['href']),
100
+ title: title_link.text.strip,
101
+ body: snippet&.text&.strip.to_s
102
+ )
103
+ end
104
+
105
+ if results.empty?
106
+ return [] if genuine_no_results?(doc)
107
+
108
+ message = diagnose_empty(doc)
109
+ raise(anomaly_modal?(doc) ? Engines::Unavailable : RuntimeError, message)
110
+ end
111
+
112
+ results
113
+ end
114
+
115
+ # True when the page is DDG's anomaly/CAPTCHA modal (IP soft-blocked).
116
+ # {.parse} uses it to pick {Engines::Unavailable} (recoverable) over
117
+ # {RuntimeError} (unknown layout).
118
+ #
119
+ # @param doc [Nokogiri::HTML::Document]
120
+ # @return [Boolean]
121
+ def self.anomaly_modal?(doc)
122
+ !!(doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description'))
123
+ end
124
+ private_class_method :anomaly_modal?
125
+
126
+ # True when a zero-node page is DDG's own "no results found" page, not an
127
+ # anomaly/CAPTCHA. Anomaly wins: if the modal divs are present it's never
128
+ # a genuine empty result, even if the text mentions "No results".
129
+ #
130
+ # @param doc [Nokogiri::HTML::Document]
131
+ # @return [Boolean]
132
+ def self.genuine_no_results?(doc)
133
+ return false if doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description')
134
+ return true if doc.at_css('div.no-results')
135
+
136
+ doc.text.include?('No results found')
137
+ end
138
+ private_class_method :genuine_no_results?
139
+
140
+ # Build an error message for a zero-match page: quote the DDG anomaly
141
+ # modal's title/description if present, else a truncated copy of the
142
+ # body's visible text (+<script>/<style>/<noscript>+ stripped).
143
+ #
144
+ # @param doc [Nokogiri::HTML::Document]
145
+ # @return [String] diagnostic to feed to +raise+
146
+ def self.diagnose_empty(doc)
147
+ title = doc.at_css('.anomaly-modal__title')&.text&.strip
148
+ desc = doc.at_css('.anomaly-modal__description')&.text&.strip
149
+ if title || desc
150
+ return "DuckDuckGo anomaly check (likely IP soft-block): #{[title, desc].compact.reject(&:empty?).join(' — ')}"
151
+ end
152
+
153
+ doc.css('script, style, noscript').remove
154
+ text = doc.text.gsub(/\s+/, ' ').strip
155
+ snippet = text.empty? ? '<empty body>' : text[0, 1500]
156
+ snippet += '…' if text.length > 1500
157
+ "DuckDuckGo returned no results. Page text: #{snippet}"
158
+ end
159
+ private_class_method :diagnose_empty
160
+
161
+ # Decode DDG's +//duckduckgo.com/l/?uddg=<encoded>+ redirect wrapper to
162
+ # the real target URL.
163
+ #
164
+ # @param href [String, nil] href from the result page
165
+ # @return [String] the decoded target, or +href+ unchanged when not a
166
+ # recognized DDG redirect or unparseable
167
+ def self.extract_url(href)
168
+ return href if href.nil? || href.empty?
169
+
170
+ uri = URI.parse(href.start_with?('//') ? "https:#{href}" : href)
171
+ return href unless uri.host&.end_with?('duckduckgo.com') && uri.path == '/l/'
172
+
173
+ params = URI.decode_www_form(uri.query.to_s).to_h
174
+ params['uddg'] || href
175
+ rescue URI::InvalidURIError
176
+ href
177
+ end
178
+ end
179
+ end
180
+ end
181
+ end