pikuri-web 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,430 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'json'
4
+
5
+ module Pikuri
6
+ module Web
7
+ # Namespace for the web-search stack behind {Web::Search}: per-provider
8
+ # modules ({DuckDuckGo}, {Brave}, {Exa}), the {Result} they return, the
9
+ # {Engines} cascade with its result caches, and the shared {RateLimiter}.
10
+ module Search
11
+ # Search-orchestration object: the cascade across selected providers, the
12
+ # result cache, and the {Unavailable} marker the cascade falls back on.
13
+ # {Web::Search.build} constructs one and wires its {#search} into a
14
+ # {Tool}. Each provider raises {Unavailable} to hand off to the next.
15
+ #
16
+ # Engines.new(engines: [:brave], brave_key: 'BSA...') # Brave alone
17
+ # Engines.new(brave_key: 'BSA...') # DuckDuckGo + Brave
18
+ # Engines.new # DuckDuckGo alone
19
+ #
20
+ # {.resolve} owns which engines that yields and which selections are
21
+ # refused; {.config_from_h} maps a config file's +search:+ block onto
22
+ # these keywords.
23
+ #
24
+ # == Provider keys are constructor config, not environment
25
+ #
26
+ # Brave and Exa are paid and need a key; DuckDuckGo needs none. An
27
+ # {Engines} is built with the keys it should use (+brave_key:+ /
28
+ # +exa_key:+, both optional) — pikuri reads no key from the environment,
29
+ # so a host supplies them. See AGENTS.md "Environment is not a secret
30
+ # store".
31
+ #
32
+ # == Page text
33
+ #
34
+ # Built with +page_text: true+, Brave and Exa also bring back each page's
35
+ # text from their own index, in the same request. {#search} stores it in
36
+ # a {PageStore} and prints the row's handle before its title, for
37
+ # +web_read_cached+ to read:
38
+ #
39
+ # [a3f9c21e] [Ruby Data](https://docs.ruby-lang.org/en/3.4/Data.html)
40
+ # Data objects are immutable.
41
+ #
42
+ # A row gets a handle only when *this* search returned its text — never a
43
+ # DuckDuckGo row — even if the store holds that URL from another search.
44
+ # So an +engines: nil+ selection leaves DuckDuckGo out in this mode: in a
45
+ # shuffled cascade it would win about every other search and leave the
46
+ # model nothing to read. A host that lists it explicitly keeps it. The
47
+ # two modes cache apart ({DISK_TEXT_CACHE}, {MEMORY_TEXT_CACHE}): they
48
+ # return different rows.
49
+ class Engines
50
+ # Subsystem logger; level via +PIKURI_LOG_ENGINES+ or +PIKURI_LOG+.
51
+ #
52
+ # @return [Logger]
53
+ LOGGER = Pikuri.logger_for('Engines')
54
+
55
+ # Every engine name accepted in an +engines:+ selection, in the order
56
+ # {#providers} reports them. Presentation only — {#search} shuffles.
57
+ #
58
+ # @return [Array<Symbol>]
59
+ ENGINE_NAMES = %i[duckduckgo brave exa].freeze
60
+
61
+ # Engine name → the constructor keyword carrying its API key. Naming
62
+ # one of these without its key is a refusal ({.resolve}); DuckDuckGo is
63
+ # absent because it needs none.
64
+ #
65
+ # @return [Hash{Symbol => Symbol}]
66
+ KEYED_ENGINES = { brave: :brave_key, exa: :exa_key }.freeze
67
+
68
+ # Engines that can bring back page text; +page_text: true+ needs one.
69
+ #
70
+ # @return [Array<Symbol>]
71
+ PAGE_TEXT_ENGINES = %i[brave exa].freeze
72
+
73
+ # The fields {.config_from_h} accepts from a config file's +search:+
74
+ # block, matching this class's keywords.
75
+ #
76
+ # @return [Array<Symbol>]
77
+ CONFIG_FIELDS = %i[engines brave_key exa_key].freeze
78
+
79
+ # Raised by a provider that is temporarily unavailable (rate-limited,
80
+ # bot-blocked, quota-exhausted — "try again later", not "your request is
81
+ # wrong"). The {#search} cascade catches it and tries the next provider;
82
+ # anything else bubbles up so genuine bugs stay visible.
83
+ class Unavailable < StandardError; end
84
+
85
+ # On-disk result cache for providers whose +disk_cacheable?+ is true, at
86
+ # class level so every engine dedupes into one directory. An entry is
87
+ # the rows as JSON, never rendered Markdown: handles are looked up at
88
+ # render time.
89
+ #
90
+ # @return [UrlCache::OnDisk]
91
+ DISK_CACHE = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_search")
92
+
93
+ # In-memory result cache for the rest (Brave, Exa: their terms forbid
94
+ # storing results), shared by every engine in the process.
95
+ #
96
+ # @return [UrlCache::InMemory]
97
+ MEMORY_CACHE = UrlCache::InMemory.new(ttl: UrlCache::DEFAULT_TTL, max_entries: 100)
98
+
99
+ # {DISK_CACHE}'s twin for +page_text: true+ engines.
100
+ #
101
+ # @return [UrlCache::OnDisk]
102
+ DISK_TEXT_CACHE = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL,
103
+ dir: "#{UrlCache::ROOT_DIR}/web_search_text")
104
+
105
+ # {MEMORY_CACHE}'s twin for +page_text: true+ engines.
106
+ #
107
+ # @return [UrlCache::InMemory]
108
+ MEMORY_TEXT_CACHE = UrlCache::InMemory.new(ttl: UrlCache::DEFAULT_TTL, max_entries: 100)
109
+
110
+ # The constructor's +disk_cache:+ default; specs swap in {UrlCache::NULL}.
111
+ #
112
+ # @param page_text [Boolean] which mode's cache
113
+ # @return [UrlCache::OnDisk, #read, #write]
114
+ def self.disk_cache(page_text: false)
115
+ page_text ? DISK_TEXT_CACHE : DISK_CACHE
116
+ end
117
+
118
+ # The constructor's +memory_cache:+ default; specs swap in
119
+ # {UrlCache::NULL}.
120
+ #
121
+ # @param page_text [Boolean] which mode's cache
122
+ # @return [UrlCache::InMemory, #read, #write]
123
+ def self.memory_cache(page_text: false)
124
+ page_text ? MEMORY_TEXT_CACHE : MEMORY_CACHE
125
+ end
126
+
127
+ # Resolve an +engines:+ selection into the engine names to build,
128
+ # refusing a selection that cannot work:
129
+ #
130
+ # resolve(engines: nil, brave_key: 'B') # => [:duckduckgo, :brave]
131
+ # resolve(engines: nil) # => [:duckduckgo]
132
+ # resolve(engines: %w[brave], brave_key: 'B') # => [:brave]
133
+ # resolve(engines: [:exa], brave_key: 'B') # raises — exa has no key
134
+ #
135
+ # +nil+ means "whatever the keys say": DuckDuckGo plus every engine
136
+ # whose key is present, which is what a host that never heard of
137
+ # +engines:+ gets. An explicit list is a **set** — it is normalized to
138
+ # {ENGINE_NAMES} order and {#search} shuffles it per call anyway, so
139
+ # +[brave, duckduckgo]+ expresses no preference for Brave. Names are
140
+ # matched case-insensitively.
141
+ #
142
+ # Supplying a key without listing its engine is fine — parking a key
143
+ # you are not using today is not a mistake.
144
+ #
145
+ # @param engines [Array<String, Symbol>, nil] engine names, or +nil+ to
146
+ # derive the set from the keys.
147
+ # @param brave_key [String, nil] Brave token; blank counts as absent.
148
+ # @param exa_key [String, nil] Exa key; blank counts as absent.
149
+ # @return [Array<Symbol>] names drawn from {ENGINE_NAMES}, in that order.
150
+ # @raise [ArgumentError] if +engines+ is not an Array, is empty, holds
151
+ # an unknown or repeated name, or names a {KEYED_ENGINES} engine
152
+ # whose key is blank.
153
+ def self.resolve(engines:, brave_key: nil, exa_key: nil)
154
+ keys = { brave: brave_key, exa: exa_key }
155
+ keyless = ->(name) { KEYED_ENGINES.key?(name) && keys[name].to_s.strip.empty? }
156
+ return ENGINE_NAMES.reject(&keyless) if engines.nil?
157
+
158
+ unless engines.is_a?(Array) && !engines.empty?
159
+ # A list of engine names is safe to echo; anything else might be a
160
+ # key the writer put in the wrong field, so show only its class.
161
+ got = engines.is_a?(Array) ? engines.inspect : engines.class
162
+ raise ArgumentError, 'search engines: expected a non-empty list of engine names ' \
163
+ "(known: #{ENGINE_NAMES.join(', ')}), got #{got}"
164
+ end
165
+
166
+ names = engines.map { |name| name.to_s.strip.downcase.to_sym }
167
+ refuse_unknown(names)
168
+ refuse_repeats(names)
169
+ refuse_keyless(names.select(&keyless))
170
+ ENGINE_NAMES & names
171
+ end
172
+
173
+ # @param names [Array<Symbol>] the requested engine names.
174
+ # @return [void]
175
+ # @raise [ArgumentError] if any name is not in {ENGINE_NAMES}.
176
+ def self.refuse_unknown(names)
177
+ unknown = names - ENGINE_NAMES
178
+ return if unknown.empty?
179
+
180
+ raise ArgumentError, "search engines: unknown engine(s) #{unknown.join(', ')} — " \
181
+ "known engines are #{ENGINE_NAMES.join(', ')}"
182
+ end
183
+
184
+ # @param names [Array<Symbol>] the requested engine names.
185
+ # @return [void]
186
+ # @raise [ArgumentError] if any name appears twice (which would put one
187
+ # provider into the cascade twice, weighting the shuffle).
188
+ def self.refuse_repeats(names)
189
+ repeated = names.tally.select { |_, count| count > 1 }.keys
190
+ return if repeated.empty?
191
+
192
+ raise ArgumentError, "search engines: #{repeated.join(', ')} listed more than once"
193
+ end
194
+
195
+ # @param keyless [Array<Symbol>] requested engines with no usable key.
196
+ # @return [void]
197
+ # @raise [ArgumentError] unless +keyless+ is empty.
198
+ def self.refuse_keyless(keyless)
199
+ return if keyless.empty?
200
+
201
+ needed = keyless.map { |name| "#{name} needs #{KEYED_ENGINES[name]}" }.join(', ')
202
+ raise ArgumentError, "search engines: #{needed} — supply the key or drop the engine from the list"
203
+ end
204
+
205
+ # Map a config file's +search:+ block onto this class's keywords, so a
206
+ # host hands over what it read:
207
+ #
208
+ # # ~/.pikuri-examples-config.yaml
209
+ # search:
210
+ # brave_key: BSA...
211
+ # exa_key: xyz
212
+ # engines: [brave, duckduckgo]
213
+ #
214
+ # Web::Search.from_h(YAML.safe_load_file(path)['search'])
215
+ #
216
+ # Nothing here knows what YAML is; a host reading TOML, JSON or its own
217
+ # settings object uses the same method. Every field is validated rather
218
+ # than coerced, because the person writing a config file cannot
219
+ # type-check it — and a mistyped engine name that silently meant "no
220
+ # Brave" would show up only as worse search results.
221
+ #
222
+ # @param config [Hash{String, Symbol => Object}, nil] the +search:+
223
+ # block, with String or Symbol keys. +nil+ — what a host reading an
224
+ # absent config key gets — yields an empty Hash, so no caller needs a
225
+ # +|| {}+.
226
+ # @return [Hash{Symbol => Object}] +engines:+ / +brave_key:+ /
227
+ # +exa_key:+ keywords for {#initialize} or {Web::Search.build}.
228
+ # @raise [ArgumentError] on a non-Hash, an unknown field, or a key field
229
+ # that is not a String.
230
+ def self.config_from_h(config)
231
+ return {} if config.nil?
232
+
233
+ unless config.is_a?(Hash)
234
+ raise ArgumentError, 'search config: expected a Hash of ' \
235
+ "#{CONFIG_FIELDS.join(', ')}, got #{config.class}"
236
+ end
237
+
238
+ fields = config.transform_keys(&:to_sym)
239
+ unknown = fields.keys - CONFIG_FIELDS
240
+ unless unknown.empty?
241
+ raise ArgumentError, "search config: unknown field(s) #{unknown.join(', ')} — " \
242
+ "known fields are #{CONFIG_FIELDS.join(', ')}"
243
+ end
244
+ KEYED_ENGINES.each_value do |field|
245
+ value = fields[field]
246
+ next if value.nil? || value.is_a?(String)
247
+
248
+ # The class, never the value: an error message about a key field is
249
+ # one paste away from a bug report.
250
+ raise ArgumentError, "search config: #{field} must be a String, got #{value.class}"
251
+ end
252
+ fields
253
+ end
254
+
255
+ # Builds the provider cascade once, from the engines {.resolve} selects.
256
+ # From here each provider is just an object answering
257
+ # +#search+/+#label+/+#disk_cacheable?+.
258
+ #
259
+ # @param engines [Array<String, Symbol>, nil] engine names; +nil+
260
+ # derives the set from the keys (see {.resolve}), minus DuckDuckGo
261
+ # under +page_text+.
262
+ # @param brave_key [String, nil] Brave token.
263
+ # @param exa_key [String, nil] Exa key.
264
+ # @param page_text [Boolean] bring back page text and print handles;
265
+ # see the class doc's +== Page text+
266
+ # @param disk_cache [UrlCache::OnDisk, #read, #write] store for a
267
+ # +disk_cacheable?+ winner's rows; defaults to {.disk_cache}.
268
+ # @param memory_cache [UrlCache::InMemory, #read, #write] store for
269
+ # every other winner's; defaults to {.memory_cache}.
270
+ # @param page_store [PageStore] where page text goes; defaults to
271
+ # {PageStore.default}
272
+ # @return [Engines]
273
+ # @raise [ArgumentError] as {.resolve}, or when +page_text+ is set but
274
+ # no selected engine can bring back page text
275
+ def initialize(engines: nil, brave_key: nil, exa_key: nil, page_text: false,
276
+ disk_cache: self.class.disk_cache(page_text: page_text),
277
+ memory_cache: self.class.memory_cache(page_text: page_text),
278
+ page_store: PageStore.default)
279
+ names = self.class.resolve(engines: engines, brave_key: brave_key, exa_key: exa_key)
280
+ if page_text
281
+ if (names & PAGE_TEXT_ENGINES).empty?
282
+ raise ArgumentError, "search engines: page_text needs #{PAGE_TEXT_ENGINES.join(' or ')}, " \
283
+ "got #{names.join(', ')}"
284
+ end
285
+ names &= PAGE_TEXT_ENGINES if engines.nil?
286
+ end
287
+
288
+ @providers = names.map do |name|
289
+ case name
290
+ when :duckduckgo then DuckDuckGo.new
291
+ when :brave then Brave.new(api_key: brave_key, page_text: page_text)
292
+ when :exa then Exa.new(api_key: exa_key, page_text: page_text)
293
+ end
294
+ end
295
+ @page_text = page_text
296
+ @disk_cache = disk_cache
297
+ @memory_cache = memory_cache
298
+ @page_store = page_store
299
+ @last_logged_providers = nil
300
+ end
301
+
302
+ # @return [Array<#search, #label, #disk_cacheable?>] the selected
303
+ # provider instances, in {ENGINE_NAMES} order (the cascade shuffles
304
+ # them per call).
305
+ attr_reader :providers
306
+
307
+ # Run +query+ through the providers in random order, falling back on each
308
+ # {Unavailable}. The shuffle spreads load so one provider isn't always
309
+ # hit (and exhausted) first. The query is whitespace-normalized first;
310
+ # the winning provider's rows are cached keyed by the cleaned query (a
311
+ # hit short-circuits the cascade) — on disk when the winner is
312
+ # +disk_cacheable?+, else in memory only — and rendered to Markdown.
313
+ # +max_results+ is *not* in the cache key, so a non-default value may
314
+ # return a previously-cached size.
315
+ #
316
+ # If every provider is unavailable, returns an +"Error: ..."+ string
317
+ # (the tool convention) rather than raising. Any non-{Unavailable}
318
+ # exception (network, parser, bad key) bubbles up.
319
+ #
320
+ # @param query [String] search query
321
+ # @param max_results [Integer] maximum result entries
322
+ # @param cancellable [Pikuri::Agent::Control::Cancellable, nil] handed
323
+ # to each provider so its {RateLimiter} pacing wait is interruptible.
324
+ # A cache hit never reaches a provider, so it never waits at all.
325
+ # @return [Tool::Result] the Markdown result list (+"Returned 3 search
326
+ # results"+), or {Tool::Result.error} when all providers are exhausted
327
+ # @raise [ArgumentError] if the query is empty after normalization
328
+ # @raise [Pikuri::Agent::Control::Cancellable::Cancelled] if
329
+ # +cancellable+ trips while a provider is pacing. It is *not* caught
330
+ # as a provider failure — the cascade stops rather than trying the
331
+ # next one, and nothing is cached.
332
+ def search(query, max_results:, cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
333
+ cleaned = query.to_s.strip.gsub(/\s+/, ' ')
334
+ raise ArgumentError, 'query is empty' if cleaned.empty?
335
+
336
+ current_providers = providers
337
+ log_providers(current_providers)
338
+
339
+ json = @memory_cache.read(cleaned) || @disk_cache.read(cleaned)
340
+ if json
341
+ LOGGER.info { "cache=hit bytes=#{json.bytesize}" }
342
+ else
343
+ json = run_cascade(cleaned, current_providers, max_results: max_results, cancellable: cancellable)
344
+ end
345
+ rows = JSON.parse(json)
346
+ Tool::Result.new(content: render(rows),
347
+ summary: "Returned #{Pikuri::Tool.pluralize(rows.size, 'search result')}")
348
+ rescue Unavailable => e
349
+ Pikuri::Tool::Result.error(e.message)
350
+ end
351
+
352
+ private
353
+
354
+ # Ask the providers in random order until one answers, store its page
355
+ # text, then cache its rows where its terms allow.
356
+ #
357
+ # @param cleaned [String] the normalized query
358
+ # @param current_providers [Array<#search, #label, #disk_cacheable?>]
359
+ # @param max_results [Integer]
360
+ # @param cancellable [Pikuri::Agent::Control::Cancellable]
361
+ # @return [String] the rows as JSON, as cached
362
+ # @raise [Unavailable] when every provider is — raised, never stored, so
363
+ # the all-unavailable message can't block this query for the TTL
364
+ def run_cascade(cleaned, current_providers, max_results:, cancellable:)
365
+ failures = []
366
+ current_providers.shuffle.each do |provider|
367
+ results = provider.search(cleaned, max_results: max_results, cancellable: cancellable)
368
+ disk = provider.disk_cacheable?
369
+ LOGGER.info { "engine=#{provider.label} results=#{results.size} cache=#{disk ? 'disk' : 'memory'}" }
370
+ return (disk ? @disk_cache : @memory_cache).write(cleaned, JSON.generate(rows_of(results)))
371
+ rescue Unavailable => e
372
+ failures << "#{provider.label} (#{e.message})"
373
+ end
374
+ raise Unavailable, "all search providers temporarily unavailable: #{failures.join('; ')}"
375
+ end
376
+
377
+ # The cacheable rows of +results+, storing each one's page text first.
378
+ # A row's +has_text+ is what earns it a handle at render time; its text
379
+ # lives only in the {PageStore}, never in a query cache.
380
+ #
381
+ # @param results [Array<Result>]
382
+ # @return [Array<Hash{String => Object}>] +url+, +title+, +snippet+,
383
+ # +has_text+ per row
384
+ def rows_of(results)
385
+ results.map do |r|
386
+ has_text = @page_text && !r.text.nil?
387
+ @page_store.store(url: r.url, title: r.title, text: r.text, indexed_at: r.indexed_at) if has_text
388
+ { 'url' => r.url, 'title' => r.title, 'snippet' => r.body, 'has_text' => has_text }
389
+ end
390
+ end
391
+
392
+ # Render rows into the smolagents-style Markdown the LLM consumes
393
+ # (+"## Search Results"+ header, then +[title](url)\nsnippet+ joined by
394
+ # blank lines), each readable row led by its handle. Empty → the +"No
395
+ # results found."+ stub.
396
+ #
397
+ # @param rows [Array<Hash{String => Object}>] as {#rows_of} made them
398
+ # @return [String] Markdown result list
399
+ def render(rows)
400
+ return "## Search Results\n\nNo results found." if rows.empty?
401
+
402
+ "## Search Results\n\n" + rows.map { |row| render_row(row) }.join("\n\n")
403
+ end
404
+
405
+ # @param row [Hash{String => Object}]
406
+ # @return [String] one row; a +has_text+ row still in the {PageStore}
407
+ # gets its handle, one evicted since renders without
408
+ def render_row(row)
409
+ line = "[#{row['title']}](#{row['url']})\n#{row['snippet']}"
410
+ handle = row['has_text'] && @page_store.handle_for(row['url'])
411
+ handle ? "[#{handle}] #{line}" : line
412
+ end
413
+
414
+ # INFO-log the currently-available providers, but only when the set
415
+ # differs from the last logged — one line per distinct configuration,
416
+ # not per search.
417
+ #
418
+ # @param current [Array<#label>] providers from {#providers}
419
+ # @return [void]
420
+ def log_providers(current)
421
+ return if @last_logged_providers == current
422
+
423
+ @last_logged_providers = current
424
+ names = current.map(&:label).join(', ')
425
+ LOGGER.info("engines available: #{names}")
426
+ end
427
+ end
428
+ end
429
+ end
430
+ end
@@ -0,0 +1,243 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'faraday'
4
+ require 'json'
5
+
6
+ module Pikuri
7
+ module Web
8
+ module Search
9
+ # Performs an Exa search via the official +/search+ endpoint, returning
10
+ # hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
11
+ # parser (.parse) so tests exercise the parser against fixture JSON.
12
+ # {Engines#search} owns the final Markdown rendering.
13
+ #
14
+ # Constructed with its API key (+Exa.new(api_key:)+); {Engines} builds one
15
+ # only when an Exa key was configured, so users who never registered don't
16
+ # spend money, then drives it through the same +#search+/+#label+
17
+ # interface as every provider. pikuri reads no key from the environment
18
+ # (AGENTS.md "Environment is not a secret store"). Paid; key at
19
+ # https://exa.ai.
20
+ #
21
+ # Requests +type: "auto"+ (Exa picks neural vs keyword) and +contents: {
22
+ # highlights: true }+ so each result carries a neural-ranked snippet
23
+ # populating {Result#body} — the analog to Brave's +description+. Built
24
+ # with +page_text: true+, the same request also asks for each page's full
25
+ # text ({Result#text}, up to {MAX_TEXT_CHARS}) — at no extra charge up to
26
+ # 10 results. Exa gives no crawl date, so {Result#indexed_at} stays +nil+.
27
+ #
28
+ # == Cache-only, never a live fetch
29
+ #
30
+ # {.request_body} pins +maxAgeHours: -1+. Omitted, Exa's crawler fetches
31
+ # an uncached result page live during the search, and since the query
32
+ # picks which pages match, an attacker owning indexed pages reads bits of
33
+ # the query from their server logs. That would break the +:tool_vouched+
34
+ # egress grade {Search} declares. A snippet needs no fresh page.
35
+ #
36
+ # == Privacy posture
37
+ #
38
+ # Exa's Privacy Policy uses Query Data to train/fine-tune its models, and
39
+ # ToS §1.2(c) grants a +perpetual and irrevocable+, sub-licensable license
40
+ # over User Input. Business customers under an MSA/DPA get carve-outs; the
41
+ # default pay-as-you-go key pikuri uses does not. Bottom line: Exa mines
42
+ # queries to train competing models. If a query would be sensitive in a
43
+ # training set, don't configure an Exa key — {Engines#providers} leaves
44
+ # Exa out of the cascade unless its key was supplied.
45
+ class Exa
46
+ # @return [String] Search endpoint (POST, JSON body)
47
+ ENDPOINT = 'https://api.exa.ai/search'
48
+ # @return [Integer] default number of results returned, matching
49
+ # {DuckDuckGo::DEFAULT_MAX_RESULTS}
50
+ DEFAULT_MAX_RESULTS = 10
51
+ # @return [RateLimiter] Exa is paid and doesn't aggressively throttle,
52
+ # so no minimum interval; the 5-minute cooldown still applies on
53
+ # {Engines::Unavailable} so the budget isn't burned on doomed retries.
54
+ LIMITER = RateLimiter.new(min_interval: 0.0, cooldown: 300.0)
55
+ # @return [Integer] per-page text cap asked for with +page_text+ — the
56
+ # +/search+ maximum, so pikuri takes all the text Exa gives and trims
57
+ # only when it is read
58
+ MAX_TEXT_CHARS = 1_000_000
59
+
60
+ # @param api_key [String] Exa API key. Required and non-blank:
61
+ # pikuri reads no key from the environment — the host supplies it
62
+ # ({Engines} only constructs an Exa when a key was configured).
63
+ # @param page_text [Boolean] also fetch each page's stored text
64
+ # @raise [ArgumentError] if +api_key+ is blank
65
+ def initialize(api_key:, page_text: false)
66
+ raise ArgumentError, 'Exa Search API key is blank' if api_key.to_s.strip.empty?
67
+
68
+ @api_key = api_key
69
+ @page_text = page_text
70
+ end
71
+
72
+ # @return [String] short provider label for {Engines} logging /
73
+ # fallback messages.
74
+ def label
75
+ 'Exa'
76
+ end
77
+
78
+ # Whether {Engines} may keep this provider's results in its on-disk
79
+ # cache. No: Exa's Terms of Service, which cover the API, forbid to
80
+ # "copy … duplicate … any information … obtained from or through, the
81
+ # Services, except for temporary files that are automatically cached
82
+ # by your web browser" (§4.2(a), https://exa.ai/terms), so they stay in
83
+ # memory ({UrlCache::InMemory}). Its Master Subscription Agreement,
84
+ # which lets a customer own the output, binds Order Form (Enterprise)
85
+ # customers, not a pay-as-you-go key.
86
+ #
87
+ # @return [Boolean] +false+
88
+ def disk_cacheable?
89
+ false
90
+ end
91
+
92
+ # Fetch results for +query+ as an +Array<Result>+. Circuit-broken for
93
+ # 5 minutes on rate-limit/unavailable (see {LIMITER}). The caller
94
+ # ({Engines#search}) normalizes the query and wraps this in a cache.
95
+ #
96
+ # @param query [String] search query (already normalized)
97
+ # @param max_results [Integer] max entries; Exa's +numResults+
98
+ # @param cancellable [Pikuri::Agent::Control::Cancellable] makes
99
+ # {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
100
+ # @return [Array<Result>] hits, possibly empty when Exa matched nothing
101
+ # @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
102
+ # back from these), or immediately if {LIMITER} is in cooldown. Other
103
+ # non-2xx (401/403 bad key) bubble up as +RuntimeError+.
104
+ # @raise [RuntimeError] for non-rate-limit HTTP failures, or an
105
+ # unrecognized empty-results shape.
106
+ def search(query, max_results: DEFAULT_MAX_RESULTS,
107
+ cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
108
+ LIMITER.call(cancellable: cancellable) do
109
+ response = Faraday.post(ENDPOINT) do |req|
110
+ req.headers['x-api-key'] = @api_key
111
+ req.headers['Content-Type'] = 'application/json'
112
+ req.headers['Accept'] = 'application/json'
113
+ req.body = JSON.dump(self.class.request_body(query, max_results: max_results,
114
+ page_text: @page_text))
115
+ end
116
+ unless response.success?
117
+ if response.status == 429 || response.status >= 500
118
+ raise Engines::Unavailable, "HTTP #{response.status}"
119
+ end
120
+
121
+ raise "Exa Search request failed: #{response.status} #{response.body}"
122
+ end
123
+
124
+ self.class.parse(response.body, max_results: max_results)
125
+ end
126
+ end
127
+
128
+ # The JSON body for a {ENDPOINT} request; cache-only, see the class doc.
129
+ #
130
+ # Exa.request_body('ruby', max_results: 5)
131
+ # # => { query: 'ruby', type: 'auto', numResults: 5,
132
+ # # contents: { highlights: true, maxAgeHours: -1 } }
133
+ # Exa.request_body('ruby', max_results: 5, page_text: true)
134
+ # # => … contents: { highlights: true, maxAgeHours: -1,
135
+ # # text: { maxCharacters: 1_000_000 } } }
136
+ #
137
+ # @param query [String] search query (already normalized)
138
+ # @param max_results [Integer] max entries; Exa's +numResults+
139
+ # @param page_text [Boolean] also ask for each page's text
140
+ # @return [Hash{Symbol => Object}]
141
+ def self.request_body(query, max_results: DEFAULT_MAX_RESULTS, page_text: false)
142
+ contents = { highlights: true, maxAgeHours: -1 }
143
+ contents[:text] = { maxCharacters: MAX_TEXT_CHARS } if page_text
144
+ { query: query, type: 'auto', numResults: max_results, contents: contents }
145
+ end
146
+
147
+ # Parse an Exa JSON response into {Result} rows, +body+ the first
148
+ # non-empty +highlights+ snippet, +text+ the page text when the response
149
+ # carries any. A genuine "no results" payload
150
+ # (+requestId+ present, empty +results+) returns +[]+ so {Engines.search}
151
+ # renders its no-results stub; any other zero-result shape raises.
152
+ #
153
+ # @param json [String] response body from {ENDPOINT}
154
+ # @param max_results [Integer] max entries
155
+ # @return [Array<Result>] hits, possibly empty on a recognized
156
+ # empty-results payload
157
+ # @raise [RuntimeError] on an unrecognized zero-result response
158
+ def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
159
+ data = JSON.parse(json)
160
+ results = Array(data['results']).take(max_results).filter_map do |r|
161
+ href = r['url'].to_s
162
+ next nil if href.empty?
163
+
164
+ Result.new(
165
+ url: href,
166
+ title: clean(r['title']) || href,
167
+ body: first_highlight(r['highlights']),
168
+ text: r['text'].is_a?(String) && !r['text'].strip.empty? ? r['text'] : nil
169
+ )
170
+ end
171
+
172
+ if results.empty?
173
+ return [] if genuine_no_results?(data)
174
+
175
+ raise diagnose_empty(data, json)
176
+ end
177
+
178
+ results
179
+ end
180
+
181
+ # Collapse whitespace and strip; +nil+ for nil/empty input so the caller
182
+ # can fall back (e.g. to the URL when a result has no title).
183
+ #
184
+ # @param text [String, nil]
185
+ # @return [String, nil] cleaned text, or +nil+ if blank
186
+ def self.clean(text)
187
+ return nil if text.nil?
188
+
189
+ cleaned = text.to_s.gsub(/\s+/, ' ').strip
190
+ cleaned.empty? ? nil : cleaned
191
+ end
192
+ private_class_method :clean
193
+
194
+ # First non-empty +highlights+ entry, cleaned. Exa sorts highlights by
195
+ # relevance; the top one populates +body+.
196
+ #
197
+ # @param highlights [Array<String>, nil] +highlights+ field
198
+ # @return [String] cleaned snippet, or empty string if none usable
199
+ def self.first_highlight(highlights)
200
+ return '' unless highlights.is_a?(Array)
201
+
202
+ highlights.each do |h|
203
+ cleaned = clean(h)
204
+ return cleaned if cleaned
205
+ end
206
+ ''
207
+ end
208
+ private_class_method :first_highlight
209
+
210
+ # True when a zero-+results+ response looks like Exa's own "ran, matched
211
+ # nothing" payload rather than an error — marked by +requestId+, which
212
+ # Exa always sets on success.
213
+ #
214
+ # @param data [Hash, Object] parsed response
215
+ # @return [Boolean]
216
+ def self.genuine_no_results?(data)
217
+ return false unless data.is_a?(Hash)
218
+ return false unless data.key?('requestId')
219
+
220
+ Array(data['results']).empty?
221
+ end
222
+ private_class_method :genuine_no_results?
223
+
224
+ # Build an error message for a zero-result response: quote Exa's
225
+ # +error+/+message+/+detail+ if present, else a truncated raw body.
226
+ #
227
+ # @param data [Hash, Object] parsed response
228
+ # @param raw [String] raw response body
229
+ # @return [String] diagnostic to feed to +raise+
230
+ def self.diagnose_empty(data, raw)
231
+ if data.is_a?(Hash) && (msg = data['error'] || data['message'] || data['detail'])
232
+ return "Exa Search returned an error: #{msg}"
233
+ end
234
+
235
+ snippet = raw.to_s[0, 800]
236
+ snippet += '…' if raw.to_s.length > 800
237
+ "Exa Search returned no results. Body: #{snippet}"
238
+ end
239
+ private_class_method :diagnose_empty
240
+ end
241
+ end
242
+ end
243
+ end