pikuri-web 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +68 -0
- data/lib/pikuri/web/fetch.rb +153 -0
- data/lib/pikuri/web/read_cached.rb +96 -0
- data/lib/pikuri/web/scrape.rb +107 -0
- data/lib/pikuri/web/scraper.rb +188 -0
- data/lib/pikuri/web/search/brave.rb +298 -0
- data/lib/pikuri/web/search/duckduckgo.rb +181 -0
- data/lib/pikuri/web/search/engines.rb +430 -0
- data/lib/pikuri/web/search/exa.rb +243 -0
- data/lib/pikuri/web/search/page_store.rb +142 -0
- data/lib/pikuri/web/search/rate_limiter.rb +114 -0
- data/lib/pikuri/web/search/result.rb +42 -0
- data/lib/pikuri/web/search.rb +229 -0
- data/lib/pikuri/web/url_cache/in_memory.rb +96 -0
- data/lib/pikuri/web/url_cache/on_disk.rb +80 -0
- data/lib/pikuri/web/url_cache.rb +63 -0
- data/lib/pikuri-web.rb +36 -0
- metadata +138 -0
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Pikuri
|
|
6
|
+
module Web
|
|
7
|
+
# Namespace for the web-search stack behind {Web::Search}: per-provider
|
|
8
|
+
# modules ({DuckDuckGo}, {Brave}, {Exa}), the {Result} they return, the
|
|
9
|
+
# {Engines} cascade with its result caches, and the shared {RateLimiter}.
|
|
10
|
+
module Search
|
|
11
|
+
# Search-orchestration object: the cascade across selected providers, the
|
|
12
|
+
# result cache, and the {Unavailable} marker the cascade falls back on.
|
|
13
|
+
# {Web::Search.build} constructs one and wires its {#search} into a
|
|
14
|
+
# {Tool}. Each provider raises {Unavailable} to hand off to the next.
|
|
15
|
+
#
|
|
16
|
+
# Engines.new(engines: [:brave], brave_key: 'BSA...') # Brave alone
|
|
17
|
+
# Engines.new(brave_key: 'BSA...') # DuckDuckGo + Brave
|
|
18
|
+
# Engines.new # DuckDuckGo alone
|
|
19
|
+
#
|
|
20
|
+
# {.resolve} owns which engines that yields and which selections are
|
|
21
|
+
# refused; {.config_from_h} maps a config file's +search:+ block onto
|
|
22
|
+
# these keywords.
|
|
23
|
+
#
|
|
24
|
+
# == Provider keys are constructor config, not environment
|
|
25
|
+
#
|
|
26
|
+
# Brave and Exa are paid and need a key; DuckDuckGo needs none. An
|
|
27
|
+
# {Engines} is built with the keys it should use (+brave_key:+ /
|
|
28
|
+
# +exa_key:+, both optional) — pikuri reads no key from the environment,
|
|
29
|
+
# so a host supplies them. See AGENTS.md "Environment is not a secret
|
|
30
|
+
# store".
|
|
31
|
+
#
|
|
32
|
+
# == Page text
|
|
33
|
+
#
|
|
34
|
+
# Built with +page_text: true+, Brave and Exa also bring back each page's
|
|
35
|
+
# text from their own index, in the same request. {#search} stores it in
|
|
36
|
+
# a {PageStore} and prints the row's handle before its title, for
|
|
37
|
+
# +web_read_cached+ to read:
|
|
38
|
+
#
|
|
39
|
+
# [a3f9c21e] [Ruby Data](https://docs.ruby-lang.org/en/3.4/Data.html)
|
|
40
|
+
# Data objects are immutable.
|
|
41
|
+
#
|
|
42
|
+
# A row gets a handle only when *this* search returned its text — never a
|
|
43
|
+
# DuckDuckGo row — even if the store holds that URL from another search.
|
|
44
|
+
# So an +engines: nil+ selection leaves DuckDuckGo out in this mode: in a
|
|
45
|
+
# shuffled cascade it would win about every other search and leave the
|
|
46
|
+
# model nothing to read. A host that lists it explicitly keeps it. The
|
|
47
|
+
# two modes cache apart ({DISK_TEXT_CACHE}, {MEMORY_TEXT_CACHE}): they
|
|
48
|
+
# return different rows.
|
|
49
|
+
class Engines
|
|
50
|
+
# Subsystem logger; level via +PIKURI_LOG_ENGINES+ or +PIKURI_LOG+.
|
|
51
|
+
#
|
|
52
|
+
# @return [Logger]
|
|
53
|
+
LOGGER = Pikuri.logger_for('Engines')
|
|
54
|
+
|
|
55
|
+
# Every engine name accepted in an +engines:+ selection, in the order
|
|
56
|
+
# {#providers} reports them. Presentation only — {#search} shuffles.
|
|
57
|
+
#
|
|
58
|
+
# @return [Array<Symbol>]
|
|
59
|
+
ENGINE_NAMES = %i[duckduckgo brave exa].freeze
|
|
60
|
+
|
|
61
|
+
# Engine name → the constructor keyword carrying its API key. Naming
|
|
62
|
+
# one of these without its key is a refusal ({.resolve}); DuckDuckGo is
|
|
63
|
+
# absent because it needs none.
|
|
64
|
+
#
|
|
65
|
+
# @return [Hash{Symbol => Symbol}]
|
|
66
|
+
KEYED_ENGINES = { brave: :brave_key, exa: :exa_key }.freeze
|
|
67
|
+
|
|
68
|
+
# Engines that can bring back page text; +page_text: true+ needs one.
|
|
69
|
+
#
|
|
70
|
+
# @return [Array<Symbol>]
|
|
71
|
+
PAGE_TEXT_ENGINES = %i[brave exa].freeze
|
|
72
|
+
|
|
73
|
+
# The fields {.config_from_h} accepts from a config file's +search:+
|
|
74
|
+
# block, matching this class's keywords.
|
|
75
|
+
#
|
|
76
|
+
# @return [Array<Symbol>]
|
|
77
|
+
CONFIG_FIELDS = %i[engines brave_key exa_key].freeze
|
|
78
|
+
|
|
79
|
+
# Raised by a provider that is temporarily unavailable (rate-limited,
|
|
80
|
+
# bot-blocked, quota-exhausted — "try again later", not "your request is
|
|
81
|
+
# wrong"). The {#search} cascade catches it and tries the next provider;
|
|
82
|
+
# anything else bubbles up so genuine bugs stay visible.
|
|
83
|
+
class Unavailable < StandardError; end
|
|
84
|
+
|
|
85
|
+
# On-disk result cache for providers whose +disk_cacheable?+ is true, at
|
|
86
|
+
# class level so every engine dedupes into one directory. An entry is
|
|
87
|
+
# the rows as JSON, never rendered Markdown: handles are looked up at
|
|
88
|
+
# render time.
|
|
89
|
+
#
|
|
90
|
+
# @return [UrlCache::OnDisk]
|
|
91
|
+
DISK_CACHE = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_search")
|
|
92
|
+
|
|
93
|
+
# In-memory result cache for the rest (Brave, Exa: their terms forbid
|
|
94
|
+
# storing results), shared by every engine in the process.
|
|
95
|
+
#
|
|
96
|
+
# @return [UrlCache::InMemory]
|
|
97
|
+
MEMORY_CACHE = UrlCache::InMemory.new(ttl: UrlCache::DEFAULT_TTL, max_entries: 100)
|
|
98
|
+
|
|
99
|
+
# {DISK_CACHE}'s twin for +page_text: true+ engines.
|
|
100
|
+
#
|
|
101
|
+
# @return [UrlCache::OnDisk]
|
|
102
|
+
DISK_TEXT_CACHE = UrlCache::OnDisk.new(ttl: UrlCache::DEFAULT_TTL,
|
|
103
|
+
dir: "#{UrlCache::ROOT_DIR}/web_search_text")
|
|
104
|
+
|
|
105
|
+
# {MEMORY_CACHE}'s twin for +page_text: true+ engines.
|
|
106
|
+
#
|
|
107
|
+
# @return [UrlCache::InMemory]
|
|
108
|
+
MEMORY_TEXT_CACHE = UrlCache::InMemory.new(ttl: UrlCache::DEFAULT_TTL, max_entries: 100)
|
|
109
|
+
|
|
110
|
+
# The constructor's +disk_cache:+ default; specs swap in {UrlCache::NULL}.
|
|
111
|
+
#
|
|
112
|
+
# @param page_text [Boolean] which mode's cache
|
|
113
|
+
# @return [UrlCache::OnDisk, #read, #write]
|
|
114
|
+
def self.disk_cache(page_text: false)
|
|
115
|
+
page_text ? DISK_TEXT_CACHE : DISK_CACHE
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# The constructor's +memory_cache:+ default; specs swap in
|
|
119
|
+
# {UrlCache::NULL}.
|
|
120
|
+
#
|
|
121
|
+
# @param page_text [Boolean] which mode's cache
|
|
122
|
+
# @return [UrlCache::InMemory, #read, #write]
|
|
123
|
+
def self.memory_cache(page_text: false)
|
|
124
|
+
page_text ? MEMORY_TEXT_CACHE : MEMORY_CACHE
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# Resolve an +engines:+ selection into the engine names to build,
|
|
128
|
+
# refusing a selection that cannot work:
|
|
129
|
+
#
|
|
130
|
+
# resolve(engines: nil, brave_key: 'B') # => [:duckduckgo, :brave]
|
|
131
|
+
# resolve(engines: nil) # => [:duckduckgo]
|
|
132
|
+
# resolve(engines: %w[brave], brave_key: 'B') # => [:brave]
|
|
133
|
+
# resolve(engines: [:exa], brave_key: 'B') # raises — exa has no key
|
|
134
|
+
#
|
|
135
|
+
# +nil+ means "whatever the keys say": DuckDuckGo plus every engine
|
|
136
|
+
# whose key is present, which is what a host that never heard of
|
|
137
|
+
# +engines:+ gets. An explicit list is a **set** — it is normalized to
|
|
138
|
+
# {ENGINE_NAMES} order and {#search} shuffles it per call anyway, so
|
|
139
|
+
# +[brave, duckduckgo]+ expresses no preference for Brave. Names are
|
|
140
|
+
# matched case-insensitively.
|
|
141
|
+
#
|
|
142
|
+
# Supplying a key without listing its engine is fine — parking a key
|
|
143
|
+
# you are not using today is not a mistake.
|
|
144
|
+
#
|
|
145
|
+
# @param engines [Array<String, Symbol>, nil] engine names, or +nil+ to
|
|
146
|
+
# derive the set from the keys.
|
|
147
|
+
# @param brave_key [String, nil] Brave token; blank counts as absent.
|
|
148
|
+
# @param exa_key [String, nil] Exa key; blank counts as absent.
|
|
149
|
+
# @return [Array<Symbol>] names drawn from {ENGINE_NAMES}, in that order.
|
|
150
|
+
# @raise [ArgumentError] if +engines+ is not an Array, is empty, holds
|
|
151
|
+
# an unknown or repeated name, or names a {KEYED_ENGINES} engine
|
|
152
|
+
# whose key is blank.
|
|
153
|
+
def self.resolve(engines:, brave_key: nil, exa_key: nil)
|
|
154
|
+
keys = { brave: brave_key, exa: exa_key }
|
|
155
|
+
keyless = ->(name) { KEYED_ENGINES.key?(name) && keys[name].to_s.strip.empty? }
|
|
156
|
+
return ENGINE_NAMES.reject(&keyless) if engines.nil?
|
|
157
|
+
|
|
158
|
+
unless engines.is_a?(Array) && !engines.empty?
|
|
159
|
+
# A list of engine names is safe to echo; anything else might be a
|
|
160
|
+
# key the writer put in the wrong field, so show only its class.
|
|
161
|
+
got = engines.is_a?(Array) ? engines.inspect : engines.class
|
|
162
|
+
raise ArgumentError, 'search engines: expected a non-empty list of engine names ' \
|
|
163
|
+
"(known: #{ENGINE_NAMES.join(', ')}), got #{got}"
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
names = engines.map { |name| name.to_s.strip.downcase.to_sym }
|
|
167
|
+
refuse_unknown(names)
|
|
168
|
+
refuse_repeats(names)
|
|
169
|
+
refuse_keyless(names.select(&keyless))
|
|
170
|
+
ENGINE_NAMES & names
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# @param names [Array<Symbol>] the requested engine names.
|
|
174
|
+
# @return [void]
|
|
175
|
+
# @raise [ArgumentError] if any name is not in {ENGINE_NAMES}.
|
|
176
|
+
def self.refuse_unknown(names)
|
|
177
|
+
unknown = names - ENGINE_NAMES
|
|
178
|
+
return if unknown.empty?
|
|
179
|
+
|
|
180
|
+
raise ArgumentError, "search engines: unknown engine(s) #{unknown.join(', ')} — " \
|
|
181
|
+
"known engines are #{ENGINE_NAMES.join(', ')}"
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# @param names [Array<Symbol>] the requested engine names.
|
|
185
|
+
# @return [void]
|
|
186
|
+
# @raise [ArgumentError] if any name appears twice (which would put one
|
|
187
|
+
# provider into the cascade twice, weighting the shuffle).
|
|
188
|
+
def self.refuse_repeats(names)
|
|
189
|
+
repeated = names.tally.select { |_, count| count > 1 }.keys
|
|
190
|
+
return if repeated.empty?
|
|
191
|
+
|
|
192
|
+
raise ArgumentError, "search engines: #{repeated.join(', ')} listed more than once"
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# @param keyless [Array<Symbol>] requested engines with no usable key.
|
|
196
|
+
# @return [void]
|
|
197
|
+
# @raise [ArgumentError] unless +keyless+ is empty.
|
|
198
|
+
def self.refuse_keyless(keyless)
|
|
199
|
+
return if keyless.empty?
|
|
200
|
+
|
|
201
|
+
needed = keyless.map { |name| "#{name} needs #{KEYED_ENGINES[name]}" }.join(', ')
|
|
202
|
+
raise ArgumentError, "search engines: #{needed} — supply the key or drop the engine from the list"
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Map a config file's +search:+ block onto this class's keywords, so a
|
|
206
|
+
# host hands over what it read:
|
|
207
|
+
#
|
|
208
|
+
# # ~/.pikuri-examples-config.yaml
|
|
209
|
+
# search:
|
|
210
|
+
# brave_key: BSA...
|
|
211
|
+
# exa_key: xyz
|
|
212
|
+
# engines: [brave, duckduckgo]
|
|
213
|
+
#
|
|
214
|
+
# Web::Search.from_h(YAML.safe_load_file(path)['search'])
|
|
215
|
+
#
|
|
216
|
+
# Nothing here knows what YAML is; a host reading TOML, JSON or its own
|
|
217
|
+
# settings object uses the same method. Every field is validated rather
|
|
218
|
+
# than coerced, because the person writing a config file cannot
|
|
219
|
+
# type-check it — and a mistyped engine name that silently meant "no
|
|
220
|
+
# Brave" would show up only as worse search results.
|
|
221
|
+
#
|
|
222
|
+
# @param config [Hash{String, Symbol => Object}, nil] the +search:+
|
|
223
|
+
# block, with String or Symbol keys. +nil+ — what a host reading an
|
|
224
|
+
# absent config key gets — yields an empty Hash, so no caller needs a
|
|
225
|
+
# +|| {}+.
|
|
226
|
+
# @return [Hash{Symbol => Object}] +engines:+ / +brave_key:+ /
|
|
227
|
+
# +exa_key:+ keywords for {#initialize} or {Web::Search.build}.
|
|
228
|
+
# @raise [ArgumentError] on a non-Hash, an unknown field, or a key field
|
|
229
|
+
# that is not a String.
|
|
230
|
+
def self.config_from_h(config)
|
|
231
|
+
return {} if config.nil?
|
|
232
|
+
|
|
233
|
+
unless config.is_a?(Hash)
|
|
234
|
+
raise ArgumentError, 'search config: expected a Hash of ' \
|
|
235
|
+
"#{CONFIG_FIELDS.join(', ')}, got #{config.class}"
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
fields = config.transform_keys(&:to_sym)
|
|
239
|
+
unknown = fields.keys - CONFIG_FIELDS
|
|
240
|
+
unless unknown.empty?
|
|
241
|
+
raise ArgumentError, "search config: unknown field(s) #{unknown.join(', ')} — " \
|
|
242
|
+
"known fields are #{CONFIG_FIELDS.join(', ')}"
|
|
243
|
+
end
|
|
244
|
+
KEYED_ENGINES.each_value do |field|
|
|
245
|
+
value = fields[field]
|
|
246
|
+
next if value.nil? || value.is_a?(String)
|
|
247
|
+
|
|
248
|
+
# The class, never the value: an error message about a key field is
|
|
249
|
+
# one paste away from a bug report.
|
|
250
|
+
raise ArgumentError, "search config: #{field} must be a String, got #{value.class}"
|
|
251
|
+
end
|
|
252
|
+
fields
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
# Builds the provider cascade once, from the engines {.resolve} selects.
|
|
256
|
+
# From here each provider is just an object answering
|
|
257
|
+
# +#search+/+#label+/+#disk_cacheable?+.
|
|
258
|
+
#
|
|
259
|
+
# @param engines [Array<String, Symbol>, nil] engine names; +nil+
|
|
260
|
+
# derives the set from the keys (see {.resolve}), minus DuckDuckGo
|
|
261
|
+
# under +page_text+.
|
|
262
|
+
# @param brave_key [String, nil] Brave token.
|
|
263
|
+
# @param exa_key [String, nil] Exa key.
|
|
264
|
+
# @param page_text [Boolean] bring back page text and print handles;
|
|
265
|
+
# see the class doc's +== Page text+
|
|
266
|
+
# @param disk_cache [UrlCache::OnDisk, #read, #write] store for a
|
|
267
|
+
# +disk_cacheable?+ winner's rows; defaults to {.disk_cache}.
|
|
268
|
+
# @param memory_cache [UrlCache::InMemory, #read, #write] store for
|
|
269
|
+
# every other winner's; defaults to {.memory_cache}.
|
|
270
|
+
# @param page_store [PageStore] where page text goes; defaults to
|
|
271
|
+
# {PageStore.default}
|
|
272
|
+
# @return [Engines]
|
|
273
|
+
# @raise [ArgumentError] as {.resolve}, or when +page_text+ is set but
|
|
274
|
+
# no selected engine can bring back page text
|
|
275
|
+
def initialize(engines: nil, brave_key: nil, exa_key: nil, page_text: false,
|
|
276
|
+
disk_cache: self.class.disk_cache(page_text: page_text),
|
|
277
|
+
memory_cache: self.class.memory_cache(page_text: page_text),
|
|
278
|
+
page_store: PageStore.default)
|
|
279
|
+
names = self.class.resolve(engines: engines, brave_key: brave_key, exa_key: exa_key)
|
|
280
|
+
if page_text
|
|
281
|
+
if (names & PAGE_TEXT_ENGINES).empty?
|
|
282
|
+
raise ArgumentError, "search engines: page_text needs #{PAGE_TEXT_ENGINES.join(' or ')}, " \
|
|
283
|
+
"got #{names.join(', ')}"
|
|
284
|
+
end
|
|
285
|
+
names &= PAGE_TEXT_ENGINES if engines.nil?
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
@providers = names.map do |name|
|
|
289
|
+
case name
|
|
290
|
+
when :duckduckgo then DuckDuckGo.new
|
|
291
|
+
when :brave then Brave.new(api_key: brave_key, page_text: page_text)
|
|
292
|
+
when :exa then Exa.new(api_key: exa_key, page_text: page_text)
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
@page_text = page_text
|
|
296
|
+
@disk_cache = disk_cache
|
|
297
|
+
@memory_cache = memory_cache
|
|
298
|
+
@page_store = page_store
|
|
299
|
+
@last_logged_providers = nil
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# @return [Array<#search, #label, #disk_cacheable?>] the selected
|
|
303
|
+
# provider instances, in {ENGINE_NAMES} order (the cascade shuffles
|
|
304
|
+
# them per call).
|
|
305
|
+
attr_reader :providers
|
|
306
|
+
|
|
307
|
+
# Run +query+ through the providers in random order, falling back on each
|
|
308
|
+
# {Unavailable}. The shuffle spreads load so one provider isn't always
|
|
309
|
+
# hit (and exhausted) first. The query is whitespace-normalized first;
|
|
310
|
+
# the winning provider's rows are cached keyed by the cleaned query (a
|
|
311
|
+
# hit short-circuits the cascade) — on disk when the winner is
|
|
312
|
+
# +disk_cacheable?+, else in memory only — and rendered to Markdown.
|
|
313
|
+
# +max_results+ is *not* in the cache key, so a non-default value may
|
|
314
|
+
# return a previously-cached size.
|
|
315
|
+
#
|
|
316
|
+
# If every provider is unavailable, returns an +"Error: ..."+ string
|
|
317
|
+
# (the tool convention) rather than raising. Any non-{Unavailable}
|
|
318
|
+
# exception (network, parser, bad key) bubbles up.
|
|
319
|
+
#
|
|
320
|
+
# @param query [String] search query
|
|
321
|
+
# @param max_results [Integer] maximum result entries
|
|
322
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable, nil] handed
|
|
323
|
+
# to each provider so its {RateLimiter} pacing wait is interruptible.
|
|
324
|
+
# A cache hit never reaches a provider, so it never waits at all.
|
|
325
|
+
# @return [Tool::Result] the Markdown result list (+"Returned 3 search
|
|
326
|
+
# results"+), or {Tool::Result.error} when all providers are exhausted
|
|
327
|
+
# @raise [ArgumentError] if the query is empty after normalization
|
|
328
|
+
# @raise [Pikuri::Agent::Control::Cancellable::Cancelled] if
|
|
329
|
+
# +cancellable+ trips while a provider is pacing. It is *not* caught
|
|
330
|
+
# as a provider failure — the cascade stops rather than trying the
|
|
331
|
+
# next one, and nothing is cached.
|
|
332
|
+
def search(query, max_results:, cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
333
|
+
cleaned = query.to_s.strip.gsub(/\s+/, ' ')
|
|
334
|
+
raise ArgumentError, 'query is empty' if cleaned.empty?
|
|
335
|
+
|
|
336
|
+
current_providers = providers
|
|
337
|
+
log_providers(current_providers)
|
|
338
|
+
|
|
339
|
+
json = @memory_cache.read(cleaned) || @disk_cache.read(cleaned)
|
|
340
|
+
if json
|
|
341
|
+
LOGGER.info { "cache=hit bytes=#{json.bytesize}" }
|
|
342
|
+
else
|
|
343
|
+
json = run_cascade(cleaned, current_providers, max_results: max_results, cancellable: cancellable)
|
|
344
|
+
end
|
|
345
|
+
rows = JSON.parse(json)
|
|
346
|
+
Tool::Result.new(content: render(rows),
|
|
347
|
+
summary: "Returned #{Pikuri::Tool.pluralize(rows.size, 'search result')}")
|
|
348
|
+
rescue Unavailable => e
|
|
349
|
+
Pikuri::Tool::Result.error(e.message)
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
private
|
|
353
|
+
|
|
354
|
+
# Ask the providers in random order until one answers, store its page
|
|
355
|
+
# text, then cache its rows where its terms allow.
|
|
356
|
+
#
|
|
357
|
+
# @param cleaned [String] the normalized query
|
|
358
|
+
# @param current_providers [Array<#search, #label, #disk_cacheable?>]
|
|
359
|
+
# @param max_results [Integer]
|
|
360
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable]
|
|
361
|
+
# @return [String] the rows as JSON, as cached
|
|
362
|
+
# @raise [Unavailable] when every provider is — raised, never stored, so
|
|
363
|
+
# the all-unavailable message can't block this query for the TTL
|
|
364
|
+
def run_cascade(cleaned, current_providers, max_results:, cancellable:)
|
|
365
|
+
failures = []
|
|
366
|
+
current_providers.shuffle.each do |provider|
|
|
367
|
+
results = provider.search(cleaned, max_results: max_results, cancellable: cancellable)
|
|
368
|
+
disk = provider.disk_cacheable?
|
|
369
|
+
LOGGER.info { "engine=#{provider.label} results=#{results.size} cache=#{disk ? 'disk' : 'memory'}" }
|
|
370
|
+
return (disk ? @disk_cache : @memory_cache).write(cleaned, JSON.generate(rows_of(results)))
|
|
371
|
+
rescue Unavailable => e
|
|
372
|
+
failures << "#{provider.label} (#{e.message})"
|
|
373
|
+
end
|
|
374
|
+
raise Unavailable, "all search providers temporarily unavailable: #{failures.join('; ')}"
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# The cacheable rows of +results+, storing each one's page text first.
|
|
378
|
+
# A row's +has_text+ is what earns it a handle at render time; its text
|
|
379
|
+
# lives only in the {PageStore}, never in a query cache.
|
|
380
|
+
#
|
|
381
|
+
# @param results [Array<Result>]
|
|
382
|
+
# @return [Array<Hash{String => Object}>] +url+, +title+, +snippet+,
|
|
383
|
+
# +has_text+ per row
|
|
384
|
+
def rows_of(results)
|
|
385
|
+
results.map do |r|
|
|
386
|
+
has_text = @page_text && !r.text.nil?
|
|
387
|
+
@page_store.store(url: r.url, title: r.title, text: r.text, indexed_at: r.indexed_at) if has_text
|
|
388
|
+
{ 'url' => r.url, 'title' => r.title, 'snippet' => r.body, 'has_text' => has_text }
|
|
389
|
+
end
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
# Render rows into the smolagents-style Markdown the LLM consumes
|
|
393
|
+
# (+"## Search Results"+ header, then +[title](url)\nsnippet+ joined by
|
|
394
|
+
# blank lines), each readable row led by its handle. Empty → the +"No
|
|
395
|
+
# results found."+ stub.
|
|
396
|
+
#
|
|
397
|
+
# @param rows [Array<Hash{String => Object}>] as {#rows_of} made them
|
|
398
|
+
# @return [String] Markdown result list
|
|
399
|
+
def render(rows)
|
|
400
|
+
return "## Search Results\n\nNo results found." if rows.empty?
|
|
401
|
+
|
|
402
|
+
"## Search Results\n\n" + rows.map { |row| render_row(row) }.join("\n\n")
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
# @param row [Hash{String => Object}]
|
|
406
|
+
# @return [String] one row; a +has_text+ row still in the {PageStore}
|
|
407
|
+
# gets its handle, one evicted since renders without
|
|
408
|
+
def render_row(row)
|
|
409
|
+
line = "[#{row['title']}](#{row['url']})\n#{row['snippet']}"
|
|
410
|
+
handle = row['has_text'] && @page_store.handle_for(row['url'])
|
|
411
|
+
handle ? "[#{handle}] #{line}" : line
|
|
412
|
+
end
|
|
413
|
+
|
|
414
|
+
# INFO-log the currently-available providers, but only when the set
|
|
415
|
+
# differs from the last logged — one line per distinct configuration,
|
|
416
|
+
# not per search.
|
|
417
|
+
#
|
|
418
|
+
# @param current [Array<#label>] providers from {#providers}
|
|
419
|
+
# @return [void]
|
|
420
|
+
def log_providers(current)
|
|
421
|
+
return if @last_logged_providers == current
|
|
422
|
+
|
|
423
|
+
@last_logged_providers = current
|
|
424
|
+
names = current.map(&:label).join(', ')
|
|
425
|
+
LOGGER.info("engines available: #{names}")
|
|
426
|
+
end
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
end
|
|
430
|
+
end
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'faraday'
|
|
4
|
+
require 'json'
|
|
5
|
+
|
|
6
|
+
module Pikuri
|
|
7
|
+
module Web
|
|
8
|
+
module Search
|
|
9
|
+
# Performs an Exa search via the official +/search+ endpoint, returning
|
|
10
|
+
# hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
|
|
11
|
+
# parser (.parse) so tests exercise the parser against fixture JSON.
|
|
12
|
+
# {Engines#search} owns the final Markdown rendering.
|
|
13
|
+
#
|
|
14
|
+
# Constructed with its API key (+Exa.new(api_key:)+); {Engines} builds one
|
|
15
|
+
# only when an Exa key was configured, so users who never registered don't
|
|
16
|
+
# spend money, then drives it through the same +#search+/+#label+
|
|
17
|
+
# interface as every provider. pikuri reads no key from the environment
|
|
18
|
+
# (AGENTS.md "Environment is not a secret store"). Paid; key at
|
|
19
|
+
# https://exa.ai.
|
|
20
|
+
#
|
|
21
|
+
# Requests +type: "auto"+ (Exa picks neural vs keyword) and +contents: {
|
|
22
|
+
# highlights: true }+ so each result carries a neural-ranked snippet
|
|
23
|
+
# populating {Result#body} — the analog to Brave's +description+. Built
|
|
24
|
+
# with +page_text: true+, the same request also asks for each page's full
|
|
25
|
+
# text ({Result#text}, up to {MAX_TEXT_CHARS}) — at no extra charge up to
|
|
26
|
+
# 10 results. Exa gives no crawl date, so {Result#indexed_at} stays +nil+.
|
|
27
|
+
#
|
|
28
|
+
# == Cache-only, never a live fetch
|
|
29
|
+
#
|
|
30
|
+
# {.request_body} pins +maxAgeHours: -1+. Omitted, Exa's crawler fetches
|
|
31
|
+
# an uncached result page live during the search, and since the query
|
|
32
|
+
# picks which pages match, an attacker owning indexed pages reads bits of
|
|
33
|
+
# the query from their server logs. That would break the +:tool_vouched+
|
|
34
|
+
# egress grade {Search} declares. A snippet needs no fresh page.
|
|
35
|
+
#
|
|
36
|
+
# == Privacy posture
|
|
37
|
+
#
|
|
38
|
+
# Exa's Privacy Policy uses Query Data to train/fine-tune its models, and
|
|
39
|
+
# ToS §1.2(c) grants a +perpetual and irrevocable+, sub-licensable license
|
|
40
|
+
# over User Input. Business customers under an MSA/DPA get carve-outs; the
|
|
41
|
+
# default pay-as-you-go key pikuri uses does not. Bottom line: Exa mines
|
|
42
|
+
# queries to train competing models. If a query would be sensitive in a
|
|
43
|
+
# training set, don't configure an Exa key — {Engines#providers} leaves
|
|
44
|
+
# Exa out of the cascade unless its key was supplied.
|
|
45
|
+
class Exa
|
|
46
|
+
# @return [String] Search endpoint (POST, JSON body)
|
|
47
|
+
ENDPOINT = 'https://api.exa.ai/search'
|
|
48
|
+
# @return [Integer] default number of results returned, matching
|
|
49
|
+
# {DuckDuckGo::DEFAULT_MAX_RESULTS}
|
|
50
|
+
DEFAULT_MAX_RESULTS = 10
|
|
51
|
+
# @return [RateLimiter] Exa is paid and doesn't aggressively throttle,
|
|
52
|
+
# so no minimum interval; the 5-minute cooldown still applies on
|
|
53
|
+
# {Engines::Unavailable} so the budget isn't burned on doomed retries.
|
|
54
|
+
LIMITER = RateLimiter.new(min_interval: 0.0, cooldown: 300.0)
|
|
55
|
+
# @return [Integer] per-page text cap asked for with +page_text+ — the
|
|
56
|
+
# +/search+ maximum, so pikuri takes all the text Exa gives and trims
|
|
57
|
+
# only when it is read
|
|
58
|
+
MAX_TEXT_CHARS = 1_000_000
|
|
59
|
+
|
|
60
|
+
# @param api_key [String] Exa API key. Required and non-blank:
|
|
61
|
+
# pikuri reads no key from the environment — the host supplies it
|
|
62
|
+
# ({Engines} only constructs an Exa when a key was configured).
|
|
63
|
+
# @param page_text [Boolean] also fetch each page's stored text
|
|
64
|
+
# @raise [ArgumentError] if +api_key+ is blank
|
|
65
|
+
def initialize(api_key:, page_text: false)
|
|
66
|
+
raise ArgumentError, 'Exa Search API key is blank' if api_key.to_s.strip.empty?
|
|
67
|
+
|
|
68
|
+
@api_key = api_key
|
|
69
|
+
@page_text = page_text
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# @return [String] short provider label for {Engines} logging /
|
|
73
|
+
# fallback messages.
|
|
74
|
+
def label
|
|
75
|
+
'Exa'
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Whether {Engines} may keep this provider's results in its on-disk
|
|
79
|
+
# cache. No: Exa's Terms of Service, which cover the API, forbid to
|
|
80
|
+
# "copy … duplicate … any information … obtained from or through, the
|
|
81
|
+
# Services, except for temporary files that are automatically cached
|
|
82
|
+
# by your web browser" (§4.2(a), https://exa.ai/terms), so they stay in
|
|
83
|
+
# memory ({UrlCache::InMemory}). Its Master Subscription Agreement,
|
|
84
|
+
# which lets a customer own the output, binds Order Form (Enterprise)
|
|
85
|
+
# customers, not a pay-as-you-go key.
|
|
86
|
+
#
|
|
87
|
+
# @return [Boolean] +false+
|
|
88
|
+
def disk_cacheable?
|
|
89
|
+
false
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# Fetch results for +query+ as an +Array<Result>+. Circuit-broken for
|
|
93
|
+
# 5 minutes on rate-limit/unavailable (see {LIMITER}). The caller
|
|
94
|
+
# ({Engines#search}) normalizes the query and wraps this in a cache.
|
|
95
|
+
#
|
|
96
|
+
# @param query [String] search query (already normalized)
|
|
97
|
+
# @param max_results [Integer] max entries; Exa's +numResults+
|
|
98
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable] makes
|
|
99
|
+
# {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
|
|
100
|
+
# @return [Array<Result>] hits, possibly empty when Exa matched nothing
|
|
101
|
+
# @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
|
|
102
|
+
# back from these), or immediately if {LIMITER} is in cooldown. Other
|
|
103
|
+
# non-2xx (401/403 bad key) bubble up as +RuntimeError+.
|
|
104
|
+
# @raise [RuntimeError] for non-rate-limit HTTP failures, or an
|
|
105
|
+
# unrecognized empty-results shape.
|
|
106
|
+
def search(query, max_results: DEFAULT_MAX_RESULTS,
|
|
107
|
+
cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
108
|
+
LIMITER.call(cancellable: cancellable) do
|
|
109
|
+
response = Faraday.post(ENDPOINT) do |req|
|
|
110
|
+
req.headers['x-api-key'] = @api_key
|
|
111
|
+
req.headers['Content-Type'] = 'application/json'
|
|
112
|
+
req.headers['Accept'] = 'application/json'
|
|
113
|
+
req.body = JSON.dump(self.class.request_body(query, max_results: max_results,
|
|
114
|
+
page_text: @page_text))
|
|
115
|
+
end
|
|
116
|
+
unless response.success?
|
|
117
|
+
if response.status == 429 || response.status >= 500
|
|
118
|
+
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
raise "Exa Search request failed: #{response.status} #{response.body}"
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
self.class.parse(response.body, max_results: max_results)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# The JSON body for a {ENDPOINT} request; cache-only, see the class doc.
|
|
129
|
+
#
|
|
130
|
+
# Exa.request_body('ruby', max_results: 5)
|
|
131
|
+
# # => { query: 'ruby', type: 'auto', numResults: 5,
|
|
132
|
+
# # contents: { highlights: true, maxAgeHours: -1 } }
|
|
133
|
+
# Exa.request_body('ruby', max_results: 5, page_text: true)
|
|
134
|
+
# # => … contents: { highlights: true, maxAgeHours: -1,
|
|
135
|
+
# # text: { maxCharacters: 1_000_000 } } }
|
|
136
|
+
#
|
|
137
|
+
# @param query [String] search query (already normalized)
|
|
138
|
+
# @param max_results [Integer] max entries; Exa's +numResults+
|
|
139
|
+
# @param page_text [Boolean] also ask for each page's text
|
|
140
|
+
# @return [Hash{Symbol => Object}]
|
|
141
|
+
def self.request_body(query, max_results: DEFAULT_MAX_RESULTS, page_text: false)
|
|
142
|
+
contents = { highlights: true, maxAgeHours: -1 }
|
|
143
|
+
contents[:text] = { maxCharacters: MAX_TEXT_CHARS } if page_text
|
|
144
|
+
{ query: query, type: 'auto', numResults: max_results, contents: contents }
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Parse an Exa JSON response into {Result} rows, +body+ the first
|
|
148
|
+
# non-empty +highlights+ snippet, +text+ the page text when the response
|
|
149
|
+
# carries any. A genuine "no results" payload
|
|
150
|
+
# (+requestId+ present, empty +results+) returns +[]+ so {Engines.search}
|
|
151
|
+
# renders its no-results stub; any other zero-result shape raises.
|
|
152
|
+
#
|
|
153
|
+
# @param json [String] response body from {ENDPOINT}
|
|
154
|
+
# @param max_results [Integer] max entries
|
|
155
|
+
# @return [Array<Result>] hits, possibly empty on a recognized
|
|
156
|
+
# empty-results payload
|
|
157
|
+
# @raise [RuntimeError] on an unrecognized zero-result response
|
|
158
|
+
def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
|
|
159
|
+
data = JSON.parse(json)
|
|
160
|
+
results = Array(data['results']).take(max_results).filter_map do |r|
|
|
161
|
+
href = r['url'].to_s
|
|
162
|
+
next nil if href.empty?
|
|
163
|
+
|
|
164
|
+
Result.new(
|
|
165
|
+
url: href,
|
|
166
|
+
title: clean(r['title']) || href,
|
|
167
|
+
body: first_highlight(r['highlights']),
|
|
168
|
+
text: r['text'].is_a?(String) && !r['text'].strip.empty? ? r['text'] : nil
|
|
169
|
+
)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
if results.empty?
|
|
173
|
+
return [] if genuine_no_results?(data)
|
|
174
|
+
|
|
175
|
+
raise diagnose_empty(data, json)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
results
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Collapse whitespace and strip; +nil+ for nil/empty input so the caller
|
|
182
|
+
# can fall back (e.g. to the URL when a result has no title).
|
|
183
|
+
#
|
|
184
|
+
# @param text [String, nil]
|
|
185
|
+
# @return [String, nil] cleaned text, or +nil+ if blank
|
|
186
|
+
def self.clean(text)
|
|
187
|
+
return nil if text.nil?
|
|
188
|
+
|
|
189
|
+
cleaned = text.to_s.gsub(/\s+/, ' ').strip
|
|
190
|
+
cleaned.empty? ? nil : cleaned
|
|
191
|
+
end
|
|
192
|
+
private_class_method :clean
|
|
193
|
+
|
|
194
|
+
# First non-empty +highlights+ entry, cleaned. Exa sorts highlights by
|
|
195
|
+
# relevance; the top one populates +body+.
|
|
196
|
+
#
|
|
197
|
+
# @param highlights [Array<String>, nil] +highlights+ field
|
|
198
|
+
# @return [String] cleaned snippet, or empty string if none usable
|
|
199
|
+
def self.first_highlight(highlights)
|
|
200
|
+
return '' unless highlights.is_a?(Array)
|
|
201
|
+
|
|
202
|
+
highlights.each do |h|
|
|
203
|
+
cleaned = clean(h)
|
|
204
|
+
return cleaned if cleaned
|
|
205
|
+
end
|
|
206
|
+
''
|
|
207
|
+
end
|
|
208
|
+
private_class_method :first_highlight
|
|
209
|
+
|
|
210
|
+
# True when a zero-+results+ response looks like Exa's own "ran, matched
|
|
211
|
+
# nothing" payload rather than an error — marked by +requestId+, which
|
|
212
|
+
# Exa always sets on success.
|
|
213
|
+
#
|
|
214
|
+
# @param data [Hash, Object] parsed response
|
|
215
|
+
# @return [Boolean]
|
|
216
|
+
def self.genuine_no_results?(data)
|
|
217
|
+
return false unless data.is_a?(Hash)
|
|
218
|
+
return false unless data.key?('requestId')
|
|
219
|
+
|
|
220
|
+
Array(data['results']).empty?
|
|
221
|
+
end
|
|
222
|
+
private_class_method :genuine_no_results?
|
|
223
|
+
|
|
224
|
+
# Build an error message for a zero-result response: quote Exa's
|
|
225
|
+
# +error+/+message+/+detail+ if present, else a truncated raw body.
|
|
226
|
+
#
|
|
227
|
+
# @param data [Hash, Object] parsed response
|
|
228
|
+
# @param raw [String] raw response body
|
|
229
|
+
# @return [String] diagnostic to feed to +raise+
|
|
230
|
+
def self.diagnose_empty(data, raw)
|
|
231
|
+
if data.is_a?(Hash) && (msg = data['error'] || data['message'] || data['detail'])
|
|
232
|
+
return "Exa Search returned an error: #{msg}"
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
snippet = raw.to_s[0, 800]
|
|
236
|
+
snippet += '…' if raw.to_s.length > 800
|
|
237
|
+
"Exa Search returned no results. Body: #{snippet}"
|
|
238
|
+
end
|
|
239
|
+
private_class_method :diagnose_empty
|
|
240
|
+
end
|
|
241
|
+
end
|
|
242
|
+
end
|
|
243
|
+
end
|