pikuri-core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -29
- data/lib/pikuri/agent/chat_transport.rb +114 -33
- data/lib/pikuri/agent/configurator.rb +191 -110
- data/lib/pikuri/agent/control/cancellable.rb +25 -16
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +87 -120
- data/lib/pikuri/agent/control.rb +9 -11
- data/lib/pikuri/agent/event.rb +73 -61
- data/lib/pikuri/agent/extension.rb +45 -81
- data/lib/pikuri/agent/history.rb +91 -497
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +115 -108
- data/lib/pikuri/agent/listener/token_log.rb +21 -55
- data/lib/pikuri/agent/listener.rb +4 -4
- data/lib/pikuri/agent/listener_list.rb +19 -13
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +624 -448
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/extractor/passthrough.rb +5 -2
- data/lib/pikuri/extractor.rb +43 -12
- data/lib/pikuri/file_type.rb +4 -4
- data/lib/pikuri/finalizers.rb +2 -2
- data/lib/pikuri/paths.rb +1 -1
- data/lib/pikuri/sanitizer.rb +4 -3
- data/lib/pikuri/subprocess.rb +17 -5
- data/lib/pikuri/testing.rb +122 -49
- data/lib/pikuri/tool/calculator.rb +134 -30
- data/lib/pikuri/tool/execute_context.rb +2 -2
- data/lib/pikuri/tool/parameters.rb +10 -0
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +52 -21
- data/lib/pikuri/tool.rb +298 -24
- data/lib/pikuri/trifecta/report.rb +28 -30
- data/lib/pikuri/trifecta.rb +1 -2
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +5 -11
- metadata +16 -73
- data/lib/pikuri/agent/context_window_detector.rb +0 -181
- data/lib/pikuri/agent/control/step_limit.rb +0 -95
- data/lib/pikuri/agent/extension_context.rb +0 -120
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -146
- data/lib/pikuri/agent/synthesizer.rb +0 -129
- data/lib/pikuri/extractor/html.rb +0 -256
- data/lib/pikuri/ruby_llm_patches.rb +0 -106
- data/lib/pikuri/tool/fetch.rb +0 -146
- data/lib/pikuri/tool/scraper.rb +0 -144
- data/lib/pikuri/tool/search/brave.rb +0 -170
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -175
- data/lib/pikuri/tool/search/engines.rb +0 -319
- data/lib/pikuri/tool/search/exa.rb +0 -195
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -115
- data/lib/pikuri/tool/search/result.rb +0 -24
- data/lib/pikuri/tool/web_scrape.rb +0 -105
- data/lib/pikuri/tool/web_search.rb +0 -133
- data/lib/pikuri/url_cache.rb +0 -96
|
@@ -1,319 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
# Namespace for the web-search stack behind {Tool::WebSearch}: per-provider
|
|
6
|
-
# modules ({DuckDuckGo}, {Brave}, {Exa}), the {Result} they return, the
|
|
7
|
-
# {Engines} cascade with its on-disk cache, and the shared {RateLimiter}.
|
|
8
|
-
module Search
|
|
9
|
-
# Search-orchestration object: the cascade across selected providers, the
|
|
10
|
-
# result cache, and the {Unavailable} marker the cascade falls back on.
|
|
11
|
-
# {Tool::WebSearch.build} constructs one and wires its {#search} into a
|
|
12
|
-
# {Tool}. Each provider raises {Unavailable} to hand off to the next.
|
|
13
|
-
#
|
|
14
|
-
# Engines.new(engines: [:brave], brave_key: 'BSA...') # Brave alone
|
|
15
|
-
# Engines.new(brave_key: 'BSA...') # DuckDuckGo + Brave
|
|
16
|
-
# Engines.new # DuckDuckGo alone
|
|
17
|
-
#
|
|
18
|
-
# {.resolve} owns which engines that yields and which selections are
|
|
19
|
-
# refused; {.config_from_h} maps a config file's +search:+ block onto
|
|
20
|
-
# these keywords.
|
|
21
|
-
#
|
|
22
|
-
# == Provider keys are constructor config, not environment
|
|
23
|
-
#
|
|
24
|
-
# Brave and Exa are paid and need a key; DuckDuckGo needs none. An
|
|
25
|
-
# {Engines} is built with the keys it should use (+brave_key:+ /
|
|
26
|
-
# +exa_key:+, both optional) — pikuri reads no key from the environment,
|
|
27
|
-
# so a host supplies them. See CLAUDE.md "Environment is not a secret
|
|
28
|
-
# store".
|
|
29
|
-
class Engines
|
|
30
|
-
# Subsystem logger; level via +PIKURI_LOG_ENGINES+ or +PIKURI_LOG+.
|
|
31
|
-
#
|
|
32
|
-
# @return [Logger]
|
|
33
|
-
LOGGER = Pikuri.logger_for('Engines')
|
|
34
|
-
|
|
35
|
-
# Every engine name accepted in an +engines:+ selection, in the order
|
|
36
|
-
# {#providers} reports them. Presentation only — {#search} shuffles.
|
|
37
|
-
#
|
|
38
|
-
# @return [Array<Symbol>]
|
|
39
|
-
ENGINE_NAMES = %i[duckduckgo brave exa].freeze
|
|
40
|
-
|
|
41
|
-
# Engine name → the constructor keyword carrying its API key. Naming
|
|
42
|
-
# one of these without its key is a refusal ({.resolve}); DuckDuckGo is
|
|
43
|
-
# absent because it needs none.
|
|
44
|
-
#
|
|
45
|
-
# @return [Hash{Symbol => Symbol}]
|
|
46
|
-
KEYED_ENGINES = { brave: :brave_key, exa: :exa_key }.freeze
|
|
47
|
-
|
|
48
|
-
# The fields {.config_from_h} accepts from a config file's +search:+
|
|
49
|
-
# block, matching this class's keywords.
|
|
50
|
-
#
|
|
51
|
-
# @return [Array<Symbol>]
|
|
52
|
-
CONFIG_FIELDS = %i[engines brave_key exa_key].freeze
|
|
53
|
-
|
|
54
|
-
# Raised by a provider that is temporarily unavailable (rate-limited,
|
|
55
|
-
# bot-blocked, quota-exhausted — "try again later", not "your request is
|
|
56
|
-
# wrong"). The {#search} cascade catches it and tries the next provider;
|
|
57
|
-
# anything else bubbles up so genuine bugs stay visible.
|
|
58
|
-
class Unavailable < StandardError; end
|
|
59
|
-
|
|
60
|
-
# Process-shared on-disk cache backing {#search}, at class level so every
|
|
61
|
-
# engine dedupes into one directory; the constructor's +cache:+ injects a
|
|
62
|
-
# different store for tests.
|
|
63
|
-
#
|
|
64
|
-
# @return [UrlCache, #fetch]
|
|
65
|
-
CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_search")
|
|
66
|
-
# Accessor for {CACHE}, the constructor's +cache:+ default; specs swap in
|
|
67
|
-
# {UrlCache::NULL}.
|
|
68
|
-
#
|
|
69
|
-
# @return [UrlCache, #fetch]
|
|
70
|
-
def self.cache
|
|
71
|
-
CACHE
|
|
72
|
-
end
|
|
73
|
-
|
|
74
|
-
# Resolve an +engines:+ selection into the engine names to build,
|
|
75
|
-
# refusing a selection that cannot work:
|
|
76
|
-
#
|
|
77
|
-
# resolve(engines: nil, brave_key: 'B') # => [:duckduckgo, :brave]
|
|
78
|
-
# resolve(engines: nil) # => [:duckduckgo]
|
|
79
|
-
# resolve(engines: %w[brave], brave_key: 'B') # => [:brave]
|
|
80
|
-
# resolve(engines: [:exa], brave_key: 'B') # raises — exa has no key
|
|
81
|
-
#
|
|
82
|
-
# +nil+ means "whatever the keys say": DuckDuckGo plus every engine
|
|
83
|
-
# whose key is present, which is what a host that never heard of
|
|
84
|
-
# +engines:+ gets. An explicit list is a **set** — it is normalized to
|
|
85
|
-
# {ENGINE_NAMES} order and {#search} shuffles it per call anyway, so
|
|
86
|
-
# +[brave, duckduckgo]+ expresses no preference for Brave. Names are
|
|
87
|
-
# matched case-insensitively.
|
|
88
|
-
#
|
|
89
|
-
# Supplying a key without listing its engine is fine — parking a key
|
|
90
|
-
# you are not using today is not a mistake.
|
|
91
|
-
#
|
|
92
|
-
# @param engines [Array<String, Symbol>, nil] engine names, or +nil+ to
|
|
93
|
-
# derive the set from the keys.
|
|
94
|
-
# @param brave_key [String, nil] Brave token; blank counts as absent.
|
|
95
|
-
# @param exa_key [String, nil] Exa key; blank counts as absent.
|
|
96
|
-
# @return [Array<Symbol>] names drawn from {ENGINE_NAMES}, in that order.
|
|
97
|
-
# @raise [ArgumentError] if +engines+ is not an Array, is empty, holds
|
|
98
|
-
# an unknown or repeated name, or names a {KEYED_ENGINES} engine
|
|
99
|
-
# whose key is blank.
|
|
100
|
-
def self.resolve(engines:, brave_key: nil, exa_key: nil)
|
|
101
|
-
keys = { brave: brave_key, exa: exa_key }
|
|
102
|
-
keyless = ->(name) { KEYED_ENGINES.key?(name) && keys[name].to_s.strip.empty? }
|
|
103
|
-
return ENGINE_NAMES.reject(&keyless) if engines.nil?
|
|
104
|
-
|
|
105
|
-
unless engines.is_a?(Array) && !engines.empty?
|
|
106
|
-
# A list of engine names is safe to echo; anything else might be a
|
|
107
|
-
# key the writer put in the wrong field, so show only its class.
|
|
108
|
-
got = engines.is_a?(Array) ? engines.inspect : engines.class
|
|
109
|
-
raise ArgumentError, 'search engines: expected a non-empty list of engine names ' \
|
|
110
|
-
"(known: #{ENGINE_NAMES.join(', ')}), got #{got}"
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
names = engines.map { |name| name.to_s.strip.downcase.to_sym }
|
|
114
|
-
refuse_unknown(names)
|
|
115
|
-
refuse_repeats(names)
|
|
116
|
-
refuse_keyless(names.select(&keyless))
|
|
117
|
-
ENGINE_NAMES & names
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
# @param names [Array<Symbol>] the requested engine names.
|
|
121
|
-
# @return [void]
|
|
122
|
-
# @raise [ArgumentError] if any name is not in {ENGINE_NAMES}.
|
|
123
|
-
def self.refuse_unknown(names)
|
|
124
|
-
unknown = names - ENGINE_NAMES
|
|
125
|
-
return if unknown.empty?
|
|
126
|
-
|
|
127
|
-
raise ArgumentError, "search engines: unknown engine(s) #{unknown.join(', ')} — " \
|
|
128
|
-
"known engines are #{ENGINE_NAMES.join(', ')}"
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
# @param names [Array<Symbol>] the requested engine names.
|
|
132
|
-
# @return [void]
|
|
133
|
-
# @raise [ArgumentError] if any name appears twice (which would put one
|
|
134
|
-
# provider into the cascade twice, weighting the shuffle).
|
|
135
|
-
def self.refuse_repeats(names)
|
|
136
|
-
repeated = names.tally.select { |_, count| count > 1 }.keys
|
|
137
|
-
return if repeated.empty?
|
|
138
|
-
|
|
139
|
-
raise ArgumentError, "search engines: #{repeated.join(', ')} listed more than once"
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
# @param keyless [Array<Symbol>] requested engines with no usable key.
|
|
143
|
-
# @return [void]
|
|
144
|
-
# @raise [ArgumentError] unless +keyless+ is empty.
|
|
145
|
-
def self.refuse_keyless(keyless)
|
|
146
|
-
return if keyless.empty?
|
|
147
|
-
|
|
148
|
-
needed = keyless.map { |name| "#{name} needs #{KEYED_ENGINES[name]}" }.join(', ')
|
|
149
|
-
raise ArgumentError, "search engines: #{needed} — supply the key or drop the engine from the list"
|
|
150
|
-
end
|
|
151
|
-
|
|
152
|
-
# Map a config file's +search:+ block onto this class's keywords, so a
|
|
153
|
-
# host hands over what it read:
|
|
154
|
-
#
|
|
155
|
-
# # ~/.pikuri-examples-config.yaml
|
|
156
|
-
# search:
|
|
157
|
-
# brave_key: BSA...
|
|
158
|
-
# exa_key: xyz
|
|
159
|
-
# engines: [brave, duckduckgo]
|
|
160
|
-
#
|
|
161
|
-
# Tool::WebSearch.from_h(YAML.safe_load_file(path)['search'])
|
|
162
|
-
#
|
|
163
|
-
# Nothing here knows what YAML is; a host reading TOML, JSON or its own
|
|
164
|
-
# settings object uses the same method. Every field is validated rather
|
|
165
|
-
# than coerced, because the person writing a config file cannot
|
|
166
|
-
# type-check it — and a mistyped engine name that silently meant "no
|
|
167
|
-
# Brave" would show up only as worse search results.
|
|
168
|
-
#
|
|
169
|
-
# @param config [Hash{String, Symbol => Object}, nil] the +search:+
|
|
170
|
-
# block, with String or Symbol keys. +nil+ — what a host reading an
|
|
171
|
-
# absent config key gets — yields an empty Hash, so no caller needs a
|
|
172
|
-
# +|| {}+.
|
|
173
|
-
# @return [Hash{Symbol => Object}] +engines:+ / +brave_key:+ /
|
|
174
|
-
# +exa_key:+ keywords for {#initialize} or {Tool::WebSearch.build}.
|
|
175
|
-
# @raise [ArgumentError] on a non-Hash, an unknown field, or a key field
|
|
176
|
-
# that is not a String.
|
|
177
|
-
def self.config_from_h(config)
|
|
178
|
-
return {} if config.nil?
|
|
179
|
-
|
|
180
|
-
unless config.is_a?(Hash)
|
|
181
|
-
raise ArgumentError, 'search config: expected a Hash of ' \
|
|
182
|
-
"#{CONFIG_FIELDS.join(', ')}, got #{config.class}"
|
|
183
|
-
end
|
|
184
|
-
|
|
185
|
-
fields = config.transform_keys(&:to_sym)
|
|
186
|
-
unknown = fields.keys - CONFIG_FIELDS
|
|
187
|
-
unless unknown.empty?
|
|
188
|
-
raise ArgumentError, "search config: unknown field(s) #{unknown.join(', ')} — " \
|
|
189
|
-
"known fields are #{CONFIG_FIELDS.join(', ')}"
|
|
190
|
-
end
|
|
191
|
-
KEYED_ENGINES.each_value do |field|
|
|
192
|
-
value = fields[field]
|
|
193
|
-
next if value.nil? || value.is_a?(String)
|
|
194
|
-
|
|
195
|
-
# The class, never the value: an error message about a key field is
|
|
196
|
-
# one paste away from a bug report.
|
|
197
|
-
raise ArgumentError, "search config: #{field} must be a String, got #{value.class}"
|
|
198
|
-
end
|
|
199
|
-
fields
|
|
200
|
-
end
|
|
201
|
-
|
|
202
|
-
# Builds the provider cascade once, from the engines {.resolve} selects.
|
|
203
|
-
# From here each provider is just an object answering
|
|
204
|
-
# +#search+/+#label+.
|
|
205
|
-
#
|
|
206
|
-
# @param engines [Array<String, Symbol>, nil] engine names; +nil+
|
|
207
|
-
# derives the set from the keys. See {.resolve}.
|
|
208
|
-
# @param brave_key [String, nil] Brave token.
|
|
209
|
-
# @param exa_key [String, nil] Exa key.
|
|
210
|
-
# @param cache [UrlCache, #fetch] result store; defaults to {.cache}.
|
|
211
|
-
# @return [Engines]
|
|
212
|
-
# @raise [ArgumentError] as {.resolve}.
|
|
213
|
-
def initialize(engines: nil, brave_key: nil, exa_key: nil, cache: self.class.cache)
|
|
214
|
-
@providers = self.class.resolve(engines: engines, brave_key: brave_key, exa_key: exa_key).map do |name|
|
|
215
|
-
case name
|
|
216
|
-
when :duckduckgo then DuckDuckGo.new
|
|
217
|
-
when :brave then Brave.new(api_key: brave_key)
|
|
218
|
-
when :exa then Exa.new(api_key: exa_key)
|
|
219
|
-
end
|
|
220
|
-
end
|
|
221
|
-
@cache = cache
|
|
222
|
-
@last_logged_providers = nil
|
|
223
|
-
end
|
|
224
|
-
|
|
225
|
-
# @return [Array<#search, #label>] the selected provider instances, in
|
|
226
|
-
# {ENGINE_NAMES} order (the cascade shuffles them per call).
|
|
227
|
-
attr_reader :providers
|
|
228
|
-
|
|
229
|
-
# Run +query+ through the providers in random order, falling back on each
|
|
230
|
-
# {Unavailable}. The shuffle spreads load so one provider isn't always
|
|
231
|
-
# hit (and exhausted) first. The query is whitespace-normalized first;
|
|
232
|
-
# the winning provider's +Array<Result>+ is rendered to Markdown and
|
|
233
|
-
# cached on disk keyed by the cleaned query (a hit short-circuits the
|
|
234
|
-
# cascade). +max_results+ is *not* in the cache key, so a non-default
|
|
235
|
-
# value may return a previously-cached size.
|
|
236
|
-
#
|
|
237
|
-
# If every provider is unavailable, returns an +"Error: ..."+ string
|
|
238
|
-
# (the tool convention) rather than raising. Any non-{Unavailable}
|
|
239
|
-
# exception (network, parser, bad key) bubbles up.
|
|
240
|
-
#
|
|
241
|
-
# @param query [String] search query
|
|
242
|
-
# @param max_results [Integer] maximum result entries
|
|
243
|
-
# @param cancellable [Pikuri::Agent::Control::Cancellable, nil] handed
|
|
244
|
-
# to each provider so its {RateLimiter} pacing wait is interruptible.
|
|
245
|
-
# A cache hit never reaches a provider, so it never waits at all.
|
|
246
|
-
# @return [String] Markdown result list, or +"Error: ..."+ when all
|
|
247
|
-
# providers are exhausted
|
|
248
|
-
# @raise [ArgumentError] if the query is empty after normalization
|
|
249
|
-
# @raise [Pikuri::Agent::Control::Cancellable::Cancelled] if
|
|
250
|
-
# +cancellable+ trips while a provider is pacing. It is *not* caught
|
|
251
|
-
# as a provider failure — the cascade stops rather than trying the
|
|
252
|
-
# next one, and nothing is cached.
|
|
253
|
-
def search(query, max_results:, cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
254
|
-
cleaned = query.to_s.strip.gsub(/\s+/, ' ')
|
|
255
|
-
raise ArgumentError, 'query is empty' if cleaned.empty?
|
|
256
|
-
|
|
257
|
-
current_providers = providers
|
|
258
|
-
log_providers(current_providers)
|
|
259
|
-
|
|
260
|
-
hit = true
|
|
261
|
-
result = @cache.fetch(cleaned) do
|
|
262
|
-
hit = false
|
|
263
|
-
failures = []
|
|
264
|
-
results = nil
|
|
265
|
-
chosen = nil
|
|
266
|
-
current_providers.shuffle.each do |provider|
|
|
267
|
-
results = provider.search(cleaned, max_results: max_results, cancellable: cancellable)
|
|
268
|
-
chosen = provider
|
|
269
|
-
break
|
|
270
|
-
rescue Unavailable => e
|
|
271
|
-
failures << "#{provider.label} (#{e.message})"
|
|
272
|
-
end
|
|
273
|
-
# Raise so {UrlCache#fetch} does NOT persist the all-unavailable
|
|
274
|
-
# message — else it would block every future search for this query
|
|
275
|
-
# until the TTL expires. The outer rescue turns it into "Error: …".
|
|
276
|
-
chosen or raise Unavailable, "all search providers temporarily unavailable: #{failures.join('; ')}"
|
|
277
|
-
|
|
278
|
-
LOGGER.info do
|
|
279
|
-
"engine=#{chosen.label} query=#{cleaned.inspect} results=#{results.size}"
|
|
280
|
-
end
|
|
281
|
-
render(results)
|
|
282
|
-
end
|
|
283
|
-
LOGGER.info { "cache=hit query=#{cleaned.inspect} bytes=#{result.bytesize}" } if hit
|
|
284
|
-
result
|
|
285
|
-
rescue Unavailable => e
|
|
286
|
-
"Error: #{e.message}"
|
|
287
|
-
end
|
|
288
|
-
|
|
289
|
-
private
|
|
290
|
-
|
|
291
|
-
# Render an +Array<Result>+ into the smolagents-style Markdown the LLM
|
|
292
|
-
# consumes (+"## Search Results"+ header, then +[title](url)\nbody+
|
|
293
|
-
# joined by blank lines). Empty → the +"No results found."+ stub.
|
|
294
|
-
#
|
|
295
|
-
# @param results [Array<Result>] hits from the winning provider
|
|
296
|
-
# @return [String] Markdown result list
|
|
297
|
-
def render(results)
|
|
298
|
-
return "## Search Results\n\nNo results found." if results.empty?
|
|
299
|
-
|
|
300
|
-
"## Search Results\n\n" + results.map { |r| "[#{r.title}](#{r.url})\n#{r.body}" }.join("\n\n")
|
|
301
|
-
end
|
|
302
|
-
|
|
303
|
-
# INFO-log the currently-available providers, but only when the set
|
|
304
|
-
# differs from the last logged — one line per distinct configuration,
|
|
305
|
-
# not per search.
|
|
306
|
-
#
|
|
307
|
-
# @param current [Array<#label>] providers from {#providers}
|
|
308
|
-
# @return [void]
|
|
309
|
-
def log_providers(current)
|
|
310
|
-
return if @last_logged_providers == current
|
|
311
|
-
|
|
312
|
-
@last_logged_providers = current
|
|
313
|
-
names = current.map(&:label).join(', ')
|
|
314
|
-
LOGGER.info("engines available: #{names}")
|
|
315
|
-
end
|
|
316
|
-
end
|
|
317
|
-
end
|
|
318
|
-
end
|
|
319
|
-
end
|
|
@@ -1,195 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'json'
|
|
5
|
-
|
|
6
|
-
module Pikuri
|
|
7
|
-
class Tool
|
|
8
|
-
module Search
|
|
9
|
-
# Performs an Exa search via the official +/search+ endpoint, returning
|
|
10
|
-
# hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
|
|
11
|
-
# parser (.parse) so tests exercise the parser against fixture JSON.
|
|
12
|
-
# {Engines#search} owns the final Markdown rendering.
|
|
13
|
-
#
|
|
14
|
-
# Constructed with its API key (+Exa.new(api_key:)+); {Engines} builds one
|
|
15
|
-
# only when an Exa key was configured, so users who never registered don't
|
|
16
|
-
# spend money, then drives it through the same +#search+/+#label+
|
|
17
|
-
# interface as every provider. pikuri reads no key from the environment
|
|
18
|
-
# (CLAUDE.md "Environment is not a secret store"). Paid; key at
|
|
19
|
-
# https://exa.ai.
|
|
20
|
-
#
|
|
21
|
-
# Requests +type: "auto"+ (Exa picks neural vs keyword) and +contents: {
|
|
22
|
-
# highlights: true }+ so each result carries a neural-ranked snippet
|
|
23
|
-
# populating {Result#body} — the analog to Brave's +description+.
|
|
24
|
-
#
|
|
25
|
-
# == Privacy posture
|
|
26
|
-
#
|
|
27
|
-
# Exa's Privacy Policy uses Query Data to train/fine-tune its models, and
|
|
28
|
-
# ToS §1.2(c) grants a +perpetual and irrevocable+, sub-licensable license
|
|
29
|
-
# over User Input. Business customers under an MSA/DPA get carve-outs; the
|
|
30
|
-
# default pay-as-you-go key pikuri uses does not. Bottom line: Exa mines
|
|
31
|
-
# queries to train competing models. If a query would be sensitive in a
|
|
32
|
-
# training set, don't configure an Exa key — {Engines#providers} leaves
|
|
33
|
-
# Exa out of the cascade unless its key was supplied.
|
|
34
|
-
class Exa
|
|
35
|
-
# @return [String] Search endpoint (POST, JSON body)
|
|
36
|
-
ENDPOINT = 'https://api.exa.ai/search'
|
|
37
|
-
# @return [Integer] default number of results returned, matching
|
|
38
|
-
# {DuckDuckGo::DEFAULT_MAX_RESULTS}
|
|
39
|
-
DEFAULT_MAX_RESULTS = 10
|
|
40
|
-
# @return [RateLimiter] Exa is paid and doesn't aggressively throttle,
|
|
41
|
-
# so no minimum interval; the 5-minute cooldown still applies on
|
|
42
|
-
# {Engines::Unavailable} so the budget isn't burned on doomed retries.
|
|
43
|
-
LIMITER = RateLimiter.new(min_interval: 0.0, cooldown: 300.0)
|
|
44
|
-
|
|
45
|
-
# @param api_key [String] Exa API key. Required and non-blank:
|
|
46
|
-
# pikuri reads no key from the environment — the host supplies it
|
|
47
|
-
# ({Engines} only constructs an Exa when a key was configured).
|
|
48
|
-
# @raise [ArgumentError] if +api_key+ is blank
|
|
49
|
-
def initialize(api_key:)
|
|
50
|
-
raise ArgumentError, 'Exa Search API key is blank' if api_key.to_s.strip.empty?
|
|
51
|
-
|
|
52
|
-
@api_key = api_key
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
# @return [String] short provider label for {Engines} logging /
|
|
56
|
-
# fallback messages.
|
|
57
|
-
def label
|
|
58
|
-
'Exa'
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# Fetch results for +query+ as an +Array<Result>+. Circuit-broken for
|
|
62
|
-
# 5 minutes on rate-limit/unavailable (see {LIMITER}). The caller
|
|
63
|
-
# ({Engines#search}) normalizes the query and wraps this in a cache.
|
|
64
|
-
#
|
|
65
|
-
# @param query [String] search query (already normalized)
|
|
66
|
-
# @param max_results [Integer] max entries; Exa's +numResults+
|
|
67
|
-
# @param cancellable [Pikuri::Agent::Control::Cancellable] makes
|
|
68
|
-
# {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
|
|
69
|
-
# @return [Array<Result>] hits, possibly empty when Exa matched nothing
|
|
70
|
-
# @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
|
|
71
|
-
# back from these), or immediately if {LIMITER} is in cooldown. Other
|
|
72
|
-
# non-2xx (401/403 bad key) bubble up as +RuntimeError+.
|
|
73
|
-
# @raise [RuntimeError] for non-rate-limit HTTP failures, or an
|
|
74
|
-
# unrecognized empty-results shape.
|
|
75
|
-
def search(query, max_results: DEFAULT_MAX_RESULTS,
|
|
76
|
-
cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
77
|
-
LIMITER.call(cancellable: cancellable) do
|
|
78
|
-
response = Faraday.post(ENDPOINT) do |req|
|
|
79
|
-
req.headers['x-api-key'] = @api_key
|
|
80
|
-
req.headers['Content-Type'] = 'application/json'
|
|
81
|
-
req.headers['Accept'] = 'application/json'
|
|
82
|
-
req.body = JSON.dump(
|
|
83
|
-
query: query,
|
|
84
|
-
type: 'auto',
|
|
85
|
-
numResults: max_results,
|
|
86
|
-
contents: { highlights: true }
|
|
87
|
-
)
|
|
88
|
-
end
|
|
89
|
-
unless response.success?
|
|
90
|
-
if response.status == 429 || response.status >= 500
|
|
91
|
-
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
raise "Exa Search request failed: #{response.status} #{response.body}"
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
self.class.parse(response.body, max_results: max_results)
|
|
98
|
-
end
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
# Parse an Exa JSON response into {Result} rows, +body+ the first
|
|
102
|
-
# non-empty +highlights+ snippet. A genuine "no results" payload
|
|
103
|
-
# (+requestId+ present, empty +results+) returns +[]+ so {Engines.search}
|
|
104
|
-
# renders its no-results stub; any other zero-result shape raises.
|
|
105
|
-
#
|
|
106
|
-
# @param json [String] response body from {ENDPOINT}
|
|
107
|
-
# @param max_results [Integer] max entries
|
|
108
|
-
# @return [Array<Result>] hits, possibly empty on a recognized
|
|
109
|
-
# empty-results payload
|
|
110
|
-
# @raise [RuntimeError] on an unrecognized zero-result response
|
|
111
|
-
def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
|
|
112
|
-
data = JSON.parse(json)
|
|
113
|
-
results = Array(data['results']).take(max_results).filter_map do |r|
|
|
114
|
-
href = r['url'].to_s
|
|
115
|
-
next nil if href.empty?
|
|
116
|
-
|
|
117
|
-
Result.new(
|
|
118
|
-
url: href,
|
|
119
|
-
title: clean(r['title']) || href,
|
|
120
|
-
body: first_highlight(r['highlights'])
|
|
121
|
-
)
|
|
122
|
-
end
|
|
123
|
-
|
|
124
|
-
if results.empty?
|
|
125
|
-
return [] if genuine_no_results?(data)
|
|
126
|
-
|
|
127
|
-
raise diagnose_empty(data, json)
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
results
|
|
131
|
-
end
|
|
132
|
-
|
|
133
|
-
# Collapse whitespace and strip; +nil+ for nil/empty input so the caller
|
|
134
|
-
# can fall back (e.g. to the URL when a result has no title).
|
|
135
|
-
#
|
|
136
|
-
# @param text [String, nil]
|
|
137
|
-
# @return [String, nil] cleaned text, or +nil+ if blank
|
|
138
|
-
def self.clean(text)
|
|
139
|
-
return nil if text.nil?
|
|
140
|
-
|
|
141
|
-
cleaned = text.to_s.gsub(/\s+/, ' ').strip
|
|
142
|
-
cleaned.empty? ? nil : cleaned
|
|
143
|
-
end
|
|
144
|
-
private_class_method :clean
|
|
145
|
-
|
|
146
|
-
# First non-empty +highlights+ entry, cleaned. Exa sorts highlights by
|
|
147
|
-
# relevance; the top one populates +body+.
|
|
148
|
-
#
|
|
149
|
-
# @param highlights [Array<String>, nil] +highlights+ field
|
|
150
|
-
# @return [String] cleaned snippet, or empty string if none usable
|
|
151
|
-
def self.first_highlight(highlights)
|
|
152
|
-
return '' unless highlights.is_a?(Array)
|
|
153
|
-
|
|
154
|
-
highlights.each do |h|
|
|
155
|
-
cleaned = clean(h)
|
|
156
|
-
return cleaned if cleaned
|
|
157
|
-
end
|
|
158
|
-
''
|
|
159
|
-
end
|
|
160
|
-
private_class_method :first_highlight
|
|
161
|
-
|
|
162
|
-
# True when a zero-+results+ response looks like Exa's own "ran, matched
|
|
163
|
-
# nothing" payload rather than an error — marked by +requestId+, which
|
|
164
|
-
# Exa always sets on success.
|
|
165
|
-
#
|
|
166
|
-
# @param data [Hash, Object] parsed response
|
|
167
|
-
# @return [Boolean]
|
|
168
|
-
def self.genuine_no_results?(data)
|
|
169
|
-
return false unless data.is_a?(Hash)
|
|
170
|
-
return false unless data.key?('requestId')
|
|
171
|
-
|
|
172
|
-
Array(data['results']).empty?
|
|
173
|
-
end
|
|
174
|
-
private_class_method :genuine_no_results?
|
|
175
|
-
|
|
176
|
-
# Build an error message for a zero-result response: quote Exa's
|
|
177
|
-
# +error+/+message+/+detail+ if present, else a truncated raw body.
|
|
178
|
-
#
|
|
179
|
-
# @param data [Hash, Object] parsed response
|
|
180
|
-
# @param raw [String] raw response body
|
|
181
|
-
# @return [String] diagnostic to feed to +raise+
|
|
182
|
-
def self.diagnose_empty(data, raw)
|
|
183
|
-
if data.is_a?(Hash) && (msg = data['error'] || data['message'] || data['detail'])
|
|
184
|
-
return "Exa Search returned an error: #{msg}"
|
|
185
|
-
end
|
|
186
|
-
|
|
187
|
-
snippet = raw.to_s[0, 800]
|
|
188
|
-
snippet += '…' if raw.to_s.length > 800
|
|
189
|
-
"Exa Search returned no results. Body: #{snippet}"
|
|
190
|
-
end
|
|
191
|
-
private_class_method :diagnose_empty
|
|
192
|
-
end
|
|
193
|
-
end
|
|
194
|
-
end
|
|
195
|
-
end
|
|
@@ -1,115 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
# Loaded after {Tool} itself is defined; the +class Tool+ reopening below
|
|
5
|
-
# assumes that order.
|
|
6
|
-
class Tool
|
|
7
|
-
module Search
|
|
8
|
-
# Thread-safe pacing + circuit-breaker wrapper for a search provider.
|
|
9
|
-
#
|
|
10
|
-
# +#call { ... }+ enforces a minimum interval between block invocations
|
|
11
|
-
# (sleeping if the previous was too recent) and watches for
|
|
12
|
-
# {Engines::Unavailable}: on one, a cooldown deadline is recorded and
|
|
13
|
-
# further calls within the window raise {Engines::Unavailable} immediately
|
|
14
|
-
# without running the block — so a rate-limited or bot-blocked provider
|
|
15
|
-
# isn't hammered with retries.
|
|
16
|
-
#
|
|
17
|
-
# Uses wall-clock {Time.now}, not monotonic: the intervals (1s–5min) are
|
|
18
|
-
# well above any NTP step, and {Time.now} keeps tests fakeable with
|
|
19
|
-
# Timecop.
|
|
20
|
-
#
|
|
21
|
-
# == Sharing
|
|
22
|
-
#
|
|
23
|
-
# +P_shared_locked+, and sharing is *mandatory*: a limiter must be scoped
|
|
24
|
-
# to the resource it protects, which here is one egress IP and one API
|
|
25
|
-
# key's quota — never one agent. Hence each provider holds its limiter in
|
|
26
|
-
# a class-level constant, so every agent in the VM queues behind the same
|
|
27
|
-
# one. Per-agent limiters would let ten agents fire ten simultaneous
|
|
28
|
-
# requests from one IP and earn exactly the ban this class exists to
|
|
29
|
-
# avoid. (Contrast {Pikuri::Agent::Listener::RateLimited}, which paces one
|
|
30
|
-
# renderer's repaints and is therefore per listener chain.)
|
|
31
|
-
#
|
|
32
|
-
# Two consequences of that scope, both correct and both surprising:
|
|
33
|
-
#
|
|
34
|
-
# * **It is a global queue, not just a pacer.** The mutex is held across
|
|
35
|
-
# the block — the HTTP call included — so ten agents searching at once
|
|
36
|
-
# serialize, with no timeout and no queue position. +cancellable:+
|
|
37
|
-
# covers the *pacing wait*, which is what the lock holder is doing; a
|
|
38
|
-
# caller still parked in +Mutex#synchronize+ waiting to acquire cannot
|
|
39
|
-
# observe its own cancel. It does still benefit, because every agent
|
|
40
|
-
# that cancels releases early and drains the queue faster. Making the
|
|
41
|
-
# acquisition itself interruptible means polling +Mutex#try_lock+ with
|
|
42
|
-
# explicit +lock+/+unlock+ — deliberately not built for a
|
|
43
|
-
# three-concurrent-agent case nothing has yet.
|
|
44
|
-
# * **The circuit breaker is global too.** One agent tripping the
|
|
45
|
-
# provider's soft-block puts every other agent into the same cooldown.
|
|
46
|
-
# Right, because the block is IP-wide and the others were blocked
|
|
47
|
-
# anyway — but from the host's side nine agents lose a provider for
|
|
48
|
-
# something the tenth did.
|
|
49
|
-
#
|
|
50
|
-
# Thread-safe.
|
|
51
|
-
class RateLimiter
|
|
52
|
-
# @param min_interval [Float] minimum seconds between consecutive
|
|
53
|
-
# block invocations. {#call} sleeps if a previous call was more
|
|
54
|
-
# recent.
|
|
55
|
-
# @param cooldown [Float] seconds to refuse calls after the block
|
|
56
|
-
# raises {Engines::Unavailable}. Calls within this window raise
|
|
57
|
-
# {Engines::Unavailable} immediately without invoking the block.
|
|
58
|
-
def initialize(min_interval:, cooldown:)
|
|
59
|
-
@min_interval = min_interval
|
|
60
|
-
@cooldown = cooldown
|
|
61
|
-
@mutex = Mutex.new
|
|
62
|
-
@last_call_at = nil
|
|
63
|
-
@cooldown_until = nil
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
# Run the block subject to the throttle and cooldown rules, with the
|
|
67
|
-
# mutex held (only one block runs at a time per limiter). If it raises
|
|
68
|
-
# {Engines::Unavailable}, cooldown is armed and the exception re-raised;
|
|
69
|
-
# any other exception bubbles up without arming cooldown.
|
|
70
|
-
#
|
|
71
|
-
# The pacing wait happens *through* +cancellable+, so the calling agent
|
|
72
|
-
# can abandon it on a user cancel. It is a per-call argument rather than
|
|
73
|
-
# constructor state because one limiter serves every agent in the VM
|
|
74
|
-
# (see +== Sharing+) while a token belongs to one of them — and it is
|
|
75
|
-
# always a real token, defaulting to
|
|
76
|
-
# {Pikuri::Agent::Control::Cancellable::NEVER} rather than +nil+, so
|
|
77
|
-
# there is one wait path instead of two.
|
|
78
|
-
#
|
|
79
|
-
# @param cancellable [Pikuri::Agent::Control::Cancellable] waited
|
|
80
|
-
# through instead of +Kernel#sleep+.
|
|
81
|
-
# @yieldreturn [Object] passed through
|
|
82
|
-
# @return [Object] whatever the block returned
|
|
83
|
-
# @raise [Engines::Unavailable] re-raised from the block, or raised
|
|
84
|
-
# directly when the limiter is in cooldown
|
|
85
|
-
# @raise [Pikuri::Agent::Control::Cancellable::Cancelled] if
|
|
86
|
-
# +cancellable+ trips before or during the pacing wait; the block does
|
|
87
|
-
# not run and the mutex is released, so the agents queued behind this
|
|
88
|
-
# one advance immediately.
|
|
89
|
-
def call(cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
90
|
-
@mutex.synchronize do
|
|
91
|
-
now = Time.now
|
|
92
|
-
if @cooldown_until && now < @cooldown_until
|
|
93
|
-
remaining = (@cooldown_until - now).ceil
|
|
94
|
-
raise Engines::Unavailable, "rate-limiter cooldown active for another #{remaining}s"
|
|
95
|
-
end
|
|
96
|
-
|
|
97
|
-
if @last_call_at
|
|
98
|
-
elapsed = now - @last_call_at
|
|
99
|
-
cancellable.sleep(@min_interval - elapsed) if elapsed < @min_interval
|
|
100
|
-
end
|
|
101
|
-
@last_call_at = Time.now
|
|
102
|
-
|
|
103
|
-
begin
|
|
104
|
-
yield
|
|
105
|
-
rescue Engines::Unavailable
|
|
106
|
-
@cooldown_until = Time.now + @cooldown
|
|
107
|
-
raise
|
|
108
|
-
end
|
|
109
|
-
end
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
end
|
|
113
|
-
end
|
|
114
|
-
end
|
|
115
|
-
end
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
module Search
|
|
6
|
-
# A single search hit from a {Tool::Search} provider ({DuckDuckGo},
|
|
7
|
-
# {Brave}, {Exa}). Providers return +Array<Result>+; {Engines.search}
|
|
8
|
-
# concatenates them into the smolagents-style Markdown the LLM sees.
|
|
9
|
-
# Structure split from rendering keeps the providers interchangeable —
|
|
10
|
-
# they need only agree on these three fields (provider extras like scores
|
|
11
|
-
# or dates are discarded, since no caller uses them).
|
|
12
|
-
#
|
|
13
|
-
# @!attribute [r] url
|
|
14
|
-
# @return [String] absolute URL of the hit
|
|
15
|
-
# @!attribute [r] title
|
|
16
|
-
# @return [String] plain-text title, provider highlight markup
|
|
17
|
-
# ({Brave}'s +<strong>+, {DuckDuckGo}'s +<b>+) already stripped
|
|
18
|
-
# @!attribute [r] body
|
|
19
|
-
# @return [String] plain-text snippet, possibly empty (an {Exa}
|
|
20
|
-
# navigational result with no highlights)
|
|
21
|
-
Result = Data.define(:url, :title, :body)
|
|
22
|
-
end
|
|
23
|
-
end
|
|
24
|
-
end
|