pikuri-core 0.0.7 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/pikuri/agent/chat_transport.rb +73 -93
- data/lib/pikuri/agent/configurator.rb +46 -106
- data/lib/pikuri/agent/context_window_detector.rb +44 -85
- data/lib/pikuri/agent/control/cancellable.rb +87 -66
- data/lib/pikuri/agent/control/interloper.rb +127 -105
- data/lib/pikuri/agent/control/step_limit.rb +25 -41
- data/lib/pikuri/agent/control.rb +14 -34
- data/lib/pikuri/agent/event.rb +123 -188
- data/lib/pikuri/agent/extension.rb +118 -94
- data/lib/pikuri/agent/extension_context.rb +50 -77
- data/lib/pikuri/agent/history.rb +653 -0
- data/lib/pikuri/agent/listener/rate_limited.rb +40 -66
- data/lib/pikuri/agent/listener/terminal.rb +143 -117
- data/lib/pikuri/agent/listener/token_log.rb +101 -140
- data/lib/pikuri/agent/listener.rb +23 -43
- data/lib/pikuri/agent/listener_list.rb +26 -47
- data/lib/pikuri/agent/synthesizer.rb +45 -87
- data/lib/pikuri/agent.rb +816 -474
- data/lib/pikuri/bundler_env.rb +68 -0
- data/lib/pikuri/extractor/html.rb +63 -110
- data/lib/pikuri/extractor/passthrough.rb +20 -30
- data/lib/pikuri/extractor.rb +93 -154
- data/lib/pikuri/file_type.rb +63 -135
- data/lib/pikuri/finalizers.rb +32 -47
- data/lib/pikuri/paths.rb +104 -13
- data/lib/pikuri/ruby_llm_patches.rb +106 -0
- data/lib/pikuri/sanitizer.rb +45 -67
- data/lib/pikuri/subprocess.rb +75 -119
- data/lib/pikuri/testing.rb +296 -0
- data/lib/pikuri/tool/calculator.rb +56 -66
- data/lib/pikuri/tool/execute_context.rb +42 -0
- data/lib/pikuri/tool/fetch.rb +51 -77
- data/lib/pikuri/tool/parameters.rb +21 -29
- data/lib/pikuri/tool/scraper.rb +55 -97
- data/lib/pikuri/tool/search/brave.rb +52 -80
- data/lib/pikuri/tool/search/duckduckgo.rb +59 -91
- data/lib/pikuri/tool/search/engines.rb +230 -97
- data/lib/pikuri/tool/search/exa.rb +56 -90
- data/lib/pikuri/tool/search/rate_limiter.rb +61 -38
- data/lib/pikuri/tool/search/result.rb +10 -15
- data/lib/pikuri/tool/trifecta_legs.rb +217 -0
- data/lib/pikuri/tool/web_scrape.rb +38 -54
- data/lib/pikuri/tool/web_search.rb +100 -24
- data/lib/pikuri/tool.rb +140 -65
- data/lib/pikuri/trifecta/contribution.rb +43 -0
- data/lib/pikuri/trifecta/node.rb +47 -0
- data/lib/pikuri/trifecta/report.rb +230 -0
- data/lib/pikuri/trifecta.rb +127 -0
- data/lib/pikuri/url_cache.rb +33 -49
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +72 -88
- data/prompts/agent-loop.txt +5 -0
- data/prompts/pikuri-chat.txt +3 -12
- metadata +14 -3
|
@@ -2,118 +2,255 @@
|
|
|
2
2
|
|
|
3
3
|
module Pikuri
|
|
4
4
|
class Tool
|
|
5
|
-
# Namespace for the web-search stack
|
|
6
|
-
#
|
|
7
|
-
#
|
|
8
|
-
# its on-disk cache, and the shared {RateLimiter} a provider can wire
|
|
9
|
-
# in to back off when a quota header says so.
|
|
5
|
+
# Namespace for the web-search stack behind {Tool::WebSearch}: per-provider
|
|
6
|
+
# modules ({DuckDuckGo}, {Brave}, {Exa}), the {Result} they return, the
|
|
7
|
+
# {Engines} cascade with its on-disk cache, and the shared {RateLimiter}.
|
|
10
8
|
module Search
|
|
11
|
-
# Search-orchestration object: the cascade across
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
9
|
+
# Search-orchestration object: the cascade across selected providers, the
|
|
10
|
+
# result cache, and the {Unavailable} marker the cascade falls back on.
|
|
11
|
+
# {Tool::WebSearch.build} constructs one and wires its {#search} into a
|
|
12
|
+
# {Tool}. Each provider raises {Unavailable} to hand off to the next.
|
|
13
|
+
#
|
|
14
|
+
# Engines.new(engines: [:brave], brave_key: 'BSA...') # Brave alone
|
|
15
|
+
# Engines.new(brave_key: 'BSA...') # DuckDuckGo + Brave
|
|
16
|
+
# Engines.new # DuckDuckGo alone
|
|
17
|
+
#
|
|
18
|
+
# {.resolve} owns which engines that yields and which selections are
|
|
19
|
+
# refused; {.config_from_h} maps a config file's +search:+ block onto
|
|
20
|
+
# these keywords.
|
|
18
21
|
#
|
|
19
22
|
# == Provider keys are constructor config, not environment
|
|
20
23
|
#
|
|
21
|
-
# Brave and Exa are paid and need
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
# configured. The host sources those keys however it likes (the
|
|
27
|
-
# bundled +bin/+ examples load a JSON config file by convention); see
|
|
28
|
-
# CLAUDE.md "Environment is not a secret store".
|
|
24
|
+
# Brave and Exa are paid and need a key; DuckDuckGo needs none. An
|
|
25
|
+
# {Engines} is built with the keys it should use (+brave_key:+ /
|
|
26
|
+
# +exa_key:+, both optional) — pikuri reads no key from the environment,
|
|
27
|
+
# so a host supplies them. See CLAUDE.md "Environment is not a secret
|
|
28
|
+
# store".
|
|
29
29
|
class Engines
|
|
30
|
-
# Subsystem logger;
|
|
31
|
-
# (e.g. +PIKURI_LOG_ENGINES=debug+) or the global +PIKURI_LOG+.
|
|
30
|
+
# Subsystem logger; level via +PIKURI_LOG_ENGINES+ or +PIKURI_LOG+.
|
|
32
31
|
#
|
|
33
32
|
# @return [Logger]
|
|
34
33
|
LOGGER = Pikuri.logger_for('Engines')
|
|
35
34
|
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
|
|
35
|
+
# Every engine name accepted in an +engines:+ selection, in the order
|
|
36
|
+
# {#providers} reports them. Presentation only — {#search} shuffles.
|
|
37
|
+
#
|
|
38
|
+
# @return [Array<Symbol>]
|
|
39
|
+
ENGINE_NAMES = %i[duckduckgo brave exa].freeze
|
|
40
|
+
|
|
41
|
+
# Engine name → the constructor keyword carrying its API key. Naming
|
|
42
|
+
# one of these without its key is a refusal ({.resolve}); DuckDuckGo is
|
|
43
|
+
# absent because it needs none.
|
|
44
|
+
#
|
|
45
|
+
# @return [Hash{Symbol => Symbol}]
|
|
46
|
+
KEYED_ENGINES = { brave: :brave_key, exa: :exa_key }.freeze
|
|
47
|
+
|
|
48
|
+
# The fields {.config_from_h} accepts from a config file's +search:+
|
|
49
|
+
# block, matching this class's keywords.
|
|
50
|
+
#
|
|
51
|
+
# @return [Array<Symbol>]
|
|
52
|
+
CONFIG_FIELDS = %i[engines brave_key exa_key].freeze
|
|
53
|
+
|
|
54
|
+
# Raised by a provider that is temporarily unavailable (rate-limited,
|
|
55
|
+
# bot-blocked, quota-exhausted — "try again later", not "your request is
|
|
56
|
+
# wrong"). The {#search} cascade catches it and tries the next provider;
|
|
57
|
+
# anything else bubbles up so genuine bugs stay visible.
|
|
41
58
|
class Unavailable < StandardError; end
|
|
42
59
|
|
|
43
|
-
# Process-shared on-disk cache backing {#search}
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
# injects a different store for tests. Exposed as a method so specs
|
|
47
|
-
# can swap it for {UrlCache::NULL} without touching the instance.
|
|
60
|
+
# Process-shared on-disk cache backing {#search}, at class level so every
|
|
61
|
+
# engine dedupes into one directory; the constructor's +cache:+ injects a
|
|
62
|
+
# different store for tests.
|
|
48
63
|
#
|
|
49
64
|
# @return [UrlCache, #fetch]
|
|
50
65
|
CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_search")
|
|
51
|
-
# Accessor for {CACHE},
|
|
52
|
-
#
|
|
66
|
+
# Accessor for {CACHE}, the constructor's +cache:+ default; specs swap in
|
|
67
|
+
# {UrlCache::NULL}.
|
|
53
68
|
#
|
|
54
69
|
# @return [UrlCache, #fetch]
|
|
55
70
|
def self.cache
|
|
56
71
|
CACHE
|
|
57
72
|
end
|
|
58
73
|
|
|
59
|
-
#
|
|
60
|
-
#
|
|
61
|
-
#
|
|
62
|
-
#
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
#
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
#
|
|
69
|
-
#
|
|
70
|
-
#
|
|
74
|
+
# Resolve an +engines:+ selection into the engine names to build,
|
|
75
|
+
# refusing a selection that cannot work:
|
|
76
|
+
#
|
|
77
|
+
# resolve(engines: nil, brave_key: 'B') # => [:duckduckgo, :brave]
|
|
78
|
+
# resolve(engines: nil) # => [:duckduckgo]
|
|
79
|
+
# resolve(engines: %w[brave], brave_key: 'B') # => [:brave]
|
|
80
|
+
# resolve(engines: [:exa], brave_key: 'B') # raises — exa has no key
|
|
81
|
+
#
|
|
82
|
+
# +nil+ means "whatever the keys say": DuckDuckGo plus every engine
|
|
83
|
+
# whose key is present, which is what a host that never heard of
|
|
84
|
+
# +engines:+ gets. An explicit list is a **set** — it is normalized to
|
|
85
|
+
# {ENGINE_NAMES} order and {#search} shuffles it per call anyway, so
|
|
86
|
+
# +[brave, duckduckgo]+ expresses no preference for Brave. Names are
|
|
87
|
+
# matched case-insensitively.
|
|
88
|
+
#
|
|
89
|
+
# Supplying a key without listing its engine is fine — parking a key
|
|
90
|
+
# you are not using today is not a mistake.
|
|
91
|
+
#
|
|
92
|
+
# @param engines [Array<String, Symbol>, nil] engine names, or +nil+ to
|
|
93
|
+
# derive the set from the keys.
|
|
94
|
+
# @param brave_key [String, nil] Brave token; blank counts as absent.
|
|
95
|
+
# @param exa_key [String, nil] Exa key; blank counts as absent.
|
|
96
|
+
# @return [Array<Symbol>] names drawn from {ENGINE_NAMES}, in that order.
|
|
97
|
+
# @raise [ArgumentError] if +engines+ is not an Array, is empty, holds
|
|
98
|
+
# an unknown or repeated name, or names a {KEYED_ENGINES} engine
|
|
99
|
+
# whose key is blank.
|
|
100
|
+
def self.resolve(engines:, brave_key: nil, exa_key: nil)
|
|
101
|
+
keys = { brave: brave_key, exa: exa_key }
|
|
102
|
+
keyless = ->(name) { KEYED_ENGINES.key?(name) && keys[name].to_s.strip.empty? }
|
|
103
|
+
return ENGINE_NAMES.reject(&keyless) if engines.nil?
|
|
104
|
+
|
|
105
|
+
unless engines.is_a?(Array) && !engines.empty?
|
|
106
|
+
# A list of engine names is safe to echo; anything else might be a
|
|
107
|
+
# key the writer put in the wrong field, so show only its class.
|
|
108
|
+
got = engines.is_a?(Array) ? engines.inspect : engines.class
|
|
109
|
+
raise ArgumentError, 'search engines: expected a non-empty list of engine names ' \
|
|
110
|
+
"(known: #{ENGINE_NAMES.join(', ')}), got #{got}"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
names = engines.map { |name| name.to_s.strip.downcase.to_sym }
|
|
114
|
+
refuse_unknown(names)
|
|
115
|
+
refuse_repeats(names)
|
|
116
|
+
refuse_keyless(names.select(&keyless))
|
|
117
|
+
ENGINE_NAMES & names
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# @param names [Array<Symbol>] the requested engine names.
|
|
121
|
+
# @return [void]
|
|
122
|
+
# @raise [ArgumentError] if any name is not in {ENGINE_NAMES}.
|
|
123
|
+
def self.refuse_unknown(names)
|
|
124
|
+
unknown = names - ENGINE_NAMES
|
|
125
|
+
return if unknown.empty?
|
|
126
|
+
|
|
127
|
+
raise ArgumentError, "search engines: unknown engine(s) #{unknown.join(', ')} — " \
|
|
128
|
+
"known engines are #{ENGINE_NAMES.join(', ')}"
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# @param names [Array<Symbol>] the requested engine names.
|
|
132
|
+
# @return [void]
|
|
133
|
+
# @raise [ArgumentError] if any name appears twice (which would put one
|
|
134
|
+
# provider into the cascade twice, weighting the shuffle).
|
|
135
|
+
def self.refuse_repeats(names)
|
|
136
|
+
repeated = names.tally.select { |_, count| count > 1 }.keys
|
|
137
|
+
return if repeated.empty?
|
|
138
|
+
|
|
139
|
+
raise ArgumentError, "search engines: #{repeated.join(', ')} listed more than once"
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# @param keyless [Array<Symbol>] requested engines with no usable key.
|
|
143
|
+
# @return [void]
|
|
144
|
+
# @raise [ArgumentError] unless +keyless+ is empty.
|
|
145
|
+
def self.refuse_keyless(keyless)
|
|
146
|
+
return if keyless.empty?
|
|
147
|
+
|
|
148
|
+
needed = keyless.map { |name| "#{name} needs #{KEYED_ENGINES[name]}" }.join(', ')
|
|
149
|
+
raise ArgumentError, "search engines: #{needed} — supply the key or drop the engine from the list"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# Map a config file's +search:+ block onto this class's keywords, so a
|
|
153
|
+
# host hands over what it read:
|
|
154
|
+
#
|
|
155
|
+
# # ~/.pikuri-examples-config.yaml
|
|
156
|
+
# search:
|
|
157
|
+
# brave_key: BSA...
|
|
158
|
+
# exa_key: xyz
|
|
159
|
+
# engines: [brave, duckduckgo]
|
|
160
|
+
#
|
|
161
|
+
# Tool::WebSearch.from_h(YAML.safe_load_file(path)['search'])
|
|
162
|
+
#
|
|
163
|
+
# Nothing here knows what YAML is; a host reading TOML, JSON or its own
|
|
164
|
+
# settings object uses the same method. Every field is validated rather
|
|
165
|
+
# than coerced, because the person writing a config file cannot
|
|
166
|
+
# type-check it — and a mistyped engine name that silently meant "no
|
|
167
|
+
# Brave" would show up only as worse search results.
|
|
168
|
+
#
|
|
169
|
+
# @param config [Hash{String, Symbol => Object}, nil] the +search:+
|
|
170
|
+
# block, with String or Symbol keys. +nil+ — what a host reading an
|
|
171
|
+
# absent config key gets — yields an empty Hash, so no caller needs a
|
|
172
|
+
# +|| {}+.
|
|
173
|
+
# @return [Hash{Symbol => Object}] +engines:+ / +brave_key:+ /
|
|
174
|
+
# +exa_key:+ keywords for {#initialize} or {Tool::WebSearch.build}.
|
|
175
|
+
# @raise [ArgumentError] on a non-Hash, an unknown field, or a key field
|
|
176
|
+
# that is not a String.
|
|
177
|
+
def self.config_from_h(config)
|
|
178
|
+
return {} if config.nil?
|
|
179
|
+
|
|
180
|
+
unless config.is_a?(Hash)
|
|
181
|
+
raise ArgumentError, 'search config: expected a Hash of ' \
|
|
182
|
+
"#{CONFIG_FIELDS.join(', ')}, got #{config.class}"
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
fields = config.transform_keys(&:to_sym)
|
|
186
|
+
unknown = fields.keys - CONFIG_FIELDS
|
|
187
|
+
unless unknown.empty?
|
|
188
|
+
raise ArgumentError, "search config: unknown field(s) #{unknown.join(', ')} — " \
|
|
189
|
+
"known fields are #{CONFIG_FIELDS.join(', ')}"
|
|
190
|
+
end
|
|
191
|
+
KEYED_ENGINES.each_value do |field|
|
|
192
|
+
value = fields[field]
|
|
193
|
+
next if value.nil? || value.is_a?(String)
|
|
194
|
+
|
|
195
|
+
# The class, never the value: an error message about a key field is
|
|
196
|
+
# one paste away from a bug report.
|
|
197
|
+
raise ArgumentError, "search config: #{field} must be a String, got #{value.class}"
|
|
198
|
+
end
|
|
199
|
+
fields
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
# Builds the provider cascade once, from the engines {.resolve} selects.
|
|
203
|
+
# From here each provider is just an object answering
|
|
204
|
+
# +#search+/+#label+.
|
|
205
|
+
#
|
|
206
|
+
# @param engines [Array<String, Symbol>, nil] engine names; +nil+
|
|
207
|
+
# derives the set from the keys. See {.resolve}.
|
|
208
|
+
# @param brave_key [String, nil] Brave token.
|
|
209
|
+
# @param exa_key [String, nil] Exa key.
|
|
210
|
+
# @param cache [UrlCache, #fetch] result store; defaults to {.cache}.
|
|
71
211
|
# @return [Engines]
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
@providers
|
|
75
|
-
|
|
212
|
+
# @raise [ArgumentError] as {.resolve}.
|
|
213
|
+
def initialize(engines: nil, brave_key: nil, exa_key: nil, cache: self.class.cache)
|
|
214
|
+
@providers = self.class.resolve(engines: engines, brave_key: brave_key, exa_key: exa_key).map do |name|
|
|
215
|
+
case name
|
|
216
|
+
when :duckduckgo then DuckDuckGo.new
|
|
217
|
+
when :brave then Brave.new(api_key: brave_key)
|
|
218
|
+
when :exa then Exa.new(api_key: exa_key)
|
|
219
|
+
end
|
|
220
|
+
end
|
|
76
221
|
@cache = cache
|
|
77
222
|
@last_logged_providers = nil
|
|
78
223
|
end
|
|
79
224
|
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
# @return [Array<#search, #label>] configured provider instances
|
|
225
|
+
# @return [Array<#search, #label>] the selected provider instances, in
|
|
226
|
+
# {ENGINE_NAMES} order (the cascade shuffles them per call).
|
|
84
227
|
attr_reader :providers
|
|
85
228
|
|
|
86
|
-
# Run +query+ through the
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
#
|
|
93
|
-
#
|
|
94
|
-
#
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
# +cache:+, keyed by the cleaned query. A cache hit short-circuits the
|
|
98
|
-
# cascade entirely (and benefits whichever provider would have
|
|
99
|
-
# answered next time too — once a query is cached, the cooldown state
|
|
100
|
-
# of the original answering provider no longer matters). +max_results+
|
|
101
|
-
# is not part of the cache key, so callers passing a non-default value
|
|
102
|
-
# may get a result rendered with the previously-cached size.
|
|
103
|
-
#
|
|
104
|
-
# If every provider reports temporary unavailability, returns an
|
|
105
|
-
# +"Error: ..."+ string instead of raising — same convention as
|
|
106
|
-
# {Tool::Calculator.calculate}, so the agent loop can feed the failure
|
|
107
|
-
# back to the model as the next observation. Any non-{Unavailable}
|
|
108
|
-
# exception (network error, parser failure, malformed response, bad
|
|
109
|
-
# API key) bubbles up to the caller.
|
|
229
|
+
# Run +query+ through the providers in random order, falling back on each
|
|
230
|
+
# {Unavailable}. The shuffle spreads load so one provider isn't always
|
|
231
|
+
# hit (and exhausted) first. The query is whitespace-normalized first;
|
|
232
|
+
# the winning provider's +Array<Result>+ is rendered to Markdown and
|
|
233
|
+
# cached on disk keyed by the cleaned query (a hit short-circuits the
|
|
234
|
+
# cascade). +max_results+ is *not* in the cache key, so a non-default
|
|
235
|
+
# value may return a previously-cached size.
|
|
236
|
+
#
|
|
237
|
+
# If every provider is unavailable, returns an +"Error: ..."+ string
|
|
238
|
+
# (the tool convention) rather than raising. Any non-{Unavailable}
|
|
239
|
+
# exception (network, parser, bad key) bubbles up.
|
|
110
240
|
#
|
|
111
241
|
# @param query [String] search query
|
|
112
|
-
# @param max_results [Integer] maximum
|
|
113
|
-
# @
|
|
114
|
-
#
|
|
242
|
+
# @param max_results [Integer] maximum result entries
|
|
243
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable, nil] handed
|
|
244
|
+
# to each provider so its {RateLimiter} pacing wait is interruptible.
|
|
245
|
+
# A cache hit never reaches a provider, so it never waits at all.
|
|
246
|
+
# @return [String] Markdown result list, or +"Error: ..."+ when all
|
|
247
|
+
# providers are exhausted
|
|
115
248
|
# @raise [ArgumentError] if the query is empty after normalization
|
|
116
|
-
|
|
249
|
+
# @raise [Pikuri::Agent::Control::Cancellable::Cancelled] if
|
|
250
|
+
# +cancellable+ trips while a provider is pacing. It is *not* caught
|
|
251
|
+
# as a provider failure — the cascade stops rather than trying the
|
|
252
|
+
# next one, and nothing is cached.
|
|
253
|
+
def search(query, max_results:, cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
117
254
|
cleaned = query.to_s.strip.gsub(/\s+/, ' ')
|
|
118
255
|
raise ArgumentError, 'query is empty' if cleaned.empty?
|
|
119
256
|
|
|
@@ -127,16 +264,15 @@ module Pikuri
|
|
|
127
264
|
results = nil
|
|
128
265
|
chosen = nil
|
|
129
266
|
current_providers.shuffle.each do |provider|
|
|
130
|
-
results = provider.search(cleaned, max_results: max_results)
|
|
267
|
+
results = provider.search(cleaned, max_results: max_results, cancellable: cancellable)
|
|
131
268
|
chosen = provider
|
|
132
269
|
break
|
|
133
270
|
rescue Unavailable => e
|
|
134
271
|
failures << "#{provider.label} (#{e.message})"
|
|
135
272
|
end
|
|
136
273
|
# Raise so {UrlCache#fetch} does NOT persist the all-unavailable
|
|
137
|
-
# message —
|
|
138
|
-
#
|
|
139
|
-
# the raise back into the calculator-style "Error: …" string.
|
|
274
|
+
# message — else it would block every future search for this query
|
|
275
|
+
# until the TTL expires. The outer rescue turns it into "Error: …".
|
|
140
276
|
chosen or raise Unavailable, "all search providers temporarily unavailable: #{failures.join('; ')}"
|
|
141
277
|
|
|
142
278
|
LOGGER.info do
|
|
@@ -152,26 +288,23 @@ module Pikuri
|
|
|
152
288
|
|
|
153
289
|
private
|
|
154
290
|
|
|
155
|
-
# Render an +Array<Result>+ into the smolagents-style Markdown the
|
|
156
|
-
#
|
|
157
|
-
#
|
|
158
|
-
# +"No results found."+ stub so the agent still gets a real
|
|
159
|
-
# observation to act on.
|
|
291
|
+
# Render an +Array<Result>+ into the smolagents-style Markdown the LLM
|
|
292
|
+
# consumes (+"## Search Results"+ header, then +[title](url)\nbody+
|
|
293
|
+
# joined by blank lines). Empty → the +"No results found."+ stub.
|
|
160
294
|
#
|
|
161
295
|
# @param results [Array<Result>] hits from the winning provider
|
|
162
|
-
# @return [String] Markdown
|
|
296
|
+
# @return [String] Markdown result list
|
|
163
297
|
def render(results)
|
|
164
298
|
return "## Search Results\n\nNo results found." if results.empty?
|
|
165
299
|
|
|
166
300
|
"## Search Results\n\n" + results.map { |r| "[#{r.title}](#{r.url})\n#{r.body}" }.join("\n\n")
|
|
167
301
|
end
|
|
168
302
|
|
|
169
|
-
#
|
|
170
|
-
#
|
|
171
|
-
#
|
|
172
|
-
# configuration rather than one per search.
|
|
303
|
+
# INFO-log the currently-available providers, but only when the set
|
|
304
|
+
# differs from the last logged — one line per distinct configuration,
|
|
305
|
+
# not per search.
|
|
173
306
|
#
|
|
174
|
-
# @param current [Array<#label>] providers
|
|
307
|
+
# @param current [Array<#label>] providers from {#providers}
|
|
175
308
|
# @return [void]
|
|
176
309
|
def log_providers(current)
|
|
177
310
|
return if @last_logged_providers == current
|
|
@@ -6,54 +6,40 @@ require 'json'
|
|
|
6
6
|
module Pikuri
|
|
7
7
|
class Tool
|
|
8
8
|
module Search
|
|
9
|
-
# Performs an Exa search via the official +/search+ endpoint
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
# cascade in {Engines#search} owns the final Markdown rendering.
|
|
9
|
+
# Performs an Exa search via the official +/search+ endpoint, returning
|
|
10
|
+
# hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
|
|
11
|
+
# parser (.parse) so tests exercise the parser against fixture JSON.
|
|
12
|
+
# {Engines#search} owns the final Markdown rendering.
|
|
14
13
|
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
20
|
-
#
|
|
21
|
-
# https://exa.ai — the service is paid.
|
|
14
|
+
# Constructed with its API key (+Exa.new(api_key:)+); {Engines} builds one
|
|
15
|
+
# only when an Exa key was configured, so users who never registered don't
|
|
16
|
+
# spend money, then drives it through the same +#search+/+#label+
|
|
17
|
+
# interface as every provider. pikuri reads no key from the environment
|
|
18
|
+
# (CLAUDE.md "Environment is not a secret store"). Paid; key at
|
|
19
|
+
# https://exa.ai.
|
|
22
20
|
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
# +description+ field, populating {Result#body} consistently across
|
|
27
|
-
# providers.
|
|
21
|
+
# Requests +type: "auto"+ (Exa picks neural vs keyword) and +contents: {
|
|
22
|
+
# highlights: true }+ so each result carries a neural-ranked snippet
|
|
23
|
+
# populating {Result#body} — the analog to Brave's +description+.
|
|
28
24
|
#
|
|
29
25
|
# == Privacy posture
|
|
30
26
|
#
|
|
31
|
-
# Exa's Privacy Policy
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
# (which is what pikuri uses) does not.
|
|
39
|
-
#
|
|
40
|
-
# Bottom line: Exa does not sell queries to data brokers, but it
|
|
41
|
-
# does mine them to train competing models, and the license it
|
|
42
|
-
# claims is effectively "do what we want with this, forever". If a
|
|
43
|
-
# query would be embarrassing or sensitive in a training set, simply
|
|
44
|
-
# don't configure an Exa key — {Engines#providers} leaves Exa out of
|
|
45
|
-
# the cascade unless its key was supplied to the constructor.
|
|
27
|
+
# Exa's Privacy Policy uses Query Data to train/fine-tune its models, and
|
|
28
|
+
# ToS §1.2(c) grants a +perpetual and irrevocable+, sub-licensable license
|
|
29
|
+
# over User Input. Business customers under an MSA/DPA get carve-outs; the
|
|
30
|
+
# default pay-as-you-go key pikuri uses does not. Bottom line: Exa mines
|
|
31
|
+
# queries to train competing models. If a query would be sensitive in a
|
|
32
|
+
# training set, don't configure an Exa key — {Engines#providers} leaves
|
|
33
|
+
# Exa out of the cascade unless its key was supplied.
|
|
46
34
|
class Exa
|
|
47
35
|
# @return [String] Search endpoint (POST, JSON body)
|
|
48
36
|
ENDPOINT = 'https://api.exa.ai/search'
|
|
49
37
|
# @return [Integer] default number of results returned, matching
|
|
50
38
|
# {DuckDuckGo::DEFAULT_MAX_RESULTS}
|
|
51
39
|
DEFAULT_MAX_RESULTS = 10
|
|
52
|
-
# @return [RateLimiter] Exa is paid and doesn't aggressively
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
# budget isn't burned on doomed retries while a 429 / 5xx
|
|
56
|
-
# condition persists.
|
|
40
|
+
# @return [RateLimiter] Exa is paid and doesn't aggressively throttle,
|
|
41
|
+
# so no minimum interval; the 5-minute cooldown still applies on
|
|
42
|
+
# {Engines::Unavailable} so the budget isn't burned on doomed retries.
|
|
57
43
|
LIMITER = RateLimiter.new(min_interval: 0.0, cooldown: 300.0)
|
|
58
44
|
|
|
59
45
|
# @param api_key [String] Exa API key. Required and non-blank:
|
|
@@ -72,28 +58,23 @@ module Pikuri
|
|
|
72
58
|
'Exa'
|
|
73
59
|
end
|
|
74
60
|
|
|
75
|
-
# Fetch results for +query+
|
|
76
|
-
#
|
|
77
|
-
#
|
|
78
|
-
# is expected to have already normalized the query and to wrap this
|
|
79
|
-
# in a result cache.
|
|
61
|
+
# Fetch results for +query+ as an +Array<Result>+. Circuit-broken for
|
|
62
|
+
# 5 minutes on rate-limit/unavailable (see {LIMITER}). The caller
|
|
63
|
+
# ({Engines#search}) normalizes the query and wraps this in a cache.
|
|
80
64
|
#
|
|
81
65
|
# @param query [String] search query (already normalized)
|
|
82
|
-
# @param max_results [Integer]
|
|
83
|
-
#
|
|
84
|
-
#
|
|
85
|
-
#
|
|
86
|
-
# @raise [Engines::Unavailable]
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
# empty-results payload.
|
|
95
|
-
def search(query, max_results: DEFAULT_MAX_RESULTS)
|
|
96
|
-
LIMITER.call do
|
|
66
|
+
# @param max_results [Integer] max entries; Exa's +numResults+
|
|
67
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable] makes
|
|
68
|
+
# {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
|
|
69
|
+
# @return [Array<Result>] hits, possibly empty when Exa matched nothing
|
|
70
|
+
# @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
|
|
71
|
+
# back from these), or immediately if {LIMITER} is in cooldown. Other
|
|
72
|
+
# non-2xx (401/403 bad key) bubble up as +RuntimeError+.
|
|
73
|
+
# @raise [RuntimeError] for non-rate-limit HTTP failures, or an
|
|
74
|
+
# unrecognized empty-results shape.
|
|
75
|
+
def search(query, max_results: DEFAULT_MAX_RESULTS,
|
|
76
|
+
cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
77
|
+
LIMITER.call(cancellable: cancellable) do
|
|
97
78
|
response = Faraday.post(ENDPOINT) do |req|
|
|
98
79
|
req.headers['x-api-key'] = @api_key
|
|
99
80
|
req.headers['Content-Type'] = 'application/json'
|
|
@@ -117,25 +98,16 @@ module Pikuri
|
|
|
117
98
|
end
|
|
118
99
|
end
|
|
119
100
|
|
|
120
|
-
# Parse an Exa
|
|
121
|
-
#
|
|
122
|
-
#
|
|
123
|
-
#
|
|
124
|
-
#
|
|
125
|
-
# When the response yields zero result entries, two cases are
|
|
126
|
-
# distinguished: a genuine "no results" payload (response carries
|
|
127
|
-
# a +requestId+ and an empty +results+ array — Exa ran the query
|
|
128
|
-
# but matched nothing) returns an empty array instead of raising,
|
|
129
|
-
# so {Engines.search} can render its standard no-results stub.
|
|
130
|
-
# Anything else (unknown shape, structured error) raises with a
|
|
131
|
-
# diagnostic so the failure surfaces.
|
|
101
|
+
# Parse an Exa JSON response into {Result} rows, +body+ the first
|
|
102
|
+
# non-empty +highlights+ snippet. A genuine "no results" payload
|
|
103
|
+
# (+requestId+ present, empty +results+) returns +[]+ so {Engines.search}
|
|
104
|
+
# renders its no-results stub; any other zero-result shape raises.
|
|
132
105
|
#
|
|
133
106
|
# @param json [String] response body from {ENDPOINT}
|
|
134
|
-
# @param max_results [Integer]
|
|
107
|
+
# @param max_results [Integer] max entries
|
|
135
108
|
# @return [Array<Result>] hits, possibly empty on a recognized
|
|
136
109
|
# empty-results payload
|
|
137
|
-
# @raise [RuntimeError]
|
|
138
|
-
# is not recognized as a genuine empty-results payload
|
|
110
|
+
# @raise [RuntimeError] on an unrecognized zero-result response
|
|
139
111
|
def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
|
|
140
112
|
data = JSON.parse(json)
|
|
141
113
|
results = Array(data['results']).take(max_results).filter_map do |r|
|
|
@@ -158,12 +130,11 @@ module Pikuri
|
|
|
158
130
|
results
|
|
159
131
|
end
|
|
160
132
|
|
|
161
|
-
# Collapse whitespace and strip;
|
|
162
|
-
#
|
|
163
|
-
# has no usable title).
|
|
133
|
+
# Collapse whitespace and strip; +nil+ for nil/empty input so the caller
|
|
134
|
+
# can fall back (e.g. to the URL when a result has no title).
|
|
164
135
|
#
|
|
165
|
-
# @param text [String, nil]
|
|
166
|
-
# @return [String, nil] cleaned text, or +nil+ if
|
|
136
|
+
# @param text [String, nil]
|
|
137
|
+
# @return [String, nil] cleaned text, or +nil+ if blank
|
|
167
138
|
def self.clean(text)
|
|
168
139
|
return nil if text.nil?
|
|
169
140
|
|
|
@@ -172,10 +143,8 @@ module Pikuri
|
|
|
172
143
|
end
|
|
173
144
|
private_class_method :clean
|
|
174
145
|
|
|
175
|
-
# First non-empty
|
|
176
|
-
#
|
|
177
|
-
# only the top one to keep the observation compact and match the
|
|
178
|
-
# one-line +body+ convention used by Brave / DuckDuckGo.
|
|
146
|
+
# First non-empty +highlights+ entry, cleaned. Exa sorts highlights by
|
|
147
|
+
# relevance; the top one populates +body+.
|
|
179
148
|
#
|
|
180
149
|
# @param highlights [Array<String>, nil] +highlights+ field
|
|
181
150
|
# @return [String] cleaned snippet, or empty string if none usable
|
|
@@ -190,10 +159,9 @@ module Pikuri
|
|
|
190
159
|
end
|
|
191
160
|
private_class_method :first_highlight
|
|
192
161
|
|
|
193
|
-
# True when a
|
|
194
|
-
#
|
|
195
|
-
#
|
|
196
|
-
# field, which Exa always sets on a successful request.
|
|
162
|
+
# True when a zero-+results+ response looks like Exa's own "ran, matched
|
|
163
|
+
# nothing" payload rather than an error — marked by +requestId+, which
|
|
164
|
+
# Exa always sets on success.
|
|
197
165
|
#
|
|
198
166
|
# @param data [Hash, Object] parsed response
|
|
199
167
|
# @return [Boolean]
|
|
@@ -205,14 +173,12 @@ module Pikuri
|
|
|
205
173
|
end
|
|
206
174
|
private_class_method :genuine_no_results?
|
|
207
175
|
|
|
208
|
-
# Build an error message for a
|
|
209
|
-
#
|
|
210
|
-
# present, otherwise truncates the raw body so the caller can see
|
|
211
|
-
# the actual payload.
|
|
176
|
+
# Build an error message for a zero-result response: quote Exa's
|
|
177
|
+
# +error+/+message+/+detail+ if present, else a truncated raw body.
|
|
212
178
|
#
|
|
213
179
|
# @param data [Hash, Object] parsed response
|
|
214
180
|
# @param raw [String] raw response body
|
|
215
|
-
# @return [String]
|
|
181
|
+
# @return [String] diagnostic to feed to +raise+
|
|
216
182
|
def self.diagnose_empty(data, raw)
|
|
217
183
|
if data.is_a?(Hash) && (msg = data['error'] || data['message'] || data['detail'])
|
|
218
184
|
return "Exa Search returned an error: #{msg}"
|