pikuri-core 0.0.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -30
- data/lib/pikuri/agent/chat_transport.rb +168 -107
- data/lib/pikuri/agent/configurator.rb +194 -173
- data/lib/pikuri/agent/control/cancellable.rb +100 -70
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +107 -118
- data/lib/pikuri/agent/control.rb +13 -35
- data/lib/pikuri/agent/event.rb +144 -197
- data/lib/pikuri/agent/extension.rb +83 -95
- data/lib/pikuri/agent/history.rb +247 -0
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +169 -136
- data/lib/pikuri/agent/listener/token_log.rb +84 -157
- data/lib/pikuri/agent/listener.rb +24 -44
- data/lib/pikuri/agent/listener_list.rb +32 -47
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +1095 -577
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/bundler_env.rb +68 -0
- data/lib/pikuri/extractor/passthrough.rb +24 -31
- data/lib/pikuri/extractor.rb +125 -155
- data/lib/pikuri/file_type.rb +64 -136
- data/lib/pikuri/finalizers.rb +32 -47
- data/lib/pikuri/paths.rb +104 -13
- data/lib/pikuri/sanitizer.rb +46 -67
- data/lib/pikuri/subprocess.rb +86 -118
- data/lib/pikuri/testing.rb +369 -0
- data/lib/pikuri/tool/calculator.rb +168 -74
- data/lib/pikuri/tool/execute_context.rb +42 -0
- data/lib/pikuri/tool/parameters.rb +31 -29
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +248 -0
- data/lib/pikuri/tool.rb +416 -67
- data/lib/pikuri/trifecta/contribution.rb +43 -0
- data/lib/pikuri/trifecta/node.rb +47 -0
- data/lib/pikuri/trifecta/report.rb +228 -0
- data/lib/pikuri/trifecta.rb +126 -0
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +70 -92
- data/prompts/agent-loop.txt +5 -0
- data/prompts/pikuri-chat.txt +3 -12
- metadata +26 -72
- data/lib/pikuri/agent/context_window_detector.rb +0 -222
- data/lib/pikuri/agent/control/step_limit.rb +0 -111
- data/lib/pikuri/agent/extension_context.rb +0 -147
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
- data/lib/pikuri/agent/synthesizer.rb +0 -171
- data/lib/pikuri/extractor/html.rb +0 -303
- data/lib/pikuri/tool/fetch.rb +0 -172
- data/lib/pikuri/tool/scraper.rb +0 -186
- data/lib/pikuri/tool/search/brave.rb +0 -198
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
- data/lib/pikuri/tool/search/engines.rb +0 -186
- data/lib/pikuri/tool/search/exa.rb +0 -229
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
- data/lib/pikuri/tool/search/result.rb +0 -29
- data/lib/pikuri/tool/web_scrape.rb +0 -121
- data/lib/pikuri/tool/web_search.rb +0 -57
- data/lib/pikuri/url_cache.rb +0 -112
|
@@ -1,207 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'uri'
|
|
6
|
-
|
|
7
|
-
module Pikuri
|
|
8
|
-
class Tool
|
|
9
|
-
module Search
|
|
10
|
-
# Performs a DuckDuckGo search by scraping +html.duckduckgo.com+ and
|
|
11
|
-
# returns the hits as a list of {Result} rows. Split into a thin HTTP
|
|
12
|
-
# fetch (#search) and a pure parser (.parse) so tests can exercise
|
|
13
|
-
# the parser against fixture HTML without hitting the network. The
|
|
14
|
-
# cascade in {Engines#search} owns the final Markdown rendering.
|
|
15
|
-
#
|
|
16
|
-
# A class (constructed with no arguments) so it shares the uniform
|
|
17
|
-
# provider shape with the keyed {Brave} / {Exa}: {Engines} holds a
|
|
18
|
-
# list of provider *instances* and calls +#search+ / +#label+ on each
|
|
19
|
-
# without caring which is which.
|
|
20
|
-
#
|
|
21
|
-
# == Privacy posture
|
|
22
|
-
#
|
|
23
|
-
# DuckDuckGo's privacy policy states +We don't save your IP address
|
|
24
|
-
# or any unique identifiers alongside your searches+ and +We have
|
|
25
|
-
# never sold any personal information+, and they proxy requests on
|
|
26
|
-
# the user's behalf so downstream content providers can't build a
|
|
27
|
-
# per-user search history. That part is real — but DDG is mainly a
|
|
28
|
-
# relay over Bing for web results, so the *query content* still
|
|
29
|
-
# reaches Microsoft for fulfillment even though DDG strips
|
|
30
|
-
# identifying info on the way out.
|
|
31
|
-
#
|
|
32
|
-
# Bottom line: DDG is a genuine privacy improvement over hitting
|
|
33
|
-
# Bing directly (your IP isn't tied to the query, no per-user
|
|
34
|
-
# profile is built on DDG's side), but query content still lands at
|
|
35
|
-
# Microsoft, who has no comparable no-training pledge. Better than
|
|
36
|
-
# Exa for sensitive queries, worse than Brave; for anything
|
|
37
|
-
# genuinely embarrassing, don't search the web at all.
|
|
38
|
-
class DuckDuckGo
|
|
39
|
-
# @return [String] HTML search endpoint
|
|
40
|
-
ENDPOINT = 'https://html.duckduckgo.com/html/'
|
|
41
|
-
# @return [String] User-Agent sent with each request; DDG often rejects
|
|
42
|
-
# requests with no UA or an obvious bot UA
|
|
43
|
-
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
|
|
44
|
-
'(KHTML, like Gecko) Chrome/120.0 Safari/537.36'
|
|
45
|
-
# @return [Integer] default number of results returned, matching smolagents
|
|
46
|
-
DEFAULT_MAX_RESULTS = 10
|
|
47
|
-
# @return [RateLimiter] paces calls (DDG bans IPs that hammer the HTML
|
|
48
|
-
# endpoint) and circuit-breaks on {Engines::Unavailable} so a
|
|
49
|
-
# soft-block response doesn't get retried for the next 5 minutes
|
|
50
|
-
LIMITER = RateLimiter.new(min_interval: 2.0, cooldown: 300.0)
|
|
51
|
-
|
|
52
|
-
# @return [String] short provider label for {Engines} logging /
|
|
53
|
-
# fallback messages. Uniform across providers (see {Brave#label}).
|
|
54
|
-
def label
|
|
55
|
-
'DuckDuckGo'
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
# Fetch results for +query+ and return them as an +Array<Result>+.
|
|
59
|
-
# Calls are throttled to one every 2s and circuit-broken for 5 minutes
|
|
60
|
-
# after a soft-block; see {LIMITER}. The caller (typically
|
|
61
|
-
# {Engines#search}) is expected to have already normalized the
|
|
62
|
-
# query and to wrap this in a result cache.
|
|
63
|
-
#
|
|
64
|
-
# @param query [String] search query (already normalized)
|
|
65
|
-
# @param max_results [Integer] maximum number of result entries
|
|
66
|
-
# @return [Array<Result>] hits, possibly empty when DDG ran the
|
|
67
|
-
# query and matched nothing
|
|
68
|
-
# @raise [Engines::Unavailable] when DDG soft-blocks the IP
|
|
69
|
-
# (anomaly/CAPTCHA page) or returns HTTP 429/5xx — i.e. "try again
|
|
70
|
-
# later" responses the cascade in {Engines#search} can fall
|
|
71
|
-
# back from. Also raised immediately if {LIMITER} is in cooldown.
|
|
72
|
-
# @raise [RuntimeError] if the HTTP call fails for other reasons or
|
|
73
|
-
# the empty-results page is in an unrecognized layout. A genuine
|
|
74
|
-
# empty-results page is *not* an error; see {.parse}.
|
|
75
|
-
def search(query, max_results: DEFAULT_MAX_RESULTS)
|
|
76
|
-
LIMITER.call do
|
|
77
|
-
response = Faraday.get(ENDPOINT, { q: query }, { 'User-Agent' => USER_AGENT })
|
|
78
|
-
unless response.success?
|
|
79
|
-
if response.status == 429 || response.status >= 500
|
|
80
|
-
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
raise "DuckDuckGo request failed: #{response.status} #{response.body}"
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
self.class.parse(response.body, max_results: max_results)
|
|
87
|
-
end
|
|
88
|
-
end
|
|
89
|
-
|
|
90
|
-
# Parse a +html.duckduckgo.com+ result page into a list of {Result}
|
|
91
|
-
# rows. +<b>+ highlights inside snippets are stripped.
|
|
92
|
-
#
|
|
93
|
-
# When the page has zero result nodes, two cases are distinguished:
|
|
94
|
-
# a genuine "no results" page (narrow query, DDG's own "No results
|
|
95
|
-
# found" indicator) returns an empty array instead of raising, so
|
|
96
|
-
# {Engines.search} can render its standard no-results stub.
|
|
97
|
-
# Anything else (anomaly modal, CAPTCHA, service-unavailable page,
|
|
98
|
-
# unknown layout) raises with the diagnostic text extracted from
|
|
99
|
-
# the body, so an IP soft-block is surfaced rather than silently
|
|
100
|
-
# masquerading as an empty search.
|
|
101
|
-
#
|
|
102
|
-
# @param html [String] HTML document body from html.duckduckgo.com
|
|
103
|
-
# @param max_results [Integer] maximum number of result entries
|
|
104
|
-
# @return [Array<Result>] hits, possibly empty on a genuine
|
|
105
|
-
# no-results page
|
|
106
|
-
# @raise [Engines::Unavailable] when the page is the DDG
|
|
107
|
-
# anomaly/CAPTCHA modal (IP soft-block) — a "try again later" the
|
|
108
|
-
# cascade can fall back from.
|
|
109
|
-
# @raise [RuntimeError] when the page contains no result nodes and is
|
|
110
|
-
# not recognized as either a genuine no-results page or the
|
|
111
|
-
# anomaly modal (likely a layout change worth surfacing loudly).
|
|
112
|
-
def self.parse(html, max_results: DEFAULT_MAX_RESULTS)
|
|
113
|
-
doc = Nokogiri::HTML(html)
|
|
114
|
-
results = doc.css('div.result.web-result').take(max_results).filter_map do |node|
|
|
115
|
-
title_link = node.at_css('a.result__a')
|
|
116
|
-
next nil if title_link.nil?
|
|
117
|
-
|
|
118
|
-
snippet = node.at_css('a.result__snippet')
|
|
119
|
-
Result.new(
|
|
120
|
-
url: extract_url(title_link['href']),
|
|
121
|
-
title: title_link.text.strip,
|
|
122
|
-
body: snippet&.text&.strip.to_s
|
|
123
|
-
)
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
if results.empty?
|
|
127
|
-
return [] if genuine_no_results?(doc)
|
|
128
|
-
|
|
129
|
-
message = diagnose_empty(doc)
|
|
130
|
-
raise(anomaly_modal?(doc) ? Engines::Unavailable : RuntimeError, message)
|
|
131
|
-
end
|
|
132
|
-
|
|
133
|
-
results
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
# True when the page contains DDG's anomaly/CAPTCHA modal — i.e. the
|
|
137
|
-
# IP has been soft-blocked. Used by {.parse} to pick between
|
|
138
|
-
# {Engines::Unavailable} (recoverable, fall back to another
|
|
139
|
-
# provider) and {RuntimeError} (unknown layout, surface loudly).
|
|
140
|
-
#
|
|
141
|
-
# @param doc [Nokogiri::HTML::Document] parsed result page
|
|
142
|
-
# @return [Boolean]
|
|
143
|
-
def self.anomaly_modal?(doc)
|
|
144
|
-
!!(doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description'))
|
|
145
|
-
end
|
|
146
|
-
private_class_method :anomaly_modal?
|
|
147
|
-
|
|
148
|
-
# True when a results page with zero result nodes looks like DDG's own
|
|
149
|
-
# "no results found" page (narrow query) rather than an anomaly/CAPTCHA
|
|
150
|
-
# or other non-results layout. Anomaly modal wins: if the modal divs
|
|
151
|
-
# are present we never treat the page as a genuine empty result, even
|
|
152
|
-
# if the surrounding text happens to mention "No results".
|
|
153
|
-
#
|
|
154
|
-
# @param doc [Nokogiri::HTML::Document] parsed result page
|
|
155
|
-
# @return [Boolean]
|
|
156
|
-
def self.genuine_no_results?(doc)
|
|
157
|
-
return false if doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description')
|
|
158
|
-
return true if doc.at_css('div.no-results')
|
|
159
|
-
|
|
160
|
-
doc.text.include?('No results found')
|
|
161
|
-
end
|
|
162
|
-
private_class_method :genuine_no_results?
|
|
163
|
-
|
|
164
|
-
# Build an error message for a results page that yielded zero matches.
|
|
165
|
-
# Recognizes the DDG anomaly modal explicitly and quotes its title and
|
|
166
|
-
# description; otherwise extracts visible text from the body (with
|
|
167
|
-
# +<script>+/+<style>+/+<noscript>+ stripped and whitespace collapsed)
|
|
168
|
-
# and includes a truncated copy so the caller can see what came back.
|
|
169
|
-
#
|
|
170
|
-
# @param doc [Nokogiri::HTML::Document] parsed result page
|
|
171
|
-
# @return [String] human-readable diagnostic to feed to +raise+
|
|
172
|
-
def self.diagnose_empty(doc)
|
|
173
|
-
title = doc.at_css('.anomaly-modal__title')&.text&.strip
|
|
174
|
-
desc = doc.at_css('.anomaly-modal__description')&.text&.strip
|
|
175
|
-
if title || desc
|
|
176
|
-
return "DuckDuckGo anomaly check (likely IP soft-block): #{[title, desc].compact.reject(&:empty?).join(' — ')}"
|
|
177
|
-
end
|
|
178
|
-
|
|
179
|
-
doc.css('script, style, noscript').remove
|
|
180
|
-
text = doc.text.gsub(/\s+/, ' ').strip
|
|
181
|
-
snippet = text.empty? ? '<empty body>' : text[0, 1500]
|
|
182
|
-
snippet += '…' if text.length > 1500
|
|
183
|
-
"DuckDuckGo returned no results. Page text: #{snippet}"
|
|
184
|
-
end
|
|
185
|
-
private_class_method :diagnose_empty
|
|
186
|
-
|
|
187
|
-
# Decode DuckDuckGo's +//duckduckgo.com/l/?uddg=<encoded>+ redirect wrapper
|
|
188
|
-
# back to the real target URL.
|
|
189
|
-
#
|
|
190
|
-
# @param href [String, nil] href as found on the search-result page
|
|
191
|
-
# @return [String] the decoded target URL, or +href+ unchanged when it is
|
|
192
|
-
# not a recognized DDG redirect or cannot be parsed
|
|
193
|
-
def self.extract_url(href)
|
|
194
|
-
return href if href.nil? || href.empty?
|
|
195
|
-
|
|
196
|
-
uri = URI.parse(href.start_with?('//') ? "https:#{href}" : href)
|
|
197
|
-
return href unless uri.host&.end_with?('duckduckgo.com') && uri.path == '/l/'
|
|
198
|
-
|
|
199
|
-
params = URI.decode_www_form(uri.query.to_s).to_h
|
|
200
|
-
params['uddg'] || href
|
|
201
|
-
rescue URI::InvalidURIError
|
|
202
|
-
href
|
|
203
|
-
end
|
|
204
|
-
end
|
|
205
|
-
end
|
|
206
|
-
end
|
|
207
|
-
end
|
|
@@ -1,186 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
# Namespace for the web-search stack used by {Tool::WebSearch}: per-
|
|
6
|
-
# provider modules ({DuckDuckGo}, {Brave}, {Exa}), the {Result} value
|
|
7
|
-
# object they all return, the cross-provider {Engines} cascade with
|
|
8
|
-
# its on-disk cache, and the shared {RateLimiter} a provider can wire
|
|
9
|
-
# in to back off when a quota header says so.
|
|
10
|
-
module Search
|
|
11
|
-
# Search-orchestration object: the cascade across configured
|
|
12
|
-
# providers, the result cache, and the {Unavailable} protocol marker
|
|
13
|
-
# the cascade uses to fall back. The LLM-facing tool itself is built
|
|
14
|
-
# by {Tool::WebSearch.build}, which constructs one of these and wires
|
|
15
|
-
# its {#search} into a {Tool}. Each {Tool::Search} provider module
|
|
16
|
-
# ({DuckDuckGo}, {Brave}, {Exa}) raises {Unavailable} when it wants
|
|
17
|
-
# the cascade to try the next one.
|
|
18
|
-
#
|
|
19
|
-
# == Provider keys are constructor config, not environment
|
|
20
|
-
#
|
|
21
|
-
# Brave and Exa are paid and need an API key; DuckDuckGo needs none.
|
|
22
|
-
# An {Engines} is constructed with the keys it should use
|
|
23
|
-
# (+brave_key:+ / +exa_key:+, both optional) — pikuri reads no key
|
|
24
|
-
# from the environment, so the only providers in the cascade are
|
|
25
|
-
# DuckDuckGo plus whichever keyed providers the host actually
|
|
26
|
-
# configured. The host sources those keys however it likes (the
|
|
27
|
-
# bundled +bin/+ examples load a JSON config file by convention); see
|
|
28
|
-
# CLAUDE.md "Environment is not a secret store".
|
|
29
|
-
class Engines
|
|
30
|
-
# Subsystem logger; set its level with +PIKURI_LOG_ENGINES+
|
|
31
|
-
# (e.g. +PIKURI_LOG_ENGINES=debug+) or the global +PIKURI_LOG+.
|
|
32
|
-
#
|
|
33
|
-
# @return [Logger]
|
|
34
|
-
LOGGER = Pikuri.logger_for('Engines')
|
|
35
|
-
|
|
36
|
-
# Raised by a provider when it is temporarily unavailable (rate-limited,
|
|
37
|
-
# bot-blocked, quota-exhausted, or otherwise saying "try again later"
|
|
38
|
-
# rather than "your request is wrong"). The cascade in {#search}
|
|
39
|
-
# catches this and tries the next provider; any other exception bubbles
|
|
40
|
-
# up unchanged so genuine bugs and config errors stay visible.
|
|
41
|
-
class Unavailable < StandardError; end
|
|
42
|
-
|
|
43
|
-
# Process-shared on-disk cache backing {#search}'s default. Kept at
|
|
44
|
-
# class level (not per-instance) so every engine dedupes answered
|
|
45
|
-
# queries into one directory; the constructor's +cache:+ parameter
|
|
46
|
-
# injects a different store for tests. Exposed as a method so specs
|
|
47
|
-
# can swap it for {UrlCache::NULL} without touching the instance.
|
|
48
|
-
#
|
|
49
|
-
# @return [UrlCache, #fetch]
|
|
50
|
-
CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_search")
|
|
51
|
-
# Accessor for {CACHE}, used as the constructor's +cache:+ default;
|
|
52
|
-
# specs override this to swap in {UrlCache::NULL}.
|
|
53
|
-
#
|
|
54
|
-
# @return [UrlCache, #fetch]
|
|
55
|
-
def self.cache
|
|
56
|
-
CACHE
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
# Builds the provider cascade once: {DuckDuckGo} always (no key
|
|
60
|
-
# needed), plus {Brave} / {Exa} when their key was supplied
|
|
61
|
-
# (non-blank). Each keyed provider is constructed with its key, so
|
|
62
|
-
# from here on every provider is just an object answering +#search+
|
|
63
|
-
# / +#label+ — the cascade in {#search} treats them uniformly.
|
|
64
|
-
#
|
|
65
|
-
# @param brave_key [String, nil] Brave Search subscription token;
|
|
66
|
-
# non-blank ⇒ Brave joins the cascade. +nil+/blank ⇒ not configured.
|
|
67
|
-
# @param exa_key [String, nil] Exa API key; non-blank ⇒ Exa joins the
|
|
68
|
-
# cascade. +nil+/blank ⇒ not configured.
|
|
69
|
-
# @param cache [UrlCache, #fetch] result store memoizing answered
|
|
70
|
-
# queries; defaults to the process-shared {.cache}.
|
|
71
|
-
# @return [Engines]
|
|
72
|
-
def initialize(brave_key: nil, exa_key: nil, cache: self.class.cache)
|
|
73
|
-
@providers = [DuckDuckGo.new]
|
|
74
|
-
@providers << Brave.new(api_key: brave_key) unless brave_key.to_s.strip.empty?
|
|
75
|
-
@providers << Exa.new(api_key: exa_key) unless exa_key.to_s.strip.empty?
|
|
76
|
-
@cache = cache
|
|
77
|
-
@last_logged_providers = nil
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
# The provider instances this engine cascades across, in
|
|
81
|
-
# declaration order (the cascade itself shuffles them per call).
|
|
82
|
-
#
|
|
83
|
-
# @return [Array<#search, #label>] configured provider instances
|
|
84
|
-
attr_reader :providers
|
|
85
|
-
|
|
86
|
-
# Run +query+ through the configured providers in random order, falling
|
|
87
|
-
# back to the next one each time a provider raises {Unavailable}. The
|
|
88
|
-
# shuffle spreads load so a single provider isn't always hit first
|
|
89
|
-
# (and exhausted first); revisit if it stops being the right default.
|
|
90
|
-
#
|
|
91
|
-
# The query is whitespace-trimmed and runs of whitespace collapsed
|
|
92
|
-
# to a single space before the cascade runs. The winning provider's
|
|
93
|
-
# +Array<Result>+ is rendered into smolagents-style Markdown here
|
|
94
|
-
# (+"## Search Results"+ header, then +[title](url)\nbody+ entries
|
|
95
|
-
# joined by blank lines; an empty array becomes +"No results found."+),
|
|
96
|
-
# and the rendered Markdown is cached on disk via {#initialize}'s
|
|
97
|
-
# +cache:+, keyed by the cleaned query. A cache hit short-circuits the
|
|
98
|
-
# cascade entirely (and benefits whichever provider would have
|
|
99
|
-
# answered next time too — once a query is cached, the cooldown state
|
|
100
|
-
# of the original answering provider no longer matters). +max_results+
|
|
101
|
-
# is not part of the cache key, so callers passing a non-default value
|
|
102
|
-
# may get a result rendered with the previously-cached size.
|
|
103
|
-
#
|
|
104
|
-
# If every provider reports temporary unavailability, returns an
|
|
105
|
-
# +"Error: ..."+ string instead of raising — same convention as
|
|
106
|
-
# {Tool::Calculator.calculate}, so the agent loop can feed the failure
|
|
107
|
-
# back to the model as the next observation. Any non-{Unavailable}
|
|
108
|
-
# exception (network error, parser failure, malformed response, bad
|
|
109
|
-
# API key) bubbles up to the caller.
|
|
110
|
-
#
|
|
111
|
-
# @param query [String] search query
|
|
112
|
-
# @param max_results [Integer] maximum number of result entries
|
|
113
|
-
# @return [String] Markdown-formatted result list, or +"Error: ..."+
|
|
114
|
-
# when all providers are exhausted
|
|
115
|
-
# @raise [ArgumentError] if the query is empty after normalization
|
|
116
|
-
def search(query, max_results:)
|
|
117
|
-
cleaned = query.to_s.strip.gsub(/\s+/, ' ')
|
|
118
|
-
raise ArgumentError, 'query is empty' if cleaned.empty?
|
|
119
|
-
|
|
120
|
-
current_providers = providers
|
|
121
|
-
log_providers(current_providers)
|
|
122
|
-
|
|
123
|
-
hit = true
|
|
124
|
-
result = @cache.fetch(cleaned) do
|
|
125
|
-
hit = false
|
|
126
|
-
failures = []
|
|
127
|
-
results = nil
|
|
128
|
-
chosen = nil
|
|
129
|
-
current_providers.shuffle.each do |provider|
|
|
130
|
-
results = provider.search(cleaned, max_results: max_results)
|
|
131
|
-
chosen = provider
|
|
132
|
-
break
|
|
133
|
-
rescue Unavailable => e
|
|
134
|
-
failures << "#{provider.label} (#{e.message})"
|
|
135
|
-
end
|
|
136
|
-
# Raise so {UrlCache#fetch} does NOT persist the all-unavailable
|
|
137
|
-
# message — otherwise that string would block every future search
|
|
138
|
-
# for this query until the TTL expires. The outer +rescue+ turns
|
|
139
|
-
# the raise back into the calculator-style "Error: …" string.
|
|
140
|
-
chosen or raise Unavailable, "all search providers temporarily unavailable: #{failures.join('; ')}"
|
|
141
|
-
|
|
142
|
-
LOGGER.info do
|
|
143
|
-
"engine=#{chosen.label} query=#{cleaned.inspect} results=#{results.size}"
|
|
144
|
-
end
|
|
145
|
-
render(results)
|
|
146
|
-
end
|
|
147
|
-
LOGGER.info { "cache=hit query=#{cleaned.inspect} bytes=#{result.bytesize}" } if hit
|
|
148
|
-
result
|
|
149
|
-
rescue Unavailable => e
|
|
150
|
-
"Error: #{e.message}"
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
private
|
|
154
|
-
|
|
155
|
-
# Render an +Array<Result>+ into the smolagents-style Markdown the
|
|
156
|
-
# LLM consumes: +"## Search Results"+ header, then +[title](url)\nbody+
|
|
157
|
-
# entries joined by blank lines. An empty array becomes the
|
|
158
|
-
# +"No results found."+ stub so the agent still gets a real
|
|
159
|
-
# observation to act on.
|
|
160
|
-
#
|
|
161
|
-
# @param results [Array<Result>] hits from the winning provider
|
|
162
|
-
# @return [String] Markdown-formatted result list
|
|
163
|
-
def render(results)
|
|
164
|
-
return "## Search Results\n\nNo results found." if results.empty?
|
|
165
|
-
|
|
166
|
-
"## Search Results\n\n" + results.map { |r| "[#{r.title}](#{r.url})\n#{r.body}" }.join("\n\n")
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
# Emit an INFO log line listing the currently-available providers,
|
|
170
|
-
# but only when the set differs from the last one this engine
|
|
171
|
-
# logged. The memo keeps the log to one line per distinct
|
|
172
|
-
# configuration rather than one per search.
|
|
173
|
-
#
|
|
174
|
-
# @param current [Array<#label>] providers returned by {#providers}
|
|
175
|
-
# @return [void]
|
|
176
|
-
def log_providers(current)
|
|
177
|
-
return if @last_logged_providers == current
|
|
178
|
-
|
|
179
|
-
@last_logged_providers = current
|
|
180
|
-
names = current.map(&:label).join(', ')
|
|
181
|
-
LOGGER.info("engines available: #{names}")
|
|
182
|
-
end
|
|
183
|
-
end
|
|
184
|
-
end
|
|
185
|
-
end
|
|
186
|
-
end
|
|
@@ -1,229 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'json'
|
|
5
|
-
|
|
6
|
-
module Pikuri
|
|
7
|
-
class Tool
|
|
8
|
-
module Search
|
|
9
|
-
# Performs an Exa search via the official +/search+ endpoint and
|
|
10
|
-
# returns the hits as a list of {Result} rows. Split into a thin HTTP
|
|
11
|
-
# fetch (#search) and a pure parser (.parse) so tests can exercise
|
|
12
|
-
# the parser against fixture JSON without hitting the network. The
|
|
13
|
-
# cascade in {Engines#search} owns the final Markdown rendering.
|
|
14
|
-
#
|
|
15
|
-
# A class constructed with the API key it should use
|
|
16
|
-
# (+Exa.new(api_key:)+); {Engines} builds one only when an Exa key was
|
|
17
|
-
# configured, so users who haven't registered never spend money on it,
|
|
18
|
-
# and then drives it through the same +#search+ / +#label+ interface
|
|
19
|
-
# as every other provider. pikuri reads no key from the environment
|
|
20
|
-
# (see CLAUDE.md "Environment is not a secret store"). Get a key at
|
|
21
|
-
# https://exa.ai — the service is paid.
|
|
22
|
-
#
|
|
23
|
-
# Calls request +type: "auto"+ (Exa picks neural vs keyword per
|
|
24
|
-
# query) and +contents: { highlights: true }+ so each result carries
|
|
25
|
-
# a short neural-ranked snippet — the closest analog to Brave's
|
|
26
|
-
# +description+ field, populating {Result#body} consistently across
|
|
27
|
-
# providers.
|
|
28
|
-
#
|
|
29
|
-
# == Privacy posture
|
|
30
|
-
#
|
|
31
|
-
# Exa's Privacy Policy states +Query Data is used to improve our
|
|
32
|
-
# products and technology, including by training and fine-tuning
|
|
33
|
-
# models that power our Services+, and the Terms of Service §1.2(c)
|
|
34
|
-
# grant Exa a +perpetual and irrevocable+, +sub-licensable+,
|
|
35
|
-
# worldwide license over User Input that can be disclosed to third
|
|
36
|
-
# parties +as needed+. Business customers under a Master Subscription
|
|
37
|
-
# Agreement / DPA get carve-outs; the default pay-as-you-go API key
|
|
38
|
-
# (which is what pikuri uses) does not.
|
|
39
|
-
#
|
|
40
|
-
# Bottom line: Exa does not sell queries to data brokers, but it
|
|
41
|
-
# does mine them to train competing models, and the license it
|
|
42
|
-
# claims is effectively "do what we want with this, forever". If a
|
|
43
|
-
# query would be embarrassing or sensitive in a training set, simply
|
|
44
|
-
# don't configure an Exa key — {Engines#providers} leaves Exa out of
|
|
45
|
-
# the cascade unless its key was supplied to the constructor.
|
|
46
|
-
class Exa
|
|
47
|
-
# @return [String] Search endpoint (POST, JSON body)
|
|
48
|
-
ENDPOINT = 'https://api.exa.ai/search'
|
|
49
|
-
# @return [Integer] default number of results returned, matching
|
|
50
|
-
# {DuckDuckGo::DEFAULT_MAX_RESULTS}
|
|
51
|
-
DEFAULT_MAX_RESULTS = 10
|
|
52
|
-
# @return [RateLimiter] Exa is paid and doesn't aggressively
|
|
53
|
-
# throttle, so no minimum interval is enforced. The 5-minute
|
|
54
|
-
# cooldown still applies on {Engines::Unavailable} so the user's
|
|
55
|
-
# budget isn't burned on doomed retries while a 429 / 5xx
|
|
56
|
-
# condition persists.
|
|
57
|
-
LIMITER = RateLimiter.new(min_interval: 0.0, cooldown: 300.0)
|
|
58
|
-
|
|
59
|
-
# @param api_key [String] Exa API key. Required and non-blank:
|
|
60
|
-
# pikuri reads no key from the environment — the host supplies it
|
|
61
|
-
# ({Engines} only constructs an Exa when a key was configured).
|
|
62
|
-
# @raise [ArgumentError] if +api_key+ is blank
|
|
63
|
-
def initialize(api_key:)
|
|
64
|
-
raise ArgumentError, 'Exa Search API key is blank' if api_key.to_s.strip.empty?
|
|
65
|
-
|
|
66
|
-
@api_key = api_key
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
# @return [String] short provider label for {Engines} logging /
|
|
70
|
-
# fallback messages.
|
|
71
|
-
def label
|
|
72
|
-
'Exa'
|
|
73
|
-
end
|
|
74
|
-
|
|
75
|
-
# Fetch results for +query+ and return them as an +Array<Result>+.
|
|
76
|
-
# Calls are circuit-broken for 5 minutes on rate-limit / unavailable
|
|
77
|
-
# responses; see {LIMITER}. The caller (typically {Engines#search})
|
|
78
|
-
# is expected to have already normalized the query and to wrap this
|
|
79
|
-
# in a result cache.
|
|
80
|
-
#
|
|
81
|
-
# @param query [String] search query (already normalized)
|
|
82
|
-
# @param max_results [Integer] maximum number of result entries;
|
|
83
|
-
# passed through as Exa's +numResults+
|
|
84
|
-
# @return [Array<Result>] hits, possibly empty when Exa ran the
|
|
85
|
-
# query and matched nothing
|
|
86
|
-
# @raise [Engines::Unavailable] when Exa returns HTTP 429
|
|
87
|
-
# (rate limit / quota exhausted) or 5xx — "try again later"
|
|
88
|
-
# responses the cascade in {Engines#search} can fall back from.
|
|
89
|
-
# Also raised immediately if {LIMITER} is in cooldown. Other
|
|
90
|
-
# non-2xx (e.g. 401/403 from a bad API key) bubble up as
|
|
91
|
-
# +RuntimeError+ so config problems stay visible.
|
|
92
|
-
# @raise [RuntimeError] for non-rate-limit HTTP failures or when the
|
|
93
|
-
# response shape contains no results and isn't a recognized
|
|
94
|
-
# empty-results payload.
|
|
95
|
-
def search(query, max_results: DEFAULT_MAX_RESULTS)
|
|
96
|
-
LIMITER.call do
|
|
97
|
-
response = Faraday.post(ENDPOINT) do |req|
|
|
98
|
-
req.headers['x-api-key'] = @api_key
|
|
99
|
-
req.headers['Content-Type'] = 'application/json'
|
|
100
|
-
req.headers['Accept'] = 'application/json'
|
|
101
|
-
req.body = JSON.dump(
|
|
102
|
-
query: query,
|
|
103
|
-
type: 'auto',
|
|
104
|
-
numResults: max_results,
|
|
105
|
-
contents: { highlights: true }
|
|
106
|
-
)
|
|
107
|
-
end
|
|
108
|
-
unless response.success?
|
|
109
|
-
if response.status == 429 || response.status >= 500
|
|
110
|
-
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
111
|
-
end
|
|
112
|
-
|
|
113
|
-
raise "Exa Search request failed: #{response.status} #{response.body}"
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
self.class.parse(response.body, max_results: max_results)
|
|
117
|
-
end
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
# Parse an Exa Search JSON response into a list of {Result} rows,
|
|
121
|
-
# where +body+ is the first non-empty +highlights+ snippet (empty
|
|
122
|
-
# when Exa returned no highlight for that result — e.g. for
|
|
123
|
-
# navigational results).
|
|
124
|
-
#
|
|
125
|
-
# When the response yields zero result entries, two cases are
|
|
126
|
-
# distinguished: a genuine "no results" payload (response carries
|
|
127
|
-
# a +requestId+ and an empty +results+ array — Exa ran the query
|
|
128
|
-
# but matched nothing) returns an empty array instead of raising,
|
|
129
|
-
# so {Engines.search} can render its standard no-results stub.
|
|
130
|
-
# Anything else (unknown shape, structured error) raises with a
|
|
131
|
-
# diagnostic so the failure surfaces.
|
|
132
|
-
#
|
|
133
|
-
# @param json [String] response body from {ENDPOINT}
|
|
134
|
-
# @param max_results [Integer] maximum number of result entries
|
|
135
|
-
# @return [Array<Result>] hits, possibly empty on a recognized
|
|
136
|
-
# empty-results payload
|
|
137
|
-
# @raise [RuntimeError] when the response yields no result entries and
|
|
138
|
-
# is not recognized as a genuine empty-results payload
|
|
139
|
-
def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
|
|
140
|
-
data = JSON.parse(json)
|
|
141
|
-
results = Array(data['results']).take(max_results).filter_map do |r|
|
|
142
|
-
href = r['url'].to_s
|
|
143
|
-
next nil if href.empty?
|
|
144
|
-
|
|
145
|
-
Result.new(
|
|
146
|
-
url: href,
|
|
147
|
-
title: clean(r['title']) || href,
|
|
148
|
-
body: first_highlight(r['highlights'])
|
|
149
|
-
)
|
|
150
|
-
end
|
|
151
|
-
|
|
152
|
-
if results.empty?
|
|
153
|
-
return [] if genuine_no_results?(data)
|
|
154
|
-
|
|
155
|
-
raise diagnose_empty(data, json)
|
|
156
|
-
end
|
|
157
|
-
|
|
158
|
-
results
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
# Collapse whitespace and strip; returns +nil+ for nil/empty input
|
|
162
|
-
# so the caller can fall back (typically to the URL when a result
|
|
163
|
-
# has no usable title).
|
|
164
|
-
#
|
|
165
|
-
# @param text [String, nil] raw text from an Exa result field
|
|
166
|
-
# @return [String, nil] cleaned text, or +nil+ if input was blank
|
|
167
|
-
def self.clean(text)
|
|
168
|
-
return nil if text.nil?
|
|
169
|
-
|
|
170
|
-
cleaned = text.to_s.gsub(/\s+/, ' ').strip
|
|
171
|
-
cleaned.empty? ? nil : cleaned
|
|
172
|
-
end
|
|
173
|
-
private_class_method :clean
|
|
174
|
-
|
|
175
|
-
# First non-empty entry from a +highlights+ array, cleaned. Exa
|
|
176
|
-
# returns highlights as an array sorted by relevance; we surface
|
|
177
|
-
# only the top one to keep the observation compact and match the
|
|
178
|
-
# one-line +body+ convention used by Brave / DuckDuckGo.
|
|
179
|
-
#
|
|
180
|
-
# @param highlights [Array<String>, nil] +highlights+ field
|
|
181
|
-
# @return [String] cleaned snippet, or empty string if none usable
|
|
182
|
-
def self.first_highlight(highlights)
|
|
183
|
-
return '' unless highlights.is_a?(Array)
|
|
184
|
-
|
|
185
|
-
highlights.each do |h|
|
|
186
|
-
cleaned = clean(h)
|
|
187
|
-
return cleaned if cleaned
|
|
188
|
-
end
|
|
189
|
-
''
|
|
190
|
-
end
|
|
191
|
-
private_class_method :first_highlight
|
|
192
|
-
|
|
193
|
-
# True when a parsed response with zero +results+ entries looks
|
|
194
|
-
# like Exa's own "search ran, nothing matched" payload rather than
|
|
195
|
-
# a malformed or error response. The marker is the +requestId+
|
|
196
|
-
# field, which Exa always sets on a successful request.
|
|
197
|
-
#
|
|
198
|
-
# @param data [Hash, Object] parsed response
|
|
199
|
-
# @return [Boolean]
|
|
200
|
-
def self.genuine_no_results?(data)
|
|
201
|
-
return false unless data.is_a?(Hash)
|
|
202
|
-
return false unless data.key?('requestId')
|
|
203
|
-
|
|
204
|
-
Array(data['results']).empty?
|
|
205
|
-
end
|
|
206
|
-
private_class_method :genuine_no_results?
|
|
207
|
-
|
|
208
|
-
# Build an error message for a parsed response that yielded zero
|
|
209
|
-
# results. Quotes Exa's +error+ / +message+ / +detail+ field if
|
|
210
|
-
# present, otherwise truncates the raw body so the caller can see
|
|
211
|
-
# the actual payload.
|
|
212
|
-
#
|
|
213
|
-
# @param data [Hash, Object] parsed response
|
|
214
|
-
# @param raw [String] raw response body
|
|
215
|
-
# @return [String] human-readable diagnostic to feed to +raise+
|
|
216
|
-
def self.diagnose_empty(data, raw)
|
|
217
|
-
if data.is_a?(Hash) && (msg = data['error'] || data['message'] || data['detail'])
|
|
218
|
-
return "Exa Search returned an error: #{msg}"
|
|
219
|
-
end
|
|
220
|
-
|
|
221
|
-
snippet = raw.to_s[0, 800]
|
|
222
|
-
snippet += '…' if raw.to_s.length > 800
|
|
223
|
-
"Exa Search returned no results. Body: #{snippet}"
|
|
224
|
-
end
|
|
225
|
-
private_class_method :diagnose_empty
|
|
226
|
-
end
|
|
227
|
-
end
|
|
228
|
-
end
|
|
229
|
-
end
|