pikuri-core 0.0.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +17 -30
  3. data/lib/pikuri/agent/chat_transport.rb +168 -107
  4. data/lib/pikuri/agent/configurator.rb +194 -173
  5. data/lib/pikuri/agent/control/cancellable.rb +100 -70
  6. data/lib/pikuri/agent/control/handoff.rb +75 -0
  7. data/lib/pikuri/agent/control/interloper.rb +107 -118
  8. data/lib/pikuri/agent/control.rb +13 -35
  9. data/lib/pikuri/agent/event.rb +144 -197
  10. data/lib/pikuri/agent/extension.rb +83 -95
  11. data/lib/pikuri/agent/history.rb +247 -0
  12. data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
  13. data/lib/pikuri/agent/listener/terminal.rb +169 -136
  14. data/lib/pikuri/agent/listener/token_log.rb +84 -157
  15. data/lib/pikuri/agent/listener.rb +24 -44
  16. data/lib/pikuri/agent/listener_list.rb +32 -47
  17. data/lib/pikuri/agent/request_instrumenter.rb +102 -0
  18. data/lib/pikuri/agent.rb +1095 -577
  19. data/lib/pikuri/attachment.rb +65 -0
  20. data/lib/pikuri/bundler_env.rb +68 -0
  21. data/lib/pikuri/extractor/passthrough.rb +24 -31
  22. data/lib/pikuri/extractor.rb +125 -155
  23. data/lib/pikuri/file_type.rb +64 -136
  24. data/lib/pikuri/finalizers.rb +32 -47
  25. data/lib/pikuri/paths.rb +104 -13
  26. data/lib/pikuri/sanitizer.rb +46 -67
  27. data/lib/pikuri/subprocess.rb +86 -118
  28. data/lib/pikuri/testing.rb +369 -0
  29. data/lib/pikuri/tool/calculator.rb +168 -74
  30. data/lib/pikuri/tool/execute_context.rb +42 -0
  31. data/lib/pikuri/tool/parameters.rb +31 -29
  32. data/lib/pikuri/tool/result.rb +164 -0
  33. data/lib/pikuri/tool/trifecta_legs.rb +248 -0
  34. data/lib/pikuri/tool.rb +416 -67
  35. data/lib/pikuri/trifecta/contribution.rb +43 -0
  36. data/lib/pikuri/trifecta/node.rb +47 -0
  37. data/lib/pikuri/trifecta/report.rb +228 -0
  38. data/lib/pikuri/trifecta.rb +126 -0
  39. data/lib/pikuri/version.rb +1 -1
  40. data/lib/pikuri-core.rb +70 -92
  41. data/prompts/agent-loop.txt +5 -0
  42. data/prompts/pikuri-chat.txt +3 -12
  43. metadata +26 -72
  44. data/lib/pikuri/agent/context_window_detector.rb +0 -222
  45. data/lib/pikuri/agent/control/step_limit.rb +0 -111
  46. data/lib/pikuri/agent/extension_context.rb +0 -147
  47. data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
  48. data/lib/pikuri/agent/synthesizer.rb +0 -171
  49. data/lib/pikuri/extractor/html.rb +0 -303
  50. data/lib/pikuri/tool/fetch.rb +0 -172
  51. data/lib/pikuri/tool/scraper.rb +0 -186
  52. data/lib/pikuri/tool/search/brave.rb +0 -198
  53. data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
  54. data/lib/pikuri/tool/search/engines.rb +0 -186
  55. data/lib/pikuri/tool/search/exa.rb +0 -229
  56. data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
  57. data/lib/pikuri/tool/search/result.rb +0 -29
  58. data/lib/pikuri/tool/web_scrape.rb +0 -121
  59. data/lib/pikuri/tool/web_search.rb +0 -57
  60. data/lib/pikuri/url_cache.rb +0 -112
@@ -1,207 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'faraday'
4
- require 'nokogiri'
5
- require 'uri'
6
-
7
- module Pikuri
8
- class Tool
9
- module Search
10
- # Performs a DuckDuckGo search by scraping +html.duckduckgo.com+ and
11
- # returns the hits as a list of {Result} rows. Split into a thin HTTP
12
- # fetch (#search) and a pure parser (.parse) so tests can exercise
13
- # the parser against fixture HTML without hitting the network. The
14
- # cascade in {Engines#search} owns the final Markdown rendering.
15
- #
16
- # A class (constructed with no arguments) so it shares the uniform
17
- # provider shape with the keyed {Brave} / {Exa}: {Engines} holds a
18
- # list of provider *instances* and calls +#search+ / +#label+ on each
19
- # without caring which is which.
20
- #
21
- # == Privacy posture
22
- #
23
- # DuckDuckGo's privacy policy states +We don't save your IP address
24
- # or any unique identifiers alongside your searches+ and +We have
25
- # never sold any personal information+, and they proxy requests on
26
- # the user's behalf so downstream content providers can't build a
27
- # per-user search history. That part is real — but DDG is mainly a
28
- # relay over Bing for web results, so the *query content* still
29
- # reaches Microsoft for fulfillment even though DDG strips
30
- # identifying info on the way out.
31
- #
32
- # Bottom line: DDG is a genuine privacy improvement over hitting
33
- # Bing directly (your IP isn't tied to the query, no per-user
34
- # profile is built on DDG's side), but query content still lands at
35
- # Microsoft, who has no comparable no-training pledge. Better than
36
- # Exa for sensitive queries, worse than Brave; for anything
37
- # genuinely embarrassing, don't search the web at all.
38
- class DuckDuckGo
39
- # @return [String] HTML search endpoint
40
- ENDPOINT = 'https://html.duckduckgo.com/html/'
41
- # @return [String] User-Agent sent with each request; DDG often rejects
42
- # requests with no UA or an obvious bot UA
43
- USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
44
- '(KHTML, like Gecko) Chrome/120.0 Safari/537.36'
45
- # @return [Integer] default number of results returned, matching smolagents
46
- DEFAULT_MAX_RESULTS = 10
47
- # @return [RateLimiter] paces calls (DDG bans IPs that hammer the HTML
48
- # endpoint) and circuit-breaks on {Engines::Unavailable} so a
49
- # soft-block response doesn't get retried for the next 5 minutes
50
- LIMITER = RateLimiter.new(min_interval: 2.0, cooldown: 300.0)
51
-
52
- # @return [String] short provider label for {Engines} logging /
53
- # fallback messages. Uniform across providers (see {Brave#label}).
54
- def label
55
- 'DuckDuckGo'
56
- end
57
-
58
- # Fetch results for +query+ and return them as an +Array<Result>+.
59
- # Calls are throttled to one every 2s and circuit-broken for 5 minutes
60
- # after a soft-block; see {LIMITER}. The caller (typically
61
- # {Engines#search}) is expected to have already normalized the
62
- # query and to wrap this in a result cache.
63
- #
64
- # @param query [String] search query (already normalized)
65
- # @param max_results [Integer] maximum number of result entries
66
- # @return [Array<Result>] hits, possibly empty when DDG ran the
67
- # query and matched nothing
68
- # @raise [Engines::Unavailable] when DDG soft-blocks the IP
69
- # (anomaly/CAPTCHA page) or returns HTTP 429/5xx — i.e. "try again
70
- # later" responses the cascade in {Engines#search} can fall
71
- # back from. Also raised immediately if {LIMITER} is in cooldown.
72
- # @raise [RuntimeError] if the HTTP call fails for other reasons or
73
- # the empty-results page is in an unrecognized layout. A genuine
74
- # empty-results page is *not* an error; see {.parse}.
75
- def search(query, max_results: DEFAULT_MAX_RESULTS)
76
- LIMITER.call do
77
- response = Faraday.get(ENDPOINT, { q: query }, { 'User-Agent' => USER_AGENT })
78
- unless response.success?
79
- if response.status == 429 || response.status >= 500
80
- raise Engines::Unavailable, "HTTP #{response.status}"
81
- end
82
-
83
- raise "DuckDuckGo request failed: #{response.status} #{response.body}"
84
- end
85
-
86
- self.class.parse(response.body, max_results: max_results)
87
- end
88
- end
89
-
90
- # Parse a +html.duckduckgo.com+ result page into a list of {Result}
91
- # rows. +<b>+ highlights inside snippets are stripped.
92
- #
93
- # When the page has zero result nodes, two cases are distinguished:
94
- # a genuine "no results" page (narrow query, DDG's own "No results
95
- # found" indicator) returns an empty array instead of raising, so
96
- # {Engines.search} can render its standard no-results stub.
97
- # Anything else (anomaly modal, CAPTCHA, service-unavailable page,
98
- # unknown layout) raises with the diagnostic text extracted from
99
- # the body, so an IP soft-block is surfaced rather than silently
100
- # masquerading as an empty search.
101
- #
102
- # @param html [String] HTML document body from html.duckduckgo.com
103
- # @param max_results [Integer] maximum number of result entries
104
- # @return [Array<Result>] hits, possibly empty on a genuine
105
- # no-results page
106
- # @raise [Engines::Unavailable] when the page is the DDG
107
- # anomaly/CAPTCHA modal (IP soft-block) — a "try again later" the
108
- # cascade can fall back from.
109
- # @raise [RuntimeError] when the page contains no result nodes and is
110
- # not recognized as either a genuine no-results page or the
111
- # anomaly modal (likely a layout change worth surfacing loudly).
112
- def self.parse(html, max_results: DEFAULT_MAX_RESULTS)
113
- doc = Nokogiri::HTML(html)
114
- results = doc.css('div.result.web-result').take(max_results).filter_map do |node|
115
- title_link = node.at_css('a.result__a')
116
- next nil if title_link.nil?
117
-
118
- snippet = node.at_css('a.result__snippet')
119
- Result.new(
120
- url: extract_url(title_link['href']),
121
- title: title_link.text.strip,
122
- body: snippet&.text&.strip.to_s
123
- )
124
- end
125
-
126
- if results.empty?
127
- return [] if genuine_no_results?(doc)
128
-
129
- message = diagnose_empty(doc)
130
- raise(anomaly_modal?(doc) ? Engines::Unavailable : RuntimeError, message)
131
- end
132
-
133
- results
134
- end
135
-
136
- # True when the page contains DDG's anomaly/CAPTCHA modal — i.e. the
137
- # IP has been soft-blocked. Used by {.parse} to pick between
138
- # {Engines::Unavailable} (recoverable, fall back to another
139
- # provider) and {RuntimeError} (unknown layout, surface loudly).
140
- #
141
- # @param doc [Nokogiri::HTML::Document] parsed result page
142
- # @return [Boolean]
143
- def self.anomaly_modal?(doc)
144
- !!(doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description'))
145
- end
146
- private_class_method :anomaly_modal?
147
-
148
- # True when a results page with zero result nodes looks like DDG's own
149
- # "no results found" page (narrow query) rather than an anomaly/CAPTCHA
150
- # or other non-results layout. Anomaly modal wins: if the modal divs
151
- # are present we never treat the page as a genuine empty result, even
152
- # if the surrounding text happens to mention "No results".
153
- #
154
- # @param doc [Nokogiri::HTML::Document] parsed result page
155
- # @return [Boolean]
156
- def self.genuine_no_results?(doc)
157
- return false if doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description')
158
- return true if doc.at_css('div.no-results')
159
-
160
- doc.text.include?('No results found')
161
- end
162
- private_class_method :genuine_no_results?
163
-
164
- # Build an error message for a results page that yielded zero matches.
165
- # Recognizes the DDG anomaly modal explicitly and quotes its title and
166
- # description; otherwise extracts visible text from the body (with
167
- # +<script>+/+<style>+/+<noscript>+ stripped and whitespace collapsed)
168
- # and includes a truncated copy so the caller can see what came back.
169
- #
170
- # @param doc [Nokogiri::HTML::Document] parsed result page
171
- # @return [String] human-readable diagnostic to feed to +raise+
172
- def self.diagnose_empty(doc)
173
- title = doc.at_css('.anomaly-modal__title')&.text&.strip
174
- desc = doc.at_css('.anomaly-modal__description')&.text&.strip
175
- if title || desc
176
- return "DuckDuckGo anomaly check (likely IP soft-block): #{[title, desc].compact.reject(&:empty?).join(' — ')}"
177
- end
178
-
179
- doc.css('script, style, noscript').remove
180
- text = doc.text.gsub(/\s+/, ' ').strip
181
- snippet = text.empty? ? '<empty body>' : text[0, 1500]
182
- snippet += '…' if text.length > 1500
183
- "DuckDuckGo returned no results. Page text: #{snippet}"
184
- end
185
- private_class_method :diagnose_empty
186
-
187
- # Decode DuckDuckGo's +//duckduckgo.com/l/?uddg=<encoded>+ redirect wrapper
188
- # back to the real target URL.
189
- #
190
- # @param href [String, nil] href as found on the search-result page
191
- # @return [String] the decoded target URL, or +href+ unchanged when it is
192
- # not a recognized DDG redirect or cannot be parsed
193
- def self.extract_url(href)
194
- return href if href.nil? || href.empty?
195
-
196
- uri = URI.parse(href.start_with?('//') ? "https:#{href}" : href)
197
- return href unless uri.host&.end_with?('duckduckgo.com') && uri.path == '/l/'
198
-
199
- params = URI.decode_www_form(uri.query.to_s).to_h
200
- params['uddg'] || href
201
- rescue URI::InvalidURIError
202
- href
203
- end
204
- end
205
- end
206
- end
207
- end
@@ -1,186 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Pikuri
4
- class Tool
5
- # Namespace for the web-search stack used by {Tool::WebSearch}: per-
6
- # provider modules ({DuckDuckGo}, {Brave}, {Exa}), the {Result} value
7
- # object they all return, the cross-provider {Engines} cascade with
8
- # its on-disk cache, and the shared {RateLimiter} a provider can wire
9
- # in to back off when a quota header says so.
10
- module Search
11
- # Search-orchestration object: the cascade across configured
12
- # providers, the result cache, and the {Unavailable} protocol marker
13
- # the cascade uses to fall back. The LLM-facing tool itself is built
14
- # by {Tool::WebSearch.build}, which constructs one of these and wires
15
- # its {#search} into a {Tool}. Each {Tool::Search} provider module
16
- # ({DuckDuckGo}, {Brave}, {Exa}) raises {Unavailable} when it wants
17
- # the cascade to try the next one.
18
- #
19
- # == Provider keys are constructor config, not environment
20
- #
21
- # Brave and Exa are paid and need an API key; DuckDuckGo needs none.
22
- # An {Engines} is constructed with the keys it should use
23
- # (+brave_key:+ / +exa_key:+, both optional) — pikuri reads no key
24
- # from the environment, so the only providers in the cascade are
25
- # DuckDuckGo plus whichever keyed providers the host actually
26
- # configured. The host sources those keys however it likes (the
27
- # bundled +bin/+ examples load a JSON config file by convention); see
28
- # CLAUDE.md "Environment is not a secret store".
29
- class Engines
30
- # Subsystem logger; set its level with +PIKURI_LOG_ENGINES+
31
- # (e.g. +PIKURI_LOG_ENGINES=debug+) or the global +PIKURI_LOG+.
32
- #
33
- # @return [Logger]
34
- LOGGER = Pikuri.logger_for('Engines')
35
-
36
- # Raised by a provider when it is temporarily unavailable (rate-limited,
37
- # bot-blocked, quota-exhausted, or otherwise saying "try again later"
38
- # rather than "your request is wrong"). The cascade in {#search}
39
- # catches this and tries the next provider; any other exception bubbles
40
- # up unchanged so genuine bugs and config errors stay visible.
41
- class Unavailable < StandardError; end
42
-
43
- # Process-shared on-disk cache backing {#search}'s default. Kept at
44
- # class level (not per-instance) so every engine dedupes answered
45
- # queries into one directory; the constructor's +cache:+ parameter
46
- # injects a different store for tests. Exposed as a method so specs
47
- # can swap it for {UrlCache::NULL} without touching the instance.
48
- #
49
- # @return [UrlCache, #fetch]
50
- CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_search")
51
- # Accessor for {CACHE}, used as the constructor's +cache:+ default;
52
- # specs override this to swap in {UrlCache::NULL}.
53
- #
54
- # @return [UrlCache, #fetch]
55
- def self.cache
56
- CACHE
57
- end
58
-
59
- # Builds the provider cascade once: {DuckDuckGo} always (no key
60
- # needed), plus {Brave} / {Exa} when their key was supplied
61
- # (non-blank). Each keyed provider is constructed with its key, so
62
- # from here on every provider is just an object answering +#search+
63
- # / +#label+ — the cascade in {#search} treats them uniformly.
64
- #
65
- # @param brave_key [String, nil] Brave Search subscription token;
66
- # non-blank ⇒ Brave joins the cascade. +nil+/blank ⇒ not configured.
67
- # @param exa_key [String, nil] Exa API key; non-blank ⇒ Exa joins the
68
- # cascade. +nil+/blank ⇒ not configured.
69
- # @param cache [UrlCache, #fetch] result store memoizing answered
70
- # queries; defaults to the process-shared {.cache}.
71
- # @return [Engines]
72
- def initialize(brave_key: nil, exa_key: nil, cache: self.class.cache)
73
- @providers = [DuckDuckGo.new]
74
- @providers << Brave.new(api_key: brave_key) unless brave_key.to_s.strip.empty?
75
- @providers << Exa.new(api_key: exa_key) unless exa_key.to_s.strip.empty?
76
- @cache = cache
77
- @last_logged_providers = nil
78
- end
79
-
80
- # The provider instances this engine cascades across, in
81
- # declaration order (the cascade itself shuffles them per call).
82
- #
83
- # @return [Array<#search, #label>] configured provider instances
84
- attr_reader :providers
85
-
86
- # Run +query+ through the configured providers in random order, falling
87
- # back to the next one each time a provider raises {Unavailable}. The
88
- # shuffle spreads load so a single provider isn't always hit first
89
- # (and exhausted first); revisit if it stops being the right default.
90
- #
91
- # The query is whitespace-trimmed and runs of whitespace collapsed
92
- # to a single space before the cascade runs. The winning provider's
93
- # +Array<Result>+ is rendered into smolagents-style Markdown here
94
- # (+"## Search Results"+ header, then +[title](url)\nbody+ entries
95
- # joined by blank lines; an empty array becomes +"No results found."+),
96
- # and the rendered Markdown is cached on disk via {#initialize}'s
97
- # +cache:+, keyed by the cleaned query. A cache hit short-circuits the
98
- # cascade entirely (and benefits whichever provider would have
99
- # answered next time too — once a query is cached, the cooldown state
100
- # of the original answering provider no longer matters). +max_results+
101
- # is not part of the cache key, so callers passing a non-default value
102
- # may get a result rendered with the previously-cached size.
103
- #
104
- # If every provider reports temporary unavailability, returns an
105
- # +"Error: ..."+ string instead of raising — same convention as
106
- # {Tool::Calculator.calculate}, so the agent loop can feed the failure
107
- # back to the model as the next observation. Any non-{Unavailable}
108
- # exception (network error, parser failure, malformed response, bad
109
- # API key) bubbles up to the caller.
110
- #
111
- # @param query [String] search query
112
- # @param max_results [Integer] maximum number of result entries
113
- # @return [String] Markdown-formatted result list, or +"Error: ..."+
114
- # when all providers are exhausted
115
- # @raise [ArgumentError] if the query is empty after normalization
116
- def search(query, max_results:)
117
- cleaned = query.to_s.strip.gsub(/\s+/, ' ')
118
- raise ArgumentError, 'query is empty' if cleaned.empty?
119
-
120
- current_providers = providers
121
- log_providers(current_providers)
122
-
123
- hit = true
124
- result = @cache.fetch(cleaned) do
125
- hit = false
126
- failures = []
127
- results = nil
128
- chosen = nil
129
- current_providers.shuffle.each do |provider|
130
- results = provider.search(cleaned, max_results: max_results)
131
- chosen = provider
132
- break
133
- rescue Unavailable => e
134
- failures << "#{provider.label} (#{e.message})"
135
- end
136
- # Raise so {UrlCache#fetch} does NOT persist the all-unavailable
137
- # message — otherwise that string would block every future search
138
- # for this query until the TTL expires. The outer +rescue+ turns
139
- # the raise back into the calculator-style "Error: …" string.
140
- chosen or raise Unavailable, "all search providers temporarily unavailable: #{failures.join('; ')}"
141
-
142
- LOGGER.info do
143
- "engine=#{chosen.label} query=#{cleaned.inspect} results=#{results.size}"
144
- end
145
- render(results)
146
- end
147
- LOGGER.info { "cache=hit query=#{cleaned.inspect} bytes=#{result.bytesize}" } if hit
148
- result
149
- rescue Unavailable => e
150
- "Error: #{e.message}"
151
- end
152
-
153
- private
154
-
155
- # Render an +Array<Result>+ into the smolagents-style Markdown the
156
- # LLM consumes: +"## Search Results"+ header, then +[title](url)\nbody+
157
- # entries joined by blank lines. An empty array becomes the
158
- # +"No results found."+ stub so the agent still gets a real
159
- # observation to act on.
160
- #
161
- # @param results [Array<Result>] hits from the winning provider
162
- # @return [String] Markdown-formatted result list
163
- def render(results)
164
- return "## Search Results\n\nNo results found." if results.empty?
165
-
166
- "## Search Results\n\n" + results.map { |r| "[#{r.title}](#{r.url})\n#{r.body}" }.join("\n\n")
167
- end
168
-
169
- # Emit an INFO log line listing the currently-available providers,
170
- # but only when the set differs from the last one this engine
171
- # logged. The memo keeps the log to one line per distinct
172
- # configuration rather than one per search.
173
- #
174
- # @param current [Array<#label>] providers returned by {#providers}
175
- # @return [void]
176
- def log_providers(current)
177
- return if @last_logged_providers == current
178
-
179
- @last_logged_providers = current
180
- names = current.map(&:label).join(', ')
181
- LOGGER.info("engines available: #{names}")
182
- end
183
- end
184
- end
185
- end
186
- end
@@ -1,229 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'faraday'
4
- require 'json'
5
-
6
- module Pikuri
7
- class Tool
8
- module Search
9
- # Performs an Exa search via the official +/search+ endpoint and
10
- # returns the hits as a list of {Result} rows. Split into a thin HTTP
11
- # fetch (#search) and a pure parser (.parse) so tests can exercise
12
- # the parser against fixture JSON without hitting the network. The
13
- # cascade in {Engines#search} owns the final Markdown rendering.
14
- #
15
- # A class constructed with the API key it should use
16
- # (+Exa.new(api_key:)+); {Engines} builds one only when an Exa key was
17
- # configured, so users who haven't registered never spend money on it,
18
- # and then drives it through the same +#search+ / +#label+ interface
19
- # as every other provider. pikuri reads no key from the environment
20
- # (see CLAUDE.md "Environment is not a secret store"). Get a key at
21
- # https://exa.ai — the service is paid.
22
- #
23
- # Calls request +type: "auto"+ (Exa picks neural vs keyword per
24
- # query) and +contents: { highlights: true }+ so each result carries
25
- # a short neural-ranked snippet — the closest analog to Brave's
26
- # +description+ field, populating {Result#body} consistently across
27
- # providers.
28
- #
29
- # == Privacy posture
30
- #
31
- # Exa's Privacy Policy states +Query Data is used to improve our
32
- # products and technology, including by training and fine-tuning
33
- # models that power our Services+, and the Terms of Service §1.2(c)
34
- # grant Exa a +perpetual and irrevocable+, +sub-licensable+,
35
- # worldwide license over User Input that can be disclosed to third
36
- # parties +as needed+. Business customers under a Master Subscription
37
- # Agreement / DPA get carve-outs; the default pay-as-you-go API key
38
- # (which is what pikuri uses) does not.
39
- #
40
- # Bottom line: Exa does not sell queries to data brokers, but it
41
- # does mine them to train competing models, and the license it
42
- # claims is effectively "do what we want with this, forever". If a
43
- # query would be embarrassing or sensitive in a training set, simply
44
- # don't configure an Exa key — {Engines#providers} leaves Exa out of
45
- # the cascade unless its key was supplied to the constructor.
46
- class Exa
47
- # @return [String] Search endpoint (POST, JSON body)
48
- ENDPOINT = 'https://api.exa.ai/search'
49
- # @return [Integer] default number of results returned, matching
50
- # {DuckDuckGo::DEFAULT_MAX_RESULTS}
51
- DEFAULT_MAX_RESULTS = 10
52
- # @return [RateLimiter] Exa is paid and doesn't aggressively
53
- # throttle, so no minimum interval is enforced. The 5-minute
54
- # cooldown still applies on {Engines::Unavailable} so the user's
55
- # budget isn't burned on doomed retries while a 429 / 5xx
56
- # condition persists.
57
- LIMITER = RateLimiter.new(min_interval: 0.0, cooldown: 300.0)
58
-
59
- # @param api_key [String] Exa API key. Required and non-blank:
60
- # pikuri reads no key from the environment — the host supplies it
61
- # ({Engines} only constructs an Exa when a key was configured).
62
- # @raise [ArgumentError] if +api_key+ is blank
63
- def initialize(api_key:)
64
- raise ArgumentError, 'Exa Search API key is blank' if api_key.to_s.strip.empty?
65
-
66
- @api_key = api_key
67
- end
68
-
69
- # @return [String] short provider label for {Engines} logging /
70
- # fallback messages.
71
- def label
72
- 'Exa'
73
- end
74
-
75
- # Fetch results for +query+ and return them as an +Array<Result>+.
76
- # Calls are circuit-broken for 5 minutes on rate-limit / unavailable
77
- # responses; see {LIMITER}. The caller (typically {Engines#search})
78
- # is expected to have already normalized the query and to wrap this
79
- # in a result cache.
80
- #
81
- # @param query [String] search query (already normalized)
82
- # @param max_results [Integer] maximum number of result entries;
83
- # passed through as Exa's +numResults+
84
- # @return [Array<Result>] hits, possibly empty when Exa ran the
85
- # query and matched nothing
86
- # @raise [Engines::Unavailable] when Exa returns HTTP 429
87
- # (rate limit / quota exhausted) or 5xx — "try again later"
88
- # responses the cascade in {Engines#search} can fall back from.
89
- # Also raised immediately if {LIMITER} is in cooldown. Other
90
- # non-2xx (e.g. 401/403 from a bad API key) bubble up as
91
- # +RuntimeError+ so config problems stay visible.
92
- # @raise [RuntimeError] for non-rate-limit HTTP failures or when the
93
- # response shape contains no results and isn't a recognized
94
- # empty-results payload.
95
- def search(query, max_results: DEFAULT_MAX_RESULTS)
96
- LIMITER.call do
97
- response = Faraday.post(ENDPOINT) do |req|
98
- req.headers['x-api-key'] = @api_key
99
- req.headers['Content-Type'] = 'application/json'
100
- req.headers['Accept'] = 'application/json'
101
- req.body = JSON.dump(
102
- query: query,
103
- type: 'auto',
104
- numResults: max_results,
105
- contents: { highlights: true }
106
- )
107
- end
108
- unless response.success?
109
- if response.status == 429 || response.status >= 500
110
- raise Engines::Unavailable, "HTTP #{response.status}"
111
- end
112
-
113
- raise "Exa Search request failed: #{response.status} #{response.body}"
114
- end
115
-
116
- self.class.parse(response.body, max_results: max_results)
117
- end
118
- end
119
-
120
- # Parse an Exa Search JSON response into a list of {Result} rows,
121
- # where +body+ is the first non-empty +highlights+ snippet (empty
122
- # when Exa returned no highlight for that result — e.g. for
123
- # navigational results).
124
- #
125
- # When the response yields zero result entries, two cases are
126
- # distinguished: a genuine "no results" payload (response carries
127
- # a +requestId+ and an empty +results+ array — Exa ran the query
128
- # but matched nothing) returns an empty array instead of raising,
129
- # so {Engines.search} can render its standard no-results stub.
130
- # Anything else (unknown shape, structured error) raises with a
131
- # diagnostic so the failure surfaces.
132
- #
133
- # @param json [String] response body from {ENDPOINT}
134
- # @param max_results [Integer] maximum number of result entries
135
- # @return [Array<Result>] hits, possibly empty on a recognized
136
- # empty-results payload
137
- # @raise [RuntimeError] when the response yields no result entries and
138
- # is not recognized as a genuine empty-results payload
139
- def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
140
- data = JSON.parse(json)
141
- results = Array(data['results']).take(max_results).filter_map do |r|
142
- href = r['url'].to_s
143
- next nil if href.empty?
144
-
145
- Result.new(
146
- url: href,
147
- title: clean(r['title']) || href,
148
- body: first_highlight(r['highlights'])
149
- )
150
- end
151
-
152
- if results.empty?
153
- return [] if genuine_no_results?(data)
154
-
155
- raise diagnose_empty(data, json)
156
- end
157
-
158
- results
159
- end
160
-
161
- # Collapse whitespace and strip; returns +nil+ for nil/empty input
162
- # so the caller can fall back (typically to the URL when a result
163
- # has no usable title).
164
- #
165
- # @param text [String, nil] raw text from an Exa result field
166
- # @return [String, nil] cleaned text, or +nil+ if input was blank
167
- def self.clean(text)
168
- return nil if text.nil?
169
-
170
- cleaned = text.to_s.gsub(/\s+/, ' ').strip
171
- cleaned.empty? ? nil : cleaned
172
- end
173
- private_class_method :clean
174
-
175
- # First non-empty entry from a +highlights+ array, cleaned. Exa
176
- # returns highlights as an array sorted by relevance; we surface
177
- # only the top one to keep the observation compact and match the
178
- # one-line +body+ convention used by Brave / DuckDuckGo.
179
- #
180
- # @param highlights [Array<String>, nil] +highlights+ field
181
- # @return [String] cleaned snippet, or empty string if none usable
182
- def self.first_highlight(highlights)
183
- return '' unless highlights.is_a?(Array)
184
-
185
- highlights.each do |h|
186
- cleaned = clean(h)
187
- return cleaned if cleaned
188
- end
189
- ''
190
- end
191
- private_class_method :first_highlight
192
-
193
- # True when a parsed response with zero +results+ entries looks
194
- # like Exa's own "search ran, nothing matched" payload rather than
195
- # a malformed or error response. The marker is the +requestId+
196
- # field, which Exa always sets on a successful request.
197
- #
198
- # @param data [Hash, Object] parsed response
199
- # @return [Boolean]
200
- def self.genuine_no_results?(data)
201
- return false unless data.is_a?(Hash)
202
- return false unless data.key?('requestId')
203
-
204
- Array(data['results']).empty?
205
- end
206
- private_class_method :genuine_no_results?
207
-
208
- # Build an error message for a parsed response that yielded zero
209
- # results. Quotes Exa's +error+ / +message+ / +detail+ field if
210
- # present, otherwise truncates the raw body so the caller can see
211
- # the actual payload.
212
- #
213
- # @param data [Hash, Object] parsed response
214
- # @param raw [String] raw response body
215
- # @return [String] human-readable diagnostic to feed to +raise+
216
- def self.diagnose_empty(data, raw)
217
- if data.is_a?(Hash) && (msg = data['error'] || data['message'] || data['detail'])
218
- return "Exa Search returned an error: #{msg}"
219
- end
220
-
221
- snippet = raw.to_s[0, 800]
222
- snippet += '…' if raw.to_s.length > 800
223
- "Exa Search returned no results. Body: #{snippet}"
224
- end
225
- private_class_method :diagnose_empty
226
- end
227
- end
228
- end
229
- end