pikuri-core 0.0.7 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +17 -30
  3. data/lib/pikuri/agent/chat_transport.rb +168 -107
  4. data/lib/pikuri/agent/configurator.rb +194 -173
  5. data/lib/pikuri/agent/control/cancellable.rb +100 -70
  6. data/lib/pikuri/agent/control/handoff.rb +75 -0
  7. data/lib/pikuri/agent/control/interloper.rb +107 -118
  8. data/lib/pikuri/agent/control.rb +13 -35
  9. data/lib/pikuri/agent/event.rb +144 -197
  10. data/lib/pikuri/agent/extension.rb +83 -95
  11. data/lib/pikuri/agent/history.rb +247 -0
  12. data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
  13. data/lib/pikuri/agent/listener/terminal.rb +169 -136
  14. data/lib/pikuri/agent/listener/token_log.rb +84 -157
  15. data/lib/pikuri/agent/listener.rb +24 -44
  16. data/lib/pikuri/agent/listener_list.rb +32 -47
  17. data/lib/pikuri/agent/request_instrumenter.rb +102 -0
  18. data/lib/pikuri/agent.rb +1095 -577
  19. data/lib/pikuri/attachment.rb +65 -0
  20. data/lib/pikuri/bundler_env.rb +68 -0
  21. data/lib/pikuri/extractor/passthrough.rb +24 -31
  22. data/lib/pikuri/extractor.rb +125 -155
  23. data/lib/pikuri/file_type.rb +64 -136
  24. data/lib/pikuri/finalizers.rb +32 -47
  25. data/lib/pikuri/paths.rb +104 -13
  26. data/lib/pikuri/sanitizer.rb +46 -67
  27. data/lib/pikuri/subprocess.rb +86 -118
  28. data/lib/pikuri/testing.rb +369 -0
  29. data/lib/pikuri/tool/calculator.rb +168 -74
  30. data/lib/pikuri/tool/execute_context.rb +42 -0
  31. data/lib/pikuri/tool/parameters.rb +31 -29
  32. data/lib/pikuri/tool/result.rb +164 -0
  33. data/lib/pikuri/tool/trifecta_legs.rb +248 -0
  34. data/lib/pikuri/tool.rb +416 -67
  35. data/lib/pikuri/trifecta/contribution.rb +43 -0
  36. data/lib/pikuri/trifecta/node.rb +47 -0
  37. data/lib/pikuri/trifecta/report.rb +228 -0
  38. data/lib/pikuri/trifecta.rb +126 -0
  39. data/lib/pikuri/version.rb +1 -1
  40. data/lib/pikuri-core.rb +70 -92
  41. data/prompts/agent-loop.txt +5 -0
  42. data/prompts/pikuri-chat.txt +3 -12
  43. metadata +26 -72
  44. data/lib/pikuri/agent/context_window_detector.rb +0 -222
  45. data/lib/pikuri/agent/control/step_limit.rb +0 -111
  46. data/lib/pikuri/agent/extension_context.rb +0 -147
  47. data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
  48. data/lib/pikuri/agent/synthesizer.rb +0 -171
  49. data/lib/pikuri/extractor/html.rb +0 -303
  50. data/lib/pikuri/tool/fetch.rb +0 -172
  51. data/lib/pikuri/tool/scraper.rb +0 -186
  52. data/lib/pikuri/tool/search/brave.rb +0 -198
  53. data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
  54. data/lib/pikuri/tool/search/engines.rb +0 -186
  55. data/lib/pikuri/tool/search/exa.rb +0 -229
  56. data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
  57. data/lib/pikuri/tool/search/result.rb +0 -29
  58. data/lib/pikuri/tool/web_scrape.rb +0 -121
  59. data/lib/pikuri/tool/web_search.rb +0 -57
  60. data/lib/pikuri/url_cache.rb +0 -112
@@ -1,172 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Pikuri
4
- class Tool
5
- # Truncation policy and Tool spec for the +fetch+ tool. The HTTP work
6
- # lives in {Tool::Scraper.fetch}; this module is a thin
7
- # wrapper that accepts only textual content-types, applies a character
8
- # cap so the LLM doesn't drown in long-form bodies, and exposes the
9
- # result to the agent loop in OpenAI tool-call shape.
10
- #
11
- # Sister of {Tool::WebScrape}, but with no extraction pass
12
- # (HTML→Markdown, or whatever plug-in extractors are registered):
13
- # bodies are returned verbatim. Useful for raw textual
14
- # data — JSON APIs, CSV files, +robots.txt+, sitemaps, source files —
15
- # where any rendering pass would corrupt the payload.
16
- module Fetch
17
- # @return [Integer] default character cap on the body returned by
18
- # {.fetch}. Smaller than {Tool::WebScrape::DEFAULT_MAX_CHARS}
19
- # because fetch's content profile is bimodal — most JSON/XML/CSV
20
- # responses are tiny, and the long-tail (large data dumps) is
21
- # better re-requested deliberately than padded into every default.
22
- DEFAULT_MAX_CHARS = 5_000
23
-
24
- # @return [Integer] hard ceiling on the +max_chars+ argument to
25
- # {.fetch}. Matches {Tool::WebScrape::MAX_MAX_CHARS}.
26
- MAX_MAX_CHARS = 100_000
27
-
28
- # Application content-types that are textual in practice and so
29
- # safe to return verbatim to the LLM, despite their +application/+
30
- # prefix making them fail the +text/*+ check. Anything outside
31
- # +text/*+ and this allowlist is refused.
32
- # @return [Array<String>]
33
- TEXTUAL_APPLICATION_TYPES = %w[
34
- application/json
35
- application/xml
36
- application/javascript
37
- application/xhtml+xml
38
- application/rss+xml
39
- application/atom+xml
40
- ].freeze
41
-
42
- # On-disk cache used by {.fetch} to memoize downloads. Defined as a
43
- # method so specs can swap it for an isolated cache or
44
- # {UrlCache::NULL} without touching the shared instance. Lives in
45
- # its own subdir under {UrlCache::ROOT_DIR} so a +fetch+ on a URL
46
- # and a +web_scrape+ on the same URL cannot collide on the same
47
- # cache file (one returns the raw body, the other returns extracted
48
- # Markdown).
49
- #
50
- # @return [UrlCache, #fetch]
51
- CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/fetch")
52
- # Accessor for {CACHE}; specs override this to swap in
53
- # {UrlCache::NULL} or an isolated cache.
54
- #
55
- # @return [UrlCache, #fetch]
56
- def self.cache
57
- CACHE
58
- end
59
-
60
- # Download +url+ via {Tool::Scraper.fetch} and return the
61
- # response body verbatim, provided the content-type is one we deem
62
- # textual (any +text/*+, plus the formats listed in
63
- # {TEXTUAL_APPLICATION_TYPES}). Anything else — PDFs, images, other
64
- # binaries — produces an +"Error: ..."+ string in the calculator-
65
- # style convention so the agent loop feeds the failure back to the
66
- # model as the next observation.
67
- #
68
- # The body is cached on disk via {.cache}, keyed by URL, so repeat
69
- # fetches within the cache TTL skip the network. +max_chars+ is not
70
- # part of the cache key — different values for the same URL share
71
- # one entry, and truncation runs after the cache lookup. The cache
72
- # is only populated on success: {Scraper::FetchError} (HTTP non-2xx,
73
- # network failure, redirect-loop exhaustion, refused content-type)
74
- # is caught outside the +cache.fetch+ block, so failure strings are
75
- # never persisted and a retry on the next call hits the network
76
- # again. Other exceptions (parser bugs in our own code) bubble up
77
- # unchanged.
78
- #
79
- # @param url [String] absolute HTTP(S) URL to download
80
- # @param max_chars [Integer] character cap on the returned body.
81
- # Clamped to +[1, {MAX_MAX_CHARS}]+; defaults to
82
- # {DEFAULT_MAX_CHARS}. When the body exceeds the cap, output is
83
- # cut and a marker noting the original length is appended.
84
- # @return [String] response body, possibly truncated, or
85
- # +"Error: ..."+ on a recoverable failure
86
- def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
87
- max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
88
- body = cache.fetch(url) { download(url) }
89
- truncate(body, max_chars)
90
- rescue Scraper::FetchError => e
91
- "Error: #{e.message}"
92
- end
93
-
94
- # GET +url+ and verify the response's content-type is textual.
95
- # Caller is responsible for caching and truncation; this method
96
- # always hits the network.
97
- #
98
- # @param url [String]
99
- # @return [String] response body
100
- # @raise [Scraper::FetchError] on HTTP non-2xx, network failure,
101
- # redirect-loop exhaustion, missing +Location+ on a 3xx, or a
102
- # non-textual content-type
103
- def self.download(url)
104
- fetched = Scraper.fetch(url)
105
- return fetched.body if textual?(fetched.content_type)
106
-
107
- raise Scraper::FetchError,
108
- "refused to fetch #{url}: content-type #{fetched.content_type.inspect} " \
109
- 'is not textual (use web_scrape for rendered pages)'
110
- end
111
-
112
- # @param content_type [String] normalized content-type (no +charset+
113
- # parameter, lowercased) as produced by {Scraper.fetch}
114
- # @return [Boolean] true when the content-type is +text/*+ or one
115
- # of {TEXTUAL_APPLICATION_TYPES}
116
- def self.textual?(content_type)
117
- content_type.start_with?('text/') ||
118
- TEXTUAL_APPLICATION_TYPES.include?(content_type)
119
- end
120
-
121
- # Cut +body+ to at most +max_chars+ characters, appending a marker
122
- # describing the original length when truncation actually happens.
123
- # Returns +body+ unchanged if it already fits. Same shape as
124
- # {Tool::WebScrape.truncate} so the LLM sees a consistent
125
- # truncation marker across both tools.
126
- #
127
- # @param body [String] full response body
128
- # @param max_chars [Integer] character cap; assumed already clamped
129
- # @return [String]
130
- def self.truncate(body, max_chars)
131
- return body if body.length <= max_chars
132
-
133
- "#{body[0, max_chars]}\n\n" \
134
- "... [truncated at #{max_chars} of #{body.length} chars; " \
135
- 'call again with a larger `max_chars` to see more]'
136
- end
137
- end
138
-
139
- # Verbatim URL download tool. Thin wrapper over {Tool::Fetch.fetch}
140
- # that exposes it to the agent loop in OpenAI tool-call shape. Use for
141
- # raw textual payloads (JSON APIs, CSV files, +robots.txt+, source
142
- # files); use {Tool::WEB_SCRAPE} for rendered web pages where
143
- # readability extraction makes the result usable.
144
- #
145
- # @return [Tool]
146
- FETCH = new(
147
- name: 'fetch',
148
- description: <<~DESC,
149
- Downloads the given URL and returns its body verbatim.
150
-
151
- Usage:
152
- - Use for raw textual payloads: JSON APIs, CSV files, robots.txt, sitemaps, source files — anywhere a rendering pass would corrupt the data.
153
- - For rendered HTML pages, use web_scrape — it extracts readable content; fetch returns the raw HTML bytes unchanged.
154
- - Accepts text/* and common textual application/* types (JSON, XML, JS, XHTML, RSS, Atom). Refuses PDFs, images, and other binaries.
155
- DESC
156
- parameters: Parameters.build { |p|
157
- p.required_string :url,
158
- 'Absolute URL to download, including the scheme, ' \
159
- 'e.g. "https://example.com/data.json".'
160
- p.optional_integer :max_chars,
161
- 'Maximum number of characters of the body to ' \
162
- 'return. Defaults to 5000; hard-capped at ' \
163
- '100000. When the body is longer than this, ' \
164
- 'output is cut and a marker reports the full ' \
165
- 'length.'
166
- },
167
- execute: ->(url:, max_chars: Fetch::DEFAULT_MAX_CHARS) {
168
- Fetch.fetch(url, max_chars: max_chars)
169
- }
170
- )
171
- end
172
- end
@@ -1,186 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'faraday'
4
- require 'stringio'
5
- require 'uri'
6
-
7
- module Pikuri
8
- class Tool
9
- # HTTP side of the web tools ({Tool::WEB_SCRAPE} and {Tool::FETCH}):
10
- # GET the URL with a real-browser User-Agent, follow redirects, and
11
- # hand the response body to {Pikuri::Extractor.extract} with the
12
- # response's +Content-Type+ as the hint. HTML/XHTML render via
13
- # {Extractor::HTML}, any other +text/*+ type passes through
14
- # verbatim, and plug-in extractors extend the set (with pikuri-pdf
15
- # registered, +application/pdf+ extracts — by header or by +%PDF-+
16
- # magic, so a PDF served under a lying header still works); the
17
- # remaining types raise {FetchError} so the LLM observes the
18
- # failure instead of receiving an empty rendering.
19
- #
20
- # Split into a thin HTTP fetch ({.fetch}) and the extraction
21
- # wrapper ({.visit}) so tests can drive each piece in isolation and
22
- # {Tool::Fetch} can reuse the HTTP half without the extraction
23
- # pass. Nothing here knows about the LLM; the tools that wrap this
24
- # module own caching and truncation and turn rendered Markdown (or
25
- # {FetchError}) into the next observation.
26
- module Scraper
27
- # Raised when a URL cannot be rendered into Markdown text — HTTP
28
- # non-2xx, network failure, redirect-loop, missing +Location+,
29
- # unsupported content-type, or a parse failure that reads as "try
30
- # a different URL" to the LLM. Catching this in
31
- # {Tool::WEB_SCRAPE} / {Tool::FETCH} turns the failure into an
32
- # +"Error: ..."+ observation; anything else bubbles up so genuine
33
- # bugs stay visible.
34
- class FetchError < StandardError; end
35
-
36
- # @return [String] User-Agent sent with each request; many sites
37
- # reject requests with no UA or an obvious bot UA
38
- USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
39
- '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
40
- # @return [String] +Accept+ header sent with each request, so
41
- # servers that content-negotiate hand back something we can use:
42
- # rendered HTML first, +application/pdf+ for hosts with a PDF
43
- # extractor registered, then any +text/*+ for the verbatim
44
- # pass-through arm.
45
- ACCEPT = 'text/html,application/xhtml+xml,application/pdf,text/*;q=0.8'
46
- # @return [Integer] maximum number of HTTP redirects to follow
47
- # before giving up
48
- MAX_REDIRECTS = 5
49
- # @return [Integer] connect timeout in seconds for the underlying
50
- # Faraday request
51
- OPEN_TIMEOUT = 10
52
- # @return [Integer] read timeout in seconds for the underlying
53
- # Faraday request
54
- READ_TIMEOUT = 20
55
-
56
- # @return [Integer] maximum number of characters of an error
57
- # response body to include in a {FetchError} message. The body is
58
- # often a multi-kilobyte HTML challenge page (Cloudflare, WAF
59
- # interstitial, etc.); a short excerpt tells the LLM what kind of
60
- # page came back without flooding the next observation.
61
- ERROR_BODY_EXCERPT = 200
62
-
63
- # Result of a successful {Scraper.fetch}: the response body, the
64
- # normalized content-type (lower-cased, with any +; charset=...+
65
- # parameters stripped), and the final URL after redirects.
66
- Fetched = Data.define(:body, :content_type, :url)
67
-
68
- # Fetch +url+ and render its main content as Markdown.
69
- #
70
- # No caching here — every call hits the network. Callers that want
71
- # to memoize results should wrap this method themselves (see
72
- # {Tool::WebScrape.visit}, which does exactly that).
73
- #
74
- # The extracted output is +String#strip+'d so the LLM never sees
75
- # a body that opens or closes with blank lines — common with
76
- # extracted PDFs' page-feed whitespace and with text bodies that
77
- # carry a trailing newline. Interior whitespace is preserved
78
- # because Markdown paragraph breaks and source-code indentation
79
- # are load-bearing.
80
- #
81
- # @param url [String] absolute HTTP(S) URL of the page to download
82
- # @return [String] full Markdown representation of the page with
83
- # leading/trailing whitespace trimmed, uncapped otherwise —
84
- # caller is responsible for any size limiting before feeding
85
- # the result back to the LLM
86
- # @raise [FetchError] on HTTP non-2xx, network failure, redirect
87
- # loop, a 3xx without a +Location+ header, a response no
88
- # extractor recognizes, or an extraction failure (malformed
89
- # PDF, ...)
90
- def self.visit(url)
91
- extract(fetch(url)).strip
92
- end
93
-
94
- # Render a {Fetched} response as Markdown via
95
- # {Pikuri::Extractor.extract}, re-raising both extraction failure
96
- # modes as {FetchError} — the single exception type the web tools
97
- # rescue. The content-type is passed verbatim (including the +""+
98
- # of a missing header, which matches no text arm — a body without
99
- # transport metadata is refused, not sniffed; only a strong magic
100
- # sniff like pikuri-pdf's +%PDF-+ overrides a wrong or missing
101
- # header, because such a sniff never misfires on text).
102
- #
103
- # @param fetched [Fetched]
104
- # @return [String] Markdown representation produced by the
105
- # matched extractor
106
- # @raise [FetchError] when no extractor matches the response's
107
- # content-type, or when extraction fails
108
- def self.extract(fetched)
109
- Pikuri::Extractor.extract(StringIO.new(fetched.body), content_type: fetched.content_type)
110
- rescue Pikuri::Extractor::Unsupported
111
- raise FetchError, "unsupported content-type #{fetched.content_type.inspect} for #{fetched.url}"
112
- rescue Pikuri::Extractor::Error => e
113
- raise FetchError, e.message
114
- end
115
-
116
- # Download the body of +url+, manually following up to
117
- # {MAX_REDIRECTS} redirects. Faraday is configured with no
118
- # middleware so behavior here mirrors the rest of the codebase
119
- # (see +Tool::Search::DuckDuckGo.search+).
120
- #
121
- # All recoverable failures — HTTP 4xx/5xx, +Faraday::Error+ network
122
- # blips, exhausted redirect budget, 3xx without a +Location+ —
123
- # surface as {FetchError} so the caller has a single exception type
124
- # to rescue. Error bodies are trimmed to {ERROR_BODY_EXCERPT}
125
- # characters with whitespace collapsed, so a Cloudflare-challenge
126
- # response doesn't dump kilobytes of inline HTML into the next LLM
127
- # observation.
128
- #
129
- # @param url [String] absolute HTTP(S) URL to fetch
130
- # @param limit [Integer] redirects remaining; recurses with
131
- # +limit - 1+ on each 3xx
132
- # @return [Fetched] body, normalized content-type, and final URL
133
- # after redirects
134
- # @raise [FetchError] on non-2xx/3xx responses, network errors,
135
- # redirect-loop exhaustion, or 3xx without a +Location+ header
136
- def self.fetch(url, limit: MAX_REDIRECTS)
137
- raise FetchError, "too many redirects fetching #{url}" if limit.zero?
138
-
139
- response = begin
140
- Faraday.new(request: { open_timeout: OPEN_TIMEOUT, timeout: READ_TIMEOUT }).get(url) do |req|
141
- req.headers['User-Agent'] = USER_AGENT
142
- req.headers['Accept'] = ACCEPT
143
- end
144
- rescue Faraday::Error => e
145
- raise FetchError, "#{e.class.name.split('::').last} fetching #{url}: #{e.message}"
146
- end
147
-
148
- case response.status
149
- when 200..299
150
- Fetched.new(body: response.body, content_type: normalize_content_type(response.headers['content-type']), url: url)
151
- when 300..399
152
- location = response.headers['location']
153
- raise FetchError, "HTTP #{response.status} from #{url} with no Location header" if location.nil? || location.empty?
154
-
155
- fetch(URI.join(url, location).to_s, limit: limit - 1)
156
- else
157
- raise FetchError, "HTTP #{response.status} fetching #{url}: #{excerpt(response.body)}"
158
- end
159
- end
160
-
161
- # Lower-case +raw+ and strip any +; charset=...+ parameters so the
162
- # extractors can match on a canonical token.
163
- #
164
- # @param raw [String, nil] raw +Content-Type+ header value
165
- # @return [String] normalized content-type, or +""+ when the
166
- # header was missing
167
- def self.normalize_content_type(raw)
168
- raw.to_s.split(';').first.to_s.strip.downcase
169
- end
170
- private_class_method :normalize_content_type
171
-
172
- # Whitespace-collapse +body+ and clip to {ERROR_BODY_EXCERPT}
173
- # characters, so the {FetchError} message stays a single readable
174
- # line even when the server returned a multi-KB HTML challenge
175
- # page.
176
- #
177
- # @param body [String, nil]
178
- # @return [String]
179
- def self.excerpt(body)
180
- text = body.to_s.gsub(/\s+/, ' ').strip
181
- text.length > ERROR_BODY_EXCERPT ? "#{text[0, ERROR_BODY_EXCERPT]}..." : text
182
- end
183
- private_class_method :excerpt
184
- end
185
- end
186
- end
@@ -1,198 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'faraday'
4
- require 'json'
5
- require 'nokogiri'
6
-
7
- module Pikuri
8
- class Tool
9
- module Search
10
- # Performs a Brave Search via the official Web Search API and returns
11
- # the hits as a list of {Result} rows. Split into a thin HTTP fetch
12
- # (#search) and a pure parser (.parse) so tests can exercise the
13
- # parser against fixture JSON without hitting the network. The
14
- # cascade in {Engines#search} owns the final Markdown rendering.
15
- #
16
- # A class constructed with the API key it should use
17
- # (+Brave.new(api_key:)+); {Engines} builds one only when a Brave key
18
- # was configured and then drives it through the same +#search+ /
19
- # +#label+ interface as every other provider. pikuri reads no key
20
- # from the environment (see CLAUDE.md "Environment is not a secret
21
- # store"). Get a key at https://api-dashboard.search.brave.com — the
22
- # free "Data for Search" tier allows 1 query/sec and ~2k queries/month.
23
- #
24
- # == Privacy posture
25
- #
26
- # Brave's API Privacy Notice retains Search Query Logs for 90 days
27
- # (billing / troubleshooting) and states +Brave does not collect any
28
- # identifiers that can link a search query to an individual or their
29
- # devices+. Brave publicly commits that the Search API does not use
30
- # query data to train its own models, and offers Zero Data Retention
31
- # — but only on the Enterprise plan, not on the free "Data for
32
- # Search" tier pikuri defaults to.
33
- #
34
- # Bottom line: of pikuri's three providers Brave has the cleanest
35
- # API-level posture — no training-on-queries, no IP linkage, capped
36
- # 90-day retention by default, real ZDR if you pay for it. Still a
37
- # logged 90-day window on the cheap tier, so not a substitute for
38
- # ZDR for genuinely sensitive queries.
39
- class Brave
40
- # @return [String] Web Search endpoint
41
- ENDPOINT = 'https://api.search.brave.com/res/v1/web/search'
42
- # @return [Integer] default number of results returned, matching
43
- # {DuckDuckGo::DEFAULT_MAX_RESULTS}
44
- DEFAULT_MAX_RESULTS = 10
45
- # @return [RateLimiter] free-tier Brave caps at 1 req/sec; the
46
- # 5-minute cooldown protects the limited monthly quota from
47
- # being burned on doomed retries when a 429 hits.
48
- LIMITER = RateLimiter.new(min_interval: 1.0, cooldown: 300.0)
49
-
50
- # @param api_key [String] Brave Search subscription token. Required
51
- # and non-blank: pikuri reads no key from the environment — the
52
- # host supplies it ({Engines} only constructs a Brave when a key
53
- # was configured).
54
- # @raise [ArgumentError] if +api_key+ is blank
55
- def initialize(api_key:)
56
- raise ArgumentError, 'Brave Search API key is blank' if api_key.to_s.strip.empty?
57
-
58
- @api_key = api_key
59
- end
60
-
61
- # @return [String] short provider label for {Engines} logging /
62
- # fallback messages.
63
- def label
64
- 'Brave'
65
- end
66
-
67
- # Fetch results for +query+ and return them as an +Array<Result>+.
68
- # Calls are throttled to one per second and circuit-broken for 5
69
- # minutes on rate-limit / quota-exhausted responses; see {LIMITER}.
70
- # The caller (typically {Engines#search}) is expected to have
71
- # already normalized the query and to wrap this in a result cache.
72
- #
73
- # @param query [String] search query (already normalized)
74
- # @param max_results [Integer] maximum number of result entries;
75
- # passed through as Brave's +count+ (1..20)
76
- # @return [Array<Result>] hits, possibly empty when Brave ran the
77
- # query and matched nothing
78
- # @raise [Engines::Unavailable] when Brave returns HTTP 429
79
- # (rate limit / quota exhausted) or 5xx — "try again later"
80
- # responses the cascade in {Engines#search} can fall back
81
- # from. Also raised immediately if {LIMITER} is in cooldown.
82
- # Other non-2xx (e.g. 401/403 from a bad API key) bubble up as
83
- # +RuntimeError+ so config problems stay visible.
84
- # @raise [RuntimeError] for non-rate-limit HTTP failures or when the
85
- # response shape contains no results.
86
- def search(query, max_results: DEFAULT_MAX_RESULTS)
87
- LIMITER.call do
88
- response = Faraday.get(
89
- ENDPOINT,
90
- { q: query, count: max_results },
91
- { 'X-Subscription-Token' => @api_key, 'Accept' => 'application/json' }
92
- )
93
- unless response.success?
94
- if response.status == 429 || response.status >= 500
95
- raise Engines::Unavailable, "HTTP #{response.status}"
96
- end
97
-
98
- raise "Brave Search request failed: #{response.status} #{response.body}"
99
- end
100
-
101
- self.class.parse(response.body, max_results: max_results)
102
- end
103
- end
104
-
105
- # Parse a Brave Web Search JSON response into a list of {Result} rows.
106
- # HTML highlight tags (+<strong>+) inside +title+ and +description+
107
- # are stripped via Nokogiri so the output is plain text.
108
- #
109
- # When the response yields zero result nodes, two cases are
110
- # distinguished: a genuine "no results" payload (recognized search
111
- # shape with empty +mixed.main+/+top+/+side+ — typically a too-narrow
112
- # query Brave couldn't match) returns an empty array instead of
113
- # raising, so {Engines.search} can render its standard no-results
114
- # stub. Anything else (unknown layout, structured error) raises
115
- # with a diagnostic so the failure surfaces.
116
- #
117
- # @param json [String] response body from {ENDPOINT}
118
- # @param max_results [Integer] maximum number of result entries
119
- # @return [Array<Result>] hits, possibly empty on a recognized
120
- # empty-results payload
121
- # @raise [RuntimeError] when the response yields no result entries and
122
- # is not recognized as a genuine empty-results payload
123
- def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
124
- data = JSON.parse(json)
125
- results = Array(data.dig('web', 'results')).take(max_results).filter_map do |r|
126
- href = r['url'].to_s
127
- next nil if href.empty?
128
-
129
- Result.new(
130
- url: href,
131
- title: strip_html(r['title']),
132
- body: strip_html(r['description'])
133
- )
134
- end
135
-
136
- if results.empty?
137
- return [] if genuine_no_results?(data)
138
-
139
- raise diagnose_empty(data, json)
140
- end
141
-
142
- results
143
- end
144
-
145
- # Strip HTML markup (notably +<strong>+ highlights Brave wraps around
146
- # query terms in titles and descriptions) and collapse whitespace.
147
- #
148
- # @param html [String, nil] raw text from a Brave result field
149
- # @return [String] plain text with tags removed; empty string for nil
150
- def self.strip_html(html)
151
- return '' if html.nil?
152
-
153
- Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
154
- end
155
- private_class_method :strip_html
156
-
157
- # True when a parsed response with zero +web.results+ entries looks
158
- # like Brave's own "search ran, nothing matched" payload (typically a
159
- # too-narrow query) rather than a malformed or error response. The
160
- # markers are the recognized +type: "search"+ envelope and an empty
161
- # +mixed+ block — Brave populates +mixed.main/top/side+ with whichever
162
- # verticals matched, so all three being empty arrays is the canonical
163
- # signal that the search itself succeeded but found nothing.
164
- #
165
- # @param data [Hash, Object] parsed response
166
- # @return [Boolean]
167
- def self.genuine_no_results?(data)
168
- return false unless data.is_a?(Hash) && data['type'] == 'search'
169
-
170
- mixed = data['mixed']
171
- return false unless mixed.is_a?(Hash)
172
-
173
- %w[main top side].all? { |k| Array(mixed[k]).empty? }
174
- end
175
- private_class_method :genuine_no_results?
176
-
177
- # Build an error message for a parsed response that yielded zero
178
- # results. Quotes Brave's +error.detail+ if present, otherwise
179
- # truncates the raw body so the caller can see the actual payload.
180
- #
181
- # @param data [Hash, Object] parsed response
182
- # @param raw [String] raw response body
183
- # @return [String] human-readable diagnostic to feed to +raise+
184
- def self.diagnose_empty(data, raw)
185
- if data.is_a?(Hash) && data['error'].is_a?(Hash)
186
- err = data['error']
187
- return "Brave Search returned an error: #{[err['code'], err['detail'] || err['meta']].compact.join(' — ')}"
188
- end
189
-
190
- snippet = raw.to_s[0, 800]
191
- snippet += '…' if raw.to_s.length > 800
192
- "Brave Search returned no results. Body: #{snippet}"
193
- end
194
- private_class_method :diagnose_empty
195
- end
196
- end
197
- end
198
- end