pikuri-core 0.0.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +17 -30
- data/lib/pikuri/agent/chat_transport.rb +168 -107
- data/lib/pikuri/agent/configurator.rb +194 -173
- data/lib/pikuri/agent/control/cancellable.rb +100 -70
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +107 -118
- data/lib/pikuri/agent/control.rb +13 -35
- data/lib/pikuri/agent/event.rb +144 -197
- data/lib/pikuri/agent/extension.rb +83 -95
- data/lib/pikuri/agent/history.rb +247 -0
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +169 -136
- data/lib/pikuri/agent/listener/token_log.rb +84 -157
- data/lib/pikuri/agent/listener.rb +24 -44
- data/lib/pikuri/agent/listener_list.rb +32 -47
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +1095 -577
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/bundler_env.rb +68 -0
- data/lib/pikuri/extractor/passthrough.rb +24 -31
- data/lib/pikuri/extractor.rb +125 -155
- data/lib/pikuri/file_type.rb +64 -136
- data/lib/pikuri/finalizers.rb +32 -47
- data/lib/pikuri/paths.rb +104 -13
- data/lib/pikuri/sanitizer.rb +46 -67
- data/lib/pikuri/subprocess.rb +86 -118
- data/lib/pikuri/testing.rb +369 -0
- data/lib/pikuri/tool/calculator.rb +168 -74
- data/lib/pikuri/tool/execute_context.rb +42 -0
- data/lib/pikuri/tool/parameters.rb +31 -29
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +248 -0
- data/lib/pikuri/tool.rb +416 -67
- data/lib/pikuri/trifecta/contribution.rb +43 -0
- data/lib/pikuri/trifecta/node.rb +47 -0
- data/lib/pikuri/trifecta/report.rb +228 -0
- data/lib/pikuri/trifecta.rb +126 -0
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +70 -92
- data/prompts/agent-loop.txt +5 -0
- data/prompts/pikuri-chat.txt +3 -12
- metadata +26 -72
- data/lib/pikuri/agent/context_window_detector.rb +0 -222
- data/lib/pikuri/agent/control/step_limit.rb +0 -111
- data/lib/pikuri/agent/extension_context.rb +0 -147
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -172
- data/lib/pikuri/agent/synthesizer.rb +0 -171
- data/lib/pikuri/extractor/html.rb +0 -303
- data/lib/pikuri/tool/fetch.rb +0 -172
- data/lib/pikuri/tool/scraper.rb +0 -186
- data/lib/pikuri/tool/search/brave.rb +0 -198
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -207
- data/lib/pikuri/tool/search/engines.rb +0 -186
- data/lib/pikuri/tool/search/exa.rb +0 -229
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -92
- data/lib/pikuri/tool/search/result.rb +0 -29
- data/lib/pikuri/tool/web_scrape.rb +0 -121
- data/lib/pikuri/tool/web_search.rb +0 -57
- data/lib/pikuri/url_cache.rb +0 -112
data/lib/pikuri/tool/fetch.rb
DELETED
|
@@ -1,172 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
# Truncation policy and Tool spec for the +fetch+ tool. The HTTP work
|
|
6
|
-
# lives in {Tool::Scraper.fetch}; this module is a thin
|
|
7
|
-
# wrapper that accepts only textual content-types, applies a character
|
|
8
|
-
# cap so the LLM doesn't drown in long-form bodies, and exposes the
|
|
9
|
-
# result to the agent loop in OpenAI tool-call shape.
|
|
10
|
-
#
|
|
11
|
-
# Sister of {Tool::WebScrape}, but with no extraction pass
|
|
12
|
-
# (HTML→Markdown, or whatever plug-in extractors are registered):
|
|
13
|
-
# bodies are returned verbatim. Useful for raw textual
|
|
14
|
-
# data — JSON APIs, CSV files, +robots.txt+, sitemaps, source files —
|
|
15
|
-
# where any rendering pass would corrupt the payload.
|
|
16
|
-
module Fetch
|
|
17
|
-
# @return [Integer] default character cap on the body returned by
|
|
18
|
-
# {.fetch}. Smaller than {Tool::WebScrape::DEFAULT_MAX_CHARS}
|
|
19
|
-
# because fetch's content profile is bimodal — most JSON/XML/CSV
|
|
20
|
-
# responses are tiny, and the long-tail (large data dumps) is
|
|
21
|
-
# better re-requested deliberately than padded into every default.
|
|
22
|
-
DEFAULT_MAX_CHARS = 5_000
|
|
23
|
-
|
|
24
|
-
# @return [Integer] hard ceiling on the +max_chars+ argument to
|
|
25
|
-
# {.fetch}. Matches {Tool::WebScrape::MAX_MAX_CHARS}.
|
|
26
|
-
MAX_MAX_CHARS = 100_000
|
|
27
|
-
|
|
28
|
-
# Application content-types that are textual in practice and so
|
|
29
|
-
# safe to return verbatim to the LLM, despite their +application/+
|
|
30
|
-
# prefix making them fail the +text/*+ check. Anything outside
|
|
31
|
-
# +text/*+ and this allowlist is refused.
|
|
32
|
-
# @return [Array<String>]
|
|
33
|
-
TEXTUAL_APPLICATION_TYPES = %w[
|
|
34
|
-
application/json
|
|
35
|
-
application/xml
|
|
36
|
-
application/javascript
|
|
37
|
-
application/xhtml+xml
|
|
38
|
-
application/rss+xml
|
|
39
|
-
application/atom+xml
|
|
40
|
-
].freeze
|
|
41
|
-
|
|
42
|
-
# On-disk cache used by {.fetch} to memoize downloads. Defined as a
|
|
43
|
-
# method so specs can swap it for an isolated cache or
|
|
44
|
-
# {UrlCache::NULL} without touching the shared instance. Lives in
|
|
45
|
-
# its own subdir under {UrlCache::ROOT_DIR} so a +fetch+ on a URL
|
|
46
|
-
# and a +web_scrape+ on the same URL cannot collide on the same
|
|
47
|
-
# cache file (one returns the raw body, the other returns extracted
|
|
48
|
-
# Markdown).
|
|
49
|
-
#
|
|
50
|
-
# @return [UrlCache, #fetch]
|
|
51
|
-
CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/fetch")
|
|
52
|
-
# Accessor for {CACHE}; specs override this to swap in
|
|
53
|
-
# {UrlCache::NULL} or an isolated cache.
|
|
54
|
-
#
|
|
55
|
-
# @return [UrlCache, #fetch]
|
|
56
|
-
def self.cache
|
|
57
|
-
CACHE
|
|
58
|
-
end
|
|
59
|
-
|
|
60
|
-
# Download +url+ via {Tool::Scraper.fetch} and return the
|
|
61
|
-
# response body verbatim, provided the content-type is one we deem
|
|
62
|
-
# textual (any +text/*+, plus the formats listed in
|
|
63
|
-
# {TEXTUAL_APPLICATION_TYPES}). Anything else — PDFs, images, other
|
|
64
|
-
# binaries — produces an +"Error: ..."+ string in the calculator-
|
|
65
|
-
# style convention so the agent loop feeds the failure back to the
|
|
66
|
-
# model as the next observation.
|
|
67
|
-
#
|
|
68
|
-
# The body is cached on disk via {.cache}, keyed by URL, so repeat
|
|
69
|
-
# fetches within the cache TTL skip the network. +max_chars+ is not
|
|
70
|
-
# part of the cache key — different values for the same URL share
|
|
71
|
-
# one entry, and truncation runs after the cache lookup. The cache
|
|
72
|
-
# is only populated on success: {Scraper::FetchError} (HTTP non-2xx,
|
|
73
|
-
# network failure, redirect-loop exhaustion, refused content-type)
|
|
74
|
-
# is caught outside the +cache.fetch+ block, so failure strings are
|
|
75
|
-
# never persisted and a retry on the next call hits the network
|
|
76
|
-
# again. Other exceptions (parser bugs in our own code) bubble up
|
|
77
|
-
# unchanged.
|
|
78
|
-
#
|
|
79
|
-
# @param url [String] absolute HTTP(S) URL to download
|
|
80
|
-
# @param max_chars [Integer] character cap on the returned body.
|
|
81
|
-
# Clamped to +[1, {MAX_MAX_CHARS}]+; defaults to
|
|
82
|
-
# {DEFAULT_MAX_CHARS}. When the body exceeds the cap, output is
|
|
83
|
-
# cut and a marker noting the original length is appended.
|
|
84
|
-
# @return [String] response body, possibly truncated, or
|
|
85
|
-
# +"Error: ..."+ on a recoverable failure
|
|
86
|
-
def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
|
|
87
|
-
max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
|
|
88
|
-
body = cache.fetch(url) { download(url) }
|
|
89
|
-
truncate(body, max_chars)
|
|
90
|
-
rescue Scraper::FetchError => e
|
|
91
|
-
"Error: #{e.message}"
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
# GET +url+ and verify the response's content-type is textual.
|
|
95
|
-
# Caller is responsible for caching and truncation; this method
|
|
96
|
-
# always hits the network.
|
|
97
|
-
#
|
|
98
|
-
# @param url [String]
|
|
99
|
-
# @return [String] response body
|
|
100
|
-
# @raise [Scraper::FetchError] on HTTP non-2xx, network failure,
|
|
101
|
-
# redirect-loop exhaustion, missing +Location+ on a 3xx, or a
|
|
102
|
-
# non-textual content-type
|
|
103
|
-
def self.download(url)
|
|
104
|
-
fetched = Scraper.fetch(url)
|
|
105
|
-
return fetched.body if textual?(fetched.content_type)
|
|
106
|
-
|
|
107
|
-
raise Scraper::FetchError,
|
|
108
|
-
"refused to fetch #{url}: content-type #{fetched.content_type.inspect} " \
|
|
109
|
-
'is not textual (use web_scrape for rendered pages)'
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
# @param content_type [String] normalized content-type (no +charset+
|
|
113
|
-
# parameter, lowercased) as produced by {Scraper.fetch}
|
|
114
|
-
# @return [Boolean] true when the content-type is +text/*+ or one
|
|
115
|
-
# of {TEXTUAL_APPLICATION_TYPES}
|
|
116
|
-
def self.textual?(content_type)
|
|
117
|
-
content_type.start_with?('text/') ||
|
|
118
|
-
TEXTUAL_APPLICATION_TYPES.include?(content_type)
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
# Cut +body+ to at most +max_chars+ characters, appending a marker
|
|
122
|
-
# describing the original length when truncation actually happens.
|
|
123
|
-
# Returns +body+ unchanged if it already fits. Same shape as
|
|
124
|
-
# {Tool::WebScrape.truncate} so the LLM sees a consistent
|
|
125
|
-
# truncation marker across both tools.
|
|
126
|
-
#
|
|
127
|
-
# @param body [String] full response body
|
|
128
|
-
# @param max_chars [Integer] character cap; assumed already clamped
|
|
129
|
-
# @return [String]
|
|
130
|
-
def self.truncate(body, max_chars)
|
|
131
|
-
return body if body.length <= max_chars
|
|
132
|
-
|
|
133
|
-
"#{body[0, max_chars]}\n\n" \
|
|
134
|
-
"... [truncated at #{max_chars} of #{body.length} chars; " \
|
|
135
|
-
'call again with a larger `max_chars` to see more]'
|
|
136
|
-
end
|
|
137
|
-
end
|
|
138
|
-
|
|
139
|
-
# Verbatim URL download tool. Thin wrapper over {Tool::Fetch.fetch}
|
|
140
|
-
# that exposes it to the agent loop in OpenAI tool-call shape. Use for
|
|
141
|
-
# raw textual payloads (JSON APIs, CSV files, +robots.txt+, source
|
|
142
|
-
# files); use {Tool::WEB_SCRAPE} for rendered web pages where
|
|
143
|
-
# readability extraction makes the result usable.
|
|
144
|
-
#
|
|
145
|
-
# @return [Tool]
|
|
146
|
-
FETCH = new(
|
|
147
|
-
name: 'fetch',
|
|
148
|
-
description: <<~DESC,
|
|
149
|
-
Downloads the given URL and returns its body verbatim.
|
|
150
|
-
|
|
151
|
-
Usage:
|
|
152
|
-
- Use for raw textual payloads: JSON APIs, CSV files, robots.txt, sitemaps, source files — anywhere a rendering pass would corrupt the data.
|
|
153
|
-
- For rendered HTML pages, use web_scrape — it extracts readable content; fetch returns the raw HTML bytes unchanged.
|
|
154
|
-
- Accepts text/* and common textual application/* types (JSON, XML, JS, XHTML, RSS, Atom). Refuses PDFs, images, and other binaries.
|
|
155
|
-
DESC
|
|
156
|
-
parameters: Parameters.build { |p|
|
|
157
|
-
p.required_string :url,
|
|
158
|
-
'Absolute URL to download, including the scheme, ' \
|
|
159
|
-
'e.g. "https://example.com/data.json".'
|
|
160
|
-
p.optional_integer :max_chars,
|
|
161
|
-
'Maximum number of characters of the body to ' \
|
|
162
|
-
'return. Defaults to 5000; hard-capped at ' \
|
|
163
|
-
'100000. When the body is longer than this, ' \
|
|
164
|
-
'output is cut and a marker reports the full ' \
|
|
165
|
-
'length.'
|
|
166
|
-
},
|
|
167
|
-
execute: ->(url:, max_chars: Fetch::DEFAULT_MAX_CHARS) {
|
|
168
|
-
Fetch.fetch(url, max_chars: max_chars)
|
|
169
|
-
}
|
|
170
|
-
)
|
|
171
|
-
end
|
|
172
|
-
end
|
data/lib/pikuri/tool/scraper.rb
DELETED
|
@@ -1,186 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'stringio'
|
|
5
|
-
require 'uri'
|
|
6
|
-
|
|
7
|
-
module Pikuri
|
|
8
|
-
class Tool
|
|
9
|
-
# HTTP side of the web tools ({Tool::WEB_SCRAPE} and {Tool::FETCH}):
|
|
10
|
-
# GET the URL with a real-browser User-Agent, follow redirects, and
|
|
11
|
-
# hand the response body to {Pikuri::Extractor.extract} with the
|
|
12
|
-
# response's +Content-Type+ as the hint. HTML/XHTML render via
|
|
13
|
-
# {Extractor::HTML}, any other +text/*+ type passes through
|
|
14
|
-
# verbatim, and plug-in extractors extend the set (with pikuri-pdf
|
|
15
|
-
# registered, +application/pdf+ extracts — by header or by +%PDF-+
|
|
16
|
-
# magic, so a PDF served under a lying header still works); the
|
|
17
|
-
# remaining types raise {FetchError} so the LLM observes the
|
|
18
|
-
# failure instead of receiving an empty rendering.
|
|
19
|
-
#
|
|
20
|
-
# Split into a thin HTTP fetch ({.fetch}) and the extraction
|
|
21
|
-
# wrapper ({.visit}) so tests can drive each piece in isolation and
|
|
22
|
-
# {Tool::Fetch} can reuse the HTTP half without the extraction
|
|
23
|
-
# pass. Nothing here knows about the LLM; the tools that wrap this
|
|
24
|
-
# module own caching and truncation and turn rendered Markdown (or
|
|
25
|
-
# {FetchError}) into the next observation.
|
|
26
|
-
module Scraper
|
|
27
|
-
# Raised when a URL cannot be rendered into Markdown text — HTTP
|
|
28
|
-
# non-2xx, network failure, redirect-loop, missing +Location+,
|
|
29
|
-
# unsupported content-type, or a parse failure that reads as "try
|
|
30
|
-
# a different URL" to the LLM. Catching this in
|
|
31
|
-
# {Tool::WEB_SCRAPE} / {Tool::FETCH} turns the failure into an
|
|
32
|
-
# +"Error: ..."+ observation; anything else bubbles up so genuine
|
|
33
|
-
# bugs stay visible.
|
|
34
|
-
class FetchError < StandardError; end
|
|
35
|
-
|
|
36
|
-
# @return [String] User-Agent sent with each request; many sites
|
|
37
|
-
# reject requests with no UA or an obvious bot UA
|
|
38
|
-
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
|
|
39
|
-
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
40
|
-
# @return [String] +Accept+ header sent with each request, so
|
|
41
|
-
# servers that content-negotiate hand back something we can use:
|
|
42
|
-
# rendered HTML first, +application/pdf+ for hosts with a PDF
|
|
43
|
-
# extractor registered, then any +text/*+ for the verbatim
|
|
44
|
-
# pass-through arm.
|
|
45
|
-
ACCEPT = 'text/html,application/xhtml+xml,application/pdf,text/*;q=0.8'
|
|
46
|
-
# @return [Integer] maximum number of HTTP redirects to follow
|
|
47
|
-
# before giving up
|
|
48
|
-
MAX_REDIRECTS = 5
|
|
49
|
-
# @return [Integer] connect timeout in seconds for the underlying
|
|
50
|
-
# Faraday request
|
|
51
|
-
OPEN_TIMEOUT = 10
|
|
52
|
-
# @return [Integer] read timeout in seconds for the underlying
|
|
53
|
-
# Faraday request
|
|
54
|
-
READ_TIMEOUT = 20
|
|
55
|
-
|
|
56
|
-
# @return [Integer] maximum number of characters of an error
|
|
57
|
-
# response body to include in a {FetchError} message. The body is
|
|
58
|
-
# often a multi-kilobyte HTML challenge page (Cloudflare, WAF
|
|
59
|
-
# interstitial, etc.); a short excerpt tells the LLM what kind of
|
|
60
|
-
# page came back without flooding the next observation.
|
|
61
|
-
ERROR_BODY_EXCERPT = 200
|
|
62
|
-
|
|
63
|
-
# Result of a successful {Scraper.fetch}: the response body, the
|
|
64
|
-
# normalized content-type (lower-cased, with any +; charset=...+
|
|
65
|
-
# parameters stripped), and the final URL after redirects.
|
|
66
|
-
Fetched = Data.define(:body, :content_type, :url)
|
|
67
|
-
|
|
68
|
-
# Fetch +url+ and render its main content as Markdown.
|
|
69
|
-
#
|
|
70
|
-
# No caching here — every call hits the network. Callers that want
|
|
71
|
-
# to memoize results should wrap this method themselves (see
|
|
72
|
-
# {Tool::WebScrape.visit}, which does exactly that).
|
|
73
|
-
#
|
|
74
|
-
# The extracted output is +String#strip+'d so the LLM never sees
|
|
75
|
-
# a body that opens or closes with blank lines — common with
|
|
76
|
-
# extracted PDFs' page-feed whitespace and with text bodies that
|
|
77
|
-
# carry a trailing newline. Interior whitespace is preserved
|
|
78
|
-
# because Markdown paragraph breaks and source-code indentation
|
|
79
|
-
# are load-bearing.
|
|
80
|
-
#
|
|
81
|
-
# @param url [String] absolute HTTP(S) URL of the page to download
|
|
82
|
-
# @return [String] full Markdown representation of the page with
|
|
83
|
-
# leading/trailing whitespace trimmed, uncapped otherwise —
|
|
84
|
-
# caller is responsible for any size limiting before feeding
|
|
85
|
-
# the result back to the LLM
|
|
86
|
-
# @raise [FetchError] on HTTP non-2xx, network failure, redirect
|
|
87
|
-
# loop, a 3xx without a +Location+ header, a response no
|
|
88
|
-
# extractor recognizes, or an extraction failure (malformed
|
|
89
|
-
# PDF, ...)
|
|
90
|
-
def self.visit(url)
|
|
91
|
-
extract(fetch(url)).strip
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
# Render a {Fetched} response as Markdown via
|
|
95
|
-
# {Pikuri::Extractor.extract}, re-raising both extraction failure
|
|
96
|
-
# modes as {FetchError} — the single exception type the web tools
|
|
97
|
-
# rescue. The content-type is passed verbatim (including the +""+
|
|
98
|
-
# of a missing header, which matches no text arm — a body without
|
|
99
|
-
# transport metadata is refused, not sniffed; only a strong magic
|
|
100
|
-
# sniff like pikuri-pdf's +%PDF-+ overrides a wrong or missing
|
|
101
|
-
# header, because such a sniff never misfires on text).
|
|
102
|
-
#
|
|
103
|
-
# @param fetched [Fetched]
|
|
104
|
-
# @return [String] Markdown representation produced by the
|
|
105
|
-
# matched extractor
|
|
106
|
-
# @raise [FetchError] when no extractor matches the response's
|
|
107
|
-
# content-type, or when extraction fails
|
|
108
|
-
def self.extract(fetched)
|
|
109
|
-
Pikuri::Extractor.extract(StringIO.new(fetched.body), content_type: fetched.content_type)
|
|
110
|
-
rescue Pikuri::Extractor::Unsupported
|
|
111
|
-
raise FetchError, "unsupported content-type #{fetched.content_type.inspect} for #{fetched.url}"
|
|
112
|
-
rescue Pikuri::Extractor::Error => e
|
|
113
|
-
raise FetchError, e.message
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
# Download the body of +url+, manually following up to
|
|
117
|
-
# {MAX_REDIRECTS} redirects. Faraday is configured with no
|
|
118
|
-
# middleware so behavior here mirrors the rest of the codebase
|
|
119
|
-
# (see +Tool::Search::DuckDuckGo.search+).
|
|
120
|
-
#
|
|
121
|
-
# All recoverable failures — HTTP 4xx/5xx, +Faraday::Error+ network
|
|
122
|
-
# blips, exhausted redirect budget, 3xx without a +Location+ —
|
|
123
|
-
# surface as {FetchError} so the caller has a single exception type
|
|
124
|
-
# to rescue. Error bodies are trimmed to {ERROR_BODY_EXCERPT}
|
|
125
|
-
# characters with whitespace collapsed, so a Cloudflare-challenge
|
|
126
|
-
# response doesn't dump kilobytes of inline HTML into the next LLM
|
|
127
|
-
# observation.
|
|
128
|
-
#
|
|
129
|
-
# @param url [String] absolute HTTP(S) URL to fetch
|
|
130
|
-
# @param limit [Integer] redirects remaining; recurses with
|
|
131
|
-
# +limit - 1+ on each 3xx
|
|
132
|
-
# @return [Fetched] body, normalized content-type, and final URL
|
|
133
|
-
# after redirects
|
|
134
|
-
# @raise [FetchError] on non-2xx/3xx responses, network errors,
|
|
135
|
-
# redirect-loop exhaustion, or 3xx without a +Location+ header
|
|
136
|
-
def self.fetch(url, limit: MAX_REDIRECTS)
|
|
137
|
-
raise FetchError, "too many redirects fetching #{url}" if limit.zero?
|
|
138
|
-
|
|
139
|
-
response = begin
|
|
140
|
-
Faraday.new(request: { open_timeout: OPEN_TIMEOUT, timeout: READ_TIMEOUT }).get(url) do |req|
|
|
141
|
-
req.headers['User-Agent'] = USER_AGENT
|
|
142
|
-
req.headers['Accept'] = ACCEPT
|
|
143
|
-
end
|
|
144
|
-
rescue Faraday::Error => e
|
|
145
|
-
raise FetchError, "#{e.class.name.split('::').last} fetching #{url}: #{e.message}"
|
|
146
|
-
end
|
|
147
|
-
|
|
148
|
-
case response.status
|
|
149
|
-
when 200..299
|
|
150
|
-
Fetched.new(body: response.body, content_type: normalize_content_type(response.headers['content-type']), url: url)
|
|
151
|
-
when 300..399
|
|
152
|
-
location = response.headers['location']
|
|
153
|
-
raise FetchError, "HTTP #{response.status} from #{url} with no Location header" if location.nil? || location.empty?
|
|
154
|
-
|
|
155
|
-
fetch(URI.join(url, location).to_s, limit: limit - 1)
|
|
156
|
-
else
|
|
157
|
-
raise FetchError, "HTTP #{response.status} fetching #{url}: #{excerpt(response.body)}"
|
|
158
|
-
end
|
|
159
|
-
end
|
|
160
|
-
|
|
161
|
-
# Lower-case +raw+ and strip any +; charset=...+ parameters so the
|
|
162
|
-
# extractors can match on a canonical token.
|
|
163
|
-
#
|
|
164
|
-
# @param raw [String, nil] raw +Content-Type+ header value
|
|
165
|
-
# @return [String] normalized content-type, or +""+ when the
|
|
166
|
-
# header was missing
|
|
167
|
-
def self.normalize_content_type(raw)
|
|
168
|
-
raw.to_s.split(';').first.to_s.strip.downcase
|
|
169
|
-
end
|
|
170
|
-
private_class_method :normalize_content_type
|
|
171
|
-
|
|
172
|
-
# Whitespace-collapse +body+ and clip to {ERROR_BODY_EXCERPT}
|
|
173
|
-
# characters, so the {FetchError} message stays a single readable
|
|
174
|
-
# line even when the server returned a multi-KB HTML challenge
|
|
175
|
-
# page.
|
|
176
|
-
#
|
|
177
|
-
# @param body [String, nil]
|
|
178
|
-
# @return [String]
|
|
179
|
-
def self.excerpt(body)
|
|
180
|
-
text = body.to_s.gsub(/\s+/, ' ').strip
|
|
181
|
-
text.length > ERROR_BODY_EXCERPT ? "#{text[0, ERROR_BODY_EXCERPT]}..." : text
|
|
182
|
-
end
|
|
183
|
-
private_class_method :excerpt
|
|
184
|
-
end
|
|
185
|
-
end
|
|
186
|
-
end
|
|
@@ -1,198 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'json'
|
|
5
|
-
require 'nokogiri'
|
|
6
|
-
|
|
7
|
-
module Pikuri
|
|
8
|
-
class Tool
|
|
9
|
-
module Search
|
|
10
|
-
# Performs a Brave Search via the official Web Search API and returns
|
|
11
|
-
# the hits as a list of {Result} rows. Split into a thin HTTP fetch
|
|
12
|
-
# (#search) and a pure parser (.parse) so tests can exercise the
|
|
13
|
-
# parser against fixture JSON without hitting the network. The
|
|
14
|
-
# cascade in {Engines#search} owns the final Markdown rendering.
|
|
15
|
-
#
|
|
16
|
-
# A class constructed with the API key it should use
|
|
17
|
-
# (+Brave.new(api_key:)+); {Engines} builds one only when a Brave key
|
|
18
|
-
# was configured and then drives it through the same +#search+ /
|
|
19
|
-
# +#label+ interface as every other provider. pikuri reads no key
|
|
20
|
-
# from the environment (see CLAUDE.md "Environment is not a secret
|
|
21
|
-
# store"). Get a key at https://api-dashboard.search.brave.com — the
|
|
22
|
-
# free "Data for Search" tier allows 1 query/sec and ~2k queries/month.
|
|
23
|
-
#
|
|
24
|
-
# == Privacy posture
|
|
25
|
-
#
|
|
26
|
-
# Brave's API Privacy Notice retains Search Query Logs for 90 days
|
|
27
|
-
# (billing / troubleshooting) and states +Brave does not collect any
|
|
28
|
-
# identifiers that can link a search query to an individual or their
|
|
29
|
-
# devices+. Brave publicly commits that the Search API does not use
|
|
30
|
-
# query data to train its own models, and offers Zero Data Retention
|
|
31
|
-
# — but only on the Enterprise plan, not on the free "Data for
|
|
32
|
-
# Search" tier pikuri defaults to.
|
|
33
|
-
#
|
|
34
|
-
# Bottom line: of pikuri's three providers Brave has the cleanest
|
|
35
|
-
# API-level posture — no training-on-queries, no IP linkage, capped
|
|
36
|
-
# 90-day retention by default, real ZDR if you pay for it. Still a
|
|
37
|
-
# logged 90-day window on the cheap tier, so not a substitute for
|
|
38
|
-
# ZDR for genuinely sensitive queries.
|
|
39
|
-
class Brave
|
|
40
|
-
# @return [String] Web Search endpoint
|
|
41
|
-
ENDPOINT = 'https://api.search.brave.com/res/v1/web/search'
|
|
42
|
-
# @return [Integer] default number of results returned, matching
|
|
43
|
-
# {DuckDuckGo::DEFAULT_MAX_RESULTS}
|
|
44
|
-
DEFAULT_MAX_RESULTS = 10
|
|
45
|
-
# @return [RateLimiter] free-tier Brave caps at 1 req/sec; the
|
|
46
|
-
# 5-minute cooldown protects the limited monthly quota from
|
|
47
|
-
# being burned on doomed retries when a 429 hits.
|
|
48
|
-
LIMITER = RateLimiter.new(min_interval: 1.0, cooldown: 300.0)
|
|
49
|
-
|
|
50
|
-
# @param api_key [String] Brave Search subscription token. Required
|
|
51
|
-
# and non-blank: pikuri reads no key from the environment — the
|
|
52
|
-
# host supplies it ({Engines} only constructs a Brave when a key
|
|
53
|
-
# was configured).
|
|
54
|
-
# @raise [ArgumentError] if +api_key+ is blank
|
|
55
|
-
def initialize(api_key:)
|
|
56
|
-
raise ArgumentError, 'Brave Search API key is blank' if api_key.to_s.strip.empty?
|
|
57
|
-
|
|
58
|
-
@api_key = api_key
|
|
59
|
-
end
|
|
60
|
-
|
|
61
|
-
# @return [String] short provider label for {Engines} logging /
|
|
62
|
-
# fallback messages.
|
|
63
|
-
def label
|
|
64
|
-
'Brave'
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
# Fetch results for +query+ and return them as an +Array<Result>+.
|
|
68
|
-
# Calls are throttled to one per second and circuit-broken for 5
|
|
69
|
-
# minutes on rate-limit / quota-exhausted responses; see {LIMITER}.
|
|
70
|
-
# The caller (typically {Engines#search}) is expected to have
|
|
71
|
-
# already normalized the query and to wrap this in a result cache.
|
|
72
|
-
#
|
|
73
|
-
# @param query [String] search query (already normalized)
|
|
74
|
-
# @param max_results [Integer] maximum number of result entries;
|
|
75
|
-
# passed through as Brave's +count+ (1..20)
|
|
76
|
-
# @return [Array<Result>] hits, possibly empty when Brave ran the
|
|
77
|
-
# query and matched nothing
|
|
78
|
-
# @raise [Engines::Unavailable] when Brave returns HTTP 429
|
|
79
|
-
# (rate limit / quota exhausted) or 5xx — "try again later"
|
|
80
|
-
# responses the cascade in {Engines#search} can fall back
|
|
81
|
-
# from. Also raised immediately if {LIMITER} is in cooldown.
|
|
82
|
-
# Other non-2xx (e.g. 401/403 from a bad API key) bubble up as
|
|
83
|
-
# +RuntimeError+ so config problems stay visible.
|
|
84
|
-
# @raise [RuntimeError] for non-rate-limit HTTP failures or when the
|
|
85
|
-
# response shape contains no results.
|
|
86
|
-
def search(query, max_results: DEFAULT_MAX_RESULTS)
|
|
87
|
-
LIMITER.call do
|
|
88
|
-
response = Faraday.get(
|
|
89
|
-
ENDPOINT,
|
|
90
|
-
{ q: query, count: max_results },
|
|
91
|
-
{ 'X-Subscription-Token' => @api_key, 'Accept' => 'application/json' }
|
|
92
|
-
)
|
|
93
|
-
unless response.success?
|
|
94
|
-
if response.status == 429 || response.status >= 500
|
|
95
|
-
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
96
|
-
end
|
|
97
|
-
|
|
98
|
-
raise "Brave Search request failed: #{response.status} #{response.body}"
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
self.class.parse(response.body, max_results: max_results)
|
|
102
|
-
end
|
|
103
|
-
end
|
|
104
|
-
|
|
105
|
-
# Parse a Brave Web Search JSON response into a list of {Result} rows.
|
|
106
|
-
# HTML highlight tags (+<strong>+) inside +title+ and +description+
|
|
107
|
-
# are stripped via Nokogiri so the output is plain text.
|
|
108
|
-
#
|
|
109
|
-
# When the response yields zero result nodes, two cases are
|
|
110
|
-
# distinguished: a genuine "no results" payload (recognized search
|
|
111
|
-
# shape with empty +mixed.main+/+top+/+side+ — typically a too-narrow
|
|
112
|
-
# query Brave couldn't match) returns an empty array instead of
|
|
113
|
-
# raising, so {Engines.search} can render its standard no-results
|
|
114
|
-
# stub. Anything else (unknown layout, structured error) raises
|
|
115
|
-
# with a diagnostic so the failure surfaces.
|
|
116
|
-
#
|
|
117
|
-
# @param json [String] response body from {ENDPOINT}
|
|
118
|
-
# @param max_results [Integer] maximum number of result entries
|
|
119
|
-
# @return [Array<Result>] hits, possibly empty on a recognized
|
|
120
|
-
# empty-results payload
|
|
121
|
-
# @raise [RuntimeError] when the response yields no result entries and
|
|
122
|
-
# is not recognized as a genuine empty-results payload
|
|
123
|
-
def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
|
|
124
|
-
data = JSON.parse(json)
|
|
125
|
-
results = Array(data.dig('web', 'results')).take(max_results).filter_map do |r|
|
|
126
|
-
href = r['url'].to_s
|
|
127
|
-
next nil if href.empty?
|
|
128
|
-
|
|
129
|
-
Result.new(
|
|
130
|
-
url: href,
|
|
131
|
-
title: strip_html(r['title']),
|
|
132
|
-
body: strip_html(r['description'])
|
|
133
|
-
)
|
|
134
|
-
end
|
|
135
|
-
|
|
136
|
-
if results.empty?
|
|
137
|
-
return [] if genuine_no_results?(data)
|
|
138
|
-
|
|
139
|
-
raise diagnose_empty(data, json)
|
|
140
|
-
end
|
|
141
|
-
|
|
142
|
-
results
|
|
143
|
-
end
|
|
144
|
-
|
|
145
|
-
# Strip HTML markup (notably +<strong>+ highlights Brave wraps around
|
|
146
|
-
# query terms in titles and descriptions) and collapse whitespace.
|
|
147
|
-
#
|
|
148
|
-
# @param html [String, nil] raw text from a Brave result field
|
|
149
|
-
# @return [String] plain text with tags removed; empty string for nil
|
|
150
|
-
def self.strip_html(html)
|
|
151
|
-
return '' if html.nil?
|
|
152
|
-
|
|
153
|
-
Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
|
|
154
|
-
end
|
|
155
|
-
private_class_method :strip_html
|
|
156
|
-
|
|
157
|
-
# True when a parsed response with zero +web.results+ entries looks
|
|
158
|
-
# like Brave's own "search ran, nothing matched" payload (typically a
|
|
159
|
-
# too-narrow query) rather than a malformed or error response. The
|
|
160
|
-
# markers are the recognized +type: "search"+ envelope and an empty
|
|
161
|
-
# +mixed+ block — Brave populates +mixed.main/top/side+ with whichever
|
|
162
|
-
# verticals matched, so all three being empty arrays is the canonical
|
|
163
|
-
# signal that the search itself succeeded but found nothing.
|
|
164
|
-
#
|
|
165
|
-
# @param data [Hash, Object] parsed response
|
|
166
|
-
# @return [Boolean]
|
|
167
|
-
def self.genuine_no_results?(data)
|
|
168
|
-
return false unless data.is_a?(Hash) && data['type'] == 'search'
|
|
169
|
-
|
|
170
|
-
mixed = data['mixed']
|
|
171
|
-
return false unless mixed.is_a?(Hash)
|
|
172
|
-
|
|
173
|
-
%w[main top side].all? { |k| Array(mixed[k]).empty? }
|
|
174
|
-
end
|
|
175
|
-
private_class_method :genuine_no_results?
|
|
176
|
-
|
|
177
|
-
# Build an error message for a parsed response that yielded zero
|
|
178
|
-
# results. Quotes Brave's +error.detail+ if present, otherwise
|
|
179
|
-
# truncates the raw body so the caller can see the actual payload.
|
|
180
|
-
#
|
|
181
|
-
# @param data [Hash, Object] parsed response
|
|
182
|
-
# @param raw [String] raw response body
|
|
183
|
-
# @return [String] human-readable diagnostic to feed to +raise+
|
|
184
|
-
def self.diagnose_empty(data, raw)
|
|
185
|
-
if data.is_a?(Hash) && data['error'].is_a?(Hash)
|
|
186
|
-
err = data['error']
|
|
187
|
-
return "Brave Search returned an error: #{[err['code'], err['detail'] || err['meta']].compact.join(' — ')}"
|
|
188
|
-
end
|
|
189
|
-
|
|
190
|
-
snippet = raw.to_s[0, 800]
|
|
191
|
-
snippet += '…' if raw.to_s.length > 800
|
|
192
|
-
"Brave Search returned no results. Body: #{snippet}"
|
|
193
|
-
end
|
|
194
|
-
private_class_method :diagnose_empty
|
|
195
|
-
end
|
|
196
|
-
end
|
|
197
|
-
end
|
|
198
|
-
end
|