pikuri-core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +16 -29
- data/lib/pikuri/agent/chat_transport.rb +114 -33
- data/lib/pikuri/agent/configurator.rb +191 -110
- data/lib/pikuri/agent/control/cancellable.rb +25 -16
- data/lib/pikuri/agent/control/handoff.rb +75 -0
- data/lib/pikuri/agent/control/interloper.rb +87 -120
- data/lib/pikuri/agent/control.rb +9 -11
- data/lib/pikuri/agent/event.rb +73 -61
- data/lib/pikuri/agent/extension.rb +45 -81
- data/lib/pikuri/agent/history.rb +91 -497
- data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
- data/lib/pikuri/agent/listener/terminal.rb +115 -108
- data/lib/pikuri/agent/listener/token_log.rb +21 -55
- data/lib/pikuri/agent/listener.rb +4 -4
- data/lib/pikuri/agent/listener_list.rb +19 -13
- data/lib/pikuri/agent/request_instrumenter.rb +102 -0
- data/lib/pikuri/agent.rb +624 -448
- data/lib/pikuri/attachment.rb +65 -0
- data/lib/pikuri/extractor/passthrough.rb +5 -2
- data/lib/pikuri/extractor.rb +43 -12
- data/lib/pikuri/file_type.rb +4 -4
- data/lib/pikuri/finalizers.rb +2 -2
- data/lib/pikuri/paths.rb +1 -1
- data/lib/pikuri/sanitizer.rb +4 -3
- data/lib/pikuri/subprocess.rb +17 -5
- data/lib/pikuri/testing.rb +122 -49
- data/lib/pikuri/tool/calculator.rb +134 -30
- data/lib/pikuri/tool/execute_context.rb +2 -2
- data/lib/pikuri/tool/parameters.rb +10 -0
- data/lib/pikuri/tool/result.rb +164 -0
- data/lib/pikuri/tool/trifecta_legs.rb +52 -21
- data/lib/pikuri/tool.rb +298 -24
- data/lib/pikuri/trifecta/report.rb +28 -30
- data/lib/pikuri/trifecta.rb +1 -2
- data/lib/pikuri/version.rb +1 -1
- data/lib/pikuri-core.rb +5 -11
- metadata +16 -73
- data/lib/pikuri/agent/context_window_detector.rb +0 -181
- data/lib/pikuri/agent/control/step_limit.rb +0 -95
- data/lib/pikuri/agent/extension_context.rb +0 -120
- data/lib/pikuri/agent/listener/rate_limited.rb +0 -146
- data/lib/pikuri/agent/synthesizer.rb +0 -129
- data/lib/pikuri/extractor/html.rb +0 -256
- data/lib/pikuri/ruby_llm_patches.rb +0 -106
- data/lib/pikuri/tool/fetch.rb +0 -146
- data/lib/pikuri/tool/scraper.rb +0 -144
- data/lib/pikuri/tool/search/brave.rb +0 -170
- data/lib/pikuri/tool/search/duckduckgo.rb +0 -175
- data/lib/pikuri/tool/search/engines.rb +0 -319
- data/lib/pikuri/tool/search/exa.rb +0 -195
- data/lib/pikuri/tool/search/rate_limiter.rb +0 -115
- data/lib/pikuri/tool/search/result.rb +0 -24
- data/lib/pikuri/tool/web_scrape.rb +0 -105
- data/lib/pikuri/tool/web_search.rb +0 -133
- data/lib/pikuri/url_cache.rb +0 -96
data/lib/pikuri/tool/fetch.rb
DELETED
|
@@ -1,146 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Pikuri
|
|
4
|
-
class Tool
|
|
5
|
-
# Truncation policy and Tool spec for the +fetch+ tool. HTTP work lives in
|
|
6
|
-
# {Tool::Scraper.fetch}; this wrapper accepts only textual content-types,
|
|
7
|
-
# applies a character cap, and exposes the result in OpenAI tool-call shape.
|
|
8
|
-
# Sister of {Tool::WebScrape} but with no extraction pass — bodies are
|
|
9
|
-
# returned verbatim, for raw textual data (JSON APIs, CSV, +robots.txt+,
|
|
10
|
-
# sitemaps, source files) where a rendering pass would corrupt the payload.
|
|
11
|
-
module Fetch
|
|
12
|
-
# @return [Integer] default character cap on the body from {.fetch}.
|
|
13
|
-
# Smaller than {Tool::WebScrape::DEFAULT_MAX_CHARS} because fetch's
|
|
14
|
-
# content is bimodal — most JSON/XML/CSV is tiny, and large dumps are
|
|
15
|
-
# better re-requested deliberately than padded into every default.
|
|
16
|
-
DEFAULT_MAX_CHARS = 5_000
|
|
17
|
-
|
|
18
|
-
# @return [Integer] hard ceiling on {.fetch}'s +max_chars+; matches
|
|
19
|
-
# {Tool::WebScrape::MAX_MAX_CHARS}.
|
|
20
|
-
MAX_MAX_CHARS = 100_000
|
|
21
|
-
|
|
22
|
-
# Application content-types that are textual in practice, so safe to
|
|
23
|
-
# return verbatim despite failing the +text/*+ check. Anything outside
|
|
24
|
-
# +text/*+ and this list is refused.
|
|
25
|
-
# @return [Array<String>]
|
|
26
|
-
TEXTUAL_APPLICATION_TYPES = %w[
|
|
27
|
-
application/json
|
|
28
|
-
application/xml
|
|
29
|
-
application/javascript
|
|
30
|
-
application/xhtml+xml
|
|
31
|
-
application/rss+xml
|
|
32
|
-
application/atom+xml
|
|
33
|
-
].freeze
|
|
34
|
-
|
|
35
|
-
# On-disk cache for {.fetch}, in its own subdir so a +fetch+ and a
|
|
36
|
-
# +web_scrape+ on the same URL don't collide (raw body vs extracted
|
|
37
|
-
# Markdown). A method so specs can swap {UrlCache::NULL}.
|
|
38
|
-
#
|
|
39
|
-
# @return [UrlCache, #fetch]
|
|
40
|
-
CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/fetch")
|
|
41
|
-
# Accessor for {CACHE}; specs swap in {UrlCache::NULL} or an isolated cache.
|
|
42
|
-
#
|
|
43
|
-
# @return [UrlCache, #fetch]
|
|
44
|
-
def self.cache
|
|
45
|
-
CACHE
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
# Download +url+ via {Tool::Scraper.fetch} and return the body verbatim,
|
|
49
|
-
# if the content-type is textual (any +text/*+ plus
|
|
50
|
-
# {TEXTUAL_APPLICATION_TYPES}); anything else (PDF, image, binary) yields
|
|
51
|
-
# an +"Error: ..."+ string. Cached on disk keyed by URL; +max_chars+ is
|
|
52
|
-
# *not* in the key (values share one entry, truncation runs after the
|
|
53
|
-
# lookup). Cached only on success — {Scraper::FetchError} is caught
|
|
54
|
-
# outside +cache.fetch+, so failures are never persisted; other
|
|
55
|
-
# exceptions bubble up.
|
|
56
|
-
#
|
|
57
|
-
# @param url [String] absolute HTTP(S) URL
|
|
58
|
-
# @param max_chars [Integer] body cap, clamped to +[1, {MAX_MAX_CHARS}]+;
|
|
59
|
-
# default {DEFAULT_MAX_CHARS}. Over the cap, output is cut with a marker.
|
|
60
|
-
# @return [String] body, possibly truncated, or +"Error: ..."+ on a
|
|
61
|
-
# recoverable failure
|
|
62
|
-
def self.fetch(url, max_chars: DEFAULT_MAX_CHARS)
|
|
63
|
-
max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
|
|
64
|
-
body = cache.fetch(url) { download(url) }
|
|
65
|
-
truncate(body, max_chars)
|
|
66
|
-
rescue Scraper::FetchError => e
|
|
67
|
-
"Error: #{e.message}"
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
# GET +url+ and verify the content-type is textual. Always hits the
|
|
71
|
-
# network (caller caches/truncates).
|
|
72
|
-
#
|
|
73
|
-
# @param url [String]
|
|
74
|
-
# @return [String] response body
|
|
75
|
-
# @raise [Scraper::FetchError] on HTTP non-2xx, network failure, redirect
|
|
76
|
-
# exhaustion, missing +Location+, or a non-textual content-type
|
|
77
|
-
def self.download(url)
|
|
78
|
-
fetched = Scraper.fetch(url)
|
|
79
|
-
return fetched.body if textual?(fetched.content_type)
|
|
80
|
-
|
|
81
|
-
raise Scraper::FetchError,
|
|
82
|
-
"refused to fetch #{url}: content-type #{fetched.content_type.inspect} " \
|
|
83
|
-
'is not textual (use web_scrape for rendered pages)'
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
# @param content_type [String] normalized content-type from {Scraper.fetch}
|
|
87
|
-
# @return [Boolean] true when +text/*+ or in {TEXTUAL_APPLICATION_TYPES}
|
|
88
|
-
def self.textual?(content_type)
|
|
89
|
-
content_type.start_with?('text/') ||
|
|
90
|
-
TEXTUAL_APPLICATION_TYPES.include?(content_type)
|
|
91
|
-
end
|
|
92
|
-
|
|
93
|
-
# Cut +body+ to at most +max_chars+, appending a marker with the original
|
|
94
|
-
# length when truncation happens. Same shape as {Tool::WebScrape.truncate}.
|
|
95
|
-
#
|
|
96
|
-
# @param body [String] full response body
|
|
97
|
-
# @param max_chars [Integer] cap; assumed already clamped
|
|
98
|
-
# @return [String]
|
|
99
|
-
def self.truncate(body, max_chars)
|
|
100
|
-
return body if body.length <= max_chars
|
|
101
|
-
|
|
102
|
-
"#{body[0, max_chars]}\n\n" \
|
|
103
|
-
"... [truncated at #{max_chars} of #{body.length} chars; " \
|
|
104
|
-
'call again with a larger `max_chars` to see more]'
|
|
105
|
-
end
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
# Verbatim URL download tool — a thin wrapper over {Tool::Fetch.fetch} in
|
|
109
|
-
# OpenAI tool-call shape. Use for raw textual payloads (JSON, CSV,
|
|
110
|
-
# +robots.txt+, source files); use {Tool::WEB_SCRAPE} for rendered pages.
|
|
111
|
-
#
|
|
112
|
-
# Sharing: +P_shared_benign+ — no per-agent state, but {Fetch::CACHE} is
|
|
113
|
-
# process-wide (and on disk, so cross-process too): agent B's +fetch+ can
|
|
114
|
-
# be answered from agent A's download within the TTL. See
|
|
115
|
-
# {Pikuri::UrlCache} for the race and why it's harmless.
|
|
116
|
-
#
|
|
117
|
-
# @return [Tool]
|
|
118
|
-
FETCH = new(
|
|
119
|
-
name: 'fetch',
|
|
120
|
-
description: <<~DESC,
|
|
121
|
-
Downloads the given URL and returns its body verbatim.
|
|
122
|
-
|
|
123
|
-
Usage:
|
|
124
|
-
- Use for raw textual payloads: JSON APIs, CSV files, robots.txt, sitemaps, source files — anywhere a rendering pass would corrupt the data.
|
|
125
|
-
- For rendered HTML pages, use web_scrape — it extracts readable content; fetch returns the raw HTML bytes unchanged.
|
|
126
|
-
- Accepts text/* and common textual application/* types (JSON, XML, JS, XHTML, RSS, Atom). Refuses PDFs, images, and other binaries.
|
|
127
|
-
DESC
|
|
128
|
-
parameters: Parameters.build { |p|
|
|
129
|
-
p.required_string :url,
|
|
130
|
-
'Absolute URL to download, including the scheme, ' \
|
|
131
|
-
'e.g. "https://example.com/data.json".'
|
|
132
|
-
p.optional_integer :max_chars,
|
|
133
|
-
'Maximum number of characters of the body to ' \
|
|
134
|
-
'return. Defaults to 5000; hard-capped at ' \
|
|
135
|
-
'100000. When the body is longer than this, ' \
|
|
136
|
-
'output is cut and a marker reports the full ' \
|
|
137
|
-
'length.'
|
|
138
|
-
},
|
|
139
|
-
execute: ->(url:, max_chars: Fetch::DEFAULT_MAX_CHARS) {
|
|
140
|
-
Fetch.fetch(url, max_chars: max_chars)
|
|
141
|
-
},
|
|
142
|
-
# As {WEB_SCRAPE}: attacker-choosable host, attacker-authored response.
|
|
143
|
-
trifecta_legs: Pikuri::Tool::TrifectaLegs::ASSUMED
|
|
144
|
-
)
|
|
145
|
-
end
|
|
146
|
-
end
|
data/lib/pikuri/tool/scraper.rb
DELETED
|
@@ -1,144 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'stringio'
|
|
5
|
-
require 'uri'
|
|
6
|
-
|
|
7
|
-
module Pikuri
|
|
8
|
-
class Tool
|
|
9
|
-
# HTTP side of the web tools ({Tool::WEB_SCRAPE} and {Tool::FETCH}): GET the
|
|
10
|
-
# URL with a real-browser User-Agent, follow redirects, and hand the body to
|
|
11
|
-
# {Pikuri::Extractor.extract} with the response +Content-Type+ as the hint.
|
|
12
|
-
# HTML/XHTML renders via {Extractor::HTML}, other +text/*+ passes through
|
|
13
|
-
# verbatim, plug-in extractors extend the set (with pikuri-pdf,
|
|
14
|
-
# +application/pdf+ extracts — by header or +%PDF-+ magic, so a lying header
|
|
15
|
-
# still works); remaining types raise {FetchError} so the LLM observes the
|
|
16
|
-
# failure rather than an empty rendering.
|
|
17
|
-
#
|
|
18
|
-
# Split into a thin HTTP fetch ({.fetch}) and the extraction wrapper
|
|
19
|
-
# ({.visit}) so tests drive each in isolation and {Tool::Fetch} can reuse the
|
|
20
|
-
# HTTP half. Nothing here knows about the LLM — the wrapping tools own
|
|
21
|
-
# caching, truncation, and turning the result into an observation.
|
|
22
|
-
module Scraper
|
|
23
|
-
# Raised when a URL can't be rendered to Markdown — HTTP non-2xx, network
|
|
24
|
-
# failure, redirect loop, missing +Location+, unsupported content-type, or
|
|
25
|
-
# a parse failure. The web tools rescue this into an +"Error: ..."+
|
|
26
|
-
# observation; anything else bubbles up so genuine bugs stay visible.
|
|
27
|
-
class FetchError < StandardError; end
|
|
28
|
-
|
|
29
|
-
# @return [String] User-Agent sent with each request; many sites reject a
|
|
30
|
-
# missing or obviously-bot UA.
|
|
31
|
-
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
|
|
32
|
-
'(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
|
|
33
|
-
# @return [String] +Accept+ header, so content-negotiating servers hand
|
|
34
|
-
# back something usable: HTML first, then +application/pdf+, then +text/*+.
|
|
35
|
-
ACCEPT = 'text/html,application/xhtml+xml,application/pdf,text/*;q=0.8'
|
|
36
|
-
# @return [Integer] maximum HTTP redirects to follow before giving up.
|
|
37
|
-
MAX_REDIRECTS = 5
|
|
38
|
-
# @return [Integer] connect timeout (s) for the Faraday request.
|
|
39
|
-
OPEN_TIMEOUT = 10
|
|
40
|
-
# @return [Integer] read timeout (s) for the Faraday request.
|
|
41
|
-
READ_TIMEOUT = 20
|
|
42
|
-
|
|
43
|
-
# @return [Integer] max chars of an error response body in a {FetchError}
|
|
44
|
-
# message — enough to show what came back (a Cloudflare/WAF page) without
|
|
45
|
-
# flooding the observation.
|
|
46
|
-
ERROR_BODY_EXCERPT = 200
|
|
47
|
-
|
|
48
|
-
# Result of a successful {Scraper.fetch}: the response body, the
|
|
49
|
-
# normalized content-type (lower-cased, with any +; charset=...+
|
|
50
|
-
# parameters stripped), and the final URL after redirects.
|
|
51
|
-
Fetched = Data.define(:body, :content_type, :url)
|
|
52
|
-
|
|
53
|
-
# Fetch +url+ and render its main content as Markdown. No caching — every
|
|
54
|
-
# call hits the network (wrap it to memoize; {Tool::WebScrape.visit}
|
|
55
|
-
# does). The output is +strip+'d (leading/trailing blank lines are common
|
|
56
|
-
# from PDF page-feeds and trailing newlines) but interior whitespace is
|
|
57
|
-
# preserved (paragraph breaks and code indentation are load-bearing).
|
|
58
|
-
#
|
|
59
|
-
# @param url [String] absolute HTTP(S) URL
|
|
60
|
-
# @return [String] Markdown, whitespace-trimmed, otherwise uncapped —
|
|
61
|
-
# caller size-limits before feeding the LLM
|
|
62
|
-
# @raise [FetchError] on HTTP non-2xx, network failure, redirect loop, a
|
|
63
|
-
# 3xx without +Location+, an unrecognized response, or extraction failure
|
|
64
|
-
def self.visit(url)
|
|
65
|
-
extract(fetch(url)).strip
|
|
66
|
-
end
|
|
67
|
-
|
|
68
|
-
# Render a {Fetched} response as Markdown via {Pikuri::Extractor.extract},
|
|
69
|
-
# re-raising both extraction failure modes as {FetchError}. The
|
|
70
|
-
# content-type is passed verbatim (including +""+ for a missing header,
|
|
71
|
-
# which matches no text arm — a body without transport metadata is
|
|
72
|
-
# refused, not sniffed; only a strong magic sniff like +%PDF-+ overrides).
|
|
73
|
-
#
|
|
74
|
-
# @param fetched [Fetched]
|
|
75
|
-
# @return [String] Markdown from the matched extractor
|
|
76
|
-
# @raise [FetchError] when no extractor matches, or extraction fails
|
|
77
|
-
def self.extract(fetched)
|
|
78
|
-
Pikuri::Extractor.extract(StringIO.new(fetched.body), content_type: fetched.content_type)
|
|
79
|
-
rescue Pikuri::Extractor::Unsupported
|
|
80
|
-
raise FetchError, "unsupported content-type #{fetched.content_type.inspect} for #{fetched.url}"
|
|
81
|
-
rescue Pikuri::Extractor::Error => e
|
|
82
|
-
raise FetchError, e.message
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
# Download +url+, manually following up to {MAX_REDIRECTS} redirects. All
|
|
86
|
-
# recoverable failures (HTTP 4xx/5xx, +Faraday::Error+, exhausted redirect
|
|
87
|
-
# budget, 3xx without +Location+) surface as {FetchError}. Error bodies
|
|
88
|
-
# are trimmed to {ERROR_BODY_EXCERPT} so a challenge page doesn't dump
|
|
89
|
-
# kilobytes into the observation.
|
|
90
|
-
#
|
|
91
|
-
# @param url [String] absolute HTTP(S) URL
|
|
92
|
-
# @param limit [Integer] redirects remaining; recurses with +limit - 1+ on 3xx
|
|
93
|
-
# @return [Fetched] body, normalized content-type, final URL
|
|
94
|
-
# @raise [FetchError] on non-2xx/3xx, network error, redirect-loop
|
|
95
|
-
# exhaustion, or 3xx without +Location+
|
|
96
|
-
def self.fetch(url, limit: MAX_REDIRECTS)
|
|
97
|
-
raise FetchError, "too many redirects fetching #{url}" if limit.zero?
|
|
98
|
-
|
|
99
|
-
response = begin
|
|
100
|
-
Faraday.new(request: { open_timeout: OPEN_TIMEOUT, timeout: READ_TIMEOUT }).get(url) do |req|
|
|
101
|
-
req.headers['User-Agent'] = USER_AGENT
|
|
102
|
-
req.headers['Accept'] = ACCEPT
|
|
103
|
-
end
|
|
104
|
-
rescue Faraday::Error => e
|
|
105
|
-
raise FetchError, "#{e.class.name.split('::').last} fetching #{url}: #{e.message}"
|
|
106
|
-
end
|
|
107
|
-
|
|
108
|
-
case response.status
|
|
109
|
-
when 200..299
|
|
110
|
-
Fetched.new(body: response.body, content_type: normalize_content_type(response.headers['content-type']), url: url)
|
|
111
|
-
when 300..399
|
|
112
|
-
location = response.headers['location']
|
|
113
|
-
raise FetchError, "HTTP #{response.status} from #{url} with no Location header" if location.nil? || location.empty?
|
|
114
|
-
|
|
115
|
-
fetch(URI.join(url, location).to_s, limit: limit - 1)
|
|
116
|
-
else
|
|
117
|
-
raise FetchError, "HTTP #{response.status} fetching #{url}: #{excerpt(response.body)}"
|
|
118
|
-
end
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
# Lower-case +raw+ and strip any +; charset=...+ parameters so the
|
|
122
|
-
# extractors can match on a canonical token.
|
|
123
|
-
#
|
|
124
|
-
# @param raw [String, nil] raw +Content-Type+ header value
|
|
125
|
-
# @return [String] normalized content-type, or +""+ when the
|
|
126
|
-
# header was missing
|
|
127
|
-
def self.normalize_content_type(raw)
|
|
128
|
-
raw.to_s.split(';').first.to_s.strip.downcase
|
|
129
|
-
end
|
|
130
|
-
private_class_method :normalize_content_type
|
|
131
|
-
|
|
132
|
-
# Whitespace-collapse +body+ and clip to {ERROR_BODY_EXCERPT}, keeping the
|
|
133
|
-
# {FetchError} message a single readable line.
|
|
134
|
-
#
|
|
135
|
-
# @param body [String, nil]
|
|
136
|
-
# @return [String]
|
|
137
|
-
def self.excerpt(body)
|
|
138
|
-
text = body.to_s.gsub(/\s+/, ' ').strip
|
|
139
|
-
text.length > ERROR_BODY_EXCERPT ? "#{text[0, ERROR_BODY_EXCERPT]}..." : text
|
|
140
|
-
end
|
|
141
|
-
private_class_method :excerpt
|
|
142
|
-
end
|
|
143
|
-
end
|
|
144
|
-
end
|
|
@@ -1,170 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'json'
|
|
5
|
-
require 'nokogiri'
|
|
6
|
-
|
|
7
|
-
module Pikuri
|
|
8
|
-
class Tool
|
|
9
|
-
module Search
|
|
10
|
-
# Performs a Brave Search via the official Web Search API, returning hits
|
|
11
|
-
# as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
|
|
12
|
-
# parser (.parse) so tests exercise the parser against fixture JSON.
|
|
13
|
-
# {Engines#search} owns the final Markdown rendering.
|
|
14
|
-
#
|
|
15
|
-
# Constructed with its API key (+Brave.new(api_key:)+); {Engines} builds
|
|
16
|
-
# one only when a Brave key was configured. pikuri reads no key from the
|
|
17
|
-
# environment (CLAUDE.md "Environment is not a secret store"). Key at
|
|
18
|
-
# https://api-dashboard.search.brave.com — the free "Data for Search" tier
|
|
19
|
-
# allows 1 query/sec, ~2k/month.
|
|
20
|
-
#
|
|
21
|
-
# == Privacy posture
|
|
22
|
-
#
|
|
23
|
-
# Brave's API notice retains Search Query Logs 90 days and collects no
|
|
24
|
-
# identifiers linking a query to a person; it publicly commits not to train
|
|
25
|
-
# on query data, and offers Zero Data Retention — but only on Enterprise,
|
|
26
|
-
# not the free tier pikuri defaults to. Bottom line: the cleanest
|
|
27
|
-
# API-level posture of pikuri's three providers (no training, no IP
|
|
28
|
-
# linkage, capped 90-day retention), but a logged 90-day window on the
|
|
29
|
-
# cheap tier, so not a ZDR substitute for genuinely sensitive queries.
|
|
30
|
-
class Brave
|
|
31
|
-
# @return [String] Web Search endpoint
|
|
32
|
-
ENDPOINT = 'https://api.search.brave.com/res/v1/web/search'
|
|
33
|
-
# @return [Integer] default number of results returned.
|
|
34
|
-
DEFAULT_MAX_RESULTS = 10
|
|
35
|
-
# @return [RateLimiter] free-tier Brave caps at 1 req/sec; the 5-minute
|
|
36
|
-
# cooldown protects the monthly quota from doomed retries on a 429.
|
|
37
|
-
LIMITER = RateLimiter.new(min_interval: 1.0, cooldown: 300.0)
|
|
38
|
-
|
|
39
|
-
# @param api_key [String] Brave Search subscription token. Required
|
|
40
|
-
# and non-blank: pikuri reads no key from the environment — the
|
|
41
|
-
# host supplies it ({Engines} only constructs a Brave when a key
|
|
42
|
-
# was configured).
|
|
43
|
-
# @raise [ArgumentError] if +api_key+ is blank
|
|
44
|
-
def initialize(api_key:)
|
|
45
|
-
raise ArgumentError, 'Brave Search API key is blank' if api_key.to_s.strip.empty?
|
|
46
|
-
|
|
47
|
-
@api_key = api_key
|
|
48
|
-
end
|
|
49
|
-
|
|
50
|
-
# @return [String] short provider label for {Engines}.
|
|
51
|
-
def label
|
|
52
|
-
'Brave'
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
# Fetch results for +query+ as an +Array<Result>+. Throttled to 1/sec and
|
|
56
|
-
# circuit-broken 5 min on rate-limit/quota (see {LIMITER}). The caller
|
|
57
|
-
# ({Engines#search}) normalizes the query and wraps this in a cache.
|
|
58
|
-
#
|
|
59
|
-
# @param query [String] search query (already normalized)
|
|
60
|
-
# @param max_results [Integer] max entries; Brave's +count+ (1..20)
|
|
61
|
-
# @param cancellable [Pikuri::Agent::Control::Cancellable] makes
|
|
62
|
-
# {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
|
|
63
|
-
# @return [Array<Result>] hits, possibly empty when Brave matched nothing
|
|
64
|
-
# @raise [Engines::Unavailable] on HTTP 429 or 5xx (the cascade falls
|
|
65
|
-
# back), or if {LIMITER} is in cooldown. Other non-2xx (401/403 bad
|
|
66
|
-
# key) bubble up as +RuntimeError+.
|
|
67
|
-
# @raise [RuntimeError] for non-rate-limit HTTP failures, or a
|
|
68
|
-
# zero-result response that isn't a recognized empty-results payload.
|
|
69
|
-
def search(query, max_results: DEFAULT_MAX_RESULTS,
|
|
70
|
-
cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
71
|
-
LIMITER.call(cancellable: cancellable) do
|
|
72
|
-
response = Faraday.get(
|
|
73
|
-
ENDPOINT,
|
|
74
|
-
{ q: query, count: max_results },
|
|
75
|
-
{ 'X-Subscription-Token' => @api_key, 'Accept' => 'application/json' }
|
|
76
|
-
)
|
|
77
|
-
unless response.success?
|
|
78
|
-
if response.status == 429 || response.status >= 500
|
|
79
|
-
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
80
|
-
end
|
|
81
|
-
|
|
82
|
-
raise "Brave Search request failed: #{response.status} #{response.body}"
|
|
83
|
-
end
|
|
84
|
-
|
|
85
|
-
self.class.parse(response.body, max_results: max_results)
|
|
86
|
-
end
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
# Parse a Brave JSON response into {Result} rows (+<strong>+ highlights
|
|
90
|
-
# stripped). A genuine "no results" payload (recognized search shape,
|
|
91
|
-
# empty +mixed.main/top/side+) returns +[]+ so {Engines.search} renders
|
|
92
|
-
# its stub; any other zero-result shape raises.
|
|
93
|
-
#
|
|
94
|
-
# @param json [String] response body from {ENDPOINT}
|
|
95
|
-
# @param max_results [Integer] max entries
|
|
96
|
-
# @return [Array<Result>] hits, possibly empty on a recognized
|
|
97
|
-
# empty-results payload
|
|
98
|
-
# @raise [RuntimeError] on an unrecognized zero-result response
|
|
99
|
-
def self.parse(json, max_results: DEFAULT_MAX_RESULTS)
|
|
100
|
-
data = JSON.parse(json)
|
|
101
|
-
results = Array(data.dig('web', 'results')).take(max_results).filter_map do |r|
|
|
102
|
-
href = r['url'].to_s
|
|
103
|
-
next nil if href.empty?
|
|
104
|
-
|
|
105
|
-
Result.new(
|
|
106
|
-
url: href,
|
|
107
|
-
title: strip_html(r['title']),
|
|
108
|
-
body: strip_html(r['description'])
|
|
109
|
-
)
|
|
110
|
-
end
|
|
111
|
-
|
|
112
|
-
if results.empty?
|
|
113
|
-
return [] if genuine_no_results?(data)
|
|
114
|
-
|
|
115
|
-
raise diagnose_empty(data, json)
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
results
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
# Strip HTML markup (notably the +<strong>+ highlights Brave wraps around
|
|
122
|
-
# query terms) and collapse whitespace.
|
|
123
|
-
#
|
|
124
|
-
# @param html [String, nil]
|
|
125
|
-
# @return [String] plain text; empty string for nil
|
|
126
|
-
def self.strip_html(html)
|
|
127
|
-
return '' if html.nil?
|
|
128
|
-
|
|
129
|
-
Nokogiri::HTML.fragment(html).text.gsub(/\s+/, ' ').strip
|
|
130
|
-
end
|
|
131
|
-
private_class_method :strip_html
|
|
132
|
-
|
|
133
|
-
# True when a zero-+web.results+ response is Brave's own "ran, matched
|
|
134
|
-
# nothing" payload, not an error. Markers: the +type: "search"+ envelope
|
|
135
|
-
# and an empty +mixed+ block (Brave fills +mixed.main/top/side+ with
|
|
136
|
-
# matched verticals, so all three empty = succeeded-but-found-nothing).
|
|
137
|
-
#
|
|
138
|
-
# @param data [Hash, Object] parsed response
|
|
139
|
-
# @return [Boolean]
|
|
140
|
-
def self.genuine_no_results?(data)
|
|
141
|
-
return false unless data.is_a?(Hash) && data['type'] == 'search'
|
|
142
|
-
|
|
143
|
-
mixed = data['mixed']
|
|
144
|
-
return false unless mixed.is_a?(Hash)
|
|
145
|
-
|
|
146
|
-
%w[main top side].all? { |k| Array(mixed[k]).empty? }
|
|
147
|
-
end
|
|
148
|
-
private_class_method :genuine_no_results?
|
|
149
|
-
|
|
150
|
-
# Build an error message for a zero-result response: quote Brave's
|
|
151
|
-
# +error.detail+ if present, else a truncated raw body.
|
|
152
|
-
#
|
|
153
|
-
# @param data [Hash, Object] parsed response
|
|
154
|
-
# @param raw [String] raw response body
|
|
155
|
-
# @return [String] diagnostic to feed to +raise+
|
|
156
|
-
def self.diagnose_empty(data, raw)
|
|
157
|
-
if data.is_a?(Hash) && data['error'].is_a?(Hash)
|
|
158
|
-
err = data['error']
|
|
159
|
-
return "Brave Search returned an error: #{[err['code'], err['detail'] || err['meta']].compact.join(' — ')}"
|
|
160
|
-
end
|
|
161
|
-
|
|
162
|
-
snippet = raw.to_s[0, 800]
|
|
163
|
-
snippet += '…' if raw.to_s.length > 800
|
|
164
|
-
"Brave Search returned no results. Body: #{snippet}"
|
|
165
|
-
end
|
|
166
|
-
private_class_method :diagnose_empty
|
|
167
|
-
end
|
|
168
|
-
end
|
|
169
|
-
end
|
|
170
|
-
end
|
|
@@ -1,175 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'faraday'
|
|
4
|
-
require 'nokogiri'
|
|
5
|
-
require 'uri'
|
|
6
|
-
|
|
7
|
-
module Pikuri
|
|
8
|
-
class Tool
|
|
9
|
-
module Search
|
|
10
|
-
# Performs a DuckDuckGo search by scraping +html.duckduckgo.com+, returning
|
|
11
|
-
# hits as {Result} rows. Split into a thin HTTP fetch (#search) and a pure
|
|
12
|
-
# parser (.parse) so tests exercise the parser against fixture HTML.
|
|
13
|
-
# {Engines#search} owns the final Markdown rendering. Constructed with no
|
|
14
|
-
# arguments, sharing the uniform +#search+/+#label+ provider shape with the
|
|
15
|
-
# keyed {Brave} / {Exa}.
|
|
16
|
-
#
|
|
17
|
-
# == Privacy posture
|
|
18
|
-
#
|
|
19
|
-
# DDG doesn't save your IP or tie a per-user profile to queries, and
|
|
20
|
-
# proxies requests so downstream providers can't build a search history —
|
|
21
|
-
# real, but DDG relays web results over Bing, so *query content* still
|
|
22
|
-
# reaches Microsoft (who has no no-training pledge). Better than Exa for
|
|
23
|
-
# sensitive queries, worse than Brave; for anything genuinely embarrassing,
|
|
24
|
-
# don't search the web at all.
|
|
25
|
-
class DuckDuckGo
|
|
26
|
-
# @return [String] HTML search endpoint
|
|
27
|
-
ENDPOINT = 'https://html.duckduckgo.com/html/'
|
|
28
|
-
# @return [String] User-Agent sent with each request; DDG often rejects
|
|
29
|
-
# a missing or obviously-bot UA.
|
|
30
|
-
USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' \
|
|
31
|
-
'(KHTML, like Gecko) Chrome/120.0 Safari/537.36'
|
|
32
|
-
# @return [Integer] default number of results returned.
|
|
33
|
-
DEFAULT_MAX_RESULTS = 10
|
|
34
|
-
# @return [RateLimiter] paces calls (DDG bans IPs that hammer the HTML
|
|
35
|
-
# endpoint) and circuit-breaks 5 min on {Engines::Unavailable}.
|
|
36
|
-
LIMITER = RateLimiter.new(min_interval: 2.0, cooldown: 300.0)
|
|
37
|
-
|
|
38
|
-
# @return [String] short provider label for {Engines}.
|
|
39
|
-
def label
|
|
40
|
-
'DuckDuckGo'
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
# Fetch results for +query+ as an +Array<Result>+. Throttled to one every
|
|
44
|
-
# 2s and circuit-broken 5 min after a soft-block (see {LIMITER}). The
|
|
45
|
-
# caller ({Engines#search}) normalizes the query and wraps this in a cache.
|
|
46
|
-
#
|
|
47
|
-
# @param query [String] search query (already normalized)
|
|
48
|
-
# @param max_results [Integer] max result entries
|
|
49
|
-
# @param cancellable [Pikuri::Agent::Control::Cancellable] makes
|
|
50
|
-
# {LIMITER}'s pacing wait interruptible; see {RateLimiter#call}.
|
|
51
|
-
# @return [Array<Result>] hits, possibly empty when DDG matched nothing
|
|
52
|
-
# @raise [Engines::Unavailable] on a soft-block (anomaly/CAPTCHA page) or
|
|
53
|
-
# HTTP 429/5xx (the cascade falls back), or if {LIMITER} is in cooldown.
|
|
54
|
-
# @raise [RuntimeError] on other HTTP failures or an unrecognized
|
|
55
|
-
# empty-results layout. A genuine empty-results page is not an error;
|
|
56
|
-
# see {.parse}.
|
|
57
|
-
def search(query, max_results: DEFAULT_MAX_RESULTS,
|
|
58
|
-
cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
59
|
-
LIMITER.call(cancellable: cancellable) do
|
|
60
|
-
response = Faraday.get(ENDPOINT, { q: query }, { 'User-Agent' => USER_AGENT })
|
|
61
|
-
unless response.success?
|
|
62
|
-
if response.status == 429 || response.status >= 500
|
|
63
|
-
raise Engines::Unavailable, "HTTP #{response.status}"
|
|
64
|
-
end
|
|
65
|
-
|
|
66
|
-
raise "DuckDuckGo request failed: #{response.status} #{response.body}"
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
self.class.parse(response.body, max_results: max_results)
|
|
70
|
-
end
|
|
71
|
-
end
|
|
72
|
-
|
|
73
|
-
# Parse a +html.duckduckgo.com+ result page into {Result} rows (+<b>+
|
|
74
|
-
# highlights stripped). On zero result nodes: a genuine "no results" page
|
|
75
|
-
# returns +[]+ (so {Engines.search} renders its stub); the anomaly/CAPTCHA
|
|
76
|
-
# modal raises {Engines::Unavailable}; anything else raises {RuntimeError}
|
|
77
|
-
# (likely a layout change) — so an IP soft-block surfaces rather than
|
|
78
|
-
# masquerading as an empty search.
|
|
79
|
-
#
|
|
80
|
-
# @param html [String] HTML from html.duckduckgo.com
|
|
81
|
-
# @param max_results [Integer] max result entries
|
|
82
|
-
# @return [Array<Result>] hits, possibly empty on a genuine no-results page
|
|
83
|
-
# @raise [Engines::Unavailable] on the anomaly/CAPTCHA modal (IP soft-block)
|
|
84
|
-
# @raise [RuntimeError] on zero results with an unrecognized layout
|
|
85
|
-
def self.parse(html, max_results: DEFAULT_MAX_RESULTS)
|
|
86
|
-
doc = Nokogiri::HTML(html)
|
|
87
|
-
results = doc.css('div.result.web-result').take(max_results).filter_map do |node|
|
|
88
|
-
title_link = node.at_css('a.result__a')
|
|
89
|
-
next nil if title_link.nil?
|
|
90
|
-
|
|
91
|
-
snippet = node.at_css('a.result__snippet')
|
|
92
|
-
Result.new(
|
|
93
|
-
url: extract_url(title_link['href']),
|
|
94
|
-
title: title_link.text.strip,
|
|
95
|
-
body: snippet&.text&.strip.to_s
|
|
96
|
-
)
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
if results.empty?
|
|
100
|
-
return [] if genuine_no_results?(doc)
|
|
101
|
-
|
|
102
|
-
message = diagnose_empty(doc)
|
|
103
|
-
raise(anomaly_modal?(doc) ? Engines::Unavailable : RuntimeError, message)
|
|
104
|
-
end
|
|
105
|
-
|
|
106
|
-
results
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
# True when the page is DDG's anomaly/CAPTCHA modal (IP soft-blocked).
|
|
110
|
-
# {.parse} uses it to pick {Engines::Unavailable} (recoverable) over
|
|
111
|
-
# {RuntimeError} (unknown layout).
|
|
112
|
-
#
|
|
113
|
-
# @param doc [Nokogiri::HTML::Document]
|
|
114
|
-
# @return [Boolean]
|
|
115
|
-
def self.anomaly_modal?(doc)
|
|
116
|
-
!!(doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description'))
|
|
117
|
-
end
|
|
118
|
-
private_class_method :anomaly_modal?
|
|
119
|
-
|
|
120
|
-
# True when a zero-node page is DDG's own "no results found" page, not an
|
|
121
|
-
# anomaly/CAPTCHA. Anomaly wins: if the modal divs are present it's never
|
|
122
|
-
# a genuine empty result, even if the text mentions "No results".
|
|
123
|
-
#
|
|
124
|
-
# @param doc [Nokogiri::HTML::Document]
|
|
125
|
-
# @return [Boolean]
|
|
126
|
-
def self.genuine_no_results?(doc)
|
|
127
|
-
return false if doc.at_css('.anomaly-modal__title') || doc.at_css('.anomaly-modal__description')
|
|
128
|
-
return true if doc.at_css('div.no-results')
|
|
129
|
-
|
|
130
|
-
doc.text.include?('No results found')
|
|
131
|
-
end
|
|
132
|
-
private_class_method :genuine_no_results?
|
|
133
|
-
|
|
134
|
-
# Build an error message for a zero-match page: quote the DDG anomaly
|
|
135
|
-
# modal's title/description if present, else a truncated copy of the
|
|
136
|
-
# body's visible text (+<script>/<style>/<noscript>+ stripped).
|
|
137
|
-
#
|
|
138
|
-
# @param doc [Nokogiri::HTML::Document]
|
|
139
|
-
# @return [String] diagnostic to feed to +raise+
|
|
140
|
-
def self.diagnose_empty(doc)
|
|
141
|
-
title = doc.at_css('.anomaly-modal__title')&.text&.strip
|
|
142
|
-
desc = doc.at_css('.anomaly-modal__description')&.text&.strip
|
|
143
|
-
if title || desc
|
|
144
|
-
return "DuckDuckGo anomaly check (likely IP soft-block): #{[title, desc].compact.reject(&:empty?).join(' — ')}"
|
|
145
|
-
end
|
|
146
|
-
|
|
147
|
-
doc.css('script, style, noscript').remove
|
|
148
|
-
text = doc.text.gsub(/\s+/, ' ').strip
|
|
149
|
-
snippet = text.empty? ? '<empty body>' : text[0, 1500]
|
|
150
|
-
snippet += '…' if text.length > 1500
|
|
151
|
-
"DuckDuckGo returned no results. Page text: #{snippet}"
|
|
152
|
-
end
|
|
153
|
-
private_class_method :diagnose_empty
|
|
154
|
-
|
|
155
|
-
# Decode DDG's +//duckduckgo.com/l/?uddg=<encoded>+ redirect wrapper to
|
|
156
|
-
# the real target URL.
|
|
157
|
-
#
|
|
158
|
-
# @param href [String, nil] href from the result page
|
|
159
|
-
# @return [String] the decoded target, or +href+ unchanged when not a
|
|
160
|
-
# recognized DDG redirect or unparseable
|
|
161
|
-
def self.extract_url(href)
|
|
162
|
-
return href if href.nil? || href.empty?
|
|
163
|
-
|
|
164
|
-
uri = URI.parse(href.start_with?('//') ? "https:#{href}" : href)
|
|
165
|
-
return href unless uri.host&.end_with?('duckduckgo.com') && uri.path == '/l/'
|
|
166
|
-
|
|
167
|
-
params = URI.decode_www_form(uri.query.to_s).to_h
|
|
168
|
-
params['uddg'] || href
|
|
169
|
-
rescue URI::InvalidURIError
|
|
170
|
-
href
|
|
171
|
-
end
|
|
172
|
-
end
|
|
173
|
-
end
|
|
174
|
-
end
|
|
175
|
-
end
|