pikuri-web 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +68 -0
- data/lib/pikuri/web/fetch.rb +153 -0
- data/lib/pikuri/web/read_cached.rb +96 -0
- data/lib/pikuri/web/scrape.rb +107 -0
- data/lib/pikuri/web/scraper.rb +188 -0
- data/lib/pikuri/web/search/brave.rb +298 -0
- data/lib/pikuri/web/search/duckduckgo.rb +181 -0
- data/lib/pikuri/web/search/engines.rb +430 -0
- data/lib/pikuri/web/search/exa.rb +243 -0
- data/lib/pikuri/web/search/page_store.rb +142 -0
- data/lib/pikuri/web/search/rate_limiter.rb +114 -0
- data/lib/pikuri/web/search/result.rb +42 -0
- data/lib/pikuri/web/search.rb +229 -0
- data/lib/pikuri/web/url_cache/in_memory.rb +96 -0
- data/lib/pikuri/web/url_cache/on_disk.rb +80 -0
- data/lib/pikuri/web/url_cache.rb +63 -0
- data/lib/pikuri-web.rb +36 -0
- metadata +138 -0
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'openssl'
|
|
5
|
+
require 'securerandom'
|
|
6
|
+
require 'time'
|
|
7
|
+
|
|
8
|
+
module Pikuri
|
|
9
|
+
module Web
|
|
10
|
+
module Search
|
|
11
|
+
# The page text a text-mode search brought back, by handle — what
|
|
12
|
+
# +web_read_cached+ reads. {Engines} stores each row that came with text
|
|
13
|
+
# and prints its handle beside the row; the reader looks the handle up:
|
|
14
|
+
#
|
|
15
|
+
# handle = PageStore.default.store(url: 'https://example.com/a', title: 'A',
|
|
16
|
+
# text: '…', indexed_at: nil) # => "a3f9c21e"
|
|
17
|
+
# PageStore.default.read('a3f9c21e') # => Page, or nil once evicted
|
|
18
|
+
#
|
|
19
|
+
# Lives in memory only, as long as the Ruby VM ({UrlCache::InMemory}): it
|
|
20
|
+
# holds Brave and Exa output, whose terms forbid storing it — see
|
|
21
|
+
# +D_engine_results_in_memory+. Bounded at {MAX_ENTRIES} pages, least
|
|
22
|
+
# recently used out first, no TTL: the text is the engine's snapshot and a
|
|
23
|
+
# later read makes it no staler. A restart empties it, so old handles then
|
|
24
|
+
# answer "search again".
|
|
25
|
+
#
|
|
26
|
+
# == Handles
|
|
27
|
+
#
|
|
28
|
+
# A handle is the first 8 hex digits of +HMAC-SHA256(secret, url)+, the
|
|
29
|
+
# secret random per process and never written anywhere. Keyed, so a
|
|
30
|
+
# crafted URL can't be made to land on a popular page's handle; derived
|
|
31
|
+
# from the URL, so a stale handle can never come back meaning another
|
|
32
|
+
# page, and the same page keeps its handle across searches. When the
|
|
33
|
+
# 8-digit handle already holds a different URL, the new one takes 12.
|
|
34
|
+
#
|
|
35
|
+
# 8 digits, not fewer: the handle is unguessable only while the space
|
|
36
|
+
# dwarfs the store. Small or sequential handles would let a model in one
|
|
37
|
+
# conversation walk through the pages another conversation in the same
|
|
38
|
+
# process read.
|
|
39
|
+
#
|
|
40
|
+
# Sharing: +P_shared_locked+ — one {.default} for every agent in the
|
|
41
|
+
# process, so a researcher sub-agent's handle reads in its parent too —
|
|
42
|
+
# no way around the delegation gate, since a handle only ever reads text
|
|
43
|
+
# the engine returned, never text a model wrote; a
|
|
44
|
+
# +Mutex+ makes the collision check and the write in {#store} one step.
|
|
45
|
+
class PageStore
|
|
46
|
+
# @return [Integer] pages kept, ~100 searches' worth
|
|
47
|
+
MAX_ENTRIES = 1_000
|
|
48
|
+
|
|
49
|
+
# @return [Integer] hex digits in a handle
|
|
50
|
+
HANDLE_LENGTH = 8
|
|
51
|
+
|
|
52
|
+
# @return [Integer] hex digits in the handle of a URL whose 8-digit one
|
|
53
|
+
# is taken by another
|
|
54
|
+
LONG_HANDLE_LENGTH = 12
|
|
55
|
+
|
|
56
|
+
# A stored page, as {#read} answers it.
|
|
57
|
+
#
|
|
58
|
+
# @!attribute [r] url
|
|
59
|
+
# @return [String]
|
|
60
|
+
# @!attribute [r] title
|
|
61
|
+
# @return [String]
|
|
62
|
+
# @!attribute [r] text
|
|
63
|
+
# @return [String] the full text, as the engine stored it
|
|
64
|
+
# @!attribute [r] indexed_at
|
|
65
|
+
# @return [Time, nil] when the engine last fetched the page, if known
|
|
66
|
+
Page = Data.define(:url, :title, :text, :indexed_at)
|
|
67
|
+
|
|
68
|
+
# @param cache [UrlCache::InMemory, #read, #write] where pages live
|
|
69
|
+
# @param secret [String] HMAC key; random per process by default
|
|
70
|
+
def initialize(cache: UrlCache::InMemory.new(ttl: Float::INFINITY, max_entries: MAX_ENTRIES),
|
|
71
|
+
secret: SecureRandom.bytes(32))
|
|
72
|
+
@cache = cache
|
|
73
|
+
@secret = secret
|
|
74
|
+
@mutex = Mutex.new
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# The process-wide store {Engines} writes and +web_read_cached+ reads.
|
|
78
|
+
# A method so specs can swap in a fresh one.
|
|
79
|
+
#
|
|
80
|
+
# @return [PageStore]
|
|
81
|
+
def self.default
|
|
82
|
+
DEFAULT
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Store a page, replacing whatever its URL held.
|
|
86
|
+
#
|
|
87
|
+
# @param url [String]
|
|
88
|
+
# @param title [String]
|
|
89
|
+
# @param text [String]
|
|
90
|
+
# @param indexed_at [Time, nil]
|
|
91
|
+
# @return [String] the page's handle
|
|
92
|
+
def store(url:, title:, text:, indexed_at:)
|
|
93
|
+
entry = JSON.generate('url' => url, 'title' => title, 'text' => text,
|
|
94
|
+
'indexed_at' => indexed_at&.utc&.iso8601)
|
|
95
|
+
@mutex.synchronize do
|
|
96
|
+
short = digest(url)[0, HANDLE_LENGTH]
|
|
97
|
+
taken = read(short)
|
|
98
|
+
handle = taken.nil? || taken.url == url ? short : digest(url)[0, LONG_HANDLE_LENGTH]
|
|
99
|
+
@cache.write(handle, entry)
|
|
100
|
+
handle
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# @param url [String]
|
|
105
|
+
# @return [String, nil] the handle +url+ is stored under, or +nil+ when
|
|
106
|
+
# it isn't (never stored, or evicted)
|
|
107
|
+
def handle_for(url)
|
|
108
|
+
full = digest(url)
|
|
109
|
+
[full[0, HANDLE_LENGTH], full[0, LONG_HANDLE_LENGTH]].find { |h| read(h)&.url == url }
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# @param handle [String] as printed in the search results; surrounding
|
|
113
|
+
# whitespace and brackets are forgiven
|
|
114
|
+
# @return [Page, nil] the page, or +nil+ for an unknown or evicted handle
|
|
115
|
+
def read(handle)
|
|
116
|
+
handle = handle.to_s.strip.delete('[]').downcase
|
|
117
|
+
return nil unless handle.match?(/\A\h{#{HANDLE_LENGTH}}(\h{#{LONG_HANDLE_LENGTH - HANDLE_LENGTH}})?\z/o)
|
|
118
|
+
|
|
119
|
+
json = @cache.read(handle)
|
|
120
|
+
return nil if json.nil?
|
|
121
|
+
|
|
122
|
+
data = JSON.parse(json)
|
|
123
|
+
Page.new(url: data['url'], title: data['title'], text: data['text'],
|
|
124
|
+
indexed_at: data['indexed_at'] && Time.iso8601(data['indexed_at']))
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
private
|
|
128
|
+
|
|
129
|
+
# @param url [String]
|
|
130
|
+
# @return [String] lowercase hex HMAC of +url+
|
|
131
|
+
def digest(url)
|
|
132
|
+
OpenSSL::HMAC.hexdigest('SHA256', @secret, url)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# The process-wide store; see {.default}.
|
|
136
|
+
#
|
|
137
|
+
# @return [PageStore]
|
|
138
|
+
DEFAULT = new
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
# Loaded after {Tool} itself is defined; the +class Tool+ reopening below
|
|
5
|
+
# assumes that order.
|
|
6
|
+
module Web
|
|
7
|
+
module Search
|
|
8
|
+
# Thread-safe pacing + circuit-breaker wrapper for a search provider.
|
|
9
|
+
#
|
|
10
|
+
# +#call { ... }+ enforces a minimum interval between block invocations
|
|
11
|
+
# (sleeping if the previous was too recent) and watches for
|
|
12
|
+
# {Engines::Unavailable}: on one, a cooldown deadline is recorded and
|
|
13
|
+
# further calls within the window raise {Engines::Unavailable} immediately
|
|
14
|
+
# without running the block — so a rate-limited or bot-blocked provider
|
|
15
|
+
# isn't hammered with retries.
|
|
16
|
+
#
|
|
17
|
+
# Uses wall-clock {Time.now}, not monotonic: the intervals (1s–5min) are
|
|
18
|
+
# well above any NTP step, and {Time.now} keeps tests fakeable with
|
|
19
|
+
# Timecop.
|
|
20
|
+
#
|
|
21
|
+
# == Sharing
|
|
22
|
+
#
|
|
23
|
+
# +P_shared_locked+, and sharing is *mandatory*: a limiter must be scoped
|
|
24
|
+
# to the resource it protects, which here is one egress IP and one API
|
|
25
|
+
# key's quota — never one agent. Hence each provider holds its limiter in
|
|
26
|
+
# a class-level constant, so every agent in the VM queues behind the same
|
|
27
|
+
# one. Per-agent limiters would let ten agents fire ten simultaneous
|
|
28
|
+
# requests from one IP and earn exactly the ban this class exists to
|
|
29
|
+
# avoid.
|
|
30
|
+
#
|
|
31
|
+
# Two consequences of that scope, both correct and both surprising:
|
|
32
|
+
#
|
|
33
|
+
# * **It is a global queue, not just a pacer.** The mutex is held across
|
|
34
|
+
# the block — the HTTP call included — so ten agents searching at once
|
|
35
|
+
# serialize, with no timeout and no queue position. +cancellable:+
|
|
36
|
+
# covers the *pacing wait*, which is what the lock holder is doing; a
|
|
37
|
+
# caller still parked in +Mutex#synchronize+ waiting to acquire cannot
|
|
38
|
+
# observe its own cancel. It does still benefit, because every agent
|
|
39
|
+
# that cancels releases early and drains the queue faster. Making the
|
|
40
|
+
# acquisition itself interruptible means polling +Mutex#try_lock+ with
|
|
41
|
+
# explicit +lock+/+unlock+ — deliberately not built for a
|
|
42
|
+
# three-concurrent-agent case nothing has yet.
|
|
43
|
+
# * **The circuit breaker is global too.** One agent tripping the
|
|
44
|
+
# provider's soft-block puts every other agent into the same cooldown.
|
|
45
|
+
# Right, because the block is IP-wide and the others were blocked
|
|
46
|
+
# anyway — but from the host's side nine agents lose a provider for
|
|
47
|
+
# something the tenth did.
|
|
48
|
+
#
|
|
49
|
+
# Thread-safe.
|
|
50
|
+
class RateLimiter
|
|
51
|
+
# @param min_interval [Float] minimum seconds between consecutive
|
|
52
|
+
# block invocations. {#call} sleeps if a previous call was more
|
|
53
|
+
# recent.
|
|
54
|
+
# @param cooldown [Float] seconds to refuse calls after the block
|
|
55
|
+
# raises {Engines::Unavailable}. Calls within this window raise
|
|
56
|
+
# {Engines::Unavailable} immediately without invoking the block.
|
|
57
|
+
def initialize(min_interval:, cooldown:)
|
|
58
|
+
@min_interval = min_interval
|
|
59
|
+
@cooldown = cooldown
|
|
60
|
+
@mutex = Mutex.new
|
|
61
|
+
@last_call_at = nil
|
|
62
|
+
@cooldown_until = nil
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Run the block subject to the throttle and cooldown rules, with the
|
|
66
|
+
# mutex held (only one block runs at a time per limiter). If it raises
|
|
67
|
+
# {Engines::Unavailable}, cooldown is armed and the exception re-raised;
|
|
68
|
+
# any other exception bubbles up without arming cooldown.
|
|
69
|
+
#
|
|
70
|
+
# The pacing wait happens *through* +cancellable+, so the calling agent
|
|
71
|
+
# can abandon it on a user cancel. It is a per-call argument rather than
|
|
72
|
+
# constructor state because one limiter serves every agent in the VM
|
|
73
|
+
# (see +== Sharing+) while a token belongs to one of them — and it is
|
|
74
|
+
# always a real token, defaulting to
|
|
75
|
+
# {Pikuri::Agent::Control::Cancellable::NEVER} rather than +nil+, so
|
|
76
|
+
# there is one wait path instead of two.
|
|
77
|
+
#
|
|
78
|
+
# @param cancellable [Pikuri::Agent::Control::Cancellable] waited
|
|
79
|
+
# through instead of +Kernel#sleep+.
|
|
80
|
+
# @yieldreturn [Object] passed through
|
|
81
|
+
# @return [Object] whatever the block returned
|
|
82
|
+
# @raise [Engines::Unavailable] re-raised from the block, or raised
|
|
83
|
+
# directly when the limiter is in cooldown
|
|
84
|
+
# @raise [Pikuri::Agent::Control::Cancellable::Cancelled] if
|
|
85
|
+
# +cancellable+ trips before or during the pacing wait; the block does
|
|
86
|
+
# not run and the mutex is released, so the agents queued behind this
|
|
87
|
+
# one advance immediately.
|
|
88
|
+
def call(cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
|
|
89
|
+
@mutex.synchronize do
|
|
90
|
+
now = Time.now
|
|
91
|
+
if @cooldown_until && now < @cooldown_until
|
|
92
|
+
remaining = (@cooldown_until - now).ceil
|
|
93
|
+
raise Engines::Unavailable, "rate-limiter cooldown active for another #{remaining}s"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
if @last_call_at
|
|
97
|
+
elapsed = now - @last_call_at
|
|
98
|
+
cancellable.sleep(@min_interval - elapsed) if elapsed < @min_interval
|
|
99
|
+
end
|
|
100
|
+
@last_call_at = Time.now
|
|
101
|
+
|
|
102
|
+
begin
|
|
103
|
+
yield
|
|
104
|
+
rescue Engines::Unavailable
|
|
105
|
+
@cooldown_until = Time.now + @cooldown
|
|
106
|
+
raise
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
module Web
|
|
5
|
+
module Search
|
|
6
|
+
# A single search hit from a {Web::Search} provider ({DuckDuckGo},
|
|
7
|
+
# {Brave}, {Exa}). Providers return +Array<Result>+; {Engines#search}
|
|
8
|
+
# renders them into the smolagents-style Markdown the LLM sees. Structure
|
|
9
|
+
# split from rendering keeps the providers interchangeable — they need
|
|
10
|
+
# only agree on these fields (provider extras like scores are discarded,
|
|
11
|
+
# since no caller uses them).
|
|
12
|
+
#
|
|
13
|
+
# Result.new(url: 'https://example.com', title: 'Example', body: 'A snippet.')
|
|
14
|
+
#
|
|
15
|
+
# @!attribute [r] url
|
|
16
|
+
# @return [String] absolute URL of the hit
|
|
17
|
+
# @!attribute [r] title
|
|
18
|
+
# @return [String] plain-text title, provider highlight markup
|
|
19
|
+
# ({Brave}'s +<strong>+, {DuckDuckGo}'s +<b>+) already stripped
|
|
20
|
+
# @!attribute [r] body
|
|
21
|
+
# @return [String] plain-text snippet, possibly empty (an {Exa}
|
|
22
|
+
# navigational result with no highlights)
|
|
23
|
+
# @!attribute [r] text
|
|
24
|
+
# @return [String, nil] the page's text as the engine stored it, only
|
|
25
|
+
# from a provider built with +page_text: true+; +nil+ otherwise, and
|
|
26
|
+
# for a row that came back without any
|
|
27
|
+
# @!attribute [r] indexed_at
|
|
28
|
+
# @return [Time, nil] when the engine last fetched the page, if it says
|
|
29
|
+
# ({Brave}'s LLM Context does; {Exa} doesn't)
|
|
30
|
+
Result = Data.define(:url, :title, :body, :text, :indexed_at) do
|
|
31
|
+
# @param url [String]
|
|
32
|
+
# @param title [String]
|
|
33
|
+
# @param body [String]
|
|
34
|
+
# @param text [String, nil]
|
|
35
|
+
# @param indexed_at [Time, nil]
|
|
36
|
+
def initialize(url:, title:, body:, text: nil, indexed_at: nil)
|
|
37
|
+
super
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
module Web
|
|
5
|
+
# Builder for the LLM-facing web-search tool, and the namespace of the
|
|
6
|
+
# engines behind it. Orchestration lives in {Engines}; this owns only the
|
|
7
|
+
# {.build} factory wiring a configured {Engines} into a {Tool}.
|
|
8
|
+
#
|
|
9
|
+
# Unlike the other web tools (shared value constants), web_search is
|
|
10
|
+
# *host-configured*: the paid providers (Brave/Exa) join the cascade only
|
|
11
|
+
# when the host passes their key, so it's built per-wiring, not a shared
|
|
12
|
+
# constant. pikuri reads no key from the
|
|
13
|
+
# environment — the host supplies them (AGENTS.md "Environment is not a
|
|
14
|
+
# secret store").
|
|
15
|
+
#
|
|
16
|
+
# == Sharing
|
|
17
|
+
#
|
|
18
|
+
# +P_shared_locked+, and the interesting case: build one per agent or share
|
|
19
|
+
# one — it makes no difference to the throttling, because the pacing state
|
|
20
|
+
# is neither per agent nor per instance. Each provider's limiter is a
|
|
21
|
+
# class-level constant ({Search::DuckDuckGo::LIMITER} and friends), so all
|
|
22
|
+
# ten agents queue behind the same one. That is the only correct scope:
|
|
23
|
+
# what it protects is the egress IP and the API key's quota, and neither is
|
|
24
|
+
# per agent.
|
|
25
|
+
#
|
|
26
|
+
# What *is* per instance is trivial — the provider list and a
|
|
27
|
+
# last-logged-set memo, which can only duplicate an INFO line. The result
|
|
28
|
+
# caches ({Engines::DISK_CACHE}, {Engines::MEMORY_CACHE} and their text
|
|
29
|
+
# twins) are class-level as well, so one agent's search answers another's
|
|
30
|
+
# within the TTL, and a +page_text+ build writes the one process-wide
|
|
31
|
+
# {Search::PageStore}, so a sub-agent's handle reads in its parent.
|
|
32
|
+
#
|
|
33
|
+
# The consequence a host should plan for is latency, not corruption: see
|
|
34
|
+
# {Search::RateLimiter} for the serialization and the shared circuit
|
|
35
|
+
# breaker. Which is why this is the one web tool that opts into the
|
|
36
|
+
# per-call {Tool::ExecuteContext} (see {Pikuri::Tool}'s +== Cancellation+):
|
|
37
|
+
# an agent queued behind another agent's pacing wait can abandon it, even
|
|
38
|
+
# though the tool itself may be shared.
|
|
39
|
+
#
|
|
40
|
+
# A build with a +confirmer:+ takes the confirmer's posture as well: a
|
|
41
|
+
# {Workspace::Confirmer::Terminal} is +P_one_agent+, so share that build
|
|
42
|
+
# only as far as the confirmer allows.
|
|
43
|
+
module Search
|
|
44
|
+
# @return [Integer] result entries per search when the model names no
|
|
45
|
+
# +max_results+
|
|
46
|
+
DEFAULT_MAX_RESULTS = 10
|
|
47
|
+
|
|
48
|
+
# Description shown to the LLM, opencode-shape (summary + +Usage:+).
|
|
49
|
+
#
|
|
50
|
+
# @return [String]
|
|
51
|
+
DESCRIPTION = <<~DESC
|
|
52
|
+
Searches the web for a query and returns the top results as a Markdown list of titles, URLs, and short snippets.
|
|
53
|
+
|
|
54
|
+
Usage:
|
|
55
|
+
- Use this to find candidate URLs, then call web_scrape on the most promising one(s) for full content. Snippets alone rarely answer a question.
|
|
56
|
+
DESC
|
|
57
|
+
|
|
58
|
+
# Description for a +page_text: true+ build, which pairs with
|
|
59
|
+
# +web_read_cached+ rather than +web_scrape+.
|
|
60
|
+
#
|
|
61
|
+
# @return [String]
|
|
62
|
+
PAGE_TEXT_DESCRIPTION = <<~DESC
|
|
63
|
+
Searches the web for a query and returns the top results as a Markdown list of titles, URLs, and short snippets.
|
|
64
|
+
|
|
65
|
+
Usage:
|
|
66
|
+
- A result led by a handle in brackets, e.g. [a3f9c21e], can be read in full: call web_read_cached with that handle. Snippets alone rarely answer a question.
|
|
67
|
+
- A result without a handle can't be read; pick another result or refine the query.
|
|
68
|
+
DESC
|
|
69
|
+
|
|
70
|
+
# Description for a +snippets_only: true+ build, which pairs with no
|
|
71
|
+
# page reader at all.
|
|
72
|
+
#
|
|
73
|
+
# @return [String]
|
|
74
|
+
SNIPPETS_ONLY_DESCRIPTION = <<~DESC
|
|
75
|
+
Searches the web for a query and returns the top results as a Markdown list of titles, URLs, and short snippets.
|
|
76
|
+
|
|
77
|
+
Usage:
|
|
78
|
+
- The snippets are all you get; you can't open the result pages. Answer from the snippets, or refine the query and search again.
|
|
79
|
+
- When a snippet answers the question, cite its URL.
|
|
80
|
+
DESC
|
|
81
|
+
|
|
82
|
+
# Build the +web_search+ tool. Calls {Search::Engines#search}, which
|
|
83
|
+
# cascades the selected providers in random order, falling back on
|
|
84
|
+
# unavailability and rendering the winner's rows to a stable Markdown
|
|
85
|
+
# shape.
|
|
86
|
+
#
|
|
87
|
+
# Search.build # DuckDuckGo alone
|
|
88
|
+
# Search.build(brave_key: 'BSA...') # DuckDuckGo + Brave
|
|
89
|
+
# Search.build(engines: [:brave], brave_key: 'BSA...') # Brave alone
|
|
90
|
+
#
|
|
91
|
+
# For a search-only agent, pair a +page_text+ build with
|
|
92
|
+
# {Web::READ_CACHED} instead of +web_scrape+: the reader makes no
|
|
93
|
+
# request, so the agent's one egress leg stays this tool's vouched one.
|
|
94
|
+
#
|
|
95
|
+
# c.add_tool Search.build(engines: [:exa], exa_key: 'xyz', page_text: true)
|
|
96
|
+
# c.add_tool Web::READ_CACHED
|
|
97
|
+
#
|
|
98
|
+
# For an agent with no page reader whatsoever, +snippets_only: true+
|
|
99
|
+
# tells the model the snippets are the whole answer, rather than sending
|
|
100
|
+
# it after a +web_scrape+ it doesn't have. Its egress is then this
|
|
101
|
+
# tool's query and nothing else, which is what makes the next knob
|
|
102
|
+
# worth having.
|
|
103
|
+
#
|
|
104
|
+
# A +confirmer:+ shows the human each query before it leaves, editable,
|
|
105
|
+
# and the edited text is what gets searched. The query is the tool's
|
|
106
|
+
# whole payload, so a confirmer that {Workspace::Confirmer#blocks_on_human?}
|
|
107
|
+
# grades the leg +:human_reviewed+ — the human sees the bytes, not a
|
|
108
|
+
# proxy. It is not an injection defense: the destination is already
|
|
109
|
+
# +:tool_vouched+, so an injected query lands in an engine's logs the
|
|
110
|
+
# attacker can't read, and the trifecta verdict doesn't move. What it
|
|
111
|
+
# buys is privacy from the engine operator, for an agent holding
|
|
112
|
+
# private data that may otherwise quote it into a query unprompted.
|
|
113
|
+
#
|
|
114
|
+
# c.add_tool Search.build(snippets_only: true, confirmer: Workspace::Confirmer::Terminal.new)
|
|
115
|
+
#
|
|
116
|
+
# @param engines [Array<String, Symbol>, nil] engine names to use —
|
|
117
|
+
# +duckduckgo+, +brave+, +exa+. +nil+ selects DuckDuckGo plus every
|
|
118
|
+
# engine whose key is supplied, DuckDuckGo left out under
|
|
119
|
+
# +page_text+. Order carries no preference; see
|
|
120
|
+
# {Search::Engines.resolve}.
|
|
121
|
+
# @param brave_key [String, nil] Brave key
|
|
122
|
+
# (https://api-dashboard.search.brave.com).
|
|
123
|
+
# @param exa_key [String, nil] Exa key (https://exa.ai).
|
|
124
|
+
# @param page_text [Boolean] bring back each result's page text from the
|
|
125
|
+
# engine's index and print a handle for {Web::READ_CACHED}; see
|
|
126
|
+
# {Search::Engines}' +== Page text+. Needs Brave or Exa among the engines.
|
|
127
|
+
# @param snippets_only [Boolean] describe the snippets as the whole
|
|
128
|
+
# answer, for an agent wired with no page reader; excludes +page_text+.
|
|
129
|
+
# @param confirmer [Workspace::Confirmer, nil] asks the human to approve
|
|
130
|
+
# (and possibly rewrite) each query before it is sent; +nil+ sends
|
|
131
|
+
# unasked.
|
|
132
|
+
# @return [Tool] the +web_search+ tool in OpenAI tool-call shape
|
|
133
|
+
# @raise [ArgumentError] on an unusable selection: an empty or repeated
|
|
134
|
+
# +engines+ list, an unknown engine name, an engine named without its
|
|
135
|
+
# key, +page_text+ without Brave or Exa, or +page_text+ together with
|
|
136
|
+
# +snippets_only+.
|
|
137
|
+
def self.build(engines: nil, brave_key: nil, exa_key: nil, page_text: false,
|
|
138
|
+
snippets_only: false, confirmer: nil)
|
|
139
|
+
raise ArgumentError, 'page_text and snippets_only exclude each other' if page_text && snippets_only
|
|
140
|
+
|
|
141
|
+
cascade = Search::Engines.new(engines: engines, brave_key: brave_key, exa_key: exa_key,
|
|
142
|
+
page_text: page_text)
|
|
143
|
+
description = if page_text then PAGE_TEXT_DESCRIPTION
|
|
144
|
+
elsif snippets_only then SNIPPETS_ONLY_DESCRIPTION
|
|
145
|
+
else DESCRIPTION
|
|
146
|
+
end
|
|
147
|
+
Tool.new(
|
|
148
|
+
name: 'web_search',
|
|
149
|
+
description: description,
|
|
150
|
+
parameters: Tool::Parameters.build { |p|
|
|
151
|
+
p.required_string :query,
|
|
152
|
+
'The search query, e.g. "BigDecimal precision Ruby".'
|
|
153
|
+
p.optional_integer :max_results,
|
|
154
|
+
'Maximum number of result entries to return. ' \
|
|
155
|
+
"Defaults to #{DEFAULT_MAX_RESULTS}, e.g. 5."
|
|
156
|
+
},
|
|
157
|
+
# +wants_context:+ opts in to the per-call {Tool::ExecuteContext}, so
|
|
158
|
+
# a shared instance still honours whichever agent is waiting.
|
|
159
|
+
wants_context: true,
|
|
160
|
+
execute: lambda { |query:, context:, max_results: DEFAULT_MAX_RESULTS|
|
|
161
|
+
asked = query
|
|
162
|
+
if confirmer
|
|
163
|
+
case decision = confirmer.ask(request: Workspace::Confirmer::Request.new(
|
|
164
|
+
question: 'Send this web search query to the search engine?',
|
|
165
|
+
detail: query, editable: true
|
|
166
|
+
), cancellable: context.cancellable)
|
|
167
|
+
in Workspace::Confirmer::Approved(new_request_detail:) then query = new_request_detail
|
|
168
|
+
in Workspace::Confirmer::Rejected | Workspace::Confirmer::Redirected
|
|
169
|
+
return decision.observation('the web search')
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
result = cascade.search(query, max_results: max_results, cancellable: context.cancellable)
|
|
173
|
+
return result if query == asked || result.content.start_with?('Error:')
|
|
174
|
+
|
|
175
|
+
# The model would otherwise read results that don't match what it
|
|
176
|
+
# asked for, and wonder why.
|
|
177
|
+
result.with(content: "The user changed the query to: #{query}\n\n#{result.content}")
|
|
178
|
+
},
|
|
179
|
+
# Snippets are written by whoever ranks for the query, and the query
|
|
180
|
+
# string itself leaves the machine — a search is an outbound channel
|
|
181
|
+
# even though nothing is "sent". Unless a human-blocking confirmer
|
|
182
|
+
# shows it first, nobody reviews that query, hence +:unreviewed+;
|
|
183
|
+
# the whole difference from {FETCH} is the destination.
|
|
184
|
+
#
|
|
185
|
+
# +:tool_vouched+ against the three clauses: the caller supplies a
|
|
186
|
+
# query, never a host, and the engine set is this file's own
|
|
187
|
+
# ({Search::Engines::ENGINE_NAMES}); a search engine does not
|
|
188
|
+
# republish the queries it receives; and the request rides TLS to a
|
|
189
|
+
# party with no relationship to the attacker. A host picks from that
|
|
190
|
+
# closed list of three and cannot extend it, so the destination is
|
|
191
|
+
# still *this code* choosing, not the model.
|
|
192
|
+
#
|
|
193
|
+
# It is not a claim that the leg is harmless. A secret can still be
|
|
194
|
+
# pushed out inside a query — it just cannot be read back, and only
|
|
195
|
+
# while this stays the agent's sole egress leg.
|
|
196
|
+
trifecta_legs: Pikuri::Tool::TrifectaLegs.new(
|
|
197
|
+
private: false, untrusted: :hard,
|
|
198
|
+
egress_payload_review: confirmer&.blocks_on_human? ? :human_reviewed : :unreviewed,
|
|
199
|
+
egress_destination: :tool_vouched
|
|
200
|
+
)
|
|
201
|
+
)
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Build the +web_search+ tool from a config file's +search:+ block, the
|
|
205
|
+
# host-facing shortcut for {build}:
|
|
206
|
+
#
|
|
207
|
+
# # ~/.pikuri-examples-config.yaml
|
|
208
|
+
# search:
|
|
209
|
+
# brave_key: BSA...
|
|
210
|
+
# engines: [brave]
|
|
211
|
+
#
|
|
212
|
+
# c.add_tool Pikuri::Web::Search.from_h(examples_config['search'])
|
|
213
|
+
#
|
|
214
|
+
# @param config [Hash{String, Symbol => Object}, nil] the +search:+ block;
|
|
215
|
+
# +nil+ (an absent key) yields the DuckDuckGo-only default.
|
|
216
|
+
# +page_text+ is a wiring choice, paired with the tools the host adds, so
|
|
217
|
+
# it never comes from the config file — a host wanting it calls {build}:
|
|
218
|
+
#
|
|
219
|
+
# Search.build(**Search::Engines.config_from_h(cfg), page_text: true)
|
|
220
|
+
#
|
|
221
|
+
# @return [Tool] the +web_search+ tool in OpenAI tool-call shape
|
|
222
|
+
# @raise [ArgumentError] as {build}, plus on a malformed block; see
|
|
223
|
+
# {Search::Engines.config_from_h}.
|
|
224
|
+
def self.from_h(config)
|
|
225
|
+
build(**Search::Engines.config_from_h(config))
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
end
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Pikuri
|
|
4
|
+
module Web
|
|
5
|
+
module UrlCache
|
|
6
|
+
# The in-memory store: a bounded, least-recently-used +Hash+ that lives as
|
|
7
|
+
# long as the Ruby VM and never touches disk. Part of {UrlCache} — see it
|
|
8
|
+
# for the API.
|
|
9
|
+
#
|
|
10
|
+
# MEMORY_CACHE = UrlCache::InMemory.new(ttl: UrlCache::DEFAULT_TTL, max_entries: 100)
|
|
11
|
+
#
|
|
12
|
+
# For payloads whose terms forbid storing them: Brave's API terms allow
|
|
13
|
+
# only "transient storage required for operation", Exa's no copy beyond a
|
|
14
|
+
# browser's display cache. A process-lifetime copy is the transient one;
|
|
15
|
+
# exit frees it. So nothing tears it down earlier — not an agent's close,
|
|
16
|
+
# a context clear or a history load.
|
|
17
|
+
#
|
|
18
|
+
# Past +max_entries+, a {#write} evicts the least recently used entry; a hit
|
|
19
|
+
# counts as a use. The TTL is a freshness rule only: an expired entry is a
|
|
20
|
+
# miss, dropped when met.
|
|
21
|
+
#
|
|
22
|
+
# Values are stored frozen, so a caller can't mutate a shared entry in
|
|
23
|
+
# place; {#write} and {#fetch} return the frozen copy, hit or miss alike.
|
|
24
|
+
#
|
|
25
|
+
# Sharing: +P_shared_locked+ — every owner keeps one in a class-level
|
|
26
|
+
# constant shared by all agents in the process; a +Mutex+ guards the hash,
|
|
27
|
+
# reads included (a hit reorders it). {#fetch}'s block runs outside the
|
|
28
|
+
# lock, so two callers racing on one cold key both compute and the later
|
|
29
|
+
# write wins — the same benign race as {OnDisk}.
|
|
30
|
+
class InMemory
|
|
31
|
+
# @param ttl [Numeric] freshness window in seconds; +Float::INFINITY+
|
|
32
|
+
# never expires, leaving eviction to +max_entries+ alone.
|
|
33
|
+
# @param max_entries [Integer] how many entries to keep
|
|
34
|
+
# @raise [ArgumentError] unless +max_entries+ is a positive Integer
|
|
35
|
+
def initialize(ttl:, max_entries:)
|
|
36
|
+
unless max_entries.is_a?(Integer) && max_entries.positive?
|
|
37
|
+
raise ArgumentError, "max_entries must be a positive Integer, got #{max_entries.inspect}"
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
@ttl = ttl
|
|
41
|
+
@max_entries = max_entries
|
|
42
|
+
@entries = {} # key => [value, stored_at]; insertion order is recency order
|
|
43
|
+
@mutex = Mutex.new
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @param key [String] a URL or any opaque string
|
|
47
|
+
# @return [String, nil] the stored (frozen) payload, or +nil+ when absent
|
|
48
|
+
# or stale
|
|
49
|
+
def read(key)
|
|
50
|
+
@mutex.synchronize do
|
|
51
|
+
entry = @entries.delete(key)
|
|
52
|
+
next nil if entry.nil? || now - entry.last >= @ttl
|
|
53
|
+
|
|
54
|
+
@entries[key] = entry
|
|
55
|
+
entry.first
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# @param key [String] a URL or any opaque string
|
|
60
|
+
# @param value [String] payload to store, replacing any entry
|
|
61
|
+
# @return [String] the stored, frozen copy of +value+
|
|
62
|
+
def write(key, value)
|
|
63
|
+
stored = value.frozen? ? value : value.dup.freeze
|
|
64
|
+
@mutex.synchronize do
|
|
65
|
+
@entries.delete(key)
|
|
66
|
+
@entries[key] = [stored, now]
|
|
67
|
+
@entries.shift while @entries.size > @max_entries
|
|
68
|
+
end
|
|
69
|
+
stored
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Return the stored payload for +key+ if fresh, else yield to compute it,
|
|
73
|
+
# store and return it. A raising block stores nothing (errors aren't cached).
|
|
74
|
+
#
|
|
75
|
+
# @param key [String] a URL or any opaque string
|
|
76
|
+
# @yieldreturn [String] payload to store and return on a miss
|
|
77
|
+
# @return [String] cached or freshly-computed payload, frozen
|
|
78
|
+
def fetch(key)
|
|
79
|
+
cached = read(key)
|
|
80
|
+
return cached unless cached.nil?
|
|
81
|
+
|
|
82
|
+
write(key, yield)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
# Monotonic, so a wall-clock jump neither expires nor revives entries.
|
|
88
|
+
#
|
|
89
|
+
# @return [Float] seconds
|
|
90
|
+
def now
|
|
91
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|