pikuri-core 0.0.7 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +1 -1
  3. data/lib/pikuri/agent/chat_transport.rb +73 -93
  4. data/lib/pikuri/agent/configurator.rb +46 -106
  5. data/lib/pikuri/agent/context_window_detector.rb +44 -85
  6. data/lib/pikuri/agent/control/cancellable.rb +87 -66
  7. data/lib/pikuri/agent/control/interloper.rb +127 -105
  8. data/lib/pikuri/agent/control/step_limit.rb +25 -41
  9. data/lib/pikuri/agent/control.rb +14 -34
  10. data/lib/pikuri/agent/event.rb +123 -188
  11. data/lib/pikuri/agent/extension.rb +118 -94
  12. data/lib/pikuri/agent/extension_context.rb +50 -77
  13. data/lib/pikuri/agent/history.rb +653 -0
  14. data/lib/pikuri/agent/listener/rate_limited.rb +40 -66
  15. data/lib/pikuri/agent/listener/terminal.rb +143 -117
  16. data/lib/pikuri/agent/listener/token_log.rb +101 -140
  17. data/lib/pikuri/agent/listener.rb +23 -43
  18. data/lib/pikuri/agent/listener_list.rb +26 -47
  19. data/lib/pikuri/agent/synthesizer.rb +45 -87
  20. data/lib/pikuri/agent.rb +816 -474
  21. data/lib/pikuri/bundler_env.rb +68 -0
  22. data/lib/pikuri/extractor/html.rb +63 -110
  23. data/lib/pikuri/extractor/passthrough.rb +20 -30
  24. data/lib/pikuri/extractor.rb +93 -154
  25. data/lib/pikuri/file_type.rb +63 -135
  26. data/lib/pikuri/finalizers.rb +32 -47
  27. data/lib/pikuri/paths.rb +104 -13
  28. data/lib/pikuri/ruby_llm_patches.rb +106 -0
  29. data/lib/pikuri/sanitizer.rb +45 -67
  30. data/lib/pikuri/subprocess.rb +75 -119
  31. data/lib/pikuri/testing.rb +296 -0
  32. data/lib/pikuri/tool/calculator.rb +56 -66
  33. data/lib/pikuri/tool/execute_context.rb +42 -0
  34. data/lib/pikuri/tool/fetch.rb +51 -77
  35. data/lib/pikuri/tool/parameters.rb +21 -29
  36. data/lib/pikuri/tool/scraper.rb +55 -97
  37. data/lib/pikuri/tool/search/brave.rb +52 -80
  38. data/lib/pikuri/tool/search/duckduckgo.rb +59 -91
  39. data/lib/pikuri/tool/search/engines.rb +230 -97
  40. data/lib/pikuri/tool/search/exa.rb +56 -90
  41. data/lib/pikuri/tool/search/rate_limiter.rb +61 -38
  42. data/lib/pikuri/tool/search/result.rb +10 -15
  43. data/lib/pikuri/tool/trifecta_legs.rb +217 -0
  44. data/lib/pikuri/tool/web_scrape.rb +38 -54
  45. data/lib/pikuri/tool/web_search.rb +100 -24
  46. data/lib/pikuri/tool.rb +140 -65
  47. data/lib/pikuri/trifecta/contribution.rb +43 -0
  48. data/lib/pikuri/trifecta/node.rb +47 -0
  49. data/lib/pikuri/trifecta/report.rb +230 -0
  50. data/lib/pikuri/trifecta.rb +127 -0
  51. data/lib/pikuri/url_cache.rb +33 -49
  52. data/lib/pikuri/version.rb +1 -1
  53. data/lib/pikuri-core.rb +72 -88
  54. data/prompts/agent-loop.txt +5 -0
  55. data/prompts/pikuri-chat.txt +3 -12
  56. metadata +14 -3
@@ -7,23 +7,47 @@ module Pikuri
7
7
  module Search
8
8
  # Thread-safe pacing + circuit-breaker wrapper for a search provider.
9
9
  #
10
- # +#call { ... }+ enforces a minimum interval between consecutive
11
- # invocations of the block (sleeping if the previous one was too
12
- # recent), and watches for {Engines::Unavailable} raised by the
13
- # block: when that happens, a cooldown deadline is recorded and
14
- # further calls within the window raise {Engines::Unavailable}
15
- # immediately without invoking the block. This stops a provider
16
- # that has been rate-limited or bot-blocked from being hammered
17
- # with retries.
10
+ # +#call { ... }+ enforces a minimum interval between block invocations
11
+ # (sleeping if the previous was too recent) and watches for
12
+ # {Engines::Unavailable}: on one, a cooldown deadline is recorded and
13
+ # further calls within the window raise {Engines::Unavailable} immediately
14
+ # without running the block — so a rate-limited or bot-blocked provider
15
+ # isn't hammered with retries.
18
16
  #
19
- # The mutex is held across the block, so concurrent callers
20
- # serialize — matching the behavior {DuckDuckGo} has always
21
- # required to keep its IP-spacing throttle correct under
22
- # concurrent agents.
17
+ # Uses wall-clock {Time.now}, not monotonic: the intervals (1s–5min) are
18
+ # well above any NTP step, and {Time.now} keeps tests fakeable with
19
+ # Timecop.
23
20
  #
24
- # Uses wall-clock {Time.now} rather than the monotonic clock; the
25
- # intervals here are 1s–5min, well above any realistic NTP step,
26
- # and {Time.now} keeps tests trivially fakeable with Timecop.
21
+ # == Sharing
22
+ #
23
+ # +P_shared_locked+, and sharing is *mandatory*: a limiter must be scoped
24
+ # to the resource it protects, which here is one egress IP and one API
25
+ # key's quota — never one agent. Hence each provider holds its limiter in
26
+ # a class-level constant, so every agent in the VM queues behind the same
27
+ # one. Per-agent limiters would let ten agents fire ten simultaneous
28
+ # requests from one IP and earn exactly the ban this class exists to
29
+ # avoid. (Contrast {Pikuri::Agent::Listener::RateLimited}, which paces one
30
+ # renderer's repaints and is therefore per listener chain.)
31
+ #
32
+ # Two consequences of that scope, both correct and both surprising:
33
+ #
34
+ # * **It is a global queue, not just a pacer.** The mutex is held across
35
+ # the block — the HTTP call included — so ten agents searching at once
36
+ # serialize, with no timeout and no queue position. +cancellable:+
37
+ # covers the *pacing wait*, which is what the lock holder is doing; a
38
+ # caller still parked in +Mutex#synchronize+ waiting to acquire cannot
39
+ # observe its own cancel. It does still benefit, because every agent
40
+ # that cancels releases early and drains the queue faster. Making the
41
+ # acquisition itself interruptible means polling +Mutex#try_lock+ with
42
+ # explicit +lock+/+unlock+ — deliberately not built for a
43
+ # three-concurrent-agent case nothing has yet.
44
+ # * **The circuit breaker is global too.** One agent tripping the
45
+ # provider's soft-block puts every other agent into the same cooldown.
46
+ # Right, because the block is IP-wide and the others were blocked
47
+ # anyway — but from the host's side nine agents lose a provider for
48
+ # something the tenth did.
49
+ #
50
+ # Thread-safe.
27
51
  class RateLimiter
28
52
  # @param min_interval [Float] minimum seconds between consecutive
29
53
  # block invocations. {#call} sleeps if a previous call was more
@@ -39,20 +63,30 @@ module Pikuri
39
63
  @cooldown_until = nil
40
64
  end
41
65
 
42
- # Run the given block subject to throttle and cooldown rules.
66
+ # Run the block subject to the throttle and cooldown rules, with the
67
+ # mutex held (only one block runs at a time per limiter). If it raises
68
+ # {Engines::Unavailable}, cooldown is armed and the exception re-raised;
69
+ # any other exception bubbles up without arming cooldown.
43
70
  #
44
- # The block is invoked with the mutex held, so concurrent calls
45
- # serialize: only one block runs at a time per limiter instance.
46
- # If the block raises {Engines::Unavailable}, the cooldown is
47
- # armed and the exception is re-raised. Any other exception
48
- # bubbles up without arming cooldown — only "try again later"
49
- # signals from the provider are treated as backoff triggers.
71
+ # The pacing wait happens *through* +cancellable+, so the calling agent
72
+ # can abandon it on a user cancel. It is a per-call argument rather than
73
+ # constructor state because one limiter serves every agent in the VM
74
+ # (see +== Sharing+) while a token belongs to one of them — and it is
75
+ # always a real token, defaulting to
76
+ # {Pikuri::Agent::Control::Cancellable::NEVER} rather than +nil+, so
77
+ # there is one wait path instead of two.
50
78
  #
51
- # @yieldreturn [Object] block's return value is passed through
79
+ # @param cancellable [Pikuri::Agent::Control::Cancellable] waited
80
+ # through instead of +Kernel#sleep+.
81
+ # @yieldreturn [Object] passed through
52
82
  # @return [Object] whatever the block returned
53
- # @raise [Engines::Unavailable] either re-raised from the block,
54
- # or raised directly when the limiter is currently in cooldown
55
- def call
83
+ # @raise [Engines::Unavailable] re-raised from the block, or raised
84
+ # directly when the limiter is in cooldown
85
+ # @raise [Pikuri::Agent::Control::Cancellable::Cancelled] if
86
+ # +cancellable+ trips before or during the pacing wait; the block does
87
+ # not run and the mutex is released, so the agents queued behind this
88
+ # one advance immediately.
89
+ def call(cancellable: Pikuri::Agent::Control::Cancellable::NEVER)
56
90
  @mutex.synchronize do
57
91
  now = Time.now
58
92
  if @cooldown_until && now < @cooldown_until
@@ -62,7 +96,7 @@ module Pikuri
62
96
 
63
97
  if @last_call_at
64
98
  elapsed = now - @last_call_at
65
- sleep_for(@min_interval - elapsed) if elapsed < @min_interval
99
+ cancellable.sleep(@min_interval - elapsed) if elapsed < @min_interval
66
100
  end
67
101
  @last_call_at = Time.now
68
102
 
@@ -75,17 +109,6 @@ module Pikuri
75
109
  end
76
110
  end
77
111
 
78
- # Sleep for +seconds+. Isolated as a private method so tests can
79
- # override it on a single instance (typically to advance a frozen
80
- # Timecop clock by the same amount) without monkey-patching the
81
- # global +sleep+.
82
- #
83
- # @param seconds [Float] non-negative duration to sleep
84
- # @return [void]
85
- def sleep_for(seconds)
86
- sleep(seconds)
87
- end
88
- private :sleep_for
89
112
  end
90
113
  end
91
114
  end
@@ -3,26 +3,21 @@
3
3
  module Pikuri
4
4
  class Tool
5
5
  module Search
6
- # A single search hit produced by a {Tool::Search} provider
7
- # ({DuckDuckGo}, {Brave}, {Exa}). Providers return +Array<Result>+
8
- # from +.parse+ / +.search+; {Engines.search} concatenates the
9
- # rows into the smolagents-style Markdown the LLM sees.
10
- #
11
- # Splitting structure from rendering keeps the three providers
12
- # interchangeable — they only have to agree on these three fields
13
- # (provider-specific extras like relevance scores or published
14
- # dates are discarded today, which is fine because no caller uses
15
- # them).
6
+ # A single search hit from a {Tool::Search} provider ({DuckDuckGo},
7
+ # {Brave}, {Exa}). Providers return +Array<Result>+; {Engines.search}
8
+ # concatenates them into the smolagents-style Markdown the LLM sees.
9
+ # Structure split from rendering keeps the providers interchangeable —
10
+ # they need only agree on these three fields (provider extras like scores
11
+ # or dates are discarded, since no caller uses them).
16
12
  #
17
13
  # @!attribute [r] url
18
14
  # @return [String] absolute URL of the hit
19
15
  # @!attribute [r] title
20
- # @return [String] plain-text title, with provider-specific
21
- # highlight markup ({Brave}'s +<strong>+, {DuckDuckGo}'s
22
- # +<b>+) already stripped
16
+ # @return [String] plain-text title, provider highlight markup
17
+ # ({Brave}'s +<strong>+, {DuckDuckGo}'s +<b>+) already stripped
23
18
  # @!attribute [r] body
24
- # @return [String] plain-text snippet, possibly empty (e.g. an
25
- # {Exa} navigational result with no highlights)
19
+ # @return [String] plain-text snippet, possibly empty (an {Exa}
20
+ # navigational result with no highlights)
26
21
  Result = Data.define(:url, :title, :body)
27
22
  end
28
23
  end
@@ -0,0 +1,217 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pikuri
4
+ class Tool
5
+ # Which of the lethal trifecta's three legs a tool contributes — private
6
+ # data in, attacker-influenceable content in, bytes out. A tool declares its
7
+ # legs in its own constructor, where every collaborator that decides them
8
+ # (workspace, sandbox, confirmer) is already a kwarg:
9
+ #
10
+ # super(name: 'read', ..., trifecta_legs: Tool::TrifectaLegs.new(
11
+ # private: fs.private?,
12
+ # untrusted: fs.trusted? ? :none : :hard,
13
+ # egress_payload_review: :no_egress))
14
+ #
15
+ # {Pikuri::Trifecta} unions these across an agent's tools and folds in its
16
+ # sub-agents to decide whether all three legs are live at one node.
17
+ #
18
+ # == Implementation details
19
+ #
20
+ # Three axes are graded rather than boolean, and this class is the only
21
+ # place their ordering lives: {#|} takes the per-axis +max+ (legs meeting at
22
+ # one node), {#cap_payload_review} the +min+ (a leg squeezed through a
23
+ # delegation channel), and {#relabel_inbound} applies the one fixed demotion
24
+ # a sub-agent boundary buys. Nothing else may compare levels — the verdict
25
+ # reads presence and top-ness only (see {Pikuri::Trifecta::Report#verdict}).
26
+ #
27
+ # The egress leg is *two* axes because they are orthogonal: an unreviewed
28
+ # query to a vouched engine and a human-reviewed send to an attacker's inbox
29
+ # are different animals, and neither dominates the other. Both carry
30
+ # +:no_egress+ at the bottom and the constructor keeps them coherent, so the
31
+ # leg is present on both or on neither ({#egress?} is the canonical read).
32
+ #
33
+ # Immutable.
34
+ #
35
+ # @!attribute [r] private
36
+ # @return [Boolean] reads data that may be sensitive. The framework cannot
37
+ # know this — sensitivity is a property of the data, not of the tool — so
38
+ # the user declares it on the workspace/corpus and the tool inherits it.
39
+ # @!attribute [r] untrusted
40
+ # @return [Symbol] +:none+, +:weakened+ (reached this agent through a
41
+ # sub-agent, whose summarization relays an injected instruction as a
42
+ # reported fact) or +:hard+ (verbatim into this agent's context).
43
+ # @!attribute [r] egress_payload_review
44
+ # Answers: did a human see the bytes that leave?
45
+ #
46
+ # @return [Symbol] +:no_egress+, +:human_reviewed+ (the payload *is* the
47
+ # reviewed artifact — a compose window, an editable task string) or
48
+ # +:unreviewed+, which covers both no human at all and a human who saw
49
+ # only a proxy: approving +curl+ approves a program, and the bytes it
50
+ # sends are computed after the click.
51
+ # @!attribute [r] egress_destination
52
+ # Answers: if an injection hijacks the model, can it reach the attacker?
53
+ #
54
+ # @return [Symbol] +:no_egress+; +:attacker_reachable+ — the caller picks
55
+ # the host, *or* a fixed sink republishes what it receives (a hard-coded
56
+ # +gist.github.com+ POST is pinned and world-readable, so pinning alone
57
+ # buys nothing); or +:tool_vouched+, declarable iff the caller cannot
58
+ # choose the destination, that destination does not republish, and the
59
+ # request is not attacker-readable in flight or at rest. Unverifiable by
60
+ # the framework: like +private+, it is a declaration it inherits.
61
+ class TrifectaLegs < Data.define(:private, :untrusted, :egress_payload_review, :egress_destination)
62
+ # Danger order, least to most: +max+ is union at a node, +min+ is the cap
63
+ # through a delegation channel.
64
+ #
65
+ # @return [Array<Symbol>]
66
+ UNTRUSTED_ORDER = %i[none weakened hard].freeze
67
+
68
+ # @return [Array<Symbol>] danger order, least to most (see {UNTRUSTED_ORDER})
69
+ EGRESS_REVIEW_ORDER = %i[no_egress human_reviewed unreviewed].freeze
70
+
71
+ # @return [Array<Symbol>] danger order, least to most (see {UNTRUSTED_ORDER})
72
+ EGRESS_DESTINATION_ORDER = %i[no_egress tool_vouched attacker_reachable].freeze
73
+
74
+ # @param private [Boolean]
75
+ # @param untrusted [Symbol] one of {UNTRUSTED_ORDER}
76
+ # @param egress_payload_review [Symbol] one of {EGRESS_REVIEW_ORDER}
77
+ # @param egress_destination [Symbol, nil] one of {EGRESS_DESTINATION_ORDER};
78
+ # omit to derive — +:no_egress+ without a leg, else the pessimistic
79
+ # +:attacker_reachable+, so silence never buys the softer grade.
80
+ # @raise [ArgumentError] on a level outside its lattice, or on the two
81
+ # egress axes disagreeing about whether the leg exists at all — a typo'd
82
+ # level would otherwise read as a missing leg, the one failure mode this
83
+ # detector exists to refuse.
84
+ def initialize(private:, untrusted:, egress_payload_review:, egress_destination: nil)
85
+ egress_destination ||= egress_payload_review == :no_egress ? :no_egress : :attacker_reachable
86
+ check!(untrusted, UNTRUSTED_ORDER, 'untrusted')
87
+ check!(egress_payload_review, EGRESS_REVIEW_ORDER, 'egress review')
88
+ check!(egress_destination, EGRESS_DESTINATION_ORDER, 'egress destination')
89
+ unless (egress_payload_review == :no_egress) == (egress_destination == :no_egress)
90
+ raise ArgumentError, 'egress axes disagree on whether the leg exists: ' \
91
+ "#{egress_payload_review.inspect} / #{egress_destination.inspect}"
92
+ end
93
+
94
+ super(private: private, untrusted: untrusted, egress_payload_review: egress_payload_review,
95
+ egress_destination: egress_destination)
96
+ end
97
+
98
+ # Union: the strongest value on each axis wins.
99
+ #
100
+ # read_legs | bash_legs # what an agent holding both ends up with
101
+ #
102
+ # @param other [TrifectaLegs]
103
+ # @return [TrifectaLegs]
104
+ def |(other)
105
+ # Per-axis, so web_search (unreviewed, vouched) beside a compose tool
106
+ # (reviewed, attacker-reachable) scores a cell neither occupies. Pairing
107
+ # them up instead would be data flow, which this detector refuses to
108
+ # model; over-warning is the declared bias.
109
+ TrifectaLegs.new(
110
+ private: private || other.private,
111
+ untrusted: [untrusted, other.untrusted].max_by { UNTRUSTED_ORDER.index(_1) },
112
+ egress_payload_review: [egress_payload_review, other.egress_payload_review]
113
+ .max_by { EGRESS_REVIEW_ORDER.index(_1) },
114
+ egress_destination: [egress_destination, other.egress_destination]
115
+ .max_by { EGRESS_DESTINATION_ORDER.index(_1) }
116
+ )
117
+ end
118
+
119
+ # @return [Boolean] whether this contributes an egress leg at all. The
120
+ # canonical presence read, since both egress axes encode absence.
121
+ def egress? = egress_payload_review != :no_egress
122
+
123
+ # These legs as they reach a *parent* across a sub-agent boundary: +:hard+
124
+ # untrusted demotes to +:weakened+, the other three axes are identity.
125
+ #
126
+ # The boundary filters *instructions*, not *content* — a faithful summary
127
+ # preserves the page's facts but has no reason to adopt an out-of-band
128
+ # imperative, so an injection arrives as a reported fact rather than as a
129
+ # command. Private data survives intact (a summary of a secret still
130
+ # contains the secret), and egress survives because the parent can simply
131
+ # ask the child to send the thing.
132
+ #
133
+ # @return [TrifectaLegs]
134
+ def relabel_inbound
135
+ # Saturating, not one-rung-down: :weakened stays :weakened. Decrementing
136
+ # per hop would launder a grandchild's untrusted content to :none at
137
+ # depth three — two summarizers are no more instruction-resistant than
138
+ # one, and that is the structural claim this design refuses to make.
139
+ with(untrusted: untrusted == :hard ? :weakened : untrusted)
140
+ end
141
+
142
+ # These legs with the *review* axis capped at what the delegation channel
143
+ # itself allows — +min+ in series, the counterpart to {#|}'s +max+ in
144
+ # parallel.
145
+ #
146
+ # A child's egress cannot exceed the channel feeding it: when the only
147
+ # bytes crossing into the child are a task string a human approved, even a
148
+ # fully poisoned child can send nothing else. That holds *only* while the
149
+ # child has no private data of its own, so {Pikuri::Trifecta.walk} owns the
150
+ # precondition and this method is the mechanical +min+.
151
+ #
152
+ # {#egress_destination} is deliberately untouched: a gate bounds *what*
153
+ # leaves and *who saw it*, never who can read it back.
154
+ #
155
+ # @param channel_level [Symbol] one of {EGRESS_REVIEW_ORDER}
156
+ # @return [TrifectaLegs] with the leg gone entirely when the channel
157
+ # carries no egress at all
158
+ # @raise [ArgumentError] on a level outside the lattice
159
+ def cap_payload_review(channel_level)
160
+ check!(channel_level, EGRESS_REVIEW_ORDER, 'egress review')
161
+ capped = [egress_payload_review, channel_level].min_by { EGRESS_REVIEW_ORDER.index(_1) }
162
+ return with(egress_payload_review: capped) unless capped == :no_egress
163
+
164
+ with(egress_payload_review: :no_egress, egress_destination: :no_egress)
165
+ end
166
+
167
+ # Symbols naming the legs actually present, in the vocabulary the docs and
168
+ # the rendered tree use. Absent legs are omitted, so a leg-free tool
169
+ # yields +[]+ rather than three +:none+ entries.
170
+ #
171
+ # TrifectaLegs.new(private: true, untrusted: :hard,
172
+ # egress_payload_review: :no_egress).to_tags
173
+ # # => [:private, :untrusted_hard]
174
+ #
175
+ # @return [Array<Symbol>]
176
+ def to_tags
177
+ tags = []
178
+ tags << :private if private
179
+ tags << :"untrusted_#{untrusted}" unless untrusted == :none
180
+ tags << egress_tag if egress?
181
+ tags
182
+ end
183
+
184
+ private
185
+
186
+ # @return [Symbol] +:egress+ at full strength, with +_reviewed+ /
187
+ # +_vouched+ appended per attenuated axis, so the four cells name
188
+ # themselves without a lookup table
189
+ def egress_tag
190
+ parts = ['egress']
191
+ parts << 'reviewed' if egress_payload_review == :human_reviewed
192
+ parts << 'vouched' if egress_destination == :tool_vouched
193
+ parts.join('_').to_sym
194
+ end
195
+
196
+ # @raise [ArgumentError] unless +value+ is a level of the +order+ lattice
197
+ def check!(value, order, name)
198
+ raise ArgumentError, "unknown #{name} level: #{value.inspect}" unless order.include?(value)
199
+ end
200
+
201
+ # @return [TrifectaLegs] no legs at all
202
+ NONE = new(private: false, untrusted: :none, egress_payload_review: :no_egress)
203
+
204
+ # What an *undeclared* tool is presumed to hold: hostile on every axis the
205
+ # framework could have determined for itself. Silence from a tool author is
206
+ # not evidence of safety.
207
+ #
208
+ # It stops short of +private: true+ deliberately — that axis is the user's
209
+ # to declare, and assuming it would make every wiring loud, turning a
210
+ # warn-only advisory into noise and getting it muted.
211
+ #
212
+ # @return [TrifectaLegs]
213
+ ASSUMED = new(private: false, untrusted: :hard, egress_payload_review: :unreviewed,
214
+ egress_destination: :attacker_reachable)
215
+ end
216
+ end
217
+ end
@@ -2,67 +2,45 @@
2
2
 
3
3
  module Pikuri
4
4
  class Tool
5
- # Truncation policy and Tool spec for the +web_scrape+ tool. The actual
6
- # scraping lives in {Tool::Scraper}; this module is a thin
7
- # wrapper that picks the scraper, applies a character cap so the LLM
8
- # doesn't drown in long-form content, and exposes the result to the
9
- # agent loop in OpenAI tool-call shape.
5
+ # Truncation policy and Tool spec for the +web_scrape+ tool. Scraping lives
6
+ # in {Tool::Scraper}; this wrapper applies a character cap and exposes the
7
+ # result in OpenAI tool-call shape.
10
8
  module WebScrape
11
- # @return [Integer] default character cap on the Markdown returned
12
- # by {.visit}. Sized to cover most post-readability article bodies
13
- # in full on the first call, so the LLM doesn't have to re-request
14
- # with a larger cap and pollute its context with the same prefix
15
- # twice. ~5K tokens at the typical char/token ratio — light even
16
- # for small local models. The genuinely long pages (long Wikipedia
17
- # entries, multi-section docs) still get cut, and the truncation
18
- # marker invites a deliberate larger {.visit} call when needed.
9
+ # @return [Integer] default character cap on the Markdown from {.visit}.
10
+ # Sized to cover most post-readability article bodies in full on the
11
+ # first call, so the LLM doesn't re-request and duplicate the prefix in
12
+ # its context. Genuinely long pages still get cut, and the marker invites
13
+ # a deliberate larger call.
19
14
  DEFAULT_MAX_CHARS = 20_000
20
15
 
21
- # @return [Integer] hard ceiling on the +max_chars+ argument to
22
- # {.visit}. Requests above this are clamped silently so the LLM
23
- # cannot dump an arbitrarily large page into the conversation.
16
+ # @return [Integer] hard ceiling on {.visit}'s +max_chars+; requests above
17
+ # it are clamped silently so the LLM can't dump an arbitrarily large page.
24
18
  MAX_MAX_CHARS = 100_000
25
19
 
26
- # On-disk cache used by {.visit} to memoize fetched pages. Defined
27
- # as a method so specs can swap it for an isolated cache or
28
- # {UrlCache::NULL} without touching the shared instance.
20
+ # On-disk cache for {.visit}. A method so specs can swap {UrlCache::NULL}.
29
21
  #
30
22
  # @return [UrlCache, #fetch]
31
23
  CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_scrape")
32
- # Accessor for {CACHE}; specs override this to swap in
33
- # {UrlCache::NULL} or an isolated cache.
24
+ # Accessor for {CACHE}; specs swap in {UrlCache::NULL} or an isolated cache.
34
25
  #
35
26
  # @return [UrlCache, #fetch]
36
27
  def self.cache
37
28
  CACHE
38
29
  end
39
30
 
40
- # Fetch +url+ via {Tool::Scraper} and truncate the rendered
41
- # Markdown to +max_chars+ characters.
31
+ # Fetch +url+ via {Tool::Scraper} and truncate the rendered Markdown to
32
+ # +max_chars+. The full Markdown is cached on disk keyed by URL (repeat
33
+ # visits skip the network and extraction); +max_chars+ is *not* in the key
34
+ # (values share one entry, truncation runs after the lookup).
35
+ # {Scraper::FetchError} is caught outside +cache.fetch+ and returned as
36
+ # +"Error: ..."+ (so the LLM can retry a different URL) — never persisted;
37
+ # other exceptions bubble up.
42
38
  #
43
- # The full extracted Markdown is cached on disk via {.cache}, keyed
44
- # by URL, so repeat visits within the cache TTL skip the network
45
- # and the extraction pass entirely. +max_chars+ is not part of the
46
- # cache key — different values for the same URL share one entry,
47
- # and truncation runs after the cache lookup.
48
- #
49
- # {Tool::Scraper::FetchError} (HTTP non-2xx, network failure,
50
- # redirect-loop, missing +Location+ header) is caught and returned
51
- # as +"Error: ..."+ in the calculator-style convention so the agent
52
- # loop feeds the failure back to the model as the next observation
53
- # instead of crashing — the LLM can then try a different URL or
54
- # search again. The rescue lives outside {.cache}'s +fetch+ block,
55
- # so failure strings are never persisted: a retry on the next call
56
- # hits the network again. Other exceptions (parser bugs in our own
57
- # code) bubble up unchanged.
58
- #
59
- # @param url [String] absolute HTTP(S) URL of the page to download
60
- # @param max_chars [Integer] character cap on the returned Markdown.
61
- # Clamped to +[1, {MAX_MAX_CHARS}]+; defaults to
62
- # {DEFAULT_MAX_CHARS}. When the full page exceeds the cap, output
63
- # is cut and a marker noting the original length is appended.
64
- # @return [String] Markdown representation of the page, possibly
65
- # truncated, or +"Error: ..."+ on a recoverable fetch failure
39
+ # @param url [String] absolute HTTP(S) URL
40
+ # @param max_chars [Integer] Markdown cap, clamped to +[1, {MAX_MAX_CHARS}]+;
41
+ # default {DEFAULT_MAX_CHARS}. Over the cap, output is cut with a marker.
42
+ # @return [String] Markdown, possibly truncated, or +"Error: ..."+ on a
43
+ # recoverable fetch failure
66
44
  def self.visit(url, max_chars: DEFAULT_MAX_CHARS)
67
45
  max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
68
46
  markdown = cache.fetch(url) { Scraper.visit(url) }
@@ -71,12 +49,11 @@ module Pikuri
71
49
  "Error: #{e.message}"
72
50
  end
73
51
 
74
- # Cut +markdown+ to at most +max_chars+ characters, appending a
75
- # marker describing the original length when truncation actually
76
- # happens. Returns +markdown+ unchanged if it already fits.
52
+ # Cut +markdown+ to at most +max_chars+, appending a marker with the
53
+ # original length when truncation happens.
77
54
  #
78
55
  # @param markdown [String] full Markdown text
79
- # @param max_chars [Integer] character cap; assumed already clamped
56
+ # @param max_chars [Integer] cap; assumed already clamped
80
57
  # @return [String]
81
58
  def self.truncate(markdown, max_chars)
82
59
  return markdown if markdown.length <= max_chars
@@ -87,9 +64,12 @@ module Pikuri
87
64
  end
88
65
  end
89
66
 
90
- # Webpage download + Markdown conversion tool. Thin wrapper over
91
- # {Tool::WebScrape.visit} that exposes it to the agent loop in OpenAI
92
- # tool-call shape.
67
+ # Webpage download + Markdown conversion tool — a thin wrapper over
68
+ # {Tool::WebScrape.visit} in OpenAI tool-call shape.
69
+ #
70
+ # Sharing: +P_shared_benign+ — as {Tool::FETCH}: no per-agent state, but
71
+ # {WebScrape::CACHE} is process-wide and on disk, so one agent's scrape
72
+ # answers another's within the TTL. See {Pikuri::UrlCache}.
93
73
  #
94
74
  # @return [Tool]
95
75
  WEB_SCRAPE = new(
@@ -115,7 +95,11 @@ module Pikuri
115
95
  },
116
96
  execute: ->(url:, max_chars: WebScrape::DEFAULT_MAX_CHARS) {
117
97
  WebScrape.visit(url, max_chars: max_chars)
118
- }
98
+ },
99
+ # Exactly {TrifectaLegs::ASSUMED}, and the archetype it was named for: the
100
+ # page is authored by whoever runs the site, and the model picks the URL,
101
+ # so the request is itself an outbound channel.
102
+ trifecta_legs: Pikuri::Tool::TrifectaLegs::ASSUMED
119
103
  )
120
104
  end
121
105
  end