pikuri 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +62 -0
  3. data/GETTING_STARTED.md +223 -0
  4. data/LICENSE +21 -0
  5. data/README.md +193 -0
  6. data/lib/pikuri/agent/chat_transport.rb +41 -0
  7. data/lib/pikuri/agent/context_window_detector.rb +101 -0
  8. data/lib/pikuri/agent/listener/in_memory_message_list.rb +33 -0
  9. data/lib/pikuri/agent/listener/message_listener.rb +93 -0
  10. data/lib/pikuri/agent/listener/step_limit.rb +97 -0
  11. data/lib/pikuri/agent/listener/terminal.rb +137 -0
  12. data/lib/pikuri/agent/listener/token_log.rb +166 -0
  13. data/lib/pikuri/agent/listener_list.rb +113 -0
  14. data/lib/pikuri/agent/message.rb +61 -0
  15. data/lib/pikuri/agent/synthesizer.rb +120 -0
  16. data/lib/pikuri/agent/tokens.rb +56 -0
  17. data/lib/pikuri/agent.rb +286 -0
  18. data/lib/pikuri/subprocess.rb +166 -0
  19. data/lib/pikuri/tool/bash.rb +272 -0
  20. data/lib/pikuri/tool/calculator.rb +82 -0
  21. data/lib/pikuri/tool/confirmer.rb +96 -0
  22. data/lib/pikuri/tool/edit.rb +196 -0
  23. data/lib/pikuri/tool/fetch.rb +167 -0
  24. data/lib/pikuri/tool/glob.rb +310 -0
  25. data/lib/pikuri/tool/grep.rb +338 -0
  26. data/lib/pikuri/tool/parameters.rb +314 -0
  27. data/lib/pikuri/tool/read.rb +254 -0
  28. data/lib/pikuri/tool/scraper/fetch_error.rb +16 -0
  29. data/lib/pikuri/tool/scraper/html.rb +285 -0
  30. data/lib/pikuri/tool/scraper/pdf.rb +54 -0
  31. data/lib/pikuri/tool/scraper/simple.rb +177 -0
  32. data/lib/pikuri/tool/search/brave.rb +184 -0
  33. data/lib/pikuri/tool/search/duckduckgo.rb +196 -0
  34. data/lib/pikuri/tool/search/engines.rb +154 -0
  35. data/lib/pikuri/tool/search/exa.rb +217 -0
  36. data/lib/pikuri/tool/search/rate_limiter.rb +92 -0
  37. data/lib/pikuri/tool/search/result.rb +29 -0
  38. data/lib/pikuri/tool/skill.rb +80 -0
  39. data/lib/pikuri/tool/skill_catalog.rb +376 -0
  40. data/lib/pikuri/tool/sub_agent.rb +102 -0
  41. data/lib/pikuri/tool/web_scrape.rb +117 -0
  42. data/lib/pikuri/tool/web_search.rb +38 -0
  43. data/lib/pikuri/tool/workspace.rb +150 -0
  44. data/lib/pikuri/tool/write.rb +170 -0
  45. data/lib/pikuri/tool.rb +118 -0
  46. data/lib/pikuri/url_cache.rb +106 -0
  47. data/lib/pikuri/version.rb +10 -0
  48. data/lib/pikuri.rb +165 -0
  49. data/prompts/coding-system-prompt.txt +28 -0
  50. data/prompts/pikuri-chat.txt +15 -0
  51. metadata +259 -0
@@ -0,0 +1,376 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'yaml'
4
+
5
+ module Pikuri
6
+ class Tool
7
+ # Discovery + validation of *skills* — Markdown files with YAML
8
+ # frontmatter that the agent loads on demand via {Tool::Skill}.
9
+ #
10
+ # A skill is a directory containing a +SKILL.md+ file. The frontmatter
11
+ # declares the skill's +name+ and +description+; everything in the
12
+ # directory (helper scripts, reference notes, templates) is treated as
13
+ # sidecar content the LLM can pull in via the regular +read+ tool
14
+ # after loading the skill. Pikuri implements the
15
+ # {https://agentskills.io/specification Agent Skills standard}
16
+ # leniently — invalid frontmatter is warned about but the skill
17
+ # still loads, except a missing +description+ which is fatal (no
18
+ # description means the LLM has no signal for when to invoke it).
19
+ #
20
+ # == Catalog vs. tool
21
+ #
22
+ # {SkillCatalog} owns the *discovery* layer: where skills live, how
23
+ # the frontmatter is parsed, what wins on a name collision. The
24
+ # {Tool::Skill} tool is the *loading* layer: it receives a catalog at
25
+ # construction and looks names up against it when the LLM invokes
26
+ # +skill+. The two are paired by {Agent} — see
27
+ # {Agent#initialize} for the auto-registration rule (non-empty
28
+ # catalog ⇒ +Tool::Skill+ is appended to the tool list and the
29
+ # catalog's prompt block is appended to the system prompt).
30
+ #
31
+ # == The seam
32
+ #
33
+ # {SkillCatalog} is an abstract base with two bundled
34
+ # implementations: {Empty} (the +EMPTY+ singleton, used by
35
+ # +pikuri-chat+ and as the default for any caller that doesn't have a
36
+ # filesystem story) and {Bundled} (the on-disk scanner). The seam
37
+ # lets future hosts (e.g. an in-editor pikuri reading skills from a
38
+ # virtual filesystem, or a synthetic test catalog) swap in without
39
+ # touching {Tool::Skill} or {Agent}.
40
+ class SkillCatalog
41
+ # Frontmatter +name+ cap per the Agent Skills standard.
42
+ MAX_NAME_LENGTH = 64
43
+
44
+ # Frontmatter +description+ cap per the Agent Skills standard.
45
+ MAX_DESCRIPTION_LENGTH = 1024
46
+
47
+ # The three skill directories pikuri scans, in precedence order.
48
+ # +.pikuri+ first (the user's own pikuri-specific skills),
49
+ # +.claude+ second (so existing Claude Code skills travel along
50
+ # for free), +.agents+ last (the cross-harness convention PI also
51
+ # honors). Same names are used for both global roots
52
+ # (under +~/+) and project roots (relative to CWD, walked up).
53
+ SKILL_DIR_NAMES = ['.pikuri/skills', '.claude/skills', '.agents/skills'].freeze
54
+
55
+ LOGGER = Pikuri.logger_for('Skills')
56
+ private_constant :LOGGER
57
+
58
+ # A loaded skill record. Holds everything the catalog discovers
59
+ # and the tool needs to satisfy a load: +name+ and +description+
60
+ # for the catalog's prompt block, +location+ (absolute path to
61
+ # the +SKILL.md+) so the LLM can reference sidecars relative to
62
+ # +File.dirname(location)+, and +body+ (the SKILL.md content
63
+ # below the frontmatter) returned verbatim when the tool fires.
64
+ #
65
+ # Bodies are eager-loaded at scan time even though only
66
+ # +name+/+description+/+location+ end up in the prompt — skill
67
+ # files are small, eager loading keeps the tool's hot path
68
+ # IO-free, and there is no fault tolerance requirement that
69
+ # warrants the complexity of lazy I/O.
70
+ Skill = Data.define(:name, :description, :location, :body)
71
+
72
+ # Skills available to the LLM, in stable insertion order (which
73
+ # equals the order they were discovered, which equals precedence).
74
+ #
75
+ # @return [Array<Skill>]
76
+ # @raise [NotImplementedError] in the abstract base
77
+ def list
78
+ raise NotImplementedError, "#{self.class}#list must be implemented"
79
+ end
80
+
81
+ # Look up a skill by name.
82
+ #
83
+ # @param name [String]
84
+ # @return [Skill, nil]
85
+ # @raise [NotImplementedError] in the abstract base
86
+ def get(name)
87
+ raise NotImplementedError, "#{self.class}#get must be implemented"
88
+ end
89
+
90
+ # @return [Boolean] true when {#list} is empty. {Agent} keys its
91
+ # auto-wiring (system-prompt augmentation + +Tool::Skill+
92
+ # registration) off this — empty catalog ⇒ no surface change.
93
+ def empty?
94
+ list.empty?
95
+ end
96
+
97
+ # System-prompt block advertising every loaded skill to the LLM.
98
+ # Follows the Agent Skills standard's XML shape (PI uses the same
99
+ # one) so a skill folder authored against any compliant harness
100
+ # renders consistently here.
101
+ #
102
+ # Returns +""+ for an empty catalog so callers can unconditionally
103
+ # concatenate without an +if+. The leading +\n\n+ separates the
104
+ # block from whatever the caller's static system prompt ends with.
105
+ #
106
+ # @return [String]
107
+ def format_for_prompt
108
+ return '' if empty?
109
+
110
+ lines = [
111
+ '',
112
+ '',
113
+ 'The following skills provide specialized instructions for specific tasks.',
114
+ "Use the `skill` tool with the skill's name to load its full instructions.",
115
+ '',
116
+ '<available_skills>'
117
+ ]
118
+ list.each do |skill|
119
+ lines << ' <skill>'
120
+ lines << " <name>#{escape_xml(skill.name)}</name>"
121
+ lines << " <description>#{escape_xml(skill.description)}</description>"
122
+ lines << " <location>#{escape_xml(skill.location)}</location>"
123
+ lines << ' </skill>'
124
+ end
125
+ lines << '</available_skills>'
126
+ lines.join("\n")
127
+ end
128
+
129
+ private
130
+
131
+ def escape_xml(str)
132
+ str.gsub('&', '&amp;')
133
+ .gsub('<', '&lt;')
134
+ .gsub('>', '&gt;')
135
+ .gsub('"', '&quot;')
136
+ .gsub("'", '&apos;')
137
+ end
138
+
139
+ # Null-object catalog. The +EMPTY+ constant is the singleton
140
+ # instance used as the default for {Agent#initialize}; constructing
141
+ # additional instances is harmless but pointless.
142
+ class Empty < SkillCatalog
143
+ # @return [Array<Skill>] always empty
144
+ def list
145
+ [].freeze
146
+ end
147
+
148
+ # @return [nil]
149
+ def get(_name)
150
+ nil
151
+ end
152
+ end
153
+
154
+ # Shared singleton for the no-skills case. Frozen.
155
+ EMPTY = Empty.new.freeze
156
+
157
+ # On-disk catalog: eager-scans the given +roots+ at construction
158
+ # and freezes itself. Use {.discover} to build one against
159
+ # pikuri's conventional locations; pass +roots:+ directly in
160
+ # tests so collision and ordering behavior can be exercised
161
+ # without staging dirs under +~/+.
162
+ #
163
+ # == Precedence
164
+ #
165
+ # +roots+ is processed left to right. The *first* skill seen for
166
+ # a given name wins; later occurrences are dropped with a warning
167
+ # logged through +Pikuri.logger_for('Skills')+. Callers that want
168
+ # "project beats global" pass project roots first.
169
+ #
170
+ # == Validation
171
+ #
172
+ # The Agent Skills standard is enforced leniently. A missing
173
+ # +description+ is the only hard failure (the skill is skipped);
174
+ # name/dir mismatch, oversized name/description, invalid name
175
+ # characters, malformed YAML — all log warnings and the skill
176
+ # still loads with whatever could be salvaged. Skills without
177
+ # a YAML frontmatter block at all are skipped silently (the file
178
+ # is presumably not meant as a skill).
179
+ class Bundled < SkillCatalog
180
+ # Build a catalog from pikuri's conventional discovery roots:
181
+ # project skills (walked from +cwd+ up to the git root or
182
+ # filesystem root) take precedence over global skills (under
183
+ # the user's home directory). Within each tier, the order in
184
+ # {SKILL_DIR_NAMES} applies (+.pikuri+ ➜ +.claude+ ➜ +.agents+).
185
+ #
186
+ # Non-existent directories are skipped silently; the loaded
187
+ # set is whatever existed at construction time.
188
+ #
189
+ # @param cwd [String, Pathname] working directory whose
190
+ # ancestor chain is searched for project skill roots.
191
+ # @return [Bundled]
192
+ def self.discover(cwd: Dir.pwd)
193
+ new(roots: discover_project_roots(cwd: cwd) + discover_global_roots)
194
+ end
195
+
196
+ # Project roots walked from +cwd+ upwards, stopping at the
197
+ # nearest ancestor containing +.git+ (inclusive) or at the
198
+ # filesystem root. The deepest directory's roots appear first,
199
+ # so closer-to-CWD skills win on name collisions.
200
+ #
201
+ # @param cwd [String, Pathname]
202
+ # @return [Array<String>] absolute paths to existing dirs
203
+ def self.discover_project_roots(cwd:)
204
+ results = []
205
+ current = File.expand_path(cwd.to_s)
206
+ loop do
207
+ SKILL_DIR_NAMES.each do |sub|
208
+ candidate = File.join(current, sub)
209
+ results << candidate if File.directory?(candidate)
210
+ end
211
+
212
+ # Stop *after* processing the git root — git roots are
213
+ # natural project boundaries; ancestors above are someone
214
+ # else's project.
215
+ break if File.directory?(File.join(current, '.git'))
216
+
217
+ parent = File.dirname(current)
218
+ break if parent == current # filesystem root
219
+
220
+ current = parent
221
+ end
222
+ results
223
+ end
224
+
225
+ # Global roots under the user's home directory.
226
+ #
227
+ # @return [Array<String>] absolute paths to existing dirs
228
+ def self.discover_global_roots
229
+ home = Dir.home
230
+ SKILL_DIR_NAMES.map { |sub| File.join(home, sub) }
231
+ .select { |dir| File.directory?(dir) }
232
+ end
233
+
234
+ # @param roots [Array<String, Pathname>] skill directories in
235
+ # precedence order — first occurrence of a given skill name
236
+ # wins.
237
+ # @return [Bundled]
238
+ def initialize(roots:)
239
+ super()
240
+ @skills = {}
241
+ roots.each { |root| scan_root(root.to_s) }
242
+ @skills.freeze
243
+ @list = @skills.values.freeze
244
+ freeze
245
+ end
246
+
247
+ # @return [Array<Skill>]
248
+ def list
249
+ @list
250
+ end
251
+
252
+ # @param name [String]
253
+ # @return [Skill, nil]
254
+ def get(name)
255
+ @skills[name]
256
+ end
257
+
258
+ private
259
+
260
+ def scan_root(root)
261
+ return unless File.directory?(root)
262
+
263
+ Dir.children(root).sort.each do |entry|
264
+ skill_dir = File.join(root, entry)
265
+ next unless File.directory?(skill_dir)
266
+
267
+ skill_md = File.join(skill_dir, 'SKILL.md')
268
+ next unless File.file?(skill_md)
269
+
270
+ skill = parse_skill(skill_md, parent_dir_name: entry)
271
+ next if skill.nil?
272
+
273
+ existing = @skills[skill.name]
274
+ if existing
275
+ LOGGER.warn(
276
+ "skill name '#{skill.name}' at #{skill_md} is shadowed by " \
277
+ "earlier entry at #{existing.location}"
278
+ )
279
+ else
280
+ @skills[skill.name] = skill
281
+ end
282
+ end
283
+ end
284
+
285
+ # Read +path+ and return a {Skill}, or +nil+ if the file is
286
+ # unusable (no frontmatter, no description, unreadable). Warnings
287
+ # for soft violations are logged but do not block loading.
288
+ def parse_skill(path, parent_dir_name:)
289
+ raw = File.read(path)
290
+ frontmatter, body = split_frontmatter(raw)
291
+
292
+ if frontmatter.nil?
293
+ LOGGER.warn("#{path}: no YAML frontmatter; skipped")
294
+ return nil
295
+ end
296
+
297
+ description = frontmatter['description'].to_s.strip
298
+ if description.empty?
299
+ LOGGER.warn("#{path}: missing required 'description'; skipped")
300
+ return nil
301
+ end
302
+
303
+ name = (frontmatter['name'] || parent_dir_name).to_s
304
+ validate_name(name, parent_dir_name: parent_dir_name, path: path)
305
+ validate_description(description, path: path)
306
+
307
+ Skill.new(
308
+ name: name,
309
+ description: description,
310
+ location: path,
311
+ body: body
312
+ )
313
+ rescue Errno::ENOENT, Errno::EACCES => e
314
+ LOGGER.warn("#{path}: #{e.class}: #{e.message}; skipped")
315
+ nil
316
+ end
317
+
318
+ # Split a SKILL.md into +[frontmatter_hash, body_string]+.
319
+ # Returns +[nil, raw]+ when there is no detectable frontmatter
320
+ # block. On YAML parse failure, logs a warning and returns
321
+ # +[{}, body]+ so the caller can still observe an empty
322
+ # frontmatter (which trips the "missing description" path
323
+ # below and discards the skill).
324
+ def split_frontmatter(content)
325
+ normalized = content.gsub(/\r\n?/, "\n")
326
+ return [nil, normalized] unless normalized.start_with?("---\n")
327
+
328
+ end_marker = normalized.index("\n---", 4)
329
+ return [nil, normalized] unless end_marker
330
+
331
+ yaml_str = normalized[4...end_marker]
332
+ body = normalized[(end_marker + 4)..].to_s
333
+ body = body.sub(/\A\n/, '')
334
+
335
+ parsed =
336
+ begin
337
+ YAML.safe_load(yaml_str) || {}
338
+ rescue Psych::SyntaxError => e
339
+ LOGGER.warn("frontmatter YAML parse error: #{e.message}")
340
+ {}
341
+ end
342
+ parsed = {} unless parsed.is_a?(Hash)
343
+
344
+ [parsed, body]
345
+ end
346
+
347
+ def validate_name(name, parent_dir_name:, path:)
348
+ if name != parent_dir_name
349
+ LOGGER.warn("#{path}: skill name '#{name}' does not match parent directory '#{parent_dir_name}'")
350
+ end
351
+ if name.length > MAX_NAME_LENGTH
352
+ LOGGER.warn("#{path}: skill name exceeds #{MAX_NAME_LENGTH} chars (#{name.length})")
353
+ end
354
+ unless /\A[a-z0-9-]+\z/.match?(name)
355
+ LOGGER.warn("#{path}: skill name '#{name}' contains invalid characters " \
356
+ "(allowed: lowercase a-z, 0-9, hyphens)")
357
+ end
358
+ if name.start_with?('-') || name.end_with?('-')
359
+ LOGGER.warn("#{path}: skill name '#{name}' starts or ends with a hyphen")
360
+ end
361
+ return unless name.include?('--')
362
+
363
+ LOGGER.warn("#{path}: skill name '#{name}' contains consecutive hyphens")
364
+ end
365
+
366
+ def validate_description(description, path:)
367
+ return unless description.length > MAX_DESCRIPTION_LENGTH
368
+
369
+ LOGGER.warn(
370
+ "#{path}: description exceeds #{MAX_DESCRIPTION_LENGTH} chars (#{description.length})"
371
+ )
372
+ end
373
+ end
374
+ end
375
+ end
376
+ end
@@ -0,0 +1,102 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pikuri
4
+ class Tool
5
+ # The +sub_agent+ tool, expressed as a {Tool} subclass: instantiating
6
+ # +Tool::SubAgent.new(parent_agent)+ produces a tool whose
7
+ # {Tool#to_ruby_llm_tool} wiring is identical to any bundled tool's,
8
+ # so ruby_llm sees nothing special about it. When the model calls it,
9
+ # the closure inside +execute+ spawns a fresh {Agent} that runs its
10
+ # own Thought / Tool-call / Observation loop on a clean message
11
+ # history, then returns only the sub-agent's final assistant message
12
+ # back as the parent's next observation.
13
+ #
14
+ # The sub-agent reuses the parent's +transport+, +system_prompt+,
15
+ # +context_window_cap+, and +name+ (as its hierarchical prefix), so
16
+ # it shares the same persona, hits the same server, and inherits the
17
+ # same context-window cap without re-probing. Its tool list is a
18
+ # snapshot of the parent's {Agent#tools} taken at construction —
19
+ # {Agent#allow_sub_agent} only appends the sub-agent tool to its own
20
+ # +@tools+ *after* this snapshot, so the sub-agent's tool list never
21
+ # contains itself (recursion guard).
22
+ #
23
+ # Its listener list comes from the parent's {Agent#listeners} via
24
+ # {Agent::ListenerList#for_sub_agent}, which forwards to each
25
+ # listener's own +for_sub_agent+ hook: +Terminal+ swaps to a padded
26
+ # fresh instance, +TokenLog+ resets its snapshot, +StepLimit+ picks
27
+ # +max_steps:+ out of the params, and listeners without the hook
28
+ # ({Agent::Listener::InMemoryMessageList}, …) are shared by reference so
29
+ # structured capture and other stateful renderers flow continuously.
30
+ #
31
+ # All parent state is captured by value at construction — the closure
32
+ # does not chase +parent_agent+ mutations later. The one piece of
33
+ # mutable state is a monotonic counter used to generate sub-agent ids:
34
+ # +"sub_agent 0"+, +"sub_agent 1"+, ... at the top level; nested
35
+ # children of +"sub_agent 0"+ are +"sub_agent 0_0"+, +"sub_agent 0_1"+,
36
+ # ... — the +"sub_agent "+ prefix appears once at the top and the
37
+ # underscore-separated counter chain records depth.
38
+ class SubAgent < Tool
39
+ # Description shown to the LLM. Follows the opencode-shape (summary
40
+ # + +Usage:+ bullets) prescribed by the project's tool-description
41
+ # convention.
42
+ #
43
+ # @return [String]
44
+ DESCRIPTION = <<~DESC
45
+ Delegate a self-contained task to a fresh sub-agent that runs its own Thought / Tool-call / Observation loop on a clean conversation, returning only its final assistant message.
46
+
47
+ Usage:
48
+ - Use to isolate side-quests — research, multi-step lookups, exploratory tool use — so intermediate observations do not clutter your own context.
49
+ - The sub-agent has your tools minus `sub_agent` itself, so it cannot recurse.
50
+ - It shares your system prompt — persona, tool-use conventions, and output format carry over. Do NOT re-explain who you are or how to use tools.
51
+ - It cannot see your conversation. Put ALL task-specific context inside `task`; the sub-agent has zero memory of what came before.
52
+ DESC
53
+
54
+ # @param parent_agent [Agent] the calling agent. Read for its
55
+ # {Agent#transport}, {Agent#system_prompt}, {Agent#tools},
56
+ # {Agent#listeners}, {Agent#context_window_cap}, and {Agent#name}.
57
+ # @param max_steps [Integer] step budget for each sub-agent run,
58
+ # forwarded as the +max_steps:+ kwarg to
59
+ # {Agent::ListenerList#for_sub_agent} (consumed by
60
+ # {Agent::Listener::StepLimit#for_sub_agent}).
61
+ # @return [SubAgent]
62
+ def initialize(parent_agent, max_steps: 10)
63
+ transport = parent_agent.transport
64
+ system_prompt = parent_agent.system_prompt
65
+ sub_tools = parent_agent.tools.dup
66
+ listeners = parent_agent.listeners
67
+ context_window = parent_agent.context_window_cap
68
+ parent_name = parent_agent.name
69
+ sub_counter = 0
70
+
71
+ super(
72
+ name: 'sub_agent',
73
+ description: DESCRIPTION,
74
+ parameters: Parameters.build { |p|
75
+ p.required_string :task,
76
+ 'Self-contained instructions for the sub-agent, ' \
77
+ 'e.g. "Find the populations of Reykjavik and ' \
78
+ 'Helsinki in 2024 and report both numbers." ' \
79
+ 'It has no access to the parent conversation, ' \
80
+ 'so include all necessary context.'
81
+ },
82
+ execute: lambda { |task:|
83
+ idx = sub_counter
84
+ sub_counter += 1
85
+ sub_name = parent_name.empty? ? "sub_agent #{idx}" : "#{parent_name}_#{idx}"
86
+
87
+ sub = Agent.new(
88
+ transport: transport,
89
+ system_prompt: system_prompt,
90
+ tools: sub_tools,
91
+ listeners: listeners.for_sub_agent(max_steps: max_steps, name: sub_name),
92
+ context_window: context_window,
93
+ name: sub_name
94
+ )
95
+ sub.run_loop(user_message: task)
96
+ sub.last_assistant_content
97
+ }
98
+ )
99
+ end
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,117 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pikuri
4
+ class Tool
5
+ # Truncation policy and Tool spec for the +web_scrape+ tool. The actual
6
+ # scraping lives in {Tool::Scraper::Simple}; this module is a thin
7
+ # wrapper that picks the scraper, applies a character cap so the LLM
8
+ # doesn't drown in long-form content, and exposes the result to the
9
+ # agent loop in OpenAI tool-call shape.
10
+ module WebScrape
11
+ # @return [Integer] default character cap on the Markdown returned
12
+ # by {.visit}. Sized to cover most post-readability article bodies
13
+ # in full on the first call, so the LLM doesn't have to re-request
14
+ # with a larger cap and pollute its context with the same prefix
15
+ # twice. ~5K tokens at the typical char/token ratio — light even
16
+ # for small local models. The genuinely long pages (long Wikipedia
17
+ # entries, multi-section docs) still get cut, and the truncation
18
+ # marker invites a deliberate larger {.visit} call when needed.
19
+ DEFAULT_MAX_CHARS = 20_000
20
+
21
+ # @return [Integer] hard ceiling on the +max_chars+ argument to
22
+ # {.visit}. Requests above this are clamped silently so the LLM
23
+ # cannot dump an arbitrarily large page into the conversation.
24
+ MAX_MAX_CHARS = 100_000
25
+
26
+ # On-disk cache used by {.visit} to memoize fetched pages. Defined
27
+ # as a method so specs can swap it for an isolated cache or
28
+ # {UrlCache::NULL} without touching the shared instance.
29
+ #
30
+ # @return [UrlCache, #fetch]
31
+ CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_scrape")
32
+ def self.cache
33
+ CACHE
34
+ end
35
+
36
+ # Fetch +url+ via {Tool::Scraper::Simple} and truncate the rendered
37
+ # Markdown to +max_chars+ characters.
38
+ #
39
+ # The full extracted Markdown is cached on disk via {.cache}, keyed
40
+ # by URL, so repeat visits within the cache TTL skip the network
41
+ # and the extraction pass entirely. +max_chars+ is not part of the
42
+ # cache key — different values for the same URL share one entry,
43
+ # and truncation runs after the cache lookup.
44
+ #
45
+ # {Tool::Scraper::FetchError} (HTTP non-2xx, network failure,
46
+ # redirect-loop, missing +Location+ header) is caught and returned
47
+ # as +"Error: ..."+ in the calculator-style convention so the agent
48
+ # loop feeds the failure back to the model as the next observation
49
+ # instead of crashing — the LLM can then try a different URL or
50
+ # search again. The rescue lives outside {.cache}'s +fetch+ block,
51
+ # so failure strings are never persisted: a retry on the next call
52
+ # hits the network again. Other exceptions (parser bugs in our own
53
+ # code) bubble up unchanged.
54
+ #
55
+ # @param url [String] absolute HTTP(S) URL of the page to download
56
+ # @param max_chars [Integer] character cap on the returned Markdown.
57
+ # Clamped to +[1, {MAX_MAX_CHARS}]+; defaults to
58
+ # {DEFAULT_MAX_CHARS}. When the full page exceeds the cap, output
59
+ # is cut and a marker noting the original length is appended.
60
+ # @return [String] Markdown representation of the page, possibly
61
+ # truncated, or +"Error: ..."+ on a recoverable fetch failure
62
+ def self.visit(url, max_chars: DEFAULT_MAX_CHARS)
63
+ max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
64
+ markdown = cache.fetch(url) { Scraper::Simple.visit(url) }
65
+ truncate(markdown, max_chars)
66
+ rescue Scraper::FetchError => e
67
+ "Error: #{e.message}"
68
+ end
69
+
70
+ # Cut +markdown+ to at most +max_chars+ characters, appending a
71
+ # marker describing the original length when truncation actually
72
+ # happens. Returns +markdown+ unchanged if it already fits.
73
+ #
74
+ # @param markdown [String] full Markdown text
75
+ # @param max_chars [Integer] character cap; assumed already clamped
76
+ # @return [String]
77
+ def self.truncate(markdown, max_chars)
78
+ return markdown if markdown.length <= max_chars
79
+
80
+ "#{markdown[0, max_chars]}\n\n" \
81
+ "... [truncated at #{max_chars} of #{markdown.length} chars; " \
82
+ 'call again with a larger `max_chars` to see more]'
83
+ end
84
+ end
85
+
86
+ # Webpage download + Markdown conversion tool. Thin wrapper over
87
+ # {Tool::WebScrape.visit} that exposes it to the agent loop in OpenAI
88
+ # tool-call shape.
89
+ #
90
+ # @return [Tool]
91
+ WEB_SCRAPE = new(
92
+ name: 'web_scrape',
93
+ description: <<~DESC,
94
+ Scrapes the rendered webpage, PDF, or text file at the given URL and returns its main content as Markdown.
95
+
96
+ Usage:
97
+ - Use for HTML pages or PDFs where you want readable content — readability extraction strips nav, sidebars, and boilerplate.
98
+ - For raw textual payloads (JSON, CSV, robots.txt, source files), use fetch instead — it returns bytes verbatim, while web_scrape would corrupt them with a Markdown pass.
99
+ - A Single Page App may return very little or no content. Do NOT retry with a larger max_chars; try a different URL instead.
100
+ DESC
101
+ parameters: Parameters.build { |p|
102
+ p.required_string :url,
103
+ 'Absolute URL of the webpage to scrape, including ' \
104
+ 'the scheme, e.g. "https://example.com/article".'
105
+ p.optional_integer :max_chars,
106
+ 'Maximum number of characters of Markdown to ' \
107
+ 'return. Defaults to 20000; hard-capped at ' \
108
+ '100000. When the page is longer than this, ' \
109
+ 'output is cut and a marker reports the full ' \
110
+ 'length.'
111
+ },
112
+ execute: ->(url:, max_chars: WebScrape::DEFAULT_MAX_CHARS) {
113
+ WebScrape.visit(url, max_chars: max_chars)
114
+ }
115
+ )
116
+ end
117
+ end
@@ -0,0 +1,38 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Pikuri
4
+ class Tool
5
+ # Namespace marker matching the file path. The actual search
6
+ # orchestration lives in {Tool::Search::Engines}; this file owns only
7
+ # the LLM-facing {Tool::WEB_SEARCH} constant below.
8
+ module WebSearch; end
9
+
10
+ # Web-search tool exposed to the agent loop in OpenAI tool-call shape.
11
+ # Calls {Tool::Search::Engines.search}, which cascades through whichever
12
+ # providers are configured (DuckDuckGo always, Brave when its API key is
13
+ # present) in random order, falling back on temporary-unavailability
14
+ # errors. Providers return structured {Tool::Search::Result} rows;
15
+ # +Engines.search+ renders the winning provider's rows into the
16
+ # smolagents-style Markdown shape the LLM sees, so the format stays
17
+ # stable regardless of which provider ran.
18
+ #
19
+ # @return [Tool]
20
+ WEB_SEARCH = new(
21
+ name: 'web_search',
22
+ description: <<~DESC,
23
+ Searches the web for a query and returns the top results as a Markdown list of titles, URLs, and short snippets.
24
+
25
+ Usage:
26
+ - Use this to find candidate URLs, then call web_scrape on the most promising one(s) for full content. Snippets alone rarely answer a question.
27
+ DESC
28
+ parameters: Parameters.build { |p|
29
+ p.required_string :query,
30
+ 'The search query, e.g. "BigDecimal precision Ruby".'
31
+ p.optional_integer :max_results,
32
+ 'Maximum number of result entries to return. ' \
33
+ 'Defaults to 10; most providers cap this at 20.'
34
+ },
35
+ execute: ->(query:, max_results: 10) { Search::Engines.search(query, max_results: max_results) }
36
+ )
37
+ end
38
+ end