pikuri-core 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +16 -29
  3. data/lib/pikuri/agent/chat_transport.rb +114 -33
  4. data/lib/pikuri/agent/configurator.rb +191 -110
  5. data/lib/pikuri/agent/control/cancellable.rb +25 -16
  6. data/lib/pikuri/agent/control/handoff.rb +75 -0
  7. data/lib/pikuri/agent/control/interloper.rb +87 -120
  8. data/lib/pikuri/agent/control.rb +9 -11
  9. data/lib/pikuri/agent/event.rb +73 -61
  10. data/lib/pikuri/agent/extension.rb +45 -81
  11. data/lib/pikuri/agent/history.rb +91 -497
  12. data/lib/pikuri/agent/listener/in_memory_event_list.rb +1 -1
  13. data/lib/pikuri/agent/listener/terminal.rb +115 -108
  14. data/lib/pikuri/agent/listener/token_log.rb +21 -55
  15. data/lib/pikuri/agent/listener.rb +4 -4
  16. data/lib/pikuri/agent/listener_list.rb +19 -13
  17. data/lib/pikuri/agent/request_instrumenter.rb +102 -0
  18. data/lib/pikuri/agent.rb +624 -448
  19. data/lib/pikuri/attachment.rb +65 -0
  20. data/lib/pikuri/extractor/passthrough.rb +5 -2
  21. data/lib/pikuri/extractor.rb +43 -12
  22. data/lib/pikuri/file_type.rb +4 -4
  23. data/lib/pikuri/finalizers.rb +2 -2
  24. data/lib/pikuri/paths.rb +1 -1
  25. data/lib/pikuri/sanitizer.rb +4 -3
  26. data/lib/pikuri/subprocess.rb +17 -5
  27. data/lib/pikuri/testing.rb +122 -49
  28. data/lib/pikuri/tool/calculator.rb +134 -30
  29. data/lib/pikuri/tool/execute_context.rb +2 -2
  30. data/lib/pikuri/tool/parameters.rb +10 -0
  31. data/lib/pikuri/tool/result.rb +164 -0
  32. data/lib/pikuri/tool/trifecta_legs.rb +52 -21
  33. data/lib/pikuri/tool.rb +298 -24
  34. data/lib/pikuri/trifecta/report.rb +28 -30
  35. data/lib/pikuri/trifecta.rb +1 -2
  36. data/lib/pikuri/version.rb +1 -1
  37. data/lib/pikuri-core.rb +5 -11
  38. metadata +16 -73
  39. data/lib/pikuri/agent/context_window_detector.rb +0 -181
  40. data/lib/pikuri/agent/control/step_limit.rb +0 -95
  41. data/lib/pikuri/agent/extension_context.rb +0 -120
  42. data/lib/pikuri/agent/listener/rate_limited.rb +0 -146
  43. data/lib/pikuri/agent/synthesizer.rb +0 -129
  44. data/lib/pikuri/extractor/html.rb +0 -256
  45. data/lib/pikuri/ruby_llm_patches.rb +0 -106
  46. data/lib/pikuri/tool/fetch.rb +0 -146
  47. data/lib/pikuri/tool/scraper.rb +0 -144
  48. data/lib/pikuri/tool/search/brave.rb +0 -170
  49. data/lib/pikuri/tool/search/duckduckgo.rb +0 -175
  50. data/lib/pikuri/tool/search/engines.rb +0 -319
  51. data/lib/pikuri/tool/search/exa.rb +0 -195
  52. data/lib/pikuri/tool/search/rate_limiter.rb +0 -115
  53. data/lib/pikuri/tool/search/result.rb +0 -24
  54. data/lib/pikuri/tool/web_scrape.rb +0 -105
  55. data/lib/pikuri/tool/web_search.rb +0 -133
  56. data/lib/pikuri/url_cache.rb +0 -96
@@ -1,105 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Pikuri
4
- class Tool
5
- # Truncation policy and Tool spec for the +web_scrape+ tool. Scraping lives
6
- # in {Tool::Scraper}; this wrapper applies a character cap and exposes the
7
- # result in OpenAI tool-call shape.
8
- module WebScrape
9
- # @return [Integer] default character cap on the Markdown from {.visit}.
10
- # Sized to cover most post-readability article bodies in full on the
11
- # first call, so the LLM doesn't re-request and duplicate the prefix in
12
- # its context. Genuinely long pages still get cut, and the marker invites
13
- # a deliberate larger call.
14
- DEFAULT_MAX_CHARS = 20_000
15
-
16
- # @return [Integer] hard ceiling on {.visit}'s +max_chars+; requests above
17
- # it are clamped silently so the LLM can't dump an arbitrarily large page.
18
- MAX_MAX_CHARS = 100_000
19
-
20
- # On-disk cache for {.visit}. A method so specs can swap {UrlCache::NULL}.
21
- #
22
- # @return [UrlCache, #fetch]
23
- CACHE = UrlCache.new(ttl: UrlCache::DEFAULT_TTL, dir: "#{UrlCache::ROOT_DIR}/web_scrape")
24
- # Accessor for {CACHE}; specs swap in {UrlCache::NULL} or an isolated cache.
25
- #
26
- # @return [UrlCache, #fetch]
27
- def self.cache
28
- CACHE
29
- end
30
-
31
- # Fetch +url+ via {Tool::Scraper} and truncate the rendered Markdown to
32
- # +max_chars+. The full Markdown is cached on disk keyed by URL (repeat
33
- # visits skip the network and extraction); +max_chars+ is *not* in the key
34
- # (values share one entry, truncation runs after the lookup).
35
- # {Scraper::FetchError} is caught outside +cache.fetch+ and returned as
36
- # +"Error: ..."+ (so the LLM can retry a different URL) — never persisted;
37
- # other exceptions bubble up.
38
- #
39
- # @param url [String] absolute HTTP(S) URL
40
- # @param max_chars [Integer] Markdown cap, clamped to +[1, {MAX_MAX_CHARS}]+;
41
- # default {DEFAULT_MAX_CHARS}. Over the cap, output is cut with a marker.
42
- # @return [String] Markdown, possibly truncated, or +"Error: ..."+ on a
43
- # recoverable fetch failure
44
- def self.visit(url, max_chars: DEFAULT_MAX_CHARS)
45
- max_chars = max_chars.clamp(1, MAX_MAX_CHARS)
46
- markdown = cache.fetch(url) { Scraper.visit(url) }
47
- truncate(markdown, max_chars)
48
- rescue Scraper::FetchError => e
49
- "Error: #{e.message}"
50
- end
51
-
52
- # Cut +markdown+ to at most +max_chars+, appending a marker with the
53
- # original length when truncation happens.
54
- #
55
- # @param markdown [String] full Markdown text
56
- # @param max_chars [Integer] cap; assumed already clamped
57
- # @return [String]
58
- def self.truncate(markdown, max_chars)
59
- return markdown if markdown.length <= max_chars
60
-
61
- "#{markdown[0, max_chars]}\n\n" \
62
- "... [truncated at #{max_chars} of #{markdown.length} chars; " \
63
- 'call again with a larger `max_chars` to see more]'
64
- end
65
- end
66
-
67
- # Webpage download + Markdown conversion tool — a thin wrapper over
68
- # {Tool::WebScrape.visit} in OpenAI tool-call shape.
69
- #
70
- # Sharing: +P_shared_benign+ — as {Tool::FETCH}: no per-agent state, but
71
- # {WebScrape::CACHE} is process-wide and on disk, so one agent's scrape
72
- # answers another's within the TTL. See {Pikuri::UrlCache}.
73
- #
74
- # @return [Tool]
75
- WEB_SCRAPE = new(
76
- name: 'web_scrape',
77
- description: <<~DESC,
78
- Scrapes the rendered webpage or text file at the given URL and returns its main content as Markdown.
79
-
80
- Usage:
81
- - Use for HTML pages where you want readable content — readability extraction strips nav, sidebars, and boilerplate.
82
- - For raw textual payloads (JSON, CSV, robots.txt, source files), use fetch instead — it returns bytes verbatim, while web_scrape would corrupt them with a Markdown pass.
83
- - A Single Page App may return very little or no content. Do NOT retry with a larger max_chars; try a different URL instead.
84
- DESC
85
- parameters: Parameters.build { |p|
86
- p.required_string :url,
87
- 'Absolute URL of the webpage to scrape, including ' \
88
- 'the scheme, e.g. "https://example.com/article".'
89
- p.optional_integer :max_chars,
90
- 'Maximum number of characters of Markdown to ' \
91
- 'return. Defaults to 20000; hard-capped at ' \
92
- '100000. When the page is longer than this, ' \
93
- 'output is cut and a marker reports the full ' \
94
- 'length.'
95
- },
96
- execute: ->(url:, max_chars: WebScrape::DEFAULT_MAX_CHARS) {
97
- WebScrape.visit(url, max_chars: max_chars)
98
- },
99
- # Exactly {TrifectaLegs::ASSUMED}, and the archetype it was named for: the
100
- # page is authored by whoever runs the site, and the model picks the URL,
101
- # so the request is itself an outbound channel.
102
- trifecta_legs: Pikuri::Tool::TrifectaLegs::ASSUMED
103
- )
104
- end
105
- end
@@ -1,133 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Pikuri
4
- class Tool
5
- # Builder for the LLM-facing web-search tool. Orchestration lives in
6
- # {Tool::Search::Engines}; this owns only the {.build} factory wiring a
7
- # configured {Search::Engines} into a {Tool}.
8
- #
9
- # Unlike the stateless bundled tools (shared value constants), web_search is
10
- # *host-configured*: the paid providers (Brave/Exa) join the cascade only
11
- # when the host passes their key, so it's built per-wiring (like
12
- # {Code::Bash}), not a shared constant. pikuri reads no key from the
13
- # environment — the host supplies them (CLAUDE.md "Environment is not a
14
- # secret store").
15
- #
16
- # == Sharing
17
- #
18
- # +P_shared_locked+, and the interesting case: build one per agent or share
19
- # one — it makes no difference to the throttling, because the pacing state
20
- # is neither per agent nor per instance. Each provider's limiter is a
21
- # class-level constant ({Search::DuckDuckGo::LIMITER} and friends), so all
22
- # ten agents queue behind the same one. That is the only correct scope:
23
- # what it protects is the egress IP and the API key's quota, and neither is
24
- # per agent.
25
- #
26
- # What *is* per instance is trivial — the provider list and a
27
- # last-logged-set memo, which can only duplicate an INFO line. The result
28
- # cache ({Engines::CACHE}) is class-level as well, so one agent's search
29
- # answers another's within the TTL.
30
- #
31
- # The consequence a host should plan for is latency, not corruption: see
32
- # {Search::RateLimiter} for the serialization and the shared circuit
33
- # breaker. Which is why this is the one bundled tool that opts into the
34
- # per-call {Tool::ExecuteContext} (see {Pikuri::Tool}'s +== Cancellation+):
35
- # an agent queued behind another agent's pacing wait can abandon it, even
36
- # though the tool itself may be shared.
37
- module WebSearch
38
- # Description shown to the LLM, opencode-shape (summary + +Usage:+).
39
- #
40
- # @return [String]
41
- DESCRIPTION = <<~DESC
42
- Searches the web for a query and returns the top results as a Markdown list of titles, URLs, and short snippets.
43
-
44
- Usage:
45
- - Use this to find candidate URLs, then call web_scrape on the most promising one(s) for full content. Snippets alone rarely answer a question.
46
- DESC
47
-
48
- # Build the +web_search+ tool. Calls {Search::Engines#search}, which
49
- # cascades the selected providers in random order, falling back on
50
- # unavailability and rendering the winner's rows to a stable Markdown
51
- # shape.
52
- #
53
- # WebSearch.build # DuckDuckGo alone
54
- # WebSearch.build(brave_key: 'BSA...') # DuckDuckGo + Brave
55
- # WebSearch.build(engines: [:brave], brave_key: 'BSA...') # Brave alone
56
- #
57
- # @param engines [Array<String, Symbol>, nil] engine names to use —
58
- # +duckduckgo+, +brave+, +exa+. +nil+ selects DuckDuckGo plus every
59
- # engine whose key is supplied. Order carries no preference; see
60
- # {Search::Engines.resolve}.
61
- # @param brave_key [String, nil] Brave key
62
- # (https://api-dashboard.search.brave.com).
63
- # @param exa_key [String, nil] Exa key (https://exa.ai).
64
- # @return [Tool] the +web_search+ tool in OpenAI tool-call shape
65
- # @raise [ArgumentError] on an unusable selection: an empty or repeated
66
- # +engines+ list, an unknown engine name, or an engine named without
67
- # its key.
68
- def self.build(engines: nil, brave_key: nil, exa_key: nil)
69
- # Validated here as well as inside the cascade, so a mis-wired host is
70
- # refused at the call it actually wrote even if this stops constructing
71
- # the cascade eagerly.
72
- Search::Engines.resolve(engines: engines, brave_key: brave_key, exa_key: exa_key)
73
- cascade = Search::Engines.new(engines: engines, brave_key: brave_key, exa_key: exa_key)
74
- Tool.new(
75
- name: 'web_search',
76
- description: DESCRIPTION,
77
- parameters: Parameters.build { |p|
78
- p.required_string :query,
79
- 'The search query, e.g. "BigDecimal precision Ruby".'
80
- p.optional_integer :max_results,
81
- 'Maximum number of result entries to return. ' \
82
- 'Defaults to 10; most providers cap this at 20.'
83
- },
84
- # +wants_context:+ opts in to the per-call {Tool::ExecuteContext}, so
85
- # a shared instance still honours whichever agent is waiting.
86
- wants_context: true,
87
- execute: lambda { |query:, max_results: 10, context:|
88
- cascade.search(query, max_results: max_results, cancellable: context.cancellable)
89
- },
90
- # Snippets are written by whoever ranks for the query, and the query
91
- # string itself leaves the machine — a search is an outbound channel
92
- # even though nothing is "sent". Nobody reviews that query, hence
93
- # +:unreviewed+; the whole difference from {FETCH} is the destination.
94
- #
95
- # +:tool_vouched+ against the three clauses: the caller supplies a
96
- # query, never a host, and the engine set is this file's own
97
- # ({Search::Engines::ENGINE_NAMES}); a search engine does not
98
- # republish the queries it receives; and the request rides TLS to a
99
- # party with no relationship to the attacker. A host picks from that
100
- # closed list of three and cannot extend it, so the destination is
101
- # still *this code* choosing, not the model.
102
- #
103
- # It is not a claim that the leg is harmless. A secret can still be
104
- # pushed out inside a query — it just cannot be read back, and only
105
- # while this stays the agent's sole egress leg.
106
- trifecta_legs: Pikuri::Tool::TrifectaLegs.new(
107
- private: false, untrusted: :hard,
108
- egress_payload_review: :unreviewed, egress_destination: :tool_vouched
109
- )
110
- )
111
- end
112
-
113
- # Build the +web_search+ tool from a config file's +search:+ block, the
114
- # host-facing shortcut for {build}:
115
- #
116
- # # ~/.pikuri-examples-config.yaml
117
- # search:
118
- # brave_key: BSA...
119
- # engines: [brave]
120
- #
121
- # c.add_tool Pikuri::Tool::WebSearch.from_h(examples_config['search'])
122
- #
123
- # @param config [Hash{String, Symbol => Object}, nil] the +search:+ block;
124
- # +nil+ (an absent key) yields the DuckDuckGo-only default.
125
- # @return [Tool] the +web_search+ tool in OpenAI tool-call shape
126
- # @raise [ArgumentError] as {build}, plus on a malformed block; see
127
- # {Search::Engines.config_from_h}.
128
- def self.from_h(config)
129
- build(**Search::Engines.config_from_h(config))
130
- end
131
- end
132
- end
133
- end
@@ -1,96 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require 'digest'
4
- require 'fileutils'
5
-
6
- module Pikuri
7
- # On-disk cache for string-keyed text payloads. The bundled tools use it to
8
- # avoid re-fetching within a TTL: {Tool::WebScrape.visit} caches a URL's
9
- # rendered Markdown, {Tool::Search::Engines.search} caches a query's result
10
- # list (keys are SHA-256 hashed, so any opaque string works).
11
- #
12
- # Each tool wires its own instance against a dedicated subdir under {ROOT_DIR},
13
- # so a +web_search+ query and a +web_scrape+ URL can't collide on one file.
14
- # No global singleton — pass a fresh instance, or {NULL} to disable caching.
15
- #
16
- # One file per entry (+<sha256>.txt+), freshness by mtime, no sidecar
17
- # metadata; a stale entry is overwritten on the next {#fetch} with that key.
18
- # +rm -rf+ the directory to clear it.
19
- #
20
- # Sharing: +P_shared_benign+ — an instance holds only its TTL and directory,
21
- # and every tool that owns one holds it in a class-level constant, so all
22
- # agents (and other pikuri processes) hit the same files. If two callers race
23
- # on the same cold key, both compute and both write the same bytes — the
24
- # accepted tradeoff for keeping this a few dozen lines (worst case: a
25
- # duplicate fetch). The cross-agent flow is deliberate: a public URL's content
26
- # is nobody's private context.
27
- class UrlCache
28
- # Root dir for per-tool cache subdirs, per XDG:
29
- # +$XDG_CACHE_HOME/pikuri/url_cache+ or +~/.cache/pikuri/url_cache+. Each
30
- # tool picks its own subdir (+"#{ROOT_DIR}/web_scrape"+) so keys can't
31
- # collide. Created lazily on first write.
32
- # @return [String]
33
- ROOT_DIR = Paths.cache.join('url_cache').to_path
34
-
35
- # Default freshness window: 2 hours. Long enough to cover an interactive
36
- # session (revisiting a page or re-running a search within the working
37
- # window hits the cache), short enough that resuming the next day doesn't
38
- # serve stale results. (opencode caches nothing; +pi-web-fetch+ uses 15 min,
39
- # +pi-web-search+ 5.)
40
- # @return [Integer]
41
- DEFAULT_TTL = 2 * 60 * 60
42
-
43
- # @param ttl [Integer] freshness window in seconds; entries with an
44
- # mtime older than this are treated as misses
45
- # @param dir [String] directory under which cache files live; created
46
- # lazily on first write
47
- def initialize(ttl:, dir:)
48
- @ttl = ttl
49
- @dir = dir
50
- end
51
-
52
- # Return the cached payload for +url+ if fresh, else yield to compute it,
53
- # persist, and return. The block runs only on a miss; if it raises, no file
54
- # is written (errors aren't cached).
55
- #
56
- # @param url [String] cache key; a URL or any opaque string
57
- # @yieldreturn [String] payload to store and return on a miss
58
- # @return [String] cached or freshly-computed payload
59
- def fetch(url)
60
- path = path_for(url)
61
- return File.read(path) if fresh?(path)
62
-
63
- content = yield
64
- FileUtils.mkdir_p(@dir)
65
- File.write(path, content)
66
- content
67
- end
68
-
69
- # @param path [String]
70
- # @return [Boolean] true when +path+ exists and was written within the
71
- # TTL window
72
- def fresh?(path)
73
- File.exist?(path) && Time.now - File.mtime(path) < @ttl
74
- end
75
-
76
- # @param url [String]
77
- # @return [String] absolute path of the cache file for +url+
78
- def path_for(url)
79
- File.join(@dir, "#{Digest::SHA256.hexdigest(url)}.txt")
80
- end
81
-
82
- # Null cache: always misses, never persists — a drop-in for tests (or
83
- # anywhere caching should be off) keeping the {UrlCache#fetch} contract.
84
- NULL = Object.new
85
- # Singleton +fetch+ on {NULL} that mirrors {UrlCache#fetch}'s shape:
86
- # ignores +_key+, always yields, never persists.
87
- #
88
- # @param _key [String] cache key; ignored
89
- # @yieldreturn [String] freshly-computed payload
90
- # @return [String] whatever the block returned
91
- def NULL.fetch(_key)
92
- yield
93
- end
94
- NULL.freeze
95
- end
96
- end