rubino-agent 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop_todo.yml +11 -2
- data/AGENTS.md +1 -1
- data/CHANGELOG.md +172 -5
- data/CONTRIBUTING.md +10 -1
- data/README.md +14 -5
- data/Rakefile +31 -0
- data/docs/agents.md +42 -23
- data/docs/architecture.md +2 -2
- data/docs/commands.md +35 -3
- data/docs/configuration.md +20 -23
- data/docs/getting-started.md +5 -3
- data/docs/security.md +16 -5
- data/docs/skills.md +31 -0
- data/docs/troubleshooting.md +1 -1
- data/exe/rubino +16 -2
- data/install.sh +721 -59
- data/lib/rubino/active_agent.rb +73 -0
- data/lib/rubino/agent/action_claim_guard.rb +881 -0
- data/lib/rubino/agent/agent_registry.rb +5 -2
- data/lib/rubino/agent/definition.rb +1 -9
- data/lib/rubino/agent/fallback_chain.rb +0 -6
- data/lib/rubino/agent/iteration_budget.rb +109 -3
- data/lib/rubino/agent/loop.rb +476 -20
- data/lib/rubino/agent/model_call_runner.rb +81 -3
- data/lib/rubino/agent/prompts/build.txt +22 -5
- data/lib/rubino/agent/response_validator.rb +8 -0
- data/lib/rubino/agent/runner.rb +133 -8
- data/lib/rubino/agent/tool_executor.rb +166 -14
- data/lib/rubino/agent/truncation_continuation.rb +4 -1
- data/lib/rubino/api/server.rb +19 -0
- data/lib/rubino/attachments/classify.rb +35 -17
- data/lib/rubino/boot/config_guard.rb +71 -0
- data/lib/rubino/cli/chat/completion_builder.rb +42 -6
- data/lib/rubino/cli/chat/idle_card_host.rb +7 -1
- data/lib/rubino/cli/chat/session_resolver.rb +87 -21
- data/lib/rubino/cli/chat_command.rb +1189 -50
- data/lib/rubino/cli/commands.rb +282 -2
- data/lib/rubino/cli/config_command.rb +68 -8
- data/lib/rubino/cli/doctor_command.rb +204 -12
- data/lib/rubino/cli/jobs_command.rb +12 -0
- data/lib/rubino/cli/memory_command.rb +53 -20
- data/lib/rubino/cli/onboarding_wizard.rb +79 -6
- data/lib/rubino/cli/session_command.rb +172 -18
- data/lib/rubino/cli/setup_command.rb +131 -8
- data/lib/rubino/cli/skills_command.rb +183 -9
- data/lib/rubino/cli/trust_gate.rb +16 -7
- data/lib/rubino/commands/built_ins.rb +2 -0
- data/lib/rubino/commands/command.rb +12 -2
- data/lib/rubino/commands/executor.rb +149 -12
- data/lib/rubino/commands/handlers/agent_switch.rb +100 -0
- data/lib/rubino/commands/handlers/agents.rb +156 -41
- data/lib/rubino/commands/handlers/config.rb +4 -1
- data/lib/rubino/commands/handlers/help.rb +113 -14
- data/lib/rubino/commands/handlers/memory.rb +15 -5
- data/lib/rubino/commands/handlers/sessions.rb +26 -3
- data/lib/rubino/commands/handlers/status.rb +9 -4
- data/lib/rubino/commands/loader.rb +12 -0
- data/lib/rubino/config/configuration.rb +86 -24
- data/lib/rubino/config/defaults.rb +140 -33
- data/lib/rubino/config/loader.rb +62 -12
- data/lib/rubino/config/validator.rb +341 -0
- data/lib/rubino/config/writer.rb +123 -31
- data/lib/rubino/context/compressor.rb +184 -22
- data/lib/rubino/context/environment_inspector.rb +2 -2
- data/lib/rubino/context/file_discovery.rb +2 -2
- data/lib/rubino/context/message_boundary.rb +27 -1
- data/lib/rubino/context/project_languages.rb +90 -0
- data/lib/rubino/context/prompt_assembler.rb +105 -22
- data/lib/rubino/context/summary_builder.rb +45 -4
- data/lib/rubino/context/token_budget.rb +36 -11
- data/lib/rubino/context/token_estimate.rb +45 -0
- data/lib/rubino/context/tool_result_pruner.rb +81 -0
- data/lib/rubino/database/connection.rb +154 -3
- data/lib/rubino/database/migrations/001_create_initial_schema.rb +314 -40
- data/lib/rubino/database/migrator.rb +98 -5
- data/lib/rubino/documents/cap_exceeded.rb +13 -0
- data/lib/rubino/documents/converters/csv.rb +4 -3
- data/lib/rubino/documents/converters/docx.rb +29 -5
- data/lib/rubino/documents/converters/html.rb +5 -1
- data/lib/rubino/documents/converters/json.rb +2 -1
- data/lib/rubino/documents/converters/pdf.rb +11 -2
- data/lib/rubino/documents/converters/plain.rb +2 -1
- data/lib/rubino/documents/converters/pptx.rb +11 -2
- data/lib/rubino/documents/converters/xlsx.rb +35 -4
- data/lib/rubino/documents/converters/xml.rb +2 -1
- data/lib/rubino/documents/limits.rb +210 -0
- data/lib/rubino/documents.rb +10 -3
- data/lib/rubino/errors.rb +36 -5
- data/lib/rubino/interaction/cancel_token.rb +19 -3
- data/lib/rubino/interaction/events.rb +13 -0
- data/lib/rubino/interaction/lifecycle.rb +99 -13
- data/lib/rubino/interaction/polishing.rb +176 -0
- data/lib/rubino/jobs/cron_job_repository.rb +5 -8
- data/lib/rubino/jobs/handlers/cleanup_sessions_job.rb +11 -0
- data/lib/rubino/jobs/handlers/distill_skill_job.rb +65 -9
- data/lib/rubino/jobs/queue.rb +63 -8
- data/lib/rubino/jobs/runner.rb +24 -6
- data/lib/rubino/jobs/worker.rb +0 -4
- data/lib/rubino/llm/adapter_response.rb +47 -4
- data/lib/rubino/llm/credential_check.rb +15 -16
- data/lib/rubino/llm/error_classifier.rb +89 -1
- data/lib/rubino/llm/inline_think_filter.rb +69 -12
- data/lib/rubino/llm/request.rb +30 -3
- data/lib/rubino/llm/ruby_llm_adapter.rb +394 -46
- data/lib/rubino/llm/tool_bridge.rb +113 -9
- data/lib/rubino/mcp/manager.rb +18 -1
- data/lib/rubino/mcp/mcp_tool_wrapper.rb +14 -3
- data/lib/rubino/memory/aux_retry.rb +107 -0
- data/lib/rubino/memory/backends/sqlite.rb +73 -44
- data/lib/rubino/memory/backends.rb +23 -7
- data/lib/rubino/memory/salience_gate.rb +103 -0
- data/lib/rubino/memory/sqlite_extraction.rb +70 -0
- data/lib/rubino/memory/sqlite_extraction_prompt.rb +11 -0
- data/lib/rubino/memory/store.rb +33 -5
- data/lib/rubino/memory/threat_scanner.rb +52 -0
- data/lib/rubino/output/cost.rb +52 -0
- data/lib/rubino/output/headless_block_latch.rb +53 -0
- data/lib/rubino/output/result_serializer.rb +222 -0
- data/lib/rubino/output/turn_recorder.rb +77 -0
- data/lib/rubino/security/approval_policy.rb +227 -32
- data/lib/rubino/security/command_allowlist.rb +79 -4
- data/lib/rubino/security/doom_loop_detector.rb +21 -2
- data/lib/rubino/security/hardline_guard.rb +189 -16
- data/lib/rubino/security/pattern_matcher.rb +28 -5
- data/lib/rubino/security/prefix_deriver.rb +25 -6
- data/lib/rubino/security/readonly_commands.rb +145 -5
- data/lib/rubino/security/secret_path.rb +134 -0
- data/lib/rubino/security/url_safety.rb +255 -0
- data/lib/rubino/session/repository.rb +212 -11
- data/lib/rubino/session/store.rb +139 -14
- data/lib/rubino/skills/installer.rb +230 -0
- data/lib/rubino/skills/prompt_index.rb +2 -2
- data/lib/rubino/skills/registry.rb +52 -1
- data/lib/rubino/skills/skill.rb +64 -3
- data/lib/rubino/skills/skill_tool.rb +16 -5
- data/lib/rubino/tools/background_tasks.rb +157 -13
- data/lib/rubino/tools/base.rb +204 -3
- data/lib/rubino/tools/edit_tool.rb +73 -18
- data/lib/rubino/tools/glob_tool.rb +48 -9
- data/lib/rubino/tools/grep_tool.rb +103 -9
- data/lib/rubino/tools/multi_edit_tool.rb +64 -9
- data/lib/rubino/tools/patch_tool.rb +5 -0
- data/lib/rubino/tools/read_attachment_tool.rb +3 -1
- data/lib/rubino/tools/read_tool.rb +33 -15
- data/lib/rubino/tools/read_tracker.rb +153 -35
- data/lib/rubino/tools/registry.rb +113 -12
- data/lib/rubino/tools/result.rb +9 -1
- data/lib/rubino/tools/ruby_tool.rb +0 -0
- data/lib/rubino/tools/shell_registry.rb +70 -0
- data/lib/rubino/tools/shell_tool.rb +40 -1
- data/lib/rubino/tools/summarize_file_tool.rb +6 -0
- data/lib/rubino/tools/task_stop_tool.rb +10 -16
- data/lib/rubino/tools/task_tool.rb +36 -8
- data/lib/rubino/tools/vision_tool.rb +5 -0
- data/lib/rubino/tools/webfetch_tool.rb +39 -7
- data/lib/rubino/tools/websearch_tool.rb +92 -30
- data/lib/rubino/tools/write_tool.rb +23 -4
- data/lib/rubino/ui/api.rb +10 -1
- data/lib/rubino/ui/base.rb +11 -0
- data/lib/rubino/ui/bottom_composer.rb +382 -74
- data/lib/rubino/ui/cli.rb +515 -83
- data/lib/rubino/ui/completion_menu.rb +11 -7
- data/lib/rubino/ui/headless_trace.rb +63 -0
- data/lib/rubino/ui/live_region.rb +70 -7
- data/lib/rubino/ui/markdown_renderer.rb +142 -7
- data/lib/rubino/ui/notifier.rb +0 -2
- data/lib/rubino/ui/null.rb +52 -5
- data/lib/rubino/ui/paste_store.rb +16 -2
- data/lib/rubino/ui/queued_indicators.rb +6 -1
- data/lib/rubino/ui/status_bar.rb +61 -7
- data/lib/rubino/ui/streaming_markdown.rb +59 -6
- data/lib/rubino/ui/subagent_view.rb +29 -4
- data/lib/rubino/ui/tool_label.rb +52 -0
- data/lib/rubino/update_check.rb +39 -4
- data/lib/rubino/util/atomic_file.rb +117 -0
- data/lib/rubino/util/ignore_rules.rb +120 -0
- data/lib/rubino/util/output.rb +229 -12
- data/lib/rubino/util/secrets_mask.rb +70 -7
- data/lib/rubino/util/spill_store.rb +153 -0
- data/lib/rubino/version.rb +1 -1
- data/lib/rubino/workspace.rb +9 -1
- data/lib/rubino.rb +191 -7
- data/rubino-agent.gemspec +1 -0
- data/skills/ruby-expert/SKILL.md +1 -0
- metadata +42 -12
- data/lib/rubino/agent/router.rb +0 -65
- data/lib/rubino/database/migrations/002_create_runs.rb +0 -45
- data/lib/rubino/database/migrations/003_create_skill_states.rb +0 -15
- data/lib/rubino/database/migrations/004_create_cron_jobs.rb +0 -36
- data/lib/rubino/database/migrations/005_create_oauth_connections.rb +0 -27
- data/lib/rubino/database/migrations/006_create_webhook_deliveries.rb +0 -34
- data/lib/rubino/database/migrations/007_create_messages_fts.rb +0 -59
- data/lib/rubino/database/migrations/008_create_memory_facts.rb +0 -75
- data/lib/rubino/database/migrations/009_create_memory_graph.rb +0 -55
- data/lib/rubino/database/migrations/010_add_owner_pid_to_sessions.rb +0 -20
data/lib/rubino/util/output.rb
CHANGED
|
@@ -19,6 +19,136 @@ module Rubino
|
|
|
19
19
|
DEFAULT_HEAD = 5
|
|
20
20
|
DEFAULT_TAIL = 10
|
|
21
21
|
|
|
22
|
+
# The NUL byte (U+0000) is the one control char that is VALID UTF-8 yet
|
|
23
|
+
# still breaks the persistence layer: the SQLite3 driver treats it as a
|
|
24
|
+
# C-string terminator and raises "unrecognized token" (the tool row never
|
|
25
|
+
# persists), and JSON re-tags the value as BINARY. String#scrub leaves it
|
|
26
|
+
# alone (it only repairs INVALID bytes), so scrub-to-UTF-8 is necessary
|
|
27
|
+
# but not sufficient — NUL has to go too.
|
|
28
|
+
NUL = "\x00"
|
|
29
|
+
|
|
30
|
+
# Coerces +text+ to a clean, persistable UTF-8 string: valid encoding AND
|
|
31
|
+
# free of NUL bytes.
|
|
32
|
+
#
|
|
33
|
+
# Tool output is captured raw from a subprocess pipe / file read / MCP
|
|
34
|
+
# response and can be binary or latin-1 (`head -c 1500 /dev/urandom`,
|
|
35
|
+
# `cat some.png`). Such bytes are tagged UTF-8 (the pipe's external
|
|
36
|
+
# encoding) but are NOT valid UTF-8, so the moment they reach
|
|
37
|
+
# JSON.generate (the LLM request, the run-event store) or the SQLite
|
|
38
|
+
# driver they raise "source sequence is illegal/malformed utf-8" /
|
|
39
|
+
# "UTF-8 passed as BINARY" / "unrecognized token" and the tool row never
|
|
40
|
+
# persists — the model loses the record on --resume. Random binary ALSO
|
|
41
|
+
# carries NUL bytes, which survive String#scrub (NUL is valid UTF-8) yet
|
|
42
|
+
# still wedge SQLite, so we strip them here too. Cleaning at the CAPTURE
|
|
43
|
+
# seam (before the bytes are ever copied into the result) means every
|
|
44
|
+
# downstream consumer sees a safe string. Idempotent on already-clean
|
|
45
|
+
# input. Pure.
|
|
46
|
+
def self.scrub_utf8(text)
|
|
47
|
+
s = scrub_encoding(text)
|
|
48
|
+
s.include?(NUL) ? s.delete(NUL) : s
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# Encoding-only repair: returns a valid-UTF-8 string, leaving control
|
|
52
|
+
# bytes (incl. NUL) in place. Split out from #scrub_utf8 because the two
|
|
53
|
+
# consumers want different things downstream of "make it valid UTF-8":
|
|
54
|
+
# the PERSIST seam (#scrub_utf8) deletes NUL outright (SQLite-fatal), but
|
|
55
|
+
# the TERMINAL render seam (#sanitize_terminal) wants every control byte
|
|
56
|
+
# turned into VISIBLE caret notation — so it scrubs encoding here, then
|
|
57
|
+
# does its own C0/C1 pass instead of pre-deleting NUL. Pure.
|
|
58
|
+
def self.scrub_encoding(text)
|
|
59
|
+
s = text.to_s
|
|
60
|
+
return s if s.encoding == Encoding::UTF_8 && s.valid_encoding?
|
|
61
|
+
|
|
62
|
+
s.dup.force_encoding(Encoding::UTF_8).scrub
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# ESC (0x1B): the introducer for ALL the dangerous sequences — CSI
|
|
66
|
+
# (cursor move, screen clear, scroll region), OSC (set window title,
|
|
67
|
+
# hyperlinks, clipboard write), DCS, etc.
|
|
68
|
+
ESC = "\e"
|
|
69
|
+
# U+009B is the single-byte CSI introducer: a terminal treats it exactly
|
|
70
|
+
# like `ESC [`, so stripping ESC alone would leave a working injection
|
|
71
|
+
# vector. It only exists AFTER UTF-8 decoding (the byte 0x9B on its own
|
|
72
|
+
# is invalid UTF-8 and scrubbed; U+0085/U+0080–U+009F arrive via valid
|
|
73
|
+
# 2-byte forms), so we strip the C1 block on the decoded string.
|
|
74
|
+
C1_RANGE = "-"
|
|
75
|
+
|
|
76
|
+
# Neutralizes terminal-control bytes in UNTRUSTED tool output before it
|
|
77
|
+
# is printed to a real terminal.
|
|
78
|
+
#
|
|
79
|
+
# Threat (CWE-150): raw `\e[2J` (clear screen), `\e[41m…\e[0m` (color),
|
|
80
|
+
# `\e]0;…\a` (set title), `\e]52;…` (clipboard write) embedded in
|
|
81
|
+
# shell/file/MCP output reach the emulator and EXECUTE — the live tool
|
|
82
|
+
# tail printed it verbatim. Following git's `core.fsmonitor`-style and
|
|
83
|
+
# dgl.cx's "sanitize at the render chokepoint" guidance, we strip every
|
|
84
|
+
# control byte that can move the cursor, repaint, or drive the terminal,
|
|
85
|
+
# and render what we removed as visible caret/<XX> notation so the user
|
|
86
|
+
# SEES that bytes were there (silent deletion hides the attack).
|
|
87
|
+
#
|
|
88
|
+
# Kept: \t (0x09) and \n (0x0A) — legitimate layout. \r is normalized to
|
|
89
|
+
# \n (a bare CR rewinds the line and lets later text overwrite what was
|
|
90
|
+
# already shown — another spoofing vector). Stripped: C0 0x00–0x1F
|
|
91
|
+
# (except \t/\n), DEL 0x7F, ESC 0x1B, and the C1 block 0x80–0x9F.
|
|
92
|
+
#
|
|
93
|
+
# rubino's OWN styling (the @pastel.dim/green wrapper applied AROUND this
|
|
94
|
+
# content) is a separate, trusted path and is never passed through here.
|
|
95
|
+
# Pure.
|
|
96
|
+
def self.sanitize_terminal(text)
|
|
97
|
+
# Encoding-scrub ONLY (keep NUL et al.) so the C0 pass below can turn
|
|
98
|
+
# every control byte into visible caret notation — silent deletion
|
|
99
|
+
# would hide that the tool tried to emit them.
|
|
100
|
+
s = scrub_encoding(text)
|
|
101
|
+
# Bare CR (not part of CRLF) → newline, so overwrite-spoofing can't
|
|
102
|
+
# rewind the rendered line. CRLF collapses to a single LF.
|
|
103
|
+
s = s.gsub(/\r\n?/, "\n")
|
|
104
|
+
s = s.gsub(/[\x00-\x08\x0B-\x1F\x7F]/) { |c| caret(c) }
|
|
105
|
+
s.gsub(/[#{C1_RANGE}]/o) { |c| "<#{format("%02X", c.ord)}>" }
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# SGR colour/style escapes (`\e[…m`) — the ONE escape class that is SAFE
|
|
109
|
+
# to keep through the sanitizer: it changes only colour/weight and cannot
|
|
110
|
+
# move the cursor, clear the screen, set the title, or write the
|
|
111
|
+
# clipboard. Matched so #sanitize_terminal_keep_sgr can preserve rubino's
|
|
112
|
+
# OWN styling (e.g. the colored /agents status glyph) while still
|
|
113
|
+
# neutralizing every dangerous control byte.
|
|
114
|
+
SGR_RE = /\e\[[0-9;]*m/
|
|
115
|
+
|
|
116
|
+
# Like #sanitize_terminal, but PRESERVES SGR colour escapes.
|
|
117
|
+
#
|
|
118
|
+
# Some sinks interpolate TRUSTED rubino styling (a pastel-colored cell,
|
|
119
|
+
# e.g. the /agents table's "● approval" status) THROUGH the same cell
|
|
120
|
+
# sanitizer that guards untrusted text. Plain #sanitize_terminal rendered
|
|
121
|
+
# those SGR bytes as visible caret notation (`^[[33m●^[[0m approval`) —
|
|
122
|
+
# the FRICTION-3 leak. Keep the (inert) SGR sequences, neutralize
|
|
123
|
+
# everything else exactly as #sanitize_terminal does, so colour survives
|
|
124
|
+
# but `\e[2J` / `\e]0;…` / cursor moves still can't reach the terminal.
|
|
125
|
+
# Callers that measure width must strip SGR first (see SGR_RE / the
|
|
126
|
+
# display-width helpers) since SGR occupies zero columns. Pure.
|
|
127
|
+
def self.sanitize_terminal_keep_sgr(text)
|
|
128
|
+
s = scrub_encoding(text)
|
|
129
|
+
# Carve out the SGR runs, sanitize the gaps, splice the SGR back in.
|
|
130
|
+
parts = []
|
|
131
|
+
last = 0
|
|
132
|
+
s.to_enum(:scan, SGR_RE).each do
|
|
133
|
+
m = Regexp.last_match
|
|
134
|
+
parts << sanitize_terminal(s[last...m.begin(0)])
|
|
135
|
+
parts << m[0]
|
|
136
|
+
last = m.end(0)
|
|
137
|
+
end
|
|
138
|
+
parts << sanitize_terminal(s[last..]) if last < s.length
|
|
139
|
+
parts.join
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# Visible, unambiguous stand-in for a stripped control byte: ESC → "^[",
|
|
143
|
+
# NUL → "^@", DEL → "^?" — the classic `cat -v` caret notation, so the
|
|
144
|
+
# user can tell exactly what the tool tried to emit.
|
|
145
|
+
def self.caret(byte)
|
|
146
|
+
code = byte.ord
|
|
147
|
+
return "^?" if code == 0x7F
|
|
148
|
+
|
|
149
|
+
"^#{(code ^ 0x40).chr}"
|
|
150
|
+
end
|
|
151
|
+
|
|
22
152
|
# Returns either the full text (when total lines <= max) or a
|
|
23
153
|
# head + marker + tail preview. Pure function — no side effects,
|
|
24
154
|
# no IO. Caller decides where to render the result.
|
|
@@ -31,17 +161,76 @@ module Rubino
|
|
|
31
161
|
def self.preview(text, max: DEFAULT_MAX, head: DEFAULT_HEAD, tail: DEFAULT_TAIL)
|
|
32
162
|
return "" if text.nil? || text.to_s.empty?
|
|
33
163
|
|
|
34
|
-
|
|
35
|
-
|
|
164
|
+
s = text.to_s
|
|
165
|
+
# Count newlines instead of materializing `s.lines` (#373): a ~1KB
|
|
166
|
+
# value with a 2-million-element single-line buffer used to allocate a
|
|
167
|
+
# 2M-element array (+ another 2M chomp'd copy via `.map(&:chomp)`) just
|
|
168
|
+
# to learn it fits — ~hundreds of MB of churn for a preview the caller
|
|
169
|
+
# may not even trim. `count("\n")` is O(n) bytes with zero allocation.
|
|
170
|
+
# total line count = newline count (+1 unless the buffer ends in \n).
|
|
171
|
+
total = line_count(s)
|
|
172
|
+
if total <= max
|
|
173
|
+
# Fits: only NOW materialize, and only to chomp the trailing newlines
|
|
174
|
+
# of the (already small) line set.
|
|
175
|
+
return s.lines.map(&:chomp).join("\n")
|
|
176
|
+
end
|
|
36
177
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
178
|
+
# Trimming: we only need the FIRST `head` and LAST `tail` lines, so
|
|
179
|
+
# take them off the head/tail SLICES of the buffer rather than splitting
|
|
180
|
+
# the whole thing into a (potentially huge) lines array. each_line with
|
|
181
|
+
# a bounded take avoids walking past what we keep on the head side.
|
|
182
|
+
head_pt = head_lines(s, head)
|
|
183
|
+
tail_pt = tail_lines(s, tail)
|
|
184
|
+
omitted = total - head_pt.size - tail_pt.size
|
|
185
|
+
marker = "… [#{omitted} more lines · full in DB] …"
|
|
41
186
|
|
|
42
187
|
(head_pt + [marker] + tail_pt).join("\n")
|
|
43
188
|
end
|
|
44
189
|
|
|
190
|
+
# First +keep+ chomp'd lines of +str+, without materializing the whole
|
|
191
|
+
# buffer into a lines array (#373). Stops scanning after +keep+ lines.
|
|
192
|
+
def self.head_lines(str, keep)
|
|
193
|
+
out = []
|
|
194
|
+
str.each_line do |line|
|
|
195
|
+
out << line.chomp
|
|
196
|
+
break if out.size >= keep
|
|
197
|
+
end
|
|
198
|
+
out
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# Line count of +str+ via a single allocation-free newline-BYTE count
|
|
202
|
+
# (#373): newlines, +1 for a final line with no trailing newline. Used by
|
|
203
|
+
# both #preview and #truncate to decide over/under cap WITHOUT splitting a
|
|
204
|
+
# potentially huge buffer into a `.lines` array. Counts on the byte view
|
|
205
|
+
# (`b`) so a raw, not-yet-scrubbed buffer (invalid UTF-8 / binary tool
|
|
206
|
+
# output) doesn't raise "invalid byte sequence" — the `\n` byte (0x0A) is
|
|
207
|
+
# unambiguous regardless of encoding, and `.b` shares the buffer (no copy).
|
|
208
|
+
def self.line_count(str)
|
|
209
|
+
return 0 if str.empty?
|
|
210
|
+
|
|
211
|
+
bytes = str.b
|
|
212
|
+
bytes.count("\n") + (bytes.end_with?("\n") ? 0 : 1)
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
# Last +keep+ chomp'd lines of +str+, found by scanning backward from the
|
|
216
|
+
# end rather than splitting the whole buffer (#373). Slices a bounded tail
|
|
217
|
+
# of the string by locating the keep-th-from-last newline.
|
|
218
|
+
def self.tail_lines(str, keep)
|
|
219
|
+
return [] if keep <= 0
|
|
220
|
+
|
|
221
|
+
idx = str.length
|
|
222
|
+
keep.times do
|
|
223
|
+
nl = str.rindex("\n", idx - 1)
|
|
224
|
+
break if nl.nil?
|
|
225
|
+
|
|
226
|
+
idx = nl
|
|
227
|
+
end
|
|
228
|
+
# idx now sits ON the newline before the kept tail (or 0 if we ran out).
|
|
229
|
+
slice = str[idx, str.length - idx]
|
|
230
|
+
slice = slice[1..] if slice.start_with?("\n")
|
|
231
|
+
slice.to_s.lines.map(&:chomp)
|
|
232
|
+
end
|
|
233
|
+
|
|
45
234
|
# Single-line elision to +max+ characters with a trailing ellipsis.
|
|
46
235
|
# Shared by the parent-note tools (AnswerChild/Task/Steer) that all
|
|
47
236
|
# carried a byte-identical private `truncate`. Pure function.
|
|
@@ -89,16 +278,44 @@ module Rubino
|
|
|
89
278
|
# spill.) Pure aside from that injected callback.
|
|
90
279
|
def self.truncate(text, max_bytes:, max_lines:, spill: nil)
|
|
91
280
|
text = text.to_s
|
|
281
|
+
# Bound PEAK cost BEFORE any whole-buffer work (#373). A 128MB tool
|
|
282
|
+
# output used to be scrubbed in full (a 128MB copy), then walked twice
|
|
283
|
+
# by `text.lines` (each a multi-million-element array) just to decide it
|
|
284
|
+
# was over-cap. Decide over/under with allocation-free passes —
|
|
285
|
+
# `bytesize` and `count("\n")` — and only ever scrub/slice a BOUNDED
|
|
286
|
+
# head+tail, never the full buffer. The model-facing cap + spill below
|
|
287
|
+
# are unchanged; this only stops the materialization blow-up.
|
|
92
288
|
over_bytes = text.bytesize > max_bytes
|
|
93
|
-
over_lines = text
|
|
94
|
-
|
|
289
|
+
over_lines = line_count(text) > max_lines
|
|
290
|
+
|
|
291
|
+
# Under both caps: scrub the (already small) buffer and return. A stray
|
|
292
|
+
# non-UTF-8 byte (printf '\xe9') OR a NUL (random binary) in SUB-cap
|
|
293
|
+
# output must still be cleaned, or it crashes JSON.generate / the SQLite
|
|
294
|
+
# driver and the tool row never persists (lost on --resume).
|
|
295
|
+
return scrub_utf8(text) unless over_bytes || over_lines
|
|
95
296
|
|
|
297
|
+
# Over cap: spill the FULL (raw) output first so nothing is lost, then
|
|
298
|
+
# shape from bounded head/tail slices. Each slice path scrubs only the
|
|
299
|
+
# bytes it keeps, so the 128MB buffer is never scrubbed whole.
|
|
96
300
|
spill_path = spill&.call(text)
|
|
97
301
|
text = tail_bias_bytes(text, max_bytes, spill_path) if over_bytes
|
|
98
|
-
|
|
302
|
+
# Re-derive the line check on whatever survived the byte pass (the byte
|
|
303
|
+
# pass already cut to ~max_bytes, so this is now a bounded count).
|
|
304
|
+
text = scrub_utf8(text) unless over_bytes
|
|
305
|
+
text = tail_bias_lines(text, max_lines, spill_path) if line_count(text) > max_lines
|
|
99
306
|
text
|
|
100
307
|
end
|
|
101
308
|
|
|
309
|
+
# Encoding-scrub + NUL-strip a BOUNDED byteslice (#373). The head/tail
|
|
310
|
+
# byte path slices BEFORE scrubbing (so the 128MB buffer is never scrubbed
|
|
311
|
+
# whole); each kept slice still has to be cleaned exactly like scrub_utf8
|
|
312
|
+
# (invalid bytes dropped, NUL deleted) so JSON/SQLite don't choke.
|
|
313
|
+
def self.clean_slice(bytes, encoding)
|
|
314
|
+
s = bytes.to_s.force_encoding(encoding).scrub("")
|
|
315
|
+
s = s.encode(Encoding::UTF_8) unless s.encoding == Encoding::UTF_8
|
|
316
|
+
s.include?(NUL) ? s.delete(NUL) : s
|
|
317
|
+
end
|
|
318
|
+
|
|
102
319
|
def self.tail_bias_bytes(text, max_bytes, spill_path = nil)
|
|
103
320
|
encoding = text.encoding
|
|
104
321
|
recover = spill_path ? " · full output saved to #{spill_path} — read it with offset/limit" : ""
|
|
@@ -111,13 +328,13 @@ module Rubino
|
|
|
111
328
|
# to a simple head truncation (old behavior). Realistic caps go
|
|
112
329
|
# through the head+tail path.
|
|
113
330
|
if tail_budget <= 0
|
|
114
|
-
truncated = text.byteslice(0, max_bytes)
|
|
331
|
+
truncated = clean_slice(text.byteslice(0, max_bytes), encoding)
|
|
115
332
|
tail_note = spill_path ? " · full output: #{spill_path}" : ""
|
|
116
333
|
return "#{truncated}\n... [truncated at #{max_bytes} bytes#{tail_note}]"
|
|
117
334
|
end
|
|
118
335
|
|
|
119
|
-
head = text.byteslice(0, head_budget)
|
|
120
|
-
tail = text.byteslice(-tail_budget, tail_budget)
|
|
336
|
+
head = clean_slice(text.byteslice(0, head_budget), encoding)
|
|
337
|
+
tail = clean_slice(text.byteslice(-tail_budget, tail_budget), encoding)
|
|
121
338
|
elided = text.bytesize - head.bytesize - tail.bytesize
|
|
122
339
|
"#{head}#{format(marker_template, elided)}#{tail}"
|
|
123
340
|
end
|
|
@@ -33,6 +33,49 @@ module Rubino
|
|
|
33
33
|
(?<val>"[^"]+"|'[^']+'|(?:Bearer\s+)?[^"'\s]+)
|
|
34
34
|
/xi
|
|
35
35
|
|
|
36
|
+
# URL userinfo credentials: `scheme://user:PASSWORD@host`. Masks ONLY the
|
|
37
|
+
# password, keeping scheme/user/host so the trace still says which
|
|
38
|
+
# service/account was touched (`postgresql://app:***@db`). The userinfo
|
|
39
|
+
# username is `[^:@/\s]+` and the password `[^@/\s]+`, both terminating at
|
|
40
|
+
# the `@`, so a bare `https://host:8080/p` (no `@`), the `host:port` that
|
|
41
|
+
# follows the `@`, and an IPv6 host `@[::1]:5432` are all left untouched —
|
|
42
|
+
# only a real `user:pass@` triggers. The unambiguous, industry-standard
|
|
43
|
+
# form (git/pip redact credentials in URLs exactly this way; RFC 3986
|
|
44
|
+
# deprecates them outright).
|
|
45
|
+
URL_USERINFO_RE = %r{
|
|
46
|
+
(?<scheme>[a-z][a-z0-9+.-]*://)
|
|
47
|
+
(?<user>[^:@/\s]+)
|
|
48
|
+
(?<sep>:)
|
|
49
|
+
(?<pass>[^@/\s]+)
|
|
50
|
+
(?<at>@)
|
|
51
|
+
}xi
|
|
52
|
+
|
|
53
|
+
# Basic-auth credential pair `-u user:pass` (curl/wget). Unambiguous: the
|
|
54
|
+
# value carries a colon-separated `user:pass`, so we mask the password half
|
|
55
|
+
# and keep the username (`-u admin:***`). Both glued (`-uadmin:pw`) and
|
|
56
|
+
# spaced (`-u admin:pw`) forms match; a bare username with no colon is left
|
|
57
|
+
# alone (no secret on the line to mask).
|
|
58
|
+
U_FLAG_CRED_RE = /
|
|
59
|
+
(?<flag>(?<![\w-])-u)
|
|
60
|
+
(?<sp>\s*)
|
|
61
|
+
(?<user>[^\s:'"]+)
|
|
62
|
+
(?<sep>:)
|
|
63
|
+
(?<pass>[^\s'"]+)
|
|
64
|
+
/x
|
|
65
|
+
|
|
66
|
+
# Glued DB-client password flag `-p<password>`, scoped to mysql/mariadb
|
|
67
|
+
# clients ONLY. `-p<val>` is a password there but a PORT/PATH/anything for
|
|
68
|
+
# most other tools (`ssh -p 22`, `kubectl -p`), so we require BOTH the
|
|
69
|
+
# value to be GLUED to the flag (`-pSECRET`, no space — mysql's own
|
|
70
|
+
# convention) AND a mysql-family client word on the same command. A
|
|
71
|
+
# generic `-p 8080` is never masked, and the spaced `mysql -p` (interactive
|
|
72
|
+
# prompt) carries no secret on the line so there is nothing to mask.
|
|
73
|
+
MYSQL_PFLAG_RE = /
|
|
74
|
+
(?<client>\b(?:mysql|mysqldump|mariadb|mariadb-dump)\b[^\n|;&]*?\s)
|
|
75
|
+
(?<flag>-p)
|
|
76
|
+
(?<pass>[^\s'"]+)
|
|
77
|
+
/xi
|
|
78
|
+
|
|
36
79
|
MASK = "***"
|
|
37
80
|
|
|
38
81
|
# True if the given key looks sensitive on its own (used when the
|
|
@@ -58,15 +101,35 @@ module Rubino
|
|
|
58
101
|
# the mask would eat a quote and the rest of the string would look
|
|
59
102
|
# like one long open string.
|
|
60
103
|
def self.mask_inline(text)
|
|
61
|
-
text.to_s.gsub(INLINE_RE) do
|
|
104
|
+
masked = text.to_s.gsub(INLINE_RE) do
|
|
62
105
|
m = Regexp.last_match
|
|
63
106
|
val = m[:val]
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
"#{m[:key]}#{m[:sep]}#{
|
|
107
|
+
inner = case val[0]
|
|
108
|
+
when '"' then %("#{MASK}")
|
|
109
|
+
when "'" then "'#{MASK}'"
|
|
110
|
+
else MASK
|
|
111
|
+
end
|
|
112
|
+
"#{m[:key]}#{m[:sep]}#{inner}"
|
|
113
|
+
end
|
|
114
|
+
mask_glued_credentials(masked)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# The glued/URL credential forms the keyed INLINE_RE can't see: URL
|
|
118
|
+
# userinfo passwords, `-u user:pass`, and mysql/mariadb `-p<password>`.
|
|
119
|
+
# Each keeps the surrounding, non-secret context (scheme/user/host, the
|
|
120
|
+
# flag, the username) so the trace stays useful while the secret is gone.
|
|
121
|
+
def self.mask_glued_credentials(text)
|
|
122
|
+
out = text.gsub(URL_USERINFO_RE) do
|
|
123
|
+
m = Regexp.last_match
|
|
124
|
+
"#{m[:scheme]}#{m[:user]}:#{MASK}@"
|
|
125
|
+
end
|
|
126
|
+
out = out.gsub(U_FLAG_CRED_RE) do
|
|
127
|
+
m = Regexp.last_match
|
|
128
|
+
"#{m[:flag]}#{m[:sp]}#{m[:user]}:#{MASK}"
|
|
129
|
+
end
|
|
130
|
+
out.gsub(MYSQL_PFLAG_RE) do
|
|
131
|
+
m = Regexp.last_match
|
|
132
|
+
"#{m[:client]}#{m[:flag]}#{MASK}"
|
|
70
133
|
end
|
|
71
134
|
end
|
|
72
135
|
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Rubino
|
|
6
|
+
module Util
|
|
7
|
+
# Lifecycle for the on-disk "spill" artifacts rubino writes outside the
|
|
8
|
+
# database (#374):
|
|
9
|
+
#
|
|
10
|
+
# * tool-result spills — <home>/tool-results/<call_id>.txt, the full
|
|
11
|
+
# pre-truncation output the model can `read` back (ToolExecutor).
|
|
12
|
+
# * oversized pastes — <home>/sessions/<id>/paste_N.txt, a big paste
|
|
13
|
+
# the model reads instead of inlining (UI::PasteStore).
|
|
14
|
+
#
|
|
15
|
+
# Both were write-only: nothing ever deleted them. A long-running session or
|
|
16
|
+
# a CI box that runs thousands of large-output tools accumulated these files
|
|
17
|
+
# FOREVER, and destroying a session (CleanupSessionsJob / Repository#destroy!)
|
|
18
|
+
# only deleted DB rows, leaving the files orphaned. This module:
|
|
19
|
+
#
|
|
20
|
+
# 1. deletes a single session's spill+paste files when it is destroyed
|
|
21
|
+
# (#destroy_session_files), and
|
|
22
|
+
# 2. evicts spill/paste files past an age and/or total-size budget
|
|
23
|
+
# (#evict!), called opportunistically and from CleanupSessionsJob.
|
|
24
|
+
#
|
|
25
|
+
# All methods are best-effort: an IO error must never take down the agent.
|
|
26
|
+
module SpillStore
|
|
27
|
+
# Default eviction policy. Tunable via the cleanup config, but these are
|
|
28
|
+
# the safe built-ins: drop anything older than the retention window, and
|
|
29
|
+
# keep the combined on-disk footprint of spills+pastes under the budget by
|
|
30
|
+
# evicting oldest-first.
|
|
31
|
+
DEFAULT_MAX_AGE_SECONDS = 7 * 86_400 # 7 days
|
|
32
|
+
DEFAULT_MAX_TOTAL_BYTES = 512 * 1024 * 1024 # 512 MB
|
|
33
|
+
|
|
34
|
+
module_function
|
|
35
|
+
|
|
36
|
+
# The directory holding per-call tool-result spills.
|
|
37
|
+
def tool_results_dir
|
|
38
|
+
File.join(Rubino.home_path, "tool-results")
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The directory holding all per-session subtrees (each session's pastes
|
|
42
|
+
# live in <sessions>/<id>/paste_N.txt).
|
|
43
|
+
def sessions_dir
|
|
44
|
+
File.join(Rubino.home_path, "sessions")
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Removes the on-disk spill + paste artifacts owned by +session_id+ when
|
|
48
|
+
# the session is destroyed (#374). Pastes are session-scoped so the whole
|
|
49
|
+
# <sessions>/<id> subtree goes; tool-result spills are keyed by call_id, so
|
|
50
|
+
# the caller passes the session's call_ids (looked up before the DB rows
|
|
51
|
+
# are deleted) and we remove the matching <tool-results>/<call_id>.txt.
|
|
52
|
+
# Best-effort; returns nil.
|
|
53
|
+
def destroy_session_files(session_id, call_ids: [])
|
|
54
|
+
return if session_id.nil? || session_id.to_s.empty?
|
|
55
|
+
|
|
56
|
+
FileUtils.rm_rf(File.join(sessions_dir, session_id.to_s))
|
|
57
|
+
Array(call_ids).each do |cid|
|
|
58
|
+
safe = sanitize_call_id(cid)
|
|
59
|
+
next if safe.nil?
|
|
60
|
+
|
|
61
|
+
FileUtils.rm_f(File.join(tool_results_dir, "#{safe}.txt"))
|
|
62
|
+
end
|
|
63
|
+
nil
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
Rubino.logger&.warn(event: "spill_store.destroy_failed", error: e.message)
|
|
66
|
+
nil
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# Evicts spill + paste files past the age and/or total-size budget. Age
|
|
70
|
+
# first (drop everything older than max_age), then size (if the survivors
|
|
71
|
+
# still exceed max_total_bytes, delete oldest-first until under budget).
|
|
72
|
+
# Empty per-session paste dirs left behind are pruned. Best-effort;
|
|
73
|
+
# returns the number of files deleted.
|
|
74
|
+
def evict!(max_age_seconds: DEFAULT_MAX_AGE_SECONDS, max_total_bytes: DEFAULT_MAX_TOTAL_BYTES,
|
|
75
|
+
now: Time.now)
|
|
76
|
+
files = collect_files
|
|
77
|
+
deleted = 0
|
|
78
|
+
|
|
79
|
+
if max_age_seconds&.positive?
|
|
80
|
+
cutoff = now - max_age_seconds
|
|
81
|
+
files.reject! do |f|
|
|
82
|
+
next false unless f[:mtime] < cutoff
|
|
83
|
+
|
|
84
|
+
FileUtils.rm_f(f[:path])
|
|
85
|
+
deleted += 1
|
|
86
|
+
true
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
if max_total_bytes&.positive?
|
|
91
|
+
total = files.sum { |f| f[:size] }
|
|
92
|
+
if total > max_total_bytes
|
|
93
|
+
# Oldest first until back under budget.
|
|
94
|
+
files.sort_by! { |f| f[:mtime] }
|
|
95
|
+
files.each do |f|
|
|
96
|
+
break if total <= max_total_bytes
|
|
97
|
+
|
|
98
|
+
FileUtils.rm_f(f[:path])
|
|
99
|
+
total -= f[:size]
|
|
100
|
+
deleted += 1
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
prune_empty_session_dirs
|
|
106
|
+
deleted
|
|
107
|
+
rescue StandardError => e
|
|
108
|
+
Rubino.logger&.warn(event: "spill_store.evict_failed", error: e.message)
|
|
109
|
+
deleted
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# All spill + paste files as {path:, size:, mtime:} records.
|
|
113
|
+
def collect_files
|
|
114
|
+
out = []
|
|
115
|
+
out.concat(stat_glob(File.join(tool_results_dir, "*.txt")))
|
|
116
|
+
out.concat(stat_glob(File.join(sessions_dir, "*", "paste_*.txt")))
|
|
117
|
+
out
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def stat_glob(pattern)
|
|
121
|
+
Dir.glob(pattern).filter_map do |path|
|
|
122
|
+
stat = File.stat(path)
|
|
123
|
+
next unless stat.file?
|
|
124
|
+
|
|
125
|
+
{ path: path, size: stat.size, mtime: stat.mtime }
|
|
126
|
+
rescue StandardError
|
|
127
|
+
nil
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Removes now-empty per-session paste dirs (a session whose only files
|
|
132
|
+
# were pastes that got evicted) so the sessions tree doesn't fill with
|
|
133
|
+
# empty directories. Never touches a dir that still has contents.
|
|
134
|
+
def prune_empty_session_dirs
|
|
135
|
+
Dir.glob(File.join(sessions_dir, "*")).each do |dir|
|
|
136
|
+
next unless File.directory?(dir)
|
|
137
|
+
next unless (Dir.entries(dir) - %w[. ..]).empty?
|
|
138
|
+
|
|
139
|
+
Dir.rmdir(dir)
|
|
140
|
+
rescue StandardError
|
|
141
|
+
nil
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# Mirrors ToolExecutor#spill_full_output's filename sanitization so the
|
|
146
|
+
# path we delete matches the path that was written.
|
|
147
|
+
def sanitize_call_id(call_id)
|
|
148
|
+
id = call_id.to_s.gsub(/[^a-zA-Z0-9_.-]/, "_")
|
|
149
|
+
id.empty? ? nil : id
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
data/lib/rubino/version.rb
CHANGED
data/lib/rubino/workspace.rb
CHANGED
|
@@ -21,8 +21,16 @@ module Rubino
|
|
|
21
21
|
# the same rule Tools::Base#workspace_root has always used, kept as the
|
|
22
22
|
# single source of truth so the @-picker, shell/test cwd, file API and
|
|
23
23
|
# attachment downloader all agree on "the" root.
|
|
24
|
+
#
|
|
25
|
+
# terminal.cwd MUST resolve to a String path: a malformed config (e.g. a
|
|
26
|
+
# YAML `terminal: { cwd: { ... } }` nested mapping) would otherwise hand a
|
|
27
|
+
# Hash to File.expand_path downstream, which raises "no implicit conversion
|
|
28
|
+
# of Hash into String" deep in a tool's #call — masking the real outcome
|
|
29
|
+
# (e.g. a write-denylist refusal) behind an opaque error. Anything that
|
|
30
|
+
# isn't a non-empty String degrades to the process cwd.
|
|
24
31
|
def primary_root
|
|
25
|
-
Rubino.configuration&.dig("terminal", "cwd")
|
|
32
|
+
configured = Rubino.configuration&.dig("terminal", "cwd")
|
|
33
|
+
configured.is_a?(String) && !configured.empty? ? configured : Dir.pwd
|
|
26
34
|
end
|
|
27
35
|
|
|
28
36
|
# Every allowed root: the primary first, then each added dir, de-duped on
|