samagotchi 0.121.0 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -1
- data/bin/chi +7 -3
- data/docs/architecture.md +37 -11
- data/docs/bundle-authoring.md +2 -1
- data/docs/cli.md +11 -4
- data/docs/configuration.md +36 -7
- data/docs/guardrails.md +63 -19
- data/docs/internals/anthropic.md +1 -1
- data/docs/internals/background-tasks.md +1 -1
- data/docs/internals/engine.md +2 -0
- data/docs/internals/participants.md +16 -8
- data/docs/internals/remote-sessions.md +15 -11
- data/docs/internals/web-push.md +22 -11
- data/docs/memory.md +33 -0
- data/docs/plugins.md +8 -3
- data/docs/sessions.md +21 -0
- data/docs/testing.md +41 -1
- data/lib/samagotchi/answer_command.rb +1 -1
- data/lib/samagotchi/bridge/turn_accumulator.rb +3 -1
- data/lib/samagotchi/bridge.rb +12 -11
- data/lib/samagotchi/bridge_client/event_stream.rb +14 -26
- data/lib/samagotchi/bridge_client/stream_open.rb +85 -0
- data/lib/samagotchi/bridge_client/stream_request.rb +8 -2
- data/lib/samagotchi/bridge_client.rb +52 -65
- data/lib/samagotchi/bridge_secret.rb +9 -7
- data/lib/samagotchi/bundle_command/check.rb +27 -2
- data/lib/samagotchi/bundles/claude-peer/manifest.yml +2 -2
- data/lib/samagotchi/bundles/claude-peer/scripts/pair +20 -6
- data/lib/samagotchi/bundles/codex-peer/manifest.yml +2 -2
- data/lib/samagotchi/bundles/codex-peer/scripts/pair +14 -3
- data/lib/samagotchi/bundles/github-pr/manifest.yml +2 -2
- data/lib/samagotchi/bundles/github-pr/plugin.rb +2 -6
- data/lib/samagotchi/bundles/system/manifest.yml +2 -2
- data/lib/samagotchi/bundles/system/memory_guide.md +13 -0
- data/lib/samagotchi/child_move.rb +5 -7
- data/lib/samagotchi/children_status.rb +2 -1
- data/lib/samagotchi/config.rb +5 -0
- data/lib/samagotchi/engine.rb +107 -1
- data/lib/samagotchi/event_fanout.rb +0 -1
- data/lib/samagotchi/guardrail_wiring.rb +24 -6
- data/lib/samagotchi/guardrails/parent_approvals.rb +9 -0
- data/lib/samagotchi/guardrails_command.rb +41 -6
- data/lib/samagotchi/image_store.rb +6 -3
- data/lib/samagotchi/interrupted_turn.rb +6 -3
- data/lib/samagotchi/live_versions.rb +6 -7
- data/lib/samagotchi/log_line.rb +1 -1
- data/lib/samagotchi/memory_bundle/profile.rb +1 -1
- data/lib/samagotchi/memory_bundle/uninstaller.rb +6 -2
- data/lib/samagotchi/memory_orphans.rb +11 -0
- data/lib/samagotchi/participant_mode.rb +11 -0
- data/lib/samagotchi/plugin/api.rb +14 -8
- data/lib/samagotchi/plugin/sessions.rb +13 -4
- data/lib/samagotchi/process_group.rb +13 -0
- data/lib/samagotchi/push_command.rb +1 -1
- data/lib/samagotchi/relay_verifier.rb +2 -3
- data/lib/samagotchi/relay_watcher.rb +2 -2
- data/lib/samagotchi/remote_attach_command.rb +1 -1
- data/lib/samagotchi/reply_wait/poll.rb +8 -2
- data/lib/samagotchi/reply_wait.rb +4 -3
- data/lib/samagotchi/review_command.rb +1 -1
- data/lib/samagotchi/review_start.rb +6 -2
- data/lib/samagotchi/run_options.rb +2 -0
- data/lib/samagotchi/session.rb +18 -1
- data/lib/samagotchi/session_access/contract.rb +1 -1
- data/lib/samagotchi/session_access/local.rb +18 -8
- data/lib/samagotchi/session_access/remote.rb +37 -12
- data/lib/samagotchi/session_access/web_bridge_client.rb +38 -19
- data/lib/samagotchi/session_access.rb +21 -9
- data/lib/samagotchi/session_continue.rb +2 -3
- data/lib/samagotchi/session_manager.rb +133 -29
- data/lib/samagotchi/session_setup.rb +8 -4
- data/lib/samagotchi/sessions_command.rb +10 -5
- data/lib/samagotchi/steer.rb +7 -6
- data/lib/samagotchi/system_one/client.rb +12 -6
- data/lib/samagotchi/system_one/fake_backend.rb +0 -2
- data/lib/samagotchi/system_one/log.rb +44 -9
- data/lib/samagotchi/system_one/self_protect.rb +39 -43
- data/lib/samagotchi/terminal_ui/attach_launcher.rb +8 -3
- data/lib/samagotchi/terminal_ui/attached_loop.rb +4 -2
- data/lib/samagotchi/terminal_ui/local_link.rb +28 -19
- data/lib/samagotchi/terminal_ui.rb +2 -1
- data/lib/samagotchi/tool_runner.rb +2 -1
- data/lib/samagotchi/toolkit.rb +95 -10
- data/lib/samagotchi/toolkit_binding.rb +115 -7
- data/lib/samagotchi/toolkit_command.rb +15 -3
- data/lib/samagotchi/toolkit_report_command.rb +318 -0
- data/lib/samagotchi/toolkit_suggest.rb +211 -0
- data/lib/samagotchi/tools/delegate.rb +1 -1
- data/lib/samagotchi/tools/delegate_relay.rb +9 -7
- data/lib/samagotchi/tools/memory.rb +1 -1
- data/lib/samagotchi/turn_checkpoint.rb +15 -9
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +127 -69
- data/lib/samagotchi/web/participant_access.rb +30 -8
- data/lib/samagotchi/web/participant_review.rb +8 -5
- data/lib/samagotchi/web/participant_view.rb +7 -3
- data/lib/samagotchi/web/participants_admin.rb +13 -0
- data/lib/samagotchi/web/principal.rb +3 -4
- data/lib/samagotchi/web/public/app.js +1 -1
- data/lib/samagotchi/web/public/bubbles.js +3 -2
- data/lib/samagotchi/web/public/notifications.js +21 -1
- data/lib/samagotchi/web/public/participant.html +3 -2
- data/lib/samagotchi/web/public/participant.js +25 -10
- data/lib/samagotchi/web/public/participants_panel.js +46 -11
- data/lib/samagotchi/web/public/push.js +4 -3
- data/lib/samagotchi/web/public/push_devices.js +17 -3
- data/lib/samagotchi/web/public/review_history.js +14 -2
- data/lib/samagotchi/web/public/review_start.js +1 -3
- data/lib/samagotchi/web/public/session_actions.js +14 -2
- data/lib/samagotchi/web/public/storage.js +16 -0
- data/lib/samagotchi/web/public/turn_events.js +2 -1
- data/lib/samagotchi/web/public/turn_html.js +1 -1
- data/lib/samagotchi/web/push/client.rb +5 -0
- data/lib/samagotchi/web/push/store.rb +16 -3
- data/lib/samagotchi/web/push_routes.rb +3 -2
- data/lib/samagotchi/web/server.rb +2 -2
- data/lib/samagotchi/web/session_access_routes.rb +28 -20
- data/lib/samagotchi/web/session_control.rb +4 -2
- data/lib/samagotchi/web/session_create.rb +4 -4
- data/lib/samagotchi/web/session_hub.rb +6 -4
- data/lib/samagotchi/web/session_summary.rb +8 -16
- data/lib/samagotchi/web/start_dir.rb +10 -5
- data/lib/samagotchi/worker.rb +17 -3
- data/lib/samagotchi/worker_link.rb +214 -0
- data/lib/samagotchi/worker_sidecar.rb +15 -37
- metadata +6 -1
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
require_relative "cli/command"
|
|
7
|
+
require_relative "cli/exit"
|
|
8
|
+
require_relative "cli/flags"
|
|
9
|
+
require_relative "config"
|
|
10
|
+
require_relative "session"
|
|
11
|
+
require_relative "system_one/log"
|
|
12
|
+
require_relative "toolkit"
|
|
13
|
+
require_relative "tool_response"
|
|
14
|
+
require_relative "toolkit_suggest"
|
|
15
|
+
|
|
16
|
+
module Samagotchi
|
|
17
|
+
# `chi toolkit report`: compare the toolkit on mode would have run (the
|
|
18
|
+
# suggestion when it would have been applied, else the default) with the
|
|
19
|
+
# tools the session called from that turn on, against every kit's rules
|
|
20
|
+
# the ask logged (ToolkitSuggest.rules).
|
|
21
|
+
class ToolkitReportCommand
|
|
22
|
+
include CLI::Command
|
|
23
|
+
|
|
24
|
+
USAGE = <<~TEXT
|
|
25
|
+
Usage: chi toolkit report [--days N]
|
|
26
|
+
Grade the kit each audit suggestion would have run (the suggested
|
|
27
|
+
kit when on mode would have applied it, else the default) against
|
|
28
|
+
the tools the session called from that turn on, call_tool targets
|
|
29
|
+
included: covers, short of a tool (named), wider than needed (a
|
|
30
|
+
narrower offered kit would also cover), or ungradable. Then a
|
|
31
|
+
per-kit tally with what on mode would have done. Includes archived
|
|
32
|
+
sessions; the report is limited to 14 days and 500 sessions.
|
|
33
|
+
TEXT
|
|
34
|
+
|
|
35
|
+
MAX_DAYS = 14
|
|
36
|
+
MAX_SESSIONS = 500
|
|
37
|
+
FLAGS = CLI::Flags.new(args: false) { |flags| flags.value "--days" }.freeze
|
|
38
|
+
ToolUse = Data.define(:name, :group, :label)
|
|
39
|
+
|
|
40
|
+
# @param argv [Array<String>] arguments after "toolkit"
|
|
41
|
+
# @param sessions_dir [String, nil] sessions location (a spec seam)
|
|
42
|
+
# @param log [SystemOne::Log, nil] suggestion log (a spec seam)
|
|
43
|
+
# @param now [Time] the report's clock (a spec seam)
|
|
44
|
+
def initialize(argv, stdout: $stdout, stderr: $stderr, sessions_dir: nil, log: nil, now: Time.now)
|
|
45
|
+
@argv = argv.dup
|
|
46
|
+
@stdout = stdout
|
|
47
|
+
@stderr = stderr
|
|
48
|
+
@sessions_dir = sessions_dir || Session.default_state_dir
|
|
49
|
+
@log = log
|
|
50
|
+
@now = now
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def run
|
|
54
|
+
@sub = @argv.first
|
|
55
|
+
if @sub.nil? || HELP_WORDS.include?(@sub)
|
|
56
|
+
@stdout.puts(USAGE)
|
|
57
|
+
return CLI::Exit::OK
|
|
58
|
+
end
|
|
59
|
+
return usage_error("unknown subcommand #{@sub}") unless @sub == "report"
|
|
60
|
+
|
|
61
|
+
parsed = parse_flags(FLAGS, @argv[1..])
|
|
62
|
+
return parsed if parsed.is_a?(Integer)
|
|
63
|
+
return usage_error("--days takes a positive whole number") if parsed.options[:days] &&
|
|
64
|
+
!parsed.options[:days].to_s.match?(/\A[1-9]\d*\z/)
|
|
65
|
+
|
|
66
|
+
report(parsed.options[:days] ? Integer(parsed.options[:days], 10) : MAX_DAYS)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def command_name = "chi toolkit report"
|
|
72
|
+
|
|
73
|
+
def report(requested_days)
|
|
74
|
+
days = effective_days(requested_days)
|
|
75
|
+
cutoff = @now - (days * 86_400)
|
|
76
|
+
max_sessions = [MAX_SESSIONS, positive_config("session.max_count", MAX_SESSIONS)].min
|
|
77
|
+
sessions = Session.list(state_dir: @sessions_dir, sort: "updated_at", order: "desc",
|
|
78
|
+
limit: max_sessions, include_archived: true)
|
|
79
|
+
session_summaries = sessions.to_h { |session| [session.id, session] }
|
|
80
|
+
records = log.records.filter_map do |entry|
|
|
81
|
+
next unless entry["kind"] == "suggestion" && session_summaries.key?(entry["session_id"])
|
|
82
|
+
next unless timestamp(entry["at"])&.>= cutoff
|
|
83
|
+
|
|
84
|
+
entry
|
|
85
|
+
end
|
|
86
|
+
if records.empty?
|
|
87
|
+
@stdout.puts("No toolkit suggestions in the last #{days} days.")
|
|
88
|
+
return CLI::Exit::OK
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
loaded = {}
|
|
92
|
+
tallies = Hash.new { |hash, kit| hash[kit] = Hash.new(0) }
|
|
93
|
+
@stdout.puts("Toolkit suggestions · last #{days} days · up to #{max_sessions} sessions")
|
|
94
|
+
records.sort_by { |entry| timestamp(entry["at"]) }.each do |entry|
|
|
95
|
+
id = entry["session_id"]
|
|
96
|
+
session = loaded[id] ||= Session.load(id, state_dir: @sessions_dir)
|
|
97
|
+
uses, window = window_uses(session, entry["turn_number"].to_i)
|
|
98
|
+
kit = graded_kit(entry)
|
|
99
|
+
grade, detail = grade(entry, kit, uses)
|
|
100
|
+
tallies[kit][grade] += 1
|
|
101
|
+
tallies[kit]["would:#{entry["would"] || "unknown"}"] += 1
|
|
102
|
+
confidence = entry["confidence"].nil? ? "n/a" : format("%.2f", entry["confidence"])
|
|
103
|
+
@stdout.puts("#{timestamp(entry["at"]).iso8601} #{id[0, 8]} turn #{entry["turn_number"]}: " \
|
|
104
|
+
"#{entry["raw"] || "no judgment"} (p=#{confidence}, would #{entry["would"] || "unknown"}) " \
|
|
105
|
+
"→ #{kit}: #{grade}#{detail}#{window}")
|
|
106
|
+
labels = uses.is_a?(Array) ? uses.map(&:label).uniq : []
|
|
107
|
+
@stdout.puts(" tools: #{labels.join(", ")}") unless labels.empty?
|
|
108
|
+
rescue StandardError => e
|
|
109
|
+
@stderr.puts("chi toolkit report: session #{id}: #{e.class}: #{e.message}; skipped")
|
|
110
|
+
end
|
|
111
|
+
print_tallies(tallies)
|
|
112
|
+
CLI::Exit::OK
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def log
|
|
116
|
+
@log ||= SystemOne::Log.new(
|
|
117
|
+
dir: File.dirname(ToolkitSuggest.log_path(@sessions_dir)),
|
|
118
|
+
file: ToolkitSuggest::FILE,
|
|
119
|
+
warn: ->(reason) { @stderr.puts("chi toolkit report: #{reason}") }
|
|
120
|
+
)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def effective_days(requested)
|
|
124
|
+
retention = Config.get("session.retention_days").to_i
|
|
125
|
+
cap = retention.positive? ? [retention, MAX_DAYS].min : MAX_DAYS
|
|
126
|
+
[requested, cap].min
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def positive_config(key, fallback)
|
|
130
|
+
value = Config.get(key).to_i
|
|
131
|
+
value.positive? ? value : fallback
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def timestamp(value)
|
|
135
|
+
Time.iso8601(value.to_s)
|
|
136
|
+
rescue ArgumentError
|
|
137
|
+
nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# The tools the session called from the suggestion's turn on: to the
|
|
141
|
+
# session's end, or only that turn when the toolkit was chosen since
|
|
142
|
+
# (/toolkit, ctx.use_toolkit: no message marks when, and a change
|
|
143
|
+
# applies from the next turn's start, so the asked turn ran on it).
|
|
144
|
+
# @return [Array(Array<ToolUse>|Symbol, String)] the uses (:ungradable
|
|
145
|
+
# when a turn's tools can't be read back) and a note on the window
|
|
146
|
+
def window_uses(session, turn_number)
|
|
147
|
+
by_turn = tool_uses_by_turn(session.messages)
|
|
148
|
+
changed = session.toolkit_origin == "chosen"
|
|
149
|
+
turns = by_turn.keys.select { |number| changed ? number == turn_number : number >= turn_number }
|
|
150
|
+
uses = turns.sort.flat_map { |number| by_turn[number] }
|
|
151
|
+
note = changed ? " (its turn only: the toolkit was chosen later)" : ""
|
|
152
|
+
[turns.empty? || uses.include?(:ungradable) ? :ungradable : uses, note]
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Tool uses grouped by the user's own prompt number; steers and merged
|
|
156
|
+
# input lines remain part of that turn rather than starting another.
|
|
157
|
+
# The chat loop saves its calls on the model's message (tool_calls);
|
|
158
|
+
# the raw-prompt loop saves only the model's text, so its calls are
|
|
159
|
+
# read back from the batch's tool_response entry (#joined_uses).
|
|
160
|
+
def tool_uses_by_turn(messages)
|
|
161
|
+
number = 0
|
|
162
|
+
current = nil
|
|
163
|
+
uses = Hash.new { |hash, key| hash[key] = [] }
|
|
164
|
+
Array(messages).each do |message|
|
|
165
|
+
case field(message, :role).to_s
|
|
166
|
+
when "user"
|
|
167
|
+
if ToolkitSuggest.owner_prompt?(message)
|
|
168
|
+
number += 1
|
|
169
|
+
current = number
|
|
170
|
+
uses[current]
|
|
171
|
+
elsif Steer.participant_author?(field(message, :author))
|
|
172
|
+
current = nil
|
|
173
|
+
end
|
|
174
|
+
when "model"
|
|
175
|
+
Array(field(message, :tool_calls)).each { |call| uses[current] << tool_use(call) } if current
|
|
176
|
+
when "tool_response"
|
|
177
|
+
uses[current].concat(joined_uses(message)) if current && !field(message, :tool_call_id)
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
uses.transform_values(&:compact)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# A ran-as line (ToolRunner, a corrected call) may precede a run's lead.
|
|
184
|
+
RUN_LEAD = /\A(?:ran as: [^\n]*\n)?\[([^\]\s]+)\](?:\n| |\z)/
|
|
185
|
+
|
|
186
|
+
# A raw-prompt batch's tools: each run's "[name]" lead (ToolResponse),
|
|
187
|
+
# a call_tool's target from its saved params line ("group/tool …").
|
|
188
|
+
# When the leads don't match the batch's run count, :ungradable.
|
|
189
|
+
def joined_uses(message)
|
|
190
|
+
parts = field(message, :content).to_s.split(ToolResponse::SEPARATOR, -1)
|
|
191
|
+
names = parts.filter_map { |part| part[RUN_LEAD, 1] }
|
|
192
|
+
ids = field(message, :tool_ids)
|
|
193
|
+
return [:ungradable] if names.empty? || (ids.is_a?(Array) && ids.size != names.size)
|
|
194
|
+
|
|
195
|
+
params = field(message, :tool_params)
|
|
196
|
+
names.each_with_index.map do |name, index|
|
|
197
|
+
next ToolUse.new(name: name, group: nil, label: name) unless name == "call_tool"
|
|
198
|
+
|
|
199
|
+
line = params.is_a?(Array) ? params[index] : nil
|
|
200
|
+
target = line.to_s.split(" ", 2).first.to_s
|
|
201
|
+
next :ungradable unless target.include?("/")
|
|
202
|
+
|
|
203
|
+
group, tool = target.split("/", 2)
|
|
204
|
+
ToolUse.new(name: tool, group: group, label: "call_tool→#{target}")
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def tool_use(call)
|
|
209
|
+
return unless call.is_a?(Hash)
|
|
210
|
+
|
|
211
|
+
function = field(call, :function)
|
|
212
|
+
name = (field(call, :name) || field(function, :name)).to_s
|
|
213
|
+
arguments = field(call, :arguments) || field(function, :arguments) || field(call, :args)
|
|
214
|
+
arguments = JSON.parse(arguments) if arguments.is_a?(String) && !arguments.empty?
|
|
215
|
+
return ToolUse.new(name: name, group: nil, label: name) unless name == "call_tool"
|
|
216
|
+
|
|
217
|
+
target = arguments.is_a?(Hash) ? (field(arguments, :tool) || field(arguments, :name)) : nil
|
|
218
|
+
return :ungradable unless target
|
|
219
|
+
|
|
220
|
+
parts = target.to_s.split("/", 2)
|
|
221
|
+
group, tool = parts.size == 2 ? parts : [nil, parts.first]
|
|
222
|
+
ToolUse.new(name: tool, group: group, label: "call_tool→#{target}")
|
|
223
|
+
rescue JSON::ParserError
|
|
224
|
+
name == "call_tool" ? :ungradable : ToolUse.new(name: name, group: nil, label: name)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# The kit on mode would have run: the suggestion only when it would
|
|
228
|
+
# have been applied, else the session's default.
|
|
229
|
+
def graded_kit(entry)
|
|
230
|
+
name = entry["would"] == "suggested" ? entry["kit"] : entry["default_kit"]
|
|
231
|
+
name || "(unknown)"
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# covers: the kit declares every tool the window used (pick ⊇ need);
|
|
235
|
+
# wider than needed: it covers, and so would a narrower logged kit;
|
|
236
|
+
# short of a tool: names what it leaves out. ungradable when the
|
|
237
|
+
# window's calls or the kit's rules can't be read.
|
|
238
|
+
def grade(entry, kit, uses)
|
|
239
|
+
kits = entry["kits"].is_a?(Hash) ? entry["kits"] : {}
|
|
240
|
+
rules = kits[kit]
|
|
241
|
+
core_tools = Array(entry["core_tools"])
|
|
242
|
+
return ["ungradable", " (no rules logged for #{kit})"] unless rules.is_a?(Hash)
|
|
243
|
+
return ["ungradable", " (tool calls not readable)"] if uses == :ungradable
|
|
244
|
+
|
|
245
|
+
missing = uses.reject { |use| covers?(rules, core_tools, use) }
|
|
246
|
+
return ["short of a tool", " (#{missing.map(&:name).uniq.join(", ")})"] unless missing.empty?
|
|
247
|
+
|
|
248
|
+
narrower = kits.filter_map do |name, other|
|
|
249
|
+
next if name == kit || !other.is_a?(Hash) || !narrower?(other, rules, core_tools)
|
|
250
|
+
|
|
251
|
+
name if uses.all? { |use| covers?(other, core_tools, use) }
|
|
252
|
+
end
|
|
253
|
+
return ["wider than needed", " (#{narrower.sort.join(", ")} would also cover)"] unless narrower.empty?
|
|
254
|
+
|
|
255
|
+
["covers", ""]
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Whether a kit's rules declare +use+ (Toolkit::Resolved#declares?): a
|
|
259
|
+
# tool-index group by the kit's groups; a built-in by its core list;
|
|
260
|
+
# any other tool (a plugin's) unless the kit removes it.
|
|
261
|
+
def covers?(rules, core_tools, use)
|
|
262
|
+
return group_reached?(rules["groups"], use.group) if use.group && use.group != "chi"
|
|
263
|
+
|
|
264
|
+
removed = Array(rules["removed"])
|
|
265
|
+
return false if removed.include?(use.name)
|
|
266
|
+
return true unless core_tools.include?(use.name)
|
|
267
|
+
|
|
268
|
+
rules["all_core"] || Array(rules["core"]).include?(use.name) || Array(rules["added"]).include?(use.name)
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def core_set(rules, core_tools)
|
|
272
|
+
declared = rules["all_core"] ? core_tools : Array(rules["core"]) + Array(rules["added"])
|
|
273
|
+
(declared & core_tools) - Array(rules["removed"])
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# +other+ declares no more built-ins and reaches no more groups than
|
|
277
|
+
# +rules+, and less of one of them.
|
|
278
|
+
def narrower?(other, rules, core_tools)
|
|
279
|
+
mine = core_set(rules, core_tools)
|
|
280
|
+
theirs = core_set(other, core_tools)
|
|
281
|
+
return false unless (theirs - mine).empty?
|
|
282
|
+
|
|
283
|
+
groups_mine = rules["groups"]
|
|
284
|
+
groups_theirs = other["groups"]
|
|
285
|
+
groups_within = groups_mine == "all" || (groups_theirs != "all" && (Array(groups_theirs) - Array(groups_mine)).empty?)
|
|
286
|
+
return false unless groups_within
|
|
287
|
+
|
|
288
|
+
theirs.size < mine.size || (groups_mine == "all" && groups_theirs != "all") ||
|
|
289
|
+
(groups_theirs != "all" && Array(groups_theirs).size < Array(groups_mine).size)
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def group_reached?(groups, group)
|
|
293
|
+
return true if ["all", :all].include?(groups)
|
|
294
|
+
|
|
295
|
+
Array(groups).any? { |selector| Toolkit.group_matches?([selector], group) }
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
GRADES = ["covers", "short of a tool", "wider than needed", "ungradable"].freeze
|
|
299
|
+
|
|
300
|
+
# Per graded kit: the grades, then what on mode would have done.
|
|
301
|
+
def print_tallies(tallies)
|
|
302
|
+
@stdout.puts("\nPer-kit")
|
|
303
|
+
@stdout.puts("kit | suggestions | #{GRADES.join(" | ")} | would")
|
|
304
|
+
tallies.sort.each do |kit, counts|
|
|
305
|
+
would = counts.select { |key, _| key.start_with?("would:") }.sort
|
|
306
|
+
.map { |key, count| "#{key.delete_prefix("would:")} #{count}" }
|
|
307
|
+
@stdout.puts("#{kit} | #{GRADES.sum { |grade| counts[grade] }} | " \
|
|
308
|
+
"#{GRADES.map { |grade| counts[grade] }.join(" | ")} | #{would.join(", ")}")
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def field(hash, key)
|
|
313
|
+
return nil unless hash.is_a?(Hash)
|
|
314
|
+
|
|
315
|
+
hash.key?(key) ? hash[key] : hash[key.to_s]
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
end
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
require_relative "message_time"
|
|
6
|
+
require_relative "steer"
|
|
7
|
+
require_relative "system_one/question"
|
|
8
|
+
|
|
9
|
+
module Samagotchi
|
|
10
|
+
# The first-prompt toolkit classifier and its append-only audit record.
|
|
11
|
+
# The Engine prepares a Snapshot on its turn thread; #ask only sees that
|
|
12
|
+
# copy, so the question cannot change the running session or tool registry.
|
|
13
|
+
class ToolkitSuggest
|
|
14
|
+
FILE = "toolkit_suggestions.jsonl"
|
|
15
|
+
QUESTION_ID = "toolkit"
|
|
16
|
+
PROMPT_MAX = 1500
|
|
17
|
+
CHARS_PER_TOKEN = 4.0
|
|
18
|
+
INSTRUCTIONS = "A user is starting a session of a coding agent with the prompt in the state. " \
|
|
19
|
+
"Which toolkit should the session start on? Pick the smallest toolkit whose tools the work will need."
|
|
20
|
+
|
|
21
|
+
# A kit as the ask saw it: its name, classifier description and #rules.
|
|
22
|
+
Candidate = Data.define(:name, :description, :rules)
|
|
23
|
+
# +default+ is a Candidate (no description needed); +core_tools+ the
|
|
24
|
+
# built-ins registered when the ask started.
|
|
25
|
+
Snapshot = Data.define(:session_id, :turn_number, :default, :candidates, :core_tools, :state)
|
|
26
|
+
Suggestion = Data.define(:kit, :raw, :confidence, :decided)
|
|
27
|
+
|
|
28
|
+
# The candidates with classifier descriptions, excluding the configured
|
|
29
|
+
# default, the participants' review boundary, the defer sentinel, and a
|
|
30
|
+
# kit that doesn't resolve (a missing base, a cycle): it is left out,
|
|
31
|
+
# not the whole ask, and the ≥ 2 clause counts what is left.
|
|
32
|
+
def self.candidates(kits, default_name:)
|
|
33
|
+
kits.values.filter_map do |kit|
|
|
34
|
+
next if [default_name.to_s, Toolkit::REVIEW, "undecided"].include?(kit.name)
|
|
35
|
+
next if kit.description.to_s.empty?
|
|
36
|
+
next unless resolves?(kit.name, kits)
|
|
37
|
+
|
|
38
|
+
kit
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def self.resolves?(name, kits)
|
|
43
|
+
Toolkit.resolve_kit(name, kits)
|
|
44
|
+
true
|
|
45
|
+
rescue Toolkit::Unresolved
|
|
46
|
+
false
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# A System 1 choice and the small, bounded state it classifies.
|
|
50
|
+
def self.question(default:, kits:, project:, prompt:)
|
|
51
|
+
candidates = candidates(kits, default_name: default.name)
|
|
52
|
+
criteria = candidates.to_h { |kit| ["kit:#{kit.name}", "#{kit.name}: #{kit.description}"] }
|
|
53
|
+
criteria["default"] = "none of the other toolkits fits: the user's everyday default"
|
|
54
|
+
criteria["undecided"] = "There is no task yet; defer this choice"
|
|
55
|
+
question = SystemOne::Question.new(id: QUESTION_ID, type: :choice, instructions: INSTRUCTIONS, criteria: criteria)
|
|
56
|
+
[question, { project: project.to_s, prompt: prompt.to_s[0, PROMPT_MAX] }]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The single turn eligibility predicate shared by audit and future on
|
|
60
|
+
# mode. A later prompt is eligible only inside an undecided defer window.
|
|
61
|
+
# +candidate_count+ and +last_suggestion+ may be callables: they are
|
|
62
|
+
# read only once the session's own clauses pass (the kits read memory
|
|
63
|
+
# files; the suggestion log is read whole), the log only on the defer
|
|
64
|
+
# path, when the session already has a model reply.
|
|
65
|
+
def self.eligible?(session:, prompt:, author:, candidate_count:, last_suggestion:, defer_turns:, defer_tokens:)
|
|
66
|
+
return false unless session&.toolkit_origin == "default"
|
|
67
|
+
return false if session.delegate? || session.review?
|
|
68
|
+
return false if Steer.participant_author?(author)
|
|
69
|
+
return false unless prompt.is_a?(String) && !prompt.strip.empty?
|
|
70
|
+
return false if value(candidate_count).to_i < 2
|
|
71
|
+
|
|
72
|
+
messages = Array(session.messages)
|
|
73
|
+
return true unless messages.any? { |message| field(message, :role).to_s == "model" }
|
|
74
|
+
return false unless field(value(last_suggestion), :raw) == "undecided"
|
|
75
|
+
|
|
76
|
+
current_turn_number = turn_number(messages)
|
|
77
|
+
current_turn_number <= defer_turns.to_i && conversation_chars(messages) / CHARS_PER_TOKEN < defer_tokens.to_i
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# The conversation's size for the defer window: its messages' text and
|
|
81
|
+
# tool calls, summed (no JSON: a byte that isn't UTF-8 can't break it).
|
|
82
|
+
def self.conversation_chars(messages)
|
|
83
|
+
Array(messages).sum do |message|
|
|
84
|
+
field(message, :content).to_s.length + field(message, :tool_calls).to_s.length
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def self.turn_number(messages) = user_prompt_count(Array(messages)) + 1
|
|
89
|
+
|
|
90
|
+
# The owner's own prompt: a user line the Engine kept for a turn (it
|
|
91
|
+
# carries the turn's id), or TurnCheckpoint's line for a turn killed
|
|
92
|
+
# before its first boundary (stamped, no id). A !command's output, a
|
|
93
|
+
# declined continue's reason and KernelLoop's recovery prompt are
|
|
94
|
+
# neither; steers, notes and reminders carry their kind.
|
|
95
|
+
def self.owner_prompt?(message)
|
|
96
|
+
field(message, :role).to_s == "user" &&
|
|
97
|
+
!Steer.participant_author?(field(message, :author)) &&
|
|
98
|
+
!%w[input steer note reminder].include?(field(message, :kind).to_s) &&
|
|
99
|
+
!field(message, :preserve_literals) &&
|
|
100
|
+
!(field(message, :turn_id) || field(message, MessageTime::KEY)).nil?
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The prior logged suggestion for this session; the log, not process
|
|
104
|
+
# memory, keeps a deferred choice alive across a worker respawn.
|
|
105
|
+
def self.last_suggestion(log, session_id)
|
|
106
|
+
log.records.reverse_each.find do |entry|
|
|
107
|
+
entry["kind"] == "suggestion" && entry["session_id"] == session_id
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Runs one already-prepared ask, then appends its raw pick, what on mode
|
|
112
|
+
# would have done, and every offered kit's #rules (the default's too),
|
|
113
|
+
# so the report can tell a narrower kit that would also have covered
|
|
114
|
+
# the turn. A missing judgment is recorded too.
|
|
115
|
+
def self.ask(client:, log:, snapshot:, threshold:)
|
|
116
|
+
question, = question(default: Toolkit.new(name: snapshot.default.name),
|
|
117
|
+
kits: snapshot.candidates.to_h { |candidate| [candidate.name, kit_for(candidate)] },
|
|
118
|
+
project: snapshot.state.fetch(:project), prompt: snapshot.state.fetch(:prompt))
|
|
119
|
+
answer = client.ask(state: snapshot.state, questions: { QUESTION_ID => question })&.fetch(QUESTION_ID, nil)
|
|
120
|
+
raw = answer&.choice
|
|
121
|
+
candidate_name = raw.to_s.delete_prefix("kit:")
|
|
122
|
+
candidate = snapshot.candidates.find { |item| item.name == candidate_name } if raw.to_s.start_with?("kit:")
|
|
123
|
+
selected = if raw == "default"
|
|
124
|
+
snapshot.default.name
|
|
125
|
+
elsif candidate
|
|
126
|
+
candidate.name
|
|
127
|
+
end
|
|
128
|
+
confidence = answer&.confidence
|
|
129
|
+
decided = raw == "default" || (candidate && confidence && confidence >= threshold.to_f)
|
|
130
|
+
would = if answer.nil?
|
|
131
|
+
"no_judgment"
|
|
132
|
+
elsif raw == "undecided"
|
|
133
|
+
"defer"
|
|
134
|
+
elsif raw == "default"
|
|
135
|
+
"default"
|
|
136
|
+
elsif candidate.nil?
|
|
137
|
+
"invalid"
|
|
138
|
+
elsif confidence.nil? || confidence < threshold.to_f
|
|
139
|
+
"below_threshold"
|
|
140
|
+
else
|
|
141
|
+
"suggested"
|
|
142
|
+
end
|
|
143
|
+
log.entry("suggestion", {
|
|
144
|
+
session_id: snapshot.session_id,
|
|
145
|
+
turn_number: snapshot.turn_number,
|
|
146
|
+
default_kit: snapshot.default.name,
|
|
147
|
+
raw: raw,
|
|
148
|
+
kit: selected,
|
|
149
|
+
confidence: confidence,
|
|
150
|
+
decided: !!decided,
|
|
151
|
+
would: would,
|
|
152
|
+
core_tools: snapshot.core_tools,
|
|
153
|
+
kits: [snapshot.default, *snapshot.candidates].to_h { |kit| [kit.name, kit.rules] }
|
|
154
|
+
})
|
|
155
|
+
Suggestion.new(kit: selected, raw: raw, confidence: confidence, decided: !!decided)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# A resolved kit (Toolkit::Resolved) as frozen, JSON-ready rules, not
|
|
159
|
+
# a tool list: plugin and MCP tools may still be loading when the ask
|
|
160
|
+
# starts, and every kit declares a non-core tool anyway
|
|
161
|
+
# (Resolved#declares?), so the report expands these against the tools
|
|
162
|
+
# a turn really called (ToolkitReportCommand).
|
|
163
|
+
# @return [Hash{String => Object}]
|
|
164
|
+
def self.rules(resolved)
|
|
165
|
+
{
|
|
166
|
+
"all_core" => resolved.all_core,
|
|
167
|
+
"core" => resolved.core.map { |name| name.to_s.dup.freeze }.freeze,
|
|
168
|
+
"added" => resolved.added.map { |name| name.to_s.dup.freeze }.freeze,
|
|
169
|
+
"removed" => resolved.removed.map { |name| name.to_s.dup.freeze }.freeze,
|
|
170
|
+
"groups" => resolved.groups == :all ? "all" : resolved.groups.map { |name| name.to_s.dup.freeze }.freeze
|
|
171
|
+
}.freeze
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def self.log_path(sessions_dir)
|
|
175
|
+
File.join(File.dirname(sessions_dir), FILE)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def self.warn(error)
|
|
179
|
+
Log.warn(:toolkit, "toolkit_suggest_failed",
|
|
180
|
+
echo: "[samagotchi:toolkit] toolkit suggestion: #{error.class}: #{error.message}",
|
|
181
|
+
error: error.class.name, msg: error.message.to_s[0, 200])
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
class << self
|
|
185
|
+
private
|
|
186
|
+
|
|
187
|
+
def value(given) = given.respond_to?(:call) ? given.call : given
|
|
188
|
+
|
|
189
|
+
def field(hash, key)
|
|
190
|
+
return nil unless hash.is_a?(Hash)
|
|
191
|
+
|
|
192
|
+
hash.key?(key) ? hash[key] : hash[key.to_s]
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def user_prompt_count(messages)
|
|
196
|
+
prompts = messages.filter_map do |message|
|
|
197
|
+
next unless owner_prompt?(message)
|
|
198
|
+
|
|
199
|
+
field(message, :turn_id) || message.object_id
|
|
200
|
+
end
|
|
201
|
+
prompts.uniq.size
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Reconstitute only the candidate metadata needed to build the question;
|
|
205
|
+
# names, descriptions and groups are copied from the Snapshot.
|
|
206
|
+
def kit_for(candidate)
|
|
207
|
+
Toolkit::Kit.new(name: candidate.name, description: candidate.description)
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
@@ -81,7 +81,7 @@ module Samagotchi
|
|
|
81
81
|
started = session.to_s.strip.empty? ? "Started a delegate session" : "Sent the follow-up to delegate #{child_id[0, 8]}"
|
|
82
82
|
"#{note}session: #{child_id}\nstatus: running\n#{started}; #{running_hint(peers)} " \
|
|
83
83
|
"It shows in chi sessions list and the web as a child of this session; the user can attach to it."
|
|
84
|
-
rescue ArgumentError => e
|
|
84
|
+
rescue ArgumentError, Toolkit::Refused => e
|
|
85
85
|
"Error: #{e.message}"
|
|
86
86
|
end
|
|
87
87
|
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "../session"
|
|
4
4
|
require_relative "../bridge_client"
|
|
5
|
+
require_relative "../worker_link"
|
|
5
6
|
require_relative "../relay"
|
|
6
7
|
require_relative "../guardrails/approval"
|
|
7
8
|
require_relative "../guardrails/parent_policy"
|
|
@@ -79,13 +80,14 @@ module Samagotchi
|
|
|
79
80
|
qid = question.id.to_s
|
|
80
81
|
desk = relay.relay_desk
|
|
81
82
|
relay_id = desk.open(child_id: child_id, child_question_id: qid)
|
|
82
|
-
client =
|
|
83
|
+
client = WorkerLink.client(child_id, session_dir: Session.session_dir(child_id, state_dir: state_dir))
|
|
83
84
|
post(client, "opened", relay_id, qid)
|
|
84
85
|
Log.info(:turn, "relay_opened", child: child_id[0, 8], id: qid)
|
|
85
86
|
|
|
86
87
|
fields = Relay.card(child, question, relay_id: relay_id, more: more)
|
|
87
88
|
answer = relay.open_question(fields, watch: watch(child_id, qid, state_dir))
|
|
88
|
-
settle(answer, fields, label: label(question), client: client, desk: desk, relay_id: relay_id, qid: qid
|
|
89
|
+
settle(answer, fields, label: label(question), client: client, desk: desk, relay_id: relay_id, qid: qid,
|
|
90
|
+
child_id: child_id)
|
|
89
91
|
ensure
|
|
90
92
|
# A relay that failed before it settled closes, so the desk (which
|
|
91
93
|
# prunes only settled ones) keeps none open; a settled one stays.
|
|
@@ -96,7 +98,7 @@ module Samagotchi
|
|
|
96
98
|
# was answered or cancelled there) or its worker is gone.
|
|
97
99
|
def watch(child_id, qid, state_dir)
|
|
98
100
|
lambda do
|
|
99
|
-
unless
|
|
101
|
+
unless WorkerLink.resolve(Session.session_dir(child_id, state_dir: state_dir), unlink: false).worker_owned?
|
|
100
102
|
next "child_gone"
|
|
101
103
|
end
|
|
102
104
|
|
|
@@ -109,7 +111,7 @@ module Samagotchi
|
|
|
109
111
|
|
|
110
112
|
# Record the parent's answer, tell the child, and say what came of it.
|
|
111
113
|
# @param label [String] the outcome line's start (#label)
|
|
112
|
-
def settle(answer, fields, label:, client:, desk:, relay_id:, qid:)
|
|
114
|
+
def settle(answer, fields, label:, client:, desk:, relay_id:, qid:, child_id:)
|
|
113
115
|
if answer.is_a?(Hash) && answer[:error] == "cancelled"
|
|
114
116
|
desk.close(relay_id)
|
|
115
117
|
return closed(answer[:reason].to_s, label, client: client, relay_id: relay_id, qid: qid)
|
|
@@ -120,7 +122,7 @@ module Samagotchi
|
|
|
120
122
|
desk.record(relay_id, selected_indices: dismissed ? [] : answer.selected_indices,
|
|
121
123
|
freeform: dismissed ? nil : answer.freeform, dismissed: dismissed, by: by)
|
|
122
124
|
response = post(client, "answered", relay_id, qid)
|
|
123
|
-
Outcome.new(line: line(label, delivered(response, answer, fields, dismissed)), stopped: false)
|
|
125
|
+
Outcome.new(line: line(label, delivered(response, answer, fields, dismissed, child_id)), stopped: false)
|
|
124
126
|
end
|
|
125
127
|
|
|
126
128
|
def closed(reason, label, client:, relay_id:, qid:)
|
|
@@ -136,12 +138,12 @@ module Samagotchi
|
|
|
136
138
|
end
|
|
137
139
|
|
|
138
140
|
# The outcome of an answer the child was told about.
|
|
139
|
-
def delivered(response, answer, fields, dismissed)
|
|
141
|
+
def delivered(response, answer, fields, dismissed, child_id)
|
|
140
142
|
return "not delivered (the child's worker is gone)" unless response
|
|
141
143
|
|
|
142
144
|
# The child's Bridge asked for a secret this worker didn't send (one
|
|
143
145
|
# from before the child's chi): not the child's guardrails refusing.
|
|
144
|
-
return "not delivered (#{BridgeSecret
|
|
146
|
+
return "not delivered (#{BridgeSecret.refused(child_id)})" if response.secret_refused?
|
|
145
147
|
|
|
146
148
|
hook = fields[:kind] == HOOK_KIND
|
|
147
149
|
case response.status
|
|
@@ -95,7 +95,7 @@ module Samagotchi
|
|
|
95
95
|
# "\nHint: …" when this project's memory folder looks orphaned
|
|
96
96
|
# (MemoryOrphans), else "".
|
|
97
97
|
def self.orphan_hint
|
|
98
|
-
message = MemoryOrphans.
|
|
98
|
+
message = MemoryOrphans.model_message
|
|
99
99
|
message ? "\nHint: #{message}" : ""
|
|
100
100
|
end
|
|
101
101
|
|