samagotchi 0.121.1 → 0.122.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -1
- data/bin/chi +4 -1
- data/docs/architecture.md +2 -1
- data/docs/cli.md +1 -0
- data/docs/configuration.md +29 -4
- data/docs/guardrails.md +15 -0
- data/docs/internals/engine.md +2 -0
- data/docs/internals/participants.md +13 -7
- data/docs/memory.md +33 -0
- data/docs/plugins.md +3 -2
- data/docs/sessions.md +8 -2
- data/lib/samagotchi/bridge/turn_accumulator.rb +3 -1
- data/lib/samagotchi/bridge.rb +4 -3
- data/lib/samagotchi/bridge_client/event_stream.rb +5 -1
- data/lib/samagotchi/bridge_client/stream_open.rb +8 -2
- data/lib/samagotchi/bundles/system/manifest.yml +2 -2
- data/lib/samagotchi/bundles/system/memory_guide.md +13 -0
- data/lib/samagotchi/config.rb +5 -0
- data/lib/samagotchi/engine.rb +107 -1
- data/lib/samagotchi/event_fanout.rb +0 -1
- data/lib/samagotchi/guardrail_wiring.rb +5 -4
- data/lib/samagotchi/log_line.rb +1 -1
- data/lib/samagotchi/memory_orphans.rb +11 -0
- data/lib/samagotchi/participant_mode.rb +11 -0
- data/lib/samagotchi/plugin/api.rb +7 -4
- data/lib/samagotchi/process_group.rb +13 -0
- data/lib/samagotchi/review_start.rb +6 -2
- data/lib/samagotchi/run_options.rb +1 -0
- data/lib/samagotchi/session.rb +14 -1
- data/lib/samagotchi/session_access/contract.rb +1 -1
- data/lib/samagotchi/session_access/remote.rb +32 -12
- data/lib/samagotchi/session_access/web_bridge_client.rb +9 -2
- data/lib/samagotchi/session_manager.rb +21 -7
- data/lib/samagotchi/session_setup.rb +8 -4
- data/lib/samagotchi/sessions_command.rb +1 -1
- data/lib/samagotchi/steer.rb +7 -6
- data/lib/samagotchi/system_one/client.rb +6 -5
- data/lib/samagotchi/system_one/log.rb +25 -5
- data/lib/samagotchi/terminal_ui.rb +2 -1
- data/lib/samagotchi/tool_runner.rb +2 -1
- data/lib/samagotchi/toolkit.rb +27 -8
- data/lib/samagotchi/toolkit_binding.rb +115 -7
- data/lib/samagotchi/toolkit_command.rb +15 -3
- data/lib/samagotchi/toolkit_report_command.rb +318 -0
- data/lib/samagotchi/toolkit_suggest.rb +211 -0
- data/lib/samagotchi/tools/memory.rb +1 -1
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +5 -5
- data/lib/samagotchi/web/participant_access.rb +4 -1
- data/lib/samagotchi/web/participant_review.rb +6 -4
- data/lib/samagotchi/web/principal.rb +3 -4
- data/lib/samagotchi/web/public/bubbles.js +3 -2
- data/lib/samagotchi/web/public/participant.js +2 -1
- data/lib/samagotchi/web/public/turn_events.js +2 -1
- data/lib/samagotchi/web/public/turn_html.js +1 -1
- data/lib/samagotchi/web/server.rb +2 -2
- data/lib/samagotchi/web/session_access_routes.rb +3 -3
- data/lib/samagotchi/web/session_control.rb +4 -2
- data/lib/samagotchi/web/start_dir.rb +10 -5
- data/lib/samagotchi/worker.rb +16 -2
- data/lib/samagotchi/worker_link.rb +1 -0
- metadata +4 -1
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "time"
|
|
5
|
+
|
|
6
|
+
require_relative "cli/command"
|
|
7
|
+
require_relative "cli/exit"
|
|
8
|
+
require_relative "cli/flags"
|
|
9
|
+
require_relative "config"
|
|
10
|
+
require_relative "session"
|
|
11
|
+
require_relative "system_one/log"
|
|
12
|
+
require_relative "toolkit"
|
|
13
|
+
require_relative "tool_response"
|
|
14
|
+
require_relative "toolkit_suggest"
|
|
15
|
+
|
|
16
|
+
module Samagotchi
|
|
17
|
+
# `chi toolkit report`: compare the toolkit on mode would have run (the
|
|
18
|
+
# suggestion when it would have been applied, else the default) with the
|
|
19
|
+
# tools the session called from that turn on, against every kit's rules
|
|
20
|
+
# the ask logged (ToolkitSuggest.rules).
|
|
21
|
+
class ToolkitReportCommand
|
|
22
|
+
include CLI::Command
|
|
23
|
+
|
|
24
|
+
USAGE = <<~TEXT
|
|
25
|
+
Usage: chi toolkit report [--days N]
|
|
26
|
+
Grade the kit each audit suggestion would have run (the suggested
|
|
27
|
+
kit when on mode would have applied it, else the default) against
|
|
28
|
+
the tools the session called from that turn on, call_tool targets
|
|
29
|
+
included: covers, short of a tool (named), wider than needed (a
|
|
30
|
+
narrower offered kit would also cover), or ungradable. Then a
|
|
31
|
+
per-kit tally with what on mode would have done. Includes archived
|
|
32
|
+
sessions; the report is limited to 14 days and 500 sessions.
|
|
33
|
+
TEXT
|
|
34
|
+
|
|
35
|
+
MAX_DAYS = 14
|
|
36
|
+
MAX_SESSIONS = 500
|
|
37
|
+
FLAGS = CLI::Flags.new(args: false) { |flags| flags.value "--days" }.freeze
|
|
38
|
+
ToolUse = Data.define(:name, :group, :label)
|
|
39
|
+
|
|
40
|
+
# @param argv [Array<String>] arguments after "toolkit"
|
|
41
|
+
# @param sessions_dir [String, nil] sessions location (a spec seam)
|
|
42
|
+
# @param log [SystemOne::Log, nil] suggestion log (a spec seam)
|
|
43
|
+
# @param now [Time] the report's clock (a spec seam)
|
|
44
|
+
def initialize(argv, stdout: $stdout, stderr: $stderr, sessions_dir: nil, log: nil, now: Time.now)
|
|
45
|
+
@argv = argv.dup
|
|
46
|
+
@stdout = stdout
|
|
47
|
+
@stderr = stderr
|
|
48
|
+
@sessions_dir = sessions_dir || Session.default_state_dir
|
|
49
|
+
@log = log
|
|
50
|
+
@now = now
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def run
|
|
54
|
+
@sub = @argv.first
|
|
55
|
+
if @sub.nil? || HELP_WORDS.include?(@sub)
|
|
56
|
+
@stdout.puts(USAGE)
|
|
57
|
+
return CLI::Exit::OK
|
|
58
|
+
end
|
|
59
|
+
return usage_error("unknown subcommand #{@sub}") unless @sub == "report"
|
|
60
|
+
|
|
61
|
+
parsed = parse_flags(FLAGS, @argv[1..])
|
|
62
|
+
return parsed if parsed.is_a?(Integer)
|
|
63
|
+
return usage_error("--days takes a positive whole number") if parsed.options[:days] &&
|
|
64
|
+
!parsed.options[:days].to_s.match?(/\A[1-9]\d*\z/)
|
|
65
|
+
|
|
66
|
+
report(parsed.options[:days] ? Integer(parsed.options[:days], 10) : MAX_DAYS)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
private
|
|
70
|
+
|
|
71
|
+
def command_name = "chi toolkit report"
|
|
72
|
+
|
|
73
|
+
def report(requested_days)
|
|
74
|
+
days = effective_days(requested_days)
|
|
75
|
+
cutoff = @now - (days * 86_400)
|
|
76
|
+
max_sessions = [MAX_SESSIONS, positive_config("session.max_count", MAX_SESSIONS)].min
|
|
77
|
+
sessions = Session.list(state_dir: @sessions_dir, sort: "updated_at", order: "desc",
|
|
78
|
+
limit: max_sessions, include_archived: true)
|
|
79
|
+
session_summaries = sessions.to_h { |session| [session.id, session] }
|
|
80
|
+
records = log.records.filter_map do |entry|
|
|
81
|
+
next unless entry["kind"] == "suggestion" && session_summaries.key?(entry["session_id"])
|
|
82
|
+
next unless timestamp(entry["at"])&.>= cutoff
|
|
83
|
+
|
|
84
|
+
entry
|
|
85
|
+
end
|
|
86
|
+
if records.empty?
|
|
87
|
+
@stdout.puts("No toolkit suggestions in the last #{days} days.")
|
|
88
|
+
return CLI::Exit::OK
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
loaded = {}
|
|
92
|
+
tallies = Hash.new { |hash, kit| hash[kit] = Hash.new(0) }
|
|
93
|
+
@stdout.puts("Toolkit suggestions · last #{days} days · up to #{max_sessions} sessions")
|
|
94
|
+
records.sort_by { |entry| timestamp(entry["at"]) }.each do |entry|
|
|
95
|
+
id = entry["session_id"]
|
|
96
|
+
session = loaded[id] ||= Session.load(id, state_dir: @sessions_dir)
|
|
97
|
+
uses, window = window_uses(session, entry["turn_number"].to_i)
|
|
98
|
+
kit = graded_kit(entry)
|
|
99
|
+
grade, detail = grade(entry, kit, uses)
|
|
100
|
+
tallies[kit][grade] += 1
|
|
101
|
+
tallies[kit]["would:#{entry["would"] || "unknown"}"] += 1
|
|
102
|
+
confidence = entry["confidence"].nil? ? "n/a" : format("%.2f", entry["confidence"])
|
|
103
|
+
@stdout.puts("#{timestamp(entry["at"]).iso8601} #{id[0, 8]} turn #{entry["turn_number"]}: " \
|
|
104
|
+
"#{entry["raw"] || "no judgment"} (p=#{confidence}, would #{entry["would"] || "unknown"}) " \
|
|
105
|
+
"→ #{kit}: #{grade}#{detail}#{window}")
|
|
106
|
+
labels = uses.is_a?(Array) ? uses.map(&:label).uniq : []
|
|
107
|
+
@stdout.puts(" tools: #{labels.join(", ")}") unless labels.empty?
|
|
108
|
+
rescue StandardError => e
|
|
109
|
+
@stderr.puts("chi toolkit report: session #{id}: #{e.class}: #{e.message}; skipped")
|
|
110
|
+
end
|
|
111
|
+
print_tallies(tallies)
|
|
112
|
+
CLI::Exit::OK
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def log
|
|
116
|
+
@log ||= SystemOne::Log.new(
|
|
117
|
+
dir: File.dirname(ToolkitSuggest.log_path(@sessions_dir)),
|
|
118
|
+
file: ToolkitSuggest::FILE,
|
|
119
|
+
warn: ->(reason) { @stderr.puts("chi toolkit report: #{reason}") }
|
|
120
|
+
)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def effective_days(requested)
|
|
124
|
+
retention = Config.get("session.retention_days").to_i
|
|
125
|
+
cap = retention.positive? ? [retention, MAX_DAYS].min : MAX_DAYS
|
|
126
|
+
[requested, cap].min
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def positive_config(key, fallback)
|
|
130
|
+
value = Config.get(key).to_i
|
|
131
|
+
value.positive? ? value : fallback
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def timestamp(value)
|
|
135
|
+
Time.iso8601(value.to_s)
|
|
136
|
+
rescue ArgumentError
|
|
137
|
+
nil
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# The tools the session called from the suggestion's turn on: to the
|
|
141
|
+
# session's end, or only that turn when the toolkit was chosen since
|
|
142
|
+
# (/toolkit, ctx.use_toolkit: no message marks when, and a change
|
|
143
|
+
# applies from the next turn's start, so the asked turn ran on it).
|
|
144
|
+
# @return [Array(Array<ToolUse>|Symbol, String)] the uses (:ungradable
|
|
145
|
+
# when a turn's tools can't be read back) and a note on the window
|
|
146
|
+
def window_uses(session, turn_number)
|
|
147
|
+
by_turn = tool_uses_by_turn(session.messages)
|
|
148
|
+
changed = session.toolkit_origin == "chosen"
|
|
149
|
+
turns = by_turn.keys.select { |number| changed ? number == turn_number : number >= turn_number }
|
|
150
|
+
uses = turns.sort.flat_map { |number| by_turn[number] }
|
|
151
|
+
note = changed ? " (its turn only: the toolkit was chosen later)" : ""
|
|
152
|
+
[turns.empty? || uses.include?(:ungradable) ? :ungradable : uses, note]
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Tool uses grouped by the user's own prompt number; steers and merged
|
|
156
|
+
# input lines remain part of that turn rather than starting another.
|
|
157
|
+
# The chat loop saves its calls on the model's message (tool_calls);
|
|
158
|
+
# the raw-prompt loop saves only the model's text, so its calls are
|
|
159
|
+
# read back from the batch's tool_response entry (#joined_uses).
|
|
160
|
+
def tool_uses_by_turn(messages)
|
|
161
|
+
number = 0
|
|
162
|
+
current = nil
|
|
163
|
+
uses = Hash.new { |hash, key| hash[key] = [] }
|
|
164
|
+
Array(messages).each do |message|
|
|
165
|
+
case field(message, :role).to_s
|
|
166
|
+
when "user"
|
|
167
|
+
if ToolkitSuggest.owner_prompt?(message)
|
|
168
|
+
number += 1
|
|
169
|
+
current = number
|
|
170
|
+
uses[current]
|
|
171
|
+
elsif Steer.participant_author?(field(message, :author))
|
|
172
|
+
current = nil
|
|
173
|
+
end
|
|
174
|
+
when "model"
|
|
175
|
+
Array(field(message, :tool_calls)).each { |call| uses[current] << tool_use(call) } if current
|
|
176
|
+
when "tool_response"
|
|
177
|
+
uses[current].concat(joined_uses(message)) if current && !field(message, :tool_call_id)
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
uses.transform_values(&:compact)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# A ran-as line (ToolRunner, a corrected call) may precede a run's lead.
|
|
184
|
+
RUN_LEAD = /\A(?:ran as: [^\n]*\n)?\[([^\]\s]+)\](?:\n| |\z)/
|
|
185
|
+
|
|
186
|
+
# A raw-prompt batch's tools: each run's "[name]" lead (ToolResponse),
|
|
187
|
+
# a call_tool's target from its saved params line ("group/tool …").
|
|
188
|
+
# When the leads don't match the batch's run count, :ungradable.
|
|
189
|
+
def joined_uses(message)
|
|
190
|
+
parts = field(message, :content).to_s.split(ToolResponse::SEPARATOR, -1)
|
|
191
|
+
names = parts.filter_map { |part| part[RUN_LEAD, 1] }
|
|
192
|
+
ids = field(message, :tool_ids)
|
|
193
|
+
return [:ungradable] if names.empty? || (ids.is_a?(Array) && ids.size != names.size)
|
|
194
|
+
|
|
195
|
+
params = field(message, :tool_params)
|
|
196
|
+
names.each_with_index.map do |name, index|
|
|
197
|
+
next ToolUse.new(name: name, group: nil, label: name) unless name == "call_tool"
|
|
198
|
+
|
|
199
|
+
line = params.is_a?(Array) ? params[index] : nil
|
|
200
|
+
target = line.to_s.split(" ", 2).first.to_s
|
|
201
|
+
next :ungradable unless target.include?("/")
|
|
202
|
+
|
|
203
|
+
group, tool = target.split("/", 2)
|
|
204
|
+
ToolUse.new(name: tool, group: group, label: "call_tool→#{target}")
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def tool_use(call)
|
|
209
|
+
return unless call.is_a?(Hash)
|
|
210
|
+
|
|
211
|
+
function = field(call, :function)
|
|
212
|
+
name = (field(call, :name) || field(function, :name)).to_s
|
|
213
|
+
arguments = field(call, :arguments) || field(function, :arguments) || field(call, :args)
|
|
214
|
+
arguments = JSON.parse(arguments) if arguments.is_a?(String) && !arguments.empty?
|
|
215
|
+
return ToolUse.new(name: name, group: nil, label: name) unless name == "call_tool"
|
|
216
|
+
|
|
217
|
+
target = arguments.is_a?(Hash) ? (field(arguments, :tool) || field(arguments, :name)) : nil
|
|
218
|
+
return :ungradable unless target
|
|
219
|
+
|
|
220
|
+
parts = target.to_s.split("/", 2)
|
|
221
|
+
group, tool = parts.size == 2 ? parts : [nil, parts.first]
|
|
222
|
+
ToolUse.new(name: tool, group: group, label: "call_tool→#{target}")
|
|
223
|
+
rescue JSON::ParserError
|
|
224
|
+
name == "call_tool" ? :ungradable : ToolUse.new(name: name, group: nil, label: name)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# The kit on mode would have run: the suggestion only when it would
|
|
228
|
+
# have been applied, else the session's default.
|
|
229
|
+
def graded_kit(entry)
|
|
230
|
+
name = entry["would"] == "suggested" ? entry["kit"] : entry["default_kit"]
|
|
231
|
+
name || "(unknown)"
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# covers: the kit declares every tool the window used (pick ⊇ need);
|
|
235
|
+
# wider than needed: it covers, and so would a narrower logged kit;
|
|
236
|
+
# short of a tool: names what it leaves out. ungradable when the
|
|
237
|
+
# window's calls or the kit's rules can't be read.
|
|
238
|
+
def grade(entry, kit, uses)
|
|
239
|
+
kits = entry["kits"].is_a?(Hash) ? entry["kits"] : {}
|
|
240
|
+
rules = kits[kit]
|
|
241
|
+
core_tools = Array(entry["core_tools"])
|
|
242
|
+
return ["ungradable", " (no rules logged for #{kit})"] unless rules.is_a?(Hash)
|
|
243
|
+
return ["ungradable", " (tool calls not readable)"] if uses == :ungradable
|
|
244
|
+
|
|
245
|
+
missing = uses.reject { |use| covers?(rules, core_tools, use) }
|
|
246
|
+
return ["short of a tool", " (#{missing.map(&:name).uniq.join(", ")})"] unless missing.empty?
|
|
247
|
+
|
|
248
|
+
narrower = kits.filter_map do |name, other|
|
|
249
|
+
next if name == kit || !other.is_a?(Hash) || !narrower?(other, rules, core_tools)
|
|
250
|
+
|
|
251
|
+
name if uses.all? { |use| covers?(other, core_tools, use) }
|
|
252
|
+
end
|
|
253
|
+
return ["wider than needed", " (#{narrower.sort.join(", ")} would also cover)"] unless narrower.empty?
|
|
254
|
+
|
|
255
|
+
["covers", ""]
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Whether a kit's rules declare +use+ (Toolkit::Resolved#declares?): a
|
|
259
|
+
# tool-index group by the kit's groups; a built-in by its core list;
|
|
260
|
+
# any other tool (a plugin's) unless the kit removes it.
|
|
261
|
+
def covers?(rules, core_tools, use)
|
|
262
|
+
return group_reached?(rules["groups"], use.group) if use.group && use.group != "chi"
|
|
263
|
+
|
|
264
|
+
removed = Array(rules["removed"])
|
|
265
|
+
return false if removed.include?(use.name)
|
|
266
|
+
return true unless core_tools.include?(use.name)
|
|
267
|
+
|
|
268
|
+
rules["all_core"] || Array(rules["core"]).include?(use.name) || Array(rules["added"]).include?(use.name)
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def core_set(rules, core_tools)
|
|
272
|
+
declared = rules["all_core"] ? core_tools : Array(rules["core"]) + Array(rules["added"])
|
|
273
|
+
(declared & core_tools) - Array(rules["removed"])
|
|
274
|
+
end
|
|
275
|
+
|
|
276
|
+
# +other+ declares no more built-ins and reaches no more groups than
|
|
277
|
+
# +rules+, and less of one of them.
|
|
278
|
+
def narrower?(other, rules, core_tools)
|
|
279
|
+
mine = core_set(rules, core_tools)
|
|
280
|
+
theirs = core_set(other, core_tools)
|
|
281
|
+
return false unless (theirs - mine).empty?
|
|
282
|
+
|
|
283
|
+
groups_mine = rules["groups"]
|
|
284
|
+
groups_theirs = other["groups"]
|
|
285
|
+
groups_within = groups_mine == "all" || (groups_theirs != "all" && (Array(groups_theirs) - Array(groups_mine)).empty?)
|
|
286
|
+
return false unless groups_within
|
|
287
|
+
|
|
288
|
+
theirs.size < mine.size || (groups_mine == "all" && groups_theirs != "all") ||
|
|
289
|
+
(groups_theirs != "all" && Array(groups_theirs).size < Array(groups_mine).size)
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def group_reached?(groups, group)
|
|
293
|
+
return true if ["all", :all].include?(groups)
|
|
294
|
+
|
|
295
|
+
Array(groups).any? { |selector| Toolkit.group_matches?([selector], group) }
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
GRADES = ["covers", "short of a tool", "wider than needed", "ungradable"].freeze
|
|
299
|
+
|
|
300
|
+
# Per graded kit: the grades, then what on mode would have done.
|
|
301
|
+
def print_tallies(tallies)
|
|
302
|
+
@stdout.puts("\nPer-kit")
|
|
303
|
+
@stdout.puts("kit | suggestions | #{GRADES.join(" | ")} | would")
|
|
304
|
+
tallies.sort.each do |kit, counts|
|
|
305
|
+
would = counts.select { |key, _| key.start_with?("would:") }.sort
|
|
306
|
+
.map { |key, count| "#{key.delete_prefix("would:")} #{count}" }
|
|
307
|
+
@stdout.puts("#{kit} | #{GRADES.sum { |grade| counts[grade] }} | " \
|
|
308
|
+
"#{GRADES.map { |grade| counts[grade] }.join(" | ")} | #{would.join(", ")}")
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def field(hash, key)
|
|
313
|
+
return nil unless hash.is_a?(Hash)
|
|
314
|
+
|
|
315
|
+
hash.key?(key) ? hash[key] : hash[key.to_s]
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
end
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
require_relative "message_time"
|
|
6
|
+
require_relative "steer"
|
|
7
|
+
require_relative "system_one/question"
|
|
8
|
+
|
|
9
|
+
module Samagotchi
|
|
10
|
+
# The first-prompt toolkit classifier and its append-only audit record.
|
|
11
|
+
# The Engine prepares a Snapshot on its turn thread; #ask only sees that
|
|
12
|
+
# copy, so the question cannot change the running session or tool registry.
|
|
13
|
+
class ToolkitSuggest
|
|
14
|
+
FILE = "toolkit_suggestions.jsonl"
|
|
15
|
+
QUESTION_ID = "toolkit"
|
|
16
|
+
PROMPT_MAX = 1500
|
|
17
|
+
CHARS_PER_TOKEN = 4.0
|
|
18
|
+
INSTRUCTIONS = "A user is starting a session of a coding agent with the prompt in the state. " \
|
|
19
|
+
"Which toolkit should the session start on? Pick the smallest toolkit whose tools the work will need."
|
|
20
|
+
|
|
21
|
+
# A kit as the ask saw it: its name, classifier description and #rules.
|
|
22
|
+
Candidate = Data.define(:name, :description, :rules)
|
|
23
|
+
# +default+ is a Candidate (no description needed); +core_tools+ the
|
|
24
|
+
# built-ins registered when the ask started.
|
|
25
|
+
Snapshot = Data.define(:session_id, :turn_number, :default, :candidates, :core_tools, :state)
|
|
26
|
+
Suggestion = Data.define(:kit, :raw, :confidence, :decided)
|
|
27
|
+
|
|
28
|
+
# The candidates with classifier descriptions, excluding the configured
|
|
29
|
+
# default, the participants' review boundary, the defer sentinel, and a
|
|
30
|
+
# kit that doesn't resolve (a missing base, a cycle): it is left out,
|
|
31
|
+
# not the whole ask, and the ≥ 2 clause counts what is left.
|
|
32
|
+
def self.candidates(kits, default_name:)
|
|
33
|
+
kits.values.filter_map do |kit|
|
|
34
|
+
next if [default_name.to_s, Toolkit::REVIEW, "undecided"].include?(kit.name)
|
|
35
|
+
next if kit.description.to_s.empty?
|
|
36
|
+
next unless resolves?(kit.name, kits)
|
|
37
|
+
|
|
38
|
+
kit
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def self.resolves?(name, kits)
|
|
43
|
+
Toolkit.resolve_kit(name, kits)
|
|
44
|
+
true
|
|
45
|
+
rescue Toolkit::Unresolved
|
|
46
|
+
false
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# A System 1 choice and the small, bounded state it classifies.
|
|
50
|
+
def self.question(default:, kits:, project:, prompt:)
|
|
51
|
+
candidates = candidates(kits, default_name: default.name)
|
|
52
|
+
criteria = candidates.to_h { |kit| ["kit:#{kit.name}", "#{kit.name}: #{kit.description}"] }
|
|
53
|
+
criteria["default"] = "none of the other toolkits fits: the user's everyday default"
|
|
54
|
+
criteria["undecided"] = "There is no task yet; defer this choice"
|
|
55
|
+
question = SystemOne::Question.new(id: QUESTION_ID, type: :choice, instructions: INSTRUCTIONS, criteria: criteria)
|
|
56
|
+
[question, { project: project.to_s, prompt: prompt.to_s[0, PROMPT_MAX] }]
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The single turn eligibility predicate shared by audit and future on
|
|
60
|
+
# mode. A later prompt is eligible only inside an undecided defer window.
|
|
61
|
+
# +candidate_count+ and +last_suggestion+ may be callables: they are
|
|
62
|
+
# read only once the session's own clauses pass (the kits read memory
|
|
63
|
+
# files; the suggestion log is read whole), the log only on the defer
|
|
64
|
+
# path, when the session already has a model reply.
|
|
65
|
+
def self.eligible?(session:, prompt:, author:, candidate_count:, last_suggestion:, defer_turns:, defer_tokens:)
|
|
66
|
+
return false unless session&.toolkit_origin == "default"
|
|
67
|
+
return false if session.delegate? || session.review?
|
|
68
|
+
return false if Steer.participant_author?(author)
|
|
69
|
+
return false unless prompt.is_a?(String) && !prompt.strip.empty?
|
|
70
|
+
return false if value(candidate_count).to_i < 2
|
|
71
|
+
|
|
72
|
+
messages = Array(session.messages)
|
|
73
|
+
return true unless messages.any? { |message| field(message, :role).to_s == "model" }
|
|
74
|
+
return false unless field(value(last_suggestion), :raw) == "undecided"
|
|
75
|
+
|
|
76
|
+
current_turn_number = turn_number(messages)
|
|
77
|
+
current_turn_number <= defer_turns.to_i && conversation_chars(messages) / CHARS_PER_TOKEN < defer_tokens.to_i
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# The conversation's size for the defer window: its messages' text and
|
|
81
|
+
# tool calls, summed (no JSON: a byte that isn't UTF-8 can't break it).
|
|
82
|
+
def self.conversation_chars(messages)
|
|
83
|
+
Array(messages).sum do |message|
|
|
84
|
+
field(message, :content).to_s.length + field(message, :tool_calls).to_s.length
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
def self.turn_number(messages) = user_prompt_count(Array(messages)) + 1
|
|
89
|
+
|
|
90
|
+
# The owner's own prompt: a user line the Engine kept for a turn (it
|
|
91
|
+
# carries the turn's id), or TurnCheckpoint's line for a turn killed
|
|
92
|
+
# before its first boundary (stamped, no id). A !command's output, a
|
|
93
|
+
# declined continue's reason and KernelLoop's recovery prompt are
|
|
94
|
+
# neither; steers, notes and reminders carry their kind.
|
|
95
|
+
def self.owner_prompt?(message)
|
|
96
|
+
field(message, :role).to_s == "user" &&
|
|
97
|
+
!Steer.participant_author?(field(message, :author)) &&
|
|
98
|
+
!%w[input steer note reminder].include?(field(message, :kind).to_s) &&
|
|
99
|
+
!field(message, :preserve_literals) &&
|
|
100
|
+
!(field(message, :turn_id) || field(message, MessageTime::KEY)).nil?
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# The prior logged suggestion for this session; the log, not process
|
|
104
|
+
# memory, keeps a deferred choice alive across a worker respawn.
|
|
105
|
+
def self.last_suggestion(log, session_id)
|
|
106
|
+
log.records.reverse_each.find do |entry|
|
|
107
|
+
entry["kind"] == "suggestion" && entry["session_id"] == session_id
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Runs one already-prepared ask, then appends its raw pick, what on mode
|
|
112
|
+
# would have done, and every offered kit's #rules (the default's too),
|
|
113
|
+
# so the report can tell a narrower kit that would also have covered
|
|
114
|
+
# the turn. A missing judgment is recorded too.
|
|
115
|
+
def self.ask(client:, log:, snapshot:, threshold:)
|
|
116
|
+
question, = question(default: Toolkit.new(name: snapshot.default.name),
|
|
117
|
+
kits: snapshot.candidates.to_h { |candidate| [candidate.name, kit_for(candidate)] },
|
|
118
|
+
project: snapshot.state.fetch(:project), prompt: snapshot.state.fetch(:prompt))
|
|
119
|
+
answer = client.ask(state: snapshot.state, questions: { QUESTION_ID => question })&.fetch(QUESTION_ID, nil)
|
|
120
|
+
raw = answer&.choice
|
|
121
|
+
candidate_name = raw.to_s.delete_prefix("kit:")
|
|
122
|
+
candidate = snapshot.candidates.find { |item| item.name == candidate_name } if raw.to_s.start_with?("kit:")
|
|
123
|
+
selected = if raw == "default"
|
|
124
|
+
snapshot.default.name
|
|
125
|
+
elsif candidate
|
|
126
|
+
candidate.name
|
|
127
|
+
end
|
|
128
|
+
confidence = answer&.confidence
|
|
129
|
+
decided = raw == "default" || (candidate && confidence && confidence >= threshold.to_f)
|
|
130
|
+
would = if answer.nil?
|
|
131
|
+
"no_judgment"
|
|
132
|
+
elsif raw == "undecided"
|
|
133
|
+
"defer"
|
|
134
|
+
elsif raw == "default"
|
|
135
|
+
"default"
|
|
136
|
+
elsif candidate.nil?
|
|
137
|
+
"invalid"
|
|
138
|
+
elsif confidence.nil? || confidence < threshold.to_f
|
|
139
|
+
"below_threshold"
|
|
140
|
+
else
|
|
141
|
+
"suggested"
|
|
142
|
+
end
|
|
143
|
+
log.entry("suggestion", {
|
|
144
|
+
session_id: snapshot.session_id,
|
|
145
|
+
turn_number: snapshot.turn_number,
|
|
146
|
+
default_kit: snapshot.default.name,
|
|
147
|
+
raw: raw,
|
|
148
|
+
kit: selected,
|
|
149
|
+
confidence: confidence,
|
|
150
|
+
decided: !!decided,
|
|
151
|
+
would: would,
|
|
152
|
+
core_tools: snapshot.core_tools,
|
|
153
|
+
kits: [snapshot.default, *snapshot.candidates].to_h { |kit| [kit.name, kit.rules] }
|
|
154
|
+
})
|
|
155
|
+
Suggestion.new(kit: selected, raw: raw, confidence: confidence, decided: !!decided)
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# A resolved kit (Toolkit::Resolved) as frozen, JSON-ready rules, not
|
|
159
|
+
# a tool list: plugin and MCP tools may still be loading when the ask
|
|
160
|
+
# starts, and every kit declares a non-core tool anyway
|
|
161
|
+
# (Resolved#declares?), so the report expands these against the tools
|
|
162
|
+
# a turn really called (ToolkitReportCommand).
|
|
163
|
+
# @return [Hash{String => Object}]
|
|
164
|
+
def self.rules(resolved)
|
|
165
|
+
{
|
|
166
|
+
"all_core" => resolved.all_core,
|
|
167
|
+
"core" => resolved.core.map { |name| name.to_s.dup.freeze }.freeze,
|
|
168
|
+
"added" => resolved.added.map { |name| name.to_s.dup.freeze }.freeze,
|
|
169
|
+
"removed" => resolved.removed.map { |name| name.to_s.dup.freeze }.freeze,
|
|
170
|
+
"groups" => resolved.groups == :all ? "all" : resolved.groups.map { |name| name.to_s.dup.freeze }.freeze
|
|
171
|
+
}.freeze
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def self.log_path(sessions_dir)
|
|
175
|
+
File.join(File.dirname(sessions_dir), FILE)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def self.warn(error)
|
|
179
|
+
Log.warn(:toolkit, "toolkit_suggest_failed",
|
|
180
|
+
echo: "[samagotchi:toolkit] toolkit suggestion: #{error.class}: #{error.message}",
|
|
181
|
+
error: error.class.name, msg: error.message.to_s[0, 200])
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
class << self
|
|
185
|
+
private
|
|
186
|
+
|
|
187
|
+
def value(given) = given.respond_to?(:call) ? given.call : given
|
|
188
|
+
|
|
189
|
+
def field(hash, key)
|
|
190
|
+
return nil unless hash.is_a?(Hash)
|
|
191
|
+
|
|
192
|
+
hash.key?(key) ? hash[key] : hash[key.to_s]
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def user_prompt_count(messages)
|
|
196
|
+
prompts = messages.filter_map do |message|
|
|
197
|
+
next unless owner_prompt?(message)
|
|
198
|
+
|
|
199
|
+
field(message, :turn_id) || message.object_id
|
|
200
|
+
end
|
|
201
|
+
prompts.uniq.size
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Reconstitute only the candidate metadata needed to build the question;
|
|
205
|
+
# names, descriptions and groups are copied from the Snapshot.
|
|
206
|
+
def kit_for(candidate)
|
|
207
|
+
Toolkit::Kit.new(name: candidate.name, description: candidate.description)
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
end
|
|
211
|
+
end
|
|
@@ -95,7 +95,7 @@ module Samagotchi
|
|
|
95
95
|
# "\nHint: …" when this project's memory folder looks orphaned
|
|
96
96
|
# (MemoryOrphans), else "".
|
|
97
97
|
def self.orphan_hint
|
|
98
|
-
message = MemoryOrphans.
|
|
98
|
+
message = MemoryOrphans.model_message
|
|
99
99
|
message ? "\nHint: #{message}" : ""
|
|
100
100
|
end
|
|
101
101
|
|
data/lib/samagotchi/version.rb
CHANGED
data/lib/samagotchi/web/app.rb
CHANGED
|
@@ -1225,7 +1225,7 @@ module Samagotchi
|
|
|
1225
1225
|
# (?full=1: as the list's).
|
|
1226
1226
|
def handle_context_show(req, id, name)
|
|
1227
1227
|
session = @session_class.load(id, state_dir: default_state_dir)
|
|
1228
|
-
attached = find_context(session, name) or return error_response(404, "
|
|
1228
|
+
attached = find_context(session, name) or return error_response(404, "context_missing", "no source #{name[0, 40]}")
|
|
1229
1229
|
own = ContextSources.session_location(id, state_dir: default_state_dir)
|
|
1230
1230
|
row = context_row(attached, own, own.subscriptions, full: truthy_param?(req.params["full"]))
|
|
1231
1231
|
json_response(200, row.merge(text: attached.snapshot.text))
|
|
@@ -1305,7 +1305,7 @@ module Samagotchi
|
|
|
1305
1305
|
return context_remove_own(id, name) if truthy_param?(req.params["own"])
|
|
1306
1306
|
|
|
1307
1307
|
session = @session_class.load(id, state_dir: default_state_dir)
|
|
1308
|
-
attached = find_context(session, name) or return error_response(404, "
|
|
1308
|
+
attached = find_context(session, name) or return error_response(404, "context_missing", "no source #{name[0, 40]}")
|
|
1309
1309
|
if attached.location.session?
|
|
1310
1310
|
attached.location.remove(attached.name)
|
|
1311
1311
|
json_response(200, { status: "detached", name: attached.name })
|
|
@@ -1320,10 +1320,10 @@ module Samagotchi
|
|
|
1320
1320
|
when :removed then json_response(200, { status: "detached", name: name })
|
|
1321
1321
|
when :project
|
|
1322
1322
|
error_response(409, "project_source", ContextCommand.project_source_message(name))
|
|
1323
|
-
else error_response(404, "
|
|
1323
|
+
else error_response(404, "context_missing", "no source #{name[0, 40]}")
|
|
1324
1324
|
end
|
|
1325
1325
|
rescue ContextSources::Invalid
|
|
1326
|
-
error_response(404, "
|
|
1326
|
+
error_response(404, "context_missing", "no source #{name[0, 40]}")
|
|
1327
1327
|
end
|
|
1328
1328
|
|
|
1329
1329
|
def attached_context(session, shadowed: false)
|
|
@@ -1530,7 +1530,7 @@ module Samagotchi
|
|
|
1530
1530
|
end
|
|
1531
1531
|
|
|
1532
1532
|
def stream_open_error(id, outcome)
|
|
1533
|
-
return error_response(503, "not_live", WorkerLink.
|
|
1533
|
+
return error_response(503, "not_live", WorkerLink.unanswered(id)) if outcome.is_a?(BridgeClient::StreamOpen::Unreachable)
|
|
1534
1534
|
return error_response(503, "not_live", WorkerLink.not_live(id)) if outcome.reason == :unknown_session
|
|
1535
1535
|
return error_response(502, "bridge_secret", BridgeSecret.refused(id)) if outcome.reason == :bridge_secret
|
|
1536
1536
|
|
|
@@ -420,7 +420,10 @@ module Samagotchi
|
|
|
420
420
|
result = session_access.wait_reply(id, **options, poll_interval: @wait_poll_interval,
|
|
421
421
|
cancelled: -> { !@server_running.call || !@grants.active?(grant.id) })
|
|
422
422
|
if result.status == :canceled
|
|
423
|
-
|
|
423
|
+
current = @grants.find(grant.id)
|
|
424
|
+
return error_response(401, *ENDED.fetch(:revoked)) unless current
|
|
425
|
+
|
|
426
|
+
ended = current.state(@grants.now)
|
|
424
427
|
return error_response(401, *ENDED.fetch(ended)) if ENDED.key?(ended)
|
|
425
428
|
|
|
426
429
|
return error_response(503, "server_stopping", "chi web is stopping; wait again")
|
|
@@ -6,6 +6,7 @@ require_relative "participant_exposure"
|
|
|
6
6
|
require_relative "../session_metrics"
|
|
7
7
|
require_relative "../review_start"
|
|
8
8
|
require_relative "../guardrails/review_paths"
|
|
9
|
+
require_relative "../participant_mode"
|
|
9
10
|
|
|
10
11
|
module Samagotchi
|
|
11
12
|
module Web
|
|
@@ -49,6 +50,7 @@ module Samagotchi
|
|
|
49
50
|
# current_turn, waiting_for_owner, queued: {mine, others}, last_event_id}
|
|
50
51
|
def handle_review(req, id)
|
|
51
52
|
session = @session_class.load(id, state_dir: default_state_dir)
|
|
53
|
+
mode = ParticipantMode.of(session).to_s
|
|
52
54
|
viewer = participant_viewer(req, session)
|
|
53
55
|
live = bridge_get_json(id, "snapshot")
|
|
54
56
|
turn_snapshot = live && live["snapshot"]
|
|
@@ -63,7 +65,7 @@ module Samagotchi
|
|
|
63
65
|
status = displayed_status(session, state)
|
|
64
66
|
status = Session::STATUS_IDLE if status == Session::STATUS_RUNNING && current.nil?
|
|
65
67
|
json_response(200, {
|
|
66
|
-
viewer: review_viewer(req, id, session: session, state: state, queued: queued, viewer: viewer),
|
|
68
|
+
viewer: review_viewer(req, id, session: session, state: state, queued: queued, viewer: viewer, mode: mode),
|
|
67
69
|
session: { id: session.id, title: participant_title(session), status: status },
|
|
68
70
|
subjects: review_subjects(session, viewer),
|
|
69
71
|
participants: participant_names(id).map { |entry| entry.slice(:id, :name, :kind) },
|
|
@@ -149,12 +151,12 @@ module Samagotchi
|
|
|
149
151
|
# participant view (no composer, no usage: the owner has /stats).
|
|
150
152
|
# +state+ the live worker's session state (nil without one), +queued+
|
|
151
153
|
# its queue.
|
|
152
|
-
def review_viewer(req, id, session:, state:, queued:, viewer:)
|
|
153
|
-
return { kind: "owner", can_comment: false } if principal(req).owner?
|
|
154
|
+
def review_viewer(req, id, session:, state:, queued:, viewer:, mode:)
|
|
155
|
+
return { kind: "owner", can_comment: false, mode: mode } if principal(req).owner?
|
|
154
156
|
|
|
155
157
|
grant = principal(req).grant
|
|
156
158
|
capped = !participant_cap_refusal(grant).nil?
|
|
157
|
-
{ kind: "participant", id: grant.id, name: grant.name, participant_kind: grant.kind, can_comment: !capped,
|
|
159
|
+
{ kind: "participant", id: grant.id, name: grant.name, participant_kind: grant.kind, mode: mode, can_comment: !capped,
|
|
158
160
|
cap_reached: capped, session_id: id,
|
|
159
161
|
usage: review_usage(grant, session: session, state: state, queued: queued, viewer: viewer) }
|
|
160
162
|
end
|
|
@@ -2,14 +2,13 @@
|
|
|
2
2
|
|
|
3
3
|
module Samagotchi
|
|
4
4
|
module Web
|
|
5
|
-
# Who a chi web request is (App#principal_for)
|
|
6
|
-
#
|
|
7
|
-
# Web::Token access token. The routes' permission column
|
|
5
|
+
# Who a chi web request is (App#principal_for): the owner or a
|
|
6
|
+
# participant admitted by a grant. The routes' permission column
|
|
8
7
|
# (App::ROUTES, :owner or :participant) is checked against it, and a
|
|
9
8
|
# turn's author comes from it, never from the request body. See
|
|
10
9
|
# docs/internals/remote-sessions.md (Trust model).
|
|
11
10
|
#
|
|
12
|
-
# +kind+ :owner; +grant+ what a
|
|
11
|
+
# +kind+ :owner or :participant; +grant+ what admitted a participant
|
|
13
12
|
# (nil for the owner).
|
|
14
13
|
Principal = Data.define(:kind, :grant) do
|
|
15
14
|
def owner? = kind == :owner
|