samagotchi 0.115.0 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +76 -1
- data/bin/chi +2 -0
- data/docs/cli.md +33 -2
- data/docs/configuration.md +13 -0
- data/docs/guardrails.md +188 -1
- data/docs/internals/background-tasks.md +1 -1
- data/docs/sessions.md +3 -3
- data/lib/samagotchi/bridge/turn_accumulator.rb +4 -0
- data/lib/samagotchi/bridge.rb +114 -14
- data/lib/samagotchi/bridge_client.rb +20 -1
- data/lib/samagotchi/bundles/system/manifest.yml +1 -1
- data/lib/samagotchi/config.rb +34 -3
- data/lib/samagotchi/engine.rb +6 -1
- data/lib/samagotchi/guardrail_wiring.rb +166 -2
- data/lib/samagotchi/guardrails/approval.rb +20 -0
- data/lib/samagotchi/guardrails/targets.rb +23 -2
- data/lib/samagotchi/guardrails/verdict.rb +23 -3
- data/lib/samagotchi/guardrails_command.rb +329 -0
- data/lib/samagotchi/hold_stop.rb +89 -0
- data/lib/samagotchi/memory_bundle/index_updater.rb +13 -0
- data/lib/samagotchi/prompt_warmup.rb +8 -3
- data/lib/samagotchi/session_access/contract.rb +1 -1
- data/lib/samagotchi/session_access/web_bridge_client.rb +9 -4
- data/lib/samagotchi/session_commands.rb +79 -2
- data/lib/samagotchi/session_manager.rb +3 -2
- data/lib/samagotchi/session_metrics.rb +7 -1
- data/lib/samagotchi/system_one/answer.rb +78 -0
- data/lib/samagotchi/system_one/approval_question.rb +113 -0
- data/lib/samagotchi/system_one/client.rb +89 -0
- data/lib/samagotchi/system_one/errors.rb +14 -0
- data/lib/samagotchi/system_one/fake_backend.rb +45 -0
- data/lib/samagotchi/system_one/http_backend.rb +129 -0
- data/lib/samagotchi/system_one/log.rb +133 -0
- data/lib/samagotchi/system_one/question.rb +86 -0
- data/lib/samagotchi/system_one/record.rb +83 -0
- data/lib/samagotchi/system_one/self_protect.rb +90 -0
- data/lib/samagotchi/system_one/similar.rb +69 -0
- data/lib/samagotchi/system_one.rb +25 -0
- data/lib/samagotchi/task_watch.rb +23 -5
- data/lib/samagotchi/terminal_ui/attached_loop.rb +145 -12
- data/lib/samagotchi/terminal_ui/event_renderer.rb +1 -0
- data/lib/samagotchi/terminal_ui/formatting.rb +14 -0
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +3 -0
- data/lib/samagotchi/web/public/activity.js +10 -0
- data/lib/samagotchi/web/public/app.js +20 -1
- data/lib/samagotchi/web/public/apple-touch-icon.png +0 -0
- data/lib/samagotchi/web/public/composer_fold.js +8 -8
- data/lib/samagotchi/web/public/composer_grip.js +65 -22
- data/lib/samagotchi/web/public/composer_size.js +40 -5
- data/lib/samagotchi/web/public/data.js +24 -0
- data/lib/samagotchi/web/public/favicon.svg +1 -1
- data/lib/samagotchi/web/public/icons/chi-180.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-192.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-512.png +0 -0
- data/lib/samagotchi/web/public/index.html +21 -0
- data/lib/samagotchi/web/public/session_actions.js +39 -6
- data/lib/samagotchi/web/public/stop_sheet.js +126 -0
- data/lib/samagotchi/web/public/system_one_wrong.js +32 -0
- data/lib/samagotchi/web/public/timing.js +9 -5
- data/lib/samagotchi/web/public/turn_events.js +1 -0
- data/lib/samagotchi/web/public/turn_html.js +13 -2
- data/lib/samagotchi/web/session_access_routes.rb +9 -6
- data/lib/samagotchi/web/session_control.rb +62 -3
- data/lib/samagotchi/web/session_show.rb +35 -5
- data/lib/samagotchi/worker.rb +57 -5
- data/lib/samagotchi/worker_idle_exit.rb +111 -9
- metadata +17 -1
|
@@ -19,8 +19,10 @@ module Samagotchi
|
|
|
19
19
|
# parent model would be writing a model's prompt); and the bundle's
|
|
20
20
|
# asks before answering chi's questions around chi answer (a parent
|
|
21
21
|
# allowing one would answer a grandchild's question as the user).
|
|
22
|
-
#
|
|
23
|
-
# ParentPolicy's user_only)
|
|
22
|
+
# A parent agent may never allow them (ParentApprovals.protected?,
|
|
23
|
+
# ParentPolicy's user_only); System 1 may, only if the user lists the
|
|
24
|
+
# rule by name in guardrails.system_one.rules, and never for the
|
|
25
|
+
# chi-answer/chi-broadcast ones (SystemOne::SelfProtect::NEVER_AUTO).
|
|
24
26
|
PROTECTED_RULES = %w[chi-config chi-hooks shell-touches-chi chi-context-cmd chi-broadcast prompt-memory-write
|
|
25
27
|
model-overlay-write chi-answer-piped chi-answer-http].freeze
|
|
26
28
|
SCOPES = %w[once session repo rule].freeze
|
|
@@ -33,6 +35,8 @@ module Samagotchi
|
|
|
33
35
|
attr_accessor :context, :targets
|
|
34
36
|
# The scope the user allowed an ask for ("once", "session", …).
|
|
35
37
|
attr_accessor :scope
|
|
38
|
+
# System 1's judgment that decided an ask, when it did: {id:, p:}.
|
|
39
|
+
attr_accessor :judgment
|
|
36
40
|
|
|
37
41
|
def initialize(call:)
|
|
38
42
|
@call = call
|
|
@@ -121,8 +125,11 @@ module Samagotchi
|
|
|
121
125
|
end
|
|
122
126
|
|
|
123
127
|
# For the activity entry; an allowed ask notes how ("approved (repo)",
|
|
124
|
-
# "approved earlier (session)")
|
|
128
|
+
# "approved earlier (session)"), one System 1 answered notes that and
|
|
129
|
+
# carries the judgment (docs/guardrails.md, "System 1").
|
|
125
130
|
def to_activity
|
|
131
|
+
return system_one_activity if @decided_by == "system one"
|
|
132
|
+
|
|
126
133
|
note = if allow? && @scope
|
|
127
134
|
"approved#{" earlier" if @decided_by == "approval"} (#{@scope})"
|
|
128
135
|
end
|
|
@@ -131,6 +138,19 @@ module Samagotchi
|
|
|
131
138
|
|
|
132
139
|
private
|
|
133
140
|
|
|
141
|
+
# The auto-allow's activity: who decided, for how long, and the
|
|
142
|
+
# judgment behind it (the p and the judgment's id), so a UI can show
|
|
143
|
+
# why the call was never asked about.
|
|
144
|
+
def system_one_activity
|
|
145
|
+
facts = @judgment || {}
|
|
146
|
+
p = facts[:p]
|
|
147
|
+
activity = { rule: @rule, verdict: @decision.to_s, decided_by: @decided_by, scope: @scope,
|
|
148
|
+
note: p ? "auto-allowed by System 1 (p=#{format("%.2f", p.to_f)})" : "auto-allowed by System 1" }
|
|
149
|
+
activity[:judgment_id] = facts[:id] if facts[:id]
|
|
150
|
+
activity[:p] = p if p
|
|
151
|
+
activity.compact
|
|
152
|
+
end
|
|
153
|
+
|
|
134
154
|
def decider
|
|
135
155
|
return [("rule #{@rule}" if @rule), @source].compact.join(", ") if @rule || @source
|
|
136
156
|
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
require_relative "cli/command"
|
|
6
|
+
require_relative "cli/exit"
|
|
7
|
+
require_relative "cli/flags"
|
|
8
|
+
require_relative "config"
|
|
9
|
+
require_relative "bridge/card_store"
|
|
10
|
+
require_relative "guardrails/approvals"
|
|
11
|
+
require_relative "guardrails/parent_policy"
|
|
12
|
+
require_relative "host_registry"
|
|
13
|
+
require_relative "session"
|
|
14
|
+
require_relative "system_one"
|
|
15
|
+
|
|
16
|
+
module Samagotchi
|
|
17
|
+
# `chi guardrails`: the guardrails' own maintenance commands, on this
|
|
18
|
+
# machine only (docs/guardrails.md, "System 1"). `replay` is the record
|
|
19
|
+
# seed: System 1 judges every approval this machine's sessions already
|
|
20
|
+
# answered, and the judgment and the user's answer go to the log as they
|
|
21
|
+
# would live. A rule's record then starts with the asks the user really
|
|
22
|
+
# decided on, instead of waiting for a few days of live ones.
|
|
23
|
+
class GuardrailsCommand
|
|
24
|
+
include CLI::Command
|
|
25
|
+
|
|
26
|
+
USAGE = <<~TEXT
|
|
27
|
+
Usage: chi guardrails <subcommand> [options]
|
|
28
|
+
replay [--dry-run] [--threshold T] [--model M]
|
|
29
|
+
Judge every approval this machine's sessions have already answered
|
|
30
|
+
(their cards, $XDG_STATE_HOME/samagotchi/sessions/<id>/cards.json)
|
|
31
|
+
through System 1 and write the judgment and the user's answer to
|
|
32
|
+
guardrails/judgments.jsonl, so a rule's record starts with the asks
|
|
33
|
+
the user really decided on. Asks with no approval card, unanswered
|
|
34
|
+
ones, and question ids a previous replay already logged are skipped.
|
|
35
|
+
A parent agent's answer is not the user's: only its deny is logged,
|
|
36
|
+
and only when the parent could have allowed (the session's
|
|
37
|
+
guardrails.parent_answers allows "permit" and the ask is not
|
|
38
|
+
user-only). --threshold overrides guardrails.system_one.threshold
|
|
39
|
+
for the table; --model overrides the configured model for this run.
|
|
40
|
+
--dry-run judges and prints the table without writing a line. Needs
|
|
41
|
+
guardrails.system_one.mode audit or on and a guardrails.system_one.host
|
|
42
|
+
(exit 2 without one).
|
|
43
|
+
TEXT
|
|
44
|
+
|
|
45
|
+
SUBCOMMANDS = %w[replay].freeze
|
|
46
|
+
FLAGS = {
|
|
47
|
+
"replay" => CLI::Flags.new(help: HELP_WORDS, args: false) do |f|
|
|
48
|
+
f.switch "--dry-run"
|
|
49
|
+
f.value "--threshold"
|
|
50
|
+
f.value "--model"
|
|
51
|
+
end
|
|
52
|
+
}.freeze
|
|
53
|
+
# The labels that mean the user let the call through.
|
|
54
|
+
ALLOW_LABELS = %w[allow_once allow_wide].freeze
|
|
55
|
+
# replay's table, in column order: the rule, how many asks it had, how
|
|
56
|
+
# many the user allowed, how many they denied, how many System 1 would
|
|
57
|
+
# have auto-allowed at the threshold, the false allows among them (the
|
|
58
|
+
# model said allow, the user did not), and the asks with no judgment.
|
|
59
|
+
COLUMNS = %w[rule asks allowed denied auto false_allows no_judgment].freeze
|
|
60
|
+
# Why an ask in a cards file was not replayed, in the words the summary
|
|
61
|
+
# uses.
|
|
62
|
+
SKIP_WORDS = {
|
|
63
|
+
already_replayed: "already replayed",
|
|
64
|
+
answered_by_parent: "answered by a parent agent",
|
|
65
|
+
parent_cannot_allow: "answered by a parent that could not allow",
|
|
66
|
+
unanswered: "unanswered",
|
|
67
|
+
no_approval: "no approval card",
|
|
68
|
+
no_question_id: "no question id"
|
|
69
|
+
}.freeze
|
|
70
|
+
|
|
71
|
+
# One ask a replay judges: the card's question id, the session it came
|
|
72
|
+
# from, its approval as the card saved it, the label its answer gives,
|
|
73
|
+
# and when the user was asked.
|
|
74
|
+
Ask = Data.define(:question_id, :session_id, :approval, :label, :asked_at)
|
|
75
|
+
|
|
76
|
+
# @param argv [Array<String>] the arguments after "guardrails"
|
|
77
|
+
# @param client [SystemOne::Client, nil] the judgment client; nil: the
|
|
78
|
+
# one config.yml describes (the seam the specs pass a fake through)
|
|
79
|
+
# @param sessions_dir [String, nil] where the sessions are; nil: this
|
|
80
|
+
# machine's ($XDG_STATE_HOME/samagotchi/sessions)
|
|
81
|
+
# @param log [SystemOne::Log, nil] the judgments log; nil: the one beside
|
|
82
|
+
# the approvals store
|
|
83
|
+
def initialize(argv, stdout: $stdout, stderr: $stderr, client: nil, sessions_dir: nil, log: nil)
|
|
84
|
+
@argv = argv.dup
|
|
85
|
+
@stdout = stdout
|
|
86
|
+
@stderr = stderr
|
|
87
|
+
@client = client
|
|
88
|
+
@sessions_dir = sessions_dir
|
|
89
|
+
@log = log
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# @return [Integer] exit status: 0 done, 2 a usage error or no System 1
|
|
93
|
+
# client to judge with
|
|
94
|
+
def run
|
|
95
|
+
@sub = @argv[0]
|
|
96
|
+
if @sub.nil? || HELP_WORDS.include?(@sub)
|
|
97
|
+
@stdout.puts(USAGE)
|
|
98
|
+
return CLI::Exit::OK
|
|
99
|
+
end
|
|
100
|
+
return usage_error("unknown subcommand #{@sub}") unless SUBCOMMANDS.include?(@sub)
|
|
101
|
+
|
|
102
|
+
parsed = parse_flags(FLAGS.fetch(@sub), @argv[1..])
|
|
103
|
+
return parsed if parsed.is_a?(Integer)
|
|
104
|
+
|
|
105
|
+
replay(parsed.options)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
private
|
|
109
|
+
|
|
110
|
+
def command_name = @sub ? "chi guardrails #{@sub}" : "chi guardrails"
|
|
111
|
+
|
|
112
|
+
def replay(options)
|
|
113
|
+
threshold = threshold(options[:threshold])
|
|
114
|
+
return usage_error(threshold) if threshold.is_a?(String)
|
|
115
|
+
|
|
116
|
+
client = @client || SystemOne::Client.from_config(host_registry: HostRegistry.new, model: options[:model])
|
|
117
|
+
if client.nil?
|
|
118
|
+
error_line("chi guardrails replay: no System 1 client — set guardrails.system_one.mode to audit or on " \
|
|
119
|
+
"and guardrails.system_one.host in config.yml")
|
|
120
|
+
return CLI::Exit::USAGE
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
sessions_dir = @sessions_dir || Session.default_state_dir
|
|
124
|
+
log = @log || SystemOne::Log.new(dir: Guardrails::Approvals.dir_for(sessions_dir))
|
|
125
|
+
replayed = File.exist?(log.path) ? log.entries.filter_map { |entry| entry["question_id"] if entry["origin"] == "replay" }.to_set : Set.new
|
|
126
|
+
@dry_run = options[:dry_run] ? true : false
|
|
127
|
+
|
|
128
|
+
asks = []
|
|
129
|
+
skipped = Hash.new(0)
|
|
130
|
+
malformed = 0
|
|
131
|
+
if Dir.exist?(sessions_dir)
|
|
132
|
+
Dir.children(sessions_dir).sort.each do |name|
|
|
133
|
+
path = File.join(sessions_dir, name, Bridge::CardStore::FILE)
|
|
134
|
+
next unless File.file?(path)
|
|
135
|
+
|
|
136
|
+
entries = sidecar(path)
|
|
137
|
+
if entries.nil?
|
|
138
|
+
malformed += 1
|
|
139
|
+
next
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
entries.each do |entry|
|
|
143
|
+
ask = ask_of(entry, name, replayed, sessions_dir)
|
|
144
|
+
next if ask.nil?
|
|
145
|
+
|
|
146
|
+
ask.is_a?(Symbol) ? skipped[ask] += 1 : asks << ask
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
stats = asks.map { |ask| judge(ask, client, log, threshold, sessions_dir) }
|
|
152
|
+
print_table(stats, threshold, client, skipped, malformed)
|
|
153
|
+
CLI::Exit::OK
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The asks of one session's cards file, or nil when it isn't one: a
|
|
157
|
+
# sidecar that doesn't parse costs one warning line, never the run.
|
|
158
|
+
def sidecar(path)
|
|
159
|
+
return nil unless File.file?(path)
|
|
160
|
+
|
|
161
|
+
data = JSON.parse(File.read(path), symbolize_names: true)
|
|
162
|
+
return nil unless data.is_a?(Hash) && data[:entries].is_a?(Array)
|
|
163
|
+
|
|
164
|
+
data[:entries]
|
|
165
|
+
rescue StandardError => e
|
|
166
|
+
error_line("chi guardrails replay: #{path}: unreadable cards (#{e.class}: #{e.message}); skipped")
|
|
167
|
+
nil
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# One card entry as an ask to judge: an Ask, a skip reason (:already_replayed,
|
|
171
|
+
# :answered_by_parent, :parent_cannot_allow, :unanswered, :no_approval,
|
|
172
|
+
# :no_question_id), or nil for an entry that is not a question at all.
|
|
173
|
+
def ask_of(entry, session_id, replayed, sessions_dir)
|
|
174
|
+
return nil unless entry.is_a?(Hash) && entry[:type] == "question"
|
|
175
|
+
|
|
176
|
+
pending = entry[:pending_question]
|
|
177
|
+
return :no_approval unless pending.is_a?(Hash) && pending[:approval].is_a?(Hash)
|
|
178
|
+
|
|
179
|
+
question_id = pending[:id].to_s
|
|
180
|
+
return :no_question_id if question_id.empty?
|
|
181
|
+
return :already_replayed if replayed.include?(question_id)
|
|
182
|
+
|
|
183
|
+
answer = entry[:answer]
|
|
184
|
+
label = if entry[:cancelled]
|
|
185
|
+
"none"
|
|
186
|
+
elsif answer.nil?
|
|
187
|
+
return :unanswered
|
|
188
|
+
elsif answer[:by] == "parent_agent"
|
|
189
|
+
# A parent's allow is not the user's decision. Its deny is a
|
|
190
|
+
# label only where the parent could have allowed: a parent
|
|
191
|
+
# that was not allowed to permit says nothing about the user
|
|
192
|
+
# (docs/guardrails.md, "System 1").
|
|
193
|
+
return :answered_by_parent unless label_for(answer, pending[:approval][:scopes]) == "deny"
|
|
194
|
+
return :parent_cannot_allow unless parent_may_permit?(pending, session_id, sessions_dir)
|
|
195
|
+
|
|
196
|
+
"deny"
|
|
197
|
+
else
|
|
198
|
+
label_for(answer, pending[:approval][:scopes])
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
Ask.new(question_id: question_id, session_id: session_id, approval: pending[:approval],
|
|
202
|
+
label: label, asked_at: pending[:created_at])
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Whether this session's parent could have allowed the ask: not a
|
|
206
|
+
# user-only ask, and a parent level that permits one. No session file:
|
|
207
|
+
# not permitted.
|
|
208
|
+
def parent_may_permit?(pending, session_id, sessions_dir)
|
|
209
|
+
return false if pending[:user_only] == true
|
|
210
|
+
|
|
211
|
+
session = session_of(session_id, sessions_dir)
|
|
212
|
+
return false if session.nil?
|
|
213
|
+
|
|
214
|
+
Guardrails::ParentPolicy.for_session(session).allow?("permit")
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# The session a card belongs to, or nil when there is no session file;
|
|
218
|
+
# read once per session for the run.
|
|
219
|
+
def session_of(session_id, sessions_dir)
|
|
220
|
+
return nil unless Session.valid_id?(session_id)
|
|
221
|
+
return nil unless File.exist?(Session.session_file(session_id, state_dir: sessions_dir))
|
|
222
|
+
|
|
223
|
+
(@sessions ||= {})[session_id] ||= begin
|
|
224
|
+
Session.load(session_id, state_dir: sessions_dir)
|
|
225
|
+
rescue StandardError
|
|
226
|
+
nil
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# What the user did, as the log's label: the scope they took, a denial
|
|
231
|
+
# when the answer is not one of the offered scopes, none for an ask that
|
|
232
|
+
# was cancelled (docs/guardrails.md, "System 1").
|
|
233
|
+
def label_for(answer, scopes)
|
|
234
|
+
index = answer[:selected_indices]&.first
|
|
235
|
+
return "deny" if index.nil?
|
|
236
|
+
|
|
237
|
+
scopes = Array(scopes)
|
|
238
|
+
if index < scopes.size
|
|
239
|
+
scopes[index] == "once" ? "allow_once" : "allow_wide"
|
|
240
|
+
else
|
|
241
|
+
"deny"
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Judge one ask, write its two lines unless this is a dry run, and return
|
|
246
|
+
# what the table counts. A judgment that never came is written with p
|
|
247
|
+
# null, exactly as the live audit writes it.
|
|
248
|
+
def judge(ask, client, log, threshold, sessions_dir)
|
|
249
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
250
|
+
answers = begin
|
|
251
|
+
client.ask(state: SystemOne::ApprovalQuestion.state(ask.approval, session_title: session_title(ask, sessions_dir)),
|
|
252
|
+
questions: { SystemOne::ApprovalQuestion::ID => SystemOne::ApprovalQuestion.question })
|
|
253
|
+
rescue StandardError => e
|
|
254
|
+
Samagotchi::Log.warn(:guardrails, "system_one_replay_ask_failed", error: "#{e.class}: #{e.message}")
|
|
255
|
+
nil
|
|
256
|
+
end
|
|
257
|
+
latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
|
|
258
|
+
p = answers && answers[SystemOne::ApprovalQuestion::ID]&.noul
|
|
259
|
+
|
|
260
|
+
unless @dry_run
|
|
261
|
+
id = log.judgment(session_id: ask.session_id, rule: ask.approval[:rule], source: ask.approval[:source],
|
|
262
|
+
tool: ask.approval[:tool], **SystemOne::ApprovalQuestion.call_fields(ask.approval),
|
|
263
|
+
repo_name: ask.approval[:repo_name], p: p, threshold: threshold, model: client.model,
|
|
264
|
+
latency_ms: latency_ms, decision: "replay", origin: "replay",
|
|
265
|
+
question_id: ask.question_id, asked_at: ask.asked_at)
|
|
266
|
+
log.label(id, ask.label)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
{ rule: ask.approval[:rule].to_s, label: ask.label, p: p, session: ask.session_id }
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# The session's title, as the ask's state carries it; nil when there is
|
|
273
|
+
# no session file (the cards alone are enough to replay).
|
|
274
|
+
def session_title(ask, sessions_dir)
|
|
275
|
+
session_of(ask.session_id, sessions_dir)&.display_title
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# The threshold for this run: --threshold, or guardrails.system_one.threshold.
|
|
279
|
+
# @return [Float, String] the number, or the usage error's message
|
|
280
|
+
def threshold(value)
|
|
281
|
+
return Config.get("guardrails.system_one.threshold").to_f if value.nil?
|
|
282
|
+
|
|
283
|
+
number = Float(value.to_s.strip)
|
|
284
|
+
return "--threshold takes a number between 0 and 1, got #{value}" if number.negative? || number > 1.0
|
|
285
|
+
|
|
286
|
+
number
|
|
287
|
+
rescue ArgumentError, TypeError
|
|
288
|
+
"--threshold takes a number between 0 and 1, got #{value}"
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def print_table(stats, threshold, client, skipped, malformed)
|
|
292
|
+
rows = stats.group_by { |row| row[:rule] }.map { |rule, asks| rule_row(rule, asks, threshold) }
|
|
293
|
+
rows.sort_by! { |rule, count, _| [-count, rule] }
|
|
294
|
+
totals = ["total", *COLUMNS[1..].each_with_index.map { |_column, offset| rows.sum { |row| row[offset + 1] } }]
|
|
295
|
+
all = rows + [totals]
|
|
296
|
+
widths = COLUMNS.each_with_index.map { |column, index| [column.size, all.map { |row| row[index].to_s.size }.max].max }
|
|
297
|
+
|
|
298
|
+
@stdout.puts(row_line(COLUMNS, widths))
|
|
299
|
+
@stdout.puts(widths.map { |width| "-" * width }.join(" "))
|
|
300
|
+
rows.each { |row| @stdout.puts(row_line(row, widths)) }
|
|
301
|
+
@stdout.puts(row_line(totals, widths))
|
|
302
|
+
|
|
303
|
+
sessions = stats.map { |row| row[:session] }.uniq.size
|
|
304
|
+
@stdout.puts("#{stats.size} asks replayed from #{sessions} session#{"s" unless sessions == 1} " \
|
|
305
|
+
"· threshold #{threshold} · model #{client.model || "the host's default"}")
|
|
306
|
+
notes = skipped.map { |reason, count| "#{count} #{SKIP_WORDS[reason] || reason}" }
|
|
307
|
+
notes << "#{malformed} unreadable cards file#{"s" unless malformed == 1}" if malformed.positive?
|
|
308
|
+
@stdout.puts("skipped: #{notes.sort.join(", ")}") unless notes.empty?
|
|
309
|
+
@stdout.puts("dry run: nothing was written") if @dry_run
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# One table row, columns left-aligned to their width.
|
|
313
|
+
def row_line(values, widths)
|
|
314
|
+
values.each_with_index.map { |value, index| value.to_s.ljust(widths[index]) }.join(" ").rstrip
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# One rule's row: asks, allowed, denied, auto-allowed at the threshold,
|
|
318
|
+
# false allows, asks with no judgment.
|
|
319
|
+
def rule_row(rule, rows, threshold)
|
|
320
|
+
allowed = rows.count { |row| ALLOW_LABELS.include?(row[:label]) }
|
|
321
|
+
denied = rows.count { |row| row[:label] == "deny" }
|
|
322
|
+
high = rows.select { |row| row[:p] && row[:p] >= threshold }
|
|
323
|
+
[rule, rows.size, allowed, denied,
|
|
324
|
+
high.count { |row| ALLOW_LABELS.include?(row[:label]) },
|
|
325
|
+
high.count { |row| row[:label] == "deny" },
|
|
326
|
+
rows.count { |row| row[:p].nil? }]
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "log"
|
|
4
|
+
require_relative "session_manager"
|
|
5
|
+
require_relative "tools/task_runtime"
|
|
6
|
+
|
|
7
|
+
module Samagotchi
|
|
8
|
+
# Stops what a session's user can stop themselves when they ask its worker
|
|
9
|
+
# to exit: the reminders its Engine holds, the background tasks it watches
|
|
10
|
+
# for a finish wake that still run, and its delegate children with a live
|
|
11
|
+
# worker (docs/sessions.md, "Worker idle exit"). Called by the exit ask's
|
|
12
|
+
# "stop them and exit" (Worker#exit_request with stop: true), and the same
|
|
13
|
+
# step the web's Stop will use.
|
|
14
|
+
#
|
|
15
|
+
# What the user can't stop from here (a turn running, prompts queued, a
|
|
16
|
+
# continue offer, another UI attached) is never touched: WorkerIdleExit
|
|
17
|
+
# refuses the exit for those before this is called.
|
|
18
|
+
class HoldStop
|
|
19
|
+
# What was stopped, by kind: reminder names, task commands, children's
|
|
20
|
+
# short ids. The 200's +stopped+ is this as JSON.
|
|
21
|
+
Stopped = Data.define(:reminders, :tasks, :delegates) do
|
|
22
|
+
def initialize(reminders: [], tasks: [], delegates: []) = super
|
|
23
|
+
|
|
24
|
+
def any? = reminders.any? || tasks.any? || delegates.any?
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# @param engine [Engine] #reminder_store
|
|
28
|
+
# @param task_watch [TaskWatch, nil] this session's finish-wake watch
|
|
29
|
+
# (nil with session.task_wakes: off)
|
|
30
|
+
# @param delegates [#call] the children to stop (ChildrenStatus::Child)
|
|
31
|
+
# @param state_dir [String] where the child sessions live
|
|
32
|
+
def initialize(engine:, task_watch:, delegates:, state_dir:)
|
|
33
|
+
@engine = engine
|
|
34
|
+
@task_watch = task_watch
|
|
35
|
+
@delegates = delegates
|
|
36
|
+
@state_dir = state_dir
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Stop every hold the user can stop, read again as it is done (one may
|
|
40
|
+
# have ended in the meantime). A stop that fails is logged and left out
|
|
41
|
+
# of the report.
|
|
42
|
+
# @return [Stopped]
|
|
43
|
+
def stop_all
|
|
44
|
+
Stopped.new(reminders: stop_reminders, tasks: stop_tasks, delegates: stop_delegates)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
# Reminders live only in this process, so clearing the store ends them;
|
|
50
|
+
# the names go to the report before it is emptied.
|
|
51
|
+
def stop_reminders
|
|
52
|
+
store = @engine.reminder_store
|
|
53
|
+
return [] unless store
|
|
54
|
+
|
|
55
|
+
names = store.reminders.values.map { |reminder| reminder[:name] }
|
|
56
|
+
store.clear_all
|
|
57
|
+
names
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The watched tasks still running, stopped the way task_stop and the
|
|
61
|
+
# web's stop-task button do (TaskRuntime.stop_task, by: "user").
|
|
62
|
+
def stop_tasks
|
|
63
|
+
return [] unless @task_watch
|
|
64
|
+
|
|
65
|
+
@task_watch.running.filter_map do |task|
|
|
66
|
+
record, error = Tools::TaskRuntime.stop_task(task.id, by: "user")
|
|
67
|
+
if record.nil?
|
|
68
|
+
Log.warn(:worker, "exit_stop_task_failed", task_id: task.id, error: error.to_s)
|
|
69
|
+
next nil
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
task.command
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# The delegate children, stopped as `chi sessions stop` stops a session
|
|
77
|
+
# (SessionManager.stop_session: its turn canceled, its worker TERMed).
|
|
78
|
+
# One owned by another terminal is left running and said so in the log.
|
|
79
|
+
def stop_delegates
|
|
80
|
+
@delegates.call.filter_map do |child|
|
|
81
|
+
SessionManager.stop_session(child.id, state_dir: @state_dir)
|
|
82
|
+
child.short_id
|
|
83
|
+
rescue StandardError => e
|
|
84
|
+
Log.warn(:worker, "exit_stop_delegate_failed", child: child.short_id, error: e.class.name, msg: e.message)
|
|
85
|
+
nil
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -88,6 +88,19 @@ module Samagotchi
|
|
|
88
88
|
end
|
|
89
89
|
end
|
|
90
90
|
|
|
91
|
+
# +entry_name+'s description in +scope+'s index.md — the text after the
|
|
92
|
+
# " — " on its managed line — or nil when the index has no line for it
|
|
93
|
+
# or that line carries no description. One small file read.
|
|
94
|
+
def self.description_for(scope, entry_name)
|
|
95
|
+
index_path = index_path_for(scope)
|
|
96
|
+
return nil unless index_path && File.exist?(index_path)
|
|
97
|
+
|
|
98
|
+
match = File.read(index_path, encoding: "UTF-8").match(managed_pattern(entry_name))
|
|
99
|
+
match ? extract_description(match[0].chomp) : nil
|
|
100
|
+
rescue StandardError
|
|
101
|
+
nil
|
|
102
|
+
end
|
|
103
|
+
|
|
91
104
|
# The bundle a managed line names ("· from <bundle>", before any
|
|
92
105
|
# " — description"), or nil.
|
|
93
106
|
def self.extract_source(line)
|
|
@@ -29,6 +29,10 @@ module Samagotchi
|
|
|
29
29
|
# restores the state. When /slots can't say, the warm-up runs.
|
|
30
30
|
class PromptWarmup
|
|
31
31
|
MODES = %w[auto off].freeze
|
|
32
|
+
# The words cache.warmup takes, in the shape YAML hands them: an unquoted
|
|
33
|
+
# on/off arrives as true/false.
|
|
34
|
+
AUTO_WORDS = %w[auto on true 1].freeze
|
|
35
|
+
OFF_WORDS = %w[off false no 0].freeze
|
|
32
36
|
# Seconds #take_pin waits for the busy-slot check (Client#slot_status
|
|
33
37
|
# gives up within 3 s).
|
|
34
38
|
DECIDE_TIMEOUT = 3
|
|
@@ -36,12 +40,13 @@ module Samagotchi
|
|
|
36
40
|
STOP_JOIN_SECONDS = 1
|
|
37
41
|
|
|
38
42
|
# Whether cache.warmup allows a warm-up (auto, the default; off, false,
|
|
39
|
-
# no or 0 turn it off;
|
|
43
|
+
# no or 0 turn it off; on/true/1 (YAML's unquoted on) are auto; anything
|
|
44
|
+
# else warns once and is auto).
|
|
40
45
|
def self.enabled?
|
|
41
46
|
value = Config.get("cache.warmup").to_s.strip.downcase
|
|
42
|
-
return false if
|
|
47
|
+
return false if OFF_WORDS.include?(value)
|
|
43
48
|
|
|
44
|
-
unless value.empty? || value
|
|
49
|
+
unless value.empty? || AUTO_WORDS.include?(value) || @warned
|
|
45
50
|
@warned = true
|
|
46
51
|
Log.warn(:config, "invalid_value", echo: "Warning: invalid value for cache.warmup: #{value.inspect} (allowed: auto, off) — using auto",
|
|
47
52
|
key: "cache.warmup")
|
|
@@ -26,7 +26,7 @@ module Samagotchi
|
|
|
26
26
|
# bin/chi subcommands that act on this machine: never sessions. chi
|
|
27
27
|
# remote (this machine's list of remotes) runs here under a remote too
|
|
28
28
|
# (Dispatch); the others are refused there.
|
|
29
|
-
MACHINE = %w[bootstrap update models desktop memory bundle remote].freeze
|
|
29
|
+
MACHINE = %w[bootstrap update models desktop memory bundle guardrails remote].freeze
|
|
30
30
|
|
|
31
31
|
# "command", "command sub" or either with a flag (its long name) →
|
|
32
32
|
# SUPPORTED or LOCAL_ONLY. A LOCAL_ONLY command or subcommand lists
|
|
@@ -139,12 +139,17 @@ module Samagotchi
|
|
|
139
139
|
end
|
|
140
140
|
|
|
141
141
|
# POST …/exit: the worker asked to exit (chi web passes client_id on
|
|
142
|
-
# only when it is an attached TUI's)
|
|
143
|
-
#
|
|
144
|
-
#
|
|
145
|
-
|
|
142
|
+
# only when it is an attached TUI's), stop: true first stopping what
|
|
143
|
+
# the user can stop, ask: false leaving the delegate children out of
|
|
144
|
+
# the list (a UI that can't ask about them).
|
|
145
|
+
# @return [BridgeClient::Response] 200 {status: exiting, discard,
|
|
146
|
+
# stopped}, 409 held {reason, tasks, holds}, 408 late, 404 an older
|
|
147
|
+
# worker's
|
|
148
|
+
def request_exit(client_id:, delete: false, stop: false, ask: true)
|
|
146
149
|
body = { client_id: client_id }
|
|
147
150
|
body[:delete] = true if delete
|
|
151
|
+
body[:stop] = "all" if stop
|
|
152
|
+
body[:ask] = false unless ask
|
|
148
153
|
post(:exit, "exit", body)
|
|
149
154
|
end
|
|
150
155
|
|
|
@@ -19,6 +19,8 @@ require_relative "reasoning_history_command"
|
|
|
19
19
|
require_relative "working_set_command"
|
|
20
20
|
require_relative "recap_command"
|
|
21
21
|
require_relative "guardrails/parent_policy"
|
|
22
|
+
require_relative "system_one/log"
|
|
23
|
+
require_relative "system_one/record"
|
|
22
24
|
|
|
23
25
|
module Samagotchi
|
|
24
26
|
# The session commands a REPL and a session worker both run: /model,
|
|
@@ -91,7 +93,7 @@ module Samagotchi
|
|
|
91
93
|
registry.register(MODELS_COMMAND, "list the hosts' models (/models <text> filters)", anytime: true) do |text|
|
|
92
94
|
reply(models_listing(text.delete_prefix(MODELS_COMMAND).strip))
|
|
93
95
|
end
|
|
94
|
-
registry.register(GUARDRAILS_COMMAND, "list
|
|
96
|
+
registry.register(GUARDRAILS_COMMAND, "list guardrails, revoke an approval or flag a System 1 auto-allow",
|
|
95
97
|
mid_turn: show_mid_turn(GUARDRAILS_COMMAND)) do |text|
|
|
96
98
|
guardrails(text.delete_prefix(GUARDRAILS_COMMAND).strip)
|
|
97
99
|
end
|
|
@@ -418,11 +420,85 @@ module Samagotchi
|
|
|
418
420
|
|
|
419
421
|
def guardrails(args)
|
|
420
422
|
return guardrails_revoke(args.delete_prefix("revoke").strip) if args.start_with?("revoke")
|
|
421
|
-
return
|
|
423
|
+
return guardrails_wrong(args.sub(/\Awrong\b/, "").strip) if args.match?(/\Awrong(?:\s|\z)/)
|
|
424
|
+
return reply("usage: /guardrails [revoke N | wrong [N]]", status: :error) unless args.empty?
|
|
422
425
|
|
|
423
426
|
reply(guardrails_listing)
|
|
424
427
|
end
|
|
425
428
|
|
|
429
|
+
def guardrails_wrong(arg)
|
|
430
|
+
unless arg.empty? || arg.match?(/\A[1-9]\d*\z/)
|
|
431
|
+
return reply("usage: /guardrails wrong [N] (N from /guardrails)", status: :error)
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
index = arg.empty? ? 1 : arg.to_i
|
|
435
|
+
entry = system_one_auto_allows[index - 1]
|
|
436
|
+
return reply("no unlabelled System 1 auto-allows in this session (see /guardrails)") unless entry
|
|
437
|
+
unless system_one_log.label(entry["id"], "should_have_asked")
|
|
438
|
+
return reply("could not label System 1 auto-allow #{index} (see the guardrails log)", status: :error)
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
call = shorten_call(entry["call"].to_s.gsub(/\s+/, " "))
|
|
442
|
+
probability = entry["p"].nil? ? "unknown" : format("%.2f", entry["p"].to_f)
|
|
443
|
+
reply("labelled auto-allow #{index}: #{entry["rule"]}: #{call} (p=#{probability}); " \
|
|
444
|
+
"the rule will ask again until its record is rebuilt")
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
def system_one_log
|
|
448
|
+
SystemOne::Log.new(dir: File.dirname(@engine.guardrail_approvals.path))
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
def system_one_auto_allows(entries = system_one_entries)
|
|
452
|
+
session_id = @engine.session&.id
|
|
453
|
+
return [] unless session_id
|
|
454
|
+
|
|
455
|
+
entries.select do |entry|
|
|
456
|
+
entry["session_id"] == session_id && entry["decision"] == "auto" && entry["answer"].nil?
|
|
457
|
+
end.last(20).reverse
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
def system_one_entries
|
|
461
|
+
log = system_one_log
|
|
462
|
+
File.file?(log.path) ? log.entries : []
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
def system_one_listing(lines)
|
|
466
|
+
entries = system_one_entries
|
|
467
|
+
mode = Config.get("guardrails.system_one.mode").to_s
|
|
468
|
+
return if mode == "off" && entries.empty?
|
|
469
|
+
|
|
470
|
+
lines << "System 1 (#{mode}):"
|
|
471
|
+
auto_allows = system_one_auto_allows(entries)
|
|
472
|
+
lines << " auto-allows (this session, newest first; /guardrails wrong N):"
|
|
473
|
+
if auto_allows.empty?
|
|
474
|
+
lines << " (none)"
|
|
475
|
+
else
|
|
476
|
+
auto_allows.each_with_index do |entry, index|
|
|
477
|
+
call = shorten_call(entry["call"].to_s.gsub(/\s+/, " "))
|
|
478
|
+
probability = entry["p"].nil? ? "unknown" : format("%.2f", entry["p"].to_f)
|
|
479
|
+
lines << " #{index + 1}. #{entry["rule"]}: #{call} (p=#{probability})"
|
|
480
|
+
end
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
rules = entries.filter_map { |entry| entry["rule"] }.uniq.sort
|
|
484
|
+
lines << " records:"
|
|
485
|
+
if rules.empty?
|
|
486
|
+
lines << " (none)"
|
|
487
|
+
else
|
|
488
|
+
rules.each do |rule|
|
|
489
|
+
summary = SystemOne::Record.new(entries, rule: rule).summary
|
|
490
|
+
counts = summary.reject { |key, _| key == :rule }.map { |key, value| "#{key}=#{value}" }.join(", ")
|
|
491
|
+
lines << " #{rule}: #{counts}"
|
|
492
|
+
end
|
|
493
|
+
end
|
|
494
|
+
end
|
|
495
|
+
|
|
496
|
+
CALL_SHOWN = 100
|
|
497
|
+
|
|
498
|
+
def shorten_call(call)
|
|
499
|
+
call.length > CALL_SHOWN ? "#{call[0, CALL_SHOWN - 1]}…" : call
|
|
500
|
+
end
|
|
501
|
+
|
|
426
502
|
def guardrails_listing
|
|
427
503
|
rules = @engine.guardrail_rules
|
|
428
504
|
approvals = @engine.guardrail_approvals.entries
|
|
@@ -467,6 +543,7 @@ module Samagotchi
|
|
|
467
543
|
lines << " (none)" if approvals.empty?
|
|
468
544
|
approvals.each_with_index { |entry, idx| lines << " #{idx + 1}. #{approval_line(entry)}" }
|
|
469
545
|
lines << "revoke one with /guardrails revoke N" unless approvals.empty?
|
|
546
|
+
system_one_listing(lines)
|
|
470
547
|
lines.join("\n")
|
|
471
548
|
end
|
|
472
549
|
|