samagotchi 0.114.0 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +72 -1
- data/bin/chi +2 -0
- data/docs/cli.md +11 -3
- data/docs/configuration.md +7 -0
- data/docs/guardrails.md +188 -1
- data/docs/internals/web-push.md +3 -2
- data/docs/sessions.md +2 -2
- data/lib/samagotchi/bridge/turn_accumulator.rb +4 -0
- data/lib/samagotchi/bridge.rb +26 -0
- data/lib/samagotchi/bridge_client.rb +7 -0
- data/lib/samagotchi/bundles/system/manifest.yml +1 -1
- data/lib/samagotchi/config.rb +15 -0
- data/lib/samagotchi/engine.rb +6 -1
- data/lib/samagotchi/guardrail_wiring.rb +166 -2
- data/lib/samagotchi/guardrails/approval.rb +20 -0
- data/lib/samagotchi/guardrails/targets.rb +23 -2
- data/lib/samagotchi/guardrails/verdict.rb +23 -3
- data/lib/samagotchi/guardrails_command.rb +329 -0
- data/lib/samagotchi/memory_bundle/index_updater.rb +13 -0
- data/lib/samagotchi/session_access/contract.rb +1 -1
- data/lib/samagotchi/session_commands.rb +79 -2
- data/lib/samagotchi/session_metrics.rb +7 -1
- data/lib/samagotchi/system_one/answer.rb +78 -0
- data/lib/samagotchi/system_one/approval_question.rb +113 -0
- data/lib/samagotchi/system_one/client.rb +89 -0
- data/lib/samagotchi/system_one/errors.rb +14 -0
- data/lib/samagotchi/system_one/fake_backend.rb +45 -0
- data/lib/samagotchi/system_one/http_backend.rb +129 -0
- data/lib/samagotchi/system_one/log.rb +133 -0
- data/lib/samagotchi/system_one/question.rb +86 -0
- data/lib/samagotchi/system_one/record.rb +83 -0
- data/lib/samagotchi/system_one/self_protect.rb +90 -0
- data/lib/samagotchi/system_one/similar.rb +69 -0
- data/lib/samagotchi/system_one.rb +25 -0
- data/lib/samagotchi/terminal_ui/attached_loop.rb +15 -4
- data/lib/samagotchi/terminal_ui/event_renderer.rb +1 -0
- data/lib/samagotchi/terminal_ui/formatting.rb +14 -0
- data/lib/samagotchi/title_command.rb +13 -4
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +1 -0
- data/lib/samagotchi/web/public/activity.js +10 -0
- data/lib/samagotchi/web/public/annotate_bar.js +22 -2
- data/lib/samagotchi/web/public/app.js +4 -0
- data/lib/samagotchi/web/public/apple-touch-icon.png +0 -0
- data/lib/samagotchi/web/public/composer_fold.js +8 -8
- data/lib/samagotchi/web/public/composer_grip.js +65 -22
- data/lib/samagotchi/web/public/composer_size.js +30 -2
- data/lib/samagotchi/web/public/data.js +10 -0
- data/lib/samagotchi/web/public/favicon.svg +1 -1
- data/lib/samagotchi/web/public/icons/chi-180.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-192.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-512.png +0 -0
- data/lib/samagotchi/web/public/index.html +7 -0
- data/lib/samagotchi/web/public/notify.js +4 -2
- data/lib/samagotchi/web/public/system_one_wrong.js +32 -0
- data/lib/samagotchi/web/public/timing.js +9 -5
- data/lib/samagotchi/web/public/turn_events.js +1 -0
- data/lib/samagotchi/web/public/turn_html.js +13 -2
- data/lib/samagotchi/web/push/attention.rb +3 -1
- data/lib/samagotchi/web/session_control.rb +22 -0
- data/lib/samagotchi/web/session_show.rb +35 -5
- metadata +15 -1
|
@@ -6,10 +6,13 @@ require_relative "config"
|
|
|
6
6
|
require_relative "log"
|
|
7
7
|
require_relative "guardrails"
|
|
8
8
|
require_relative "guardrails/parent_approvals"
|
|
9
|
+
require_relative "guardrails/parent_policy"
|
|
10
|
+
require_relative "model_match"
|
|
9
11
|
require_relative "memory_bundle/manifest"
|
|
10
12
|
require_relative "version"
|
|
11
13
|
require_relative "hooks"
|
|
12
14
|
require_relative "edit_preview"
|
|
15
|
+
require_relative "system_one"
|
|
13
16
|
require_relative "tool_activity"
|
|
14
17
|
require_relative "turn_origin"
|
|
15
18
|
|
|
@@ -38,8 +41,11 @@ module Samagotchi
|
|
|
38
41
|
# @param ask [#call] (fields) → the answer; the Engine's question flow
|
|
39
42
|
# @param notify [#call] (message) → shows a guardrails line in every UI
|
|
40
43
|
# (a bundle's rules kept because it needs a newer chi)
|
|
44
|
+
# @param system_one [#call] → SystemOne::Client or nil: the judgment client
|
|
45
|
+
# (nil while `guardrails.system_one.mode` is off or no host is set)
|
|
46
|
+
# @param cancel_controller [#call] → the running turn's CancellationController, nil
|
|
41
47
|
def initialize(scratch:, hooks:, tools:, session:, model_key:, cancelled:, ask:, model_name: -> {},
|
|
42
|
-
notify: ->(_message) {})
|
|
48
|
+
notify: ->(_message) {}, system_one: -> {}, cancel_controller: -> {})
|
|
43
49
|
@scratch = scratch
|
|
44
50
|
@hooks_lookup = hooks
|
|
45
51
|
@tools_lookup = tools
|
|
@@ -49,6 +55,8 @@ module Samagotchi
|
|
|
49
55
|
@cancelled_lookup = cancelled
|
|
50
56
|
@ask = ask
|
|
51
57
|
@notify = notify
|
|
58
|
+
@system_one = system_one
|
|
59
|
+
@cancel_controller = cancel_controller
|
|
52
60
|
# Bundle name → the rules last loaded from it whole, kept while the
|
|
53
61
|
# installed version needs a newer chi; [name, version] said so once.
|
|
54
62
|
@loaded_bundle_rules = {}
|
|
@@ -82,6 +90,7 @@ module Samagotchi
|
|
|
82
90
|
def state_dir=(state_dir)
|
|
83
91
|
@approvals = Guardrails::Approvals.new(dir: Guardrails::Approvals.dir_for(state_dir))
|
|
84
92
|
@approvals = Guardrails::Approvals::InMemory.new(@approvals) if @scratch
|
|
93
|
+
@system_one_log = nil
|
|
85
94
|
@protected = nil
|
|
86
95
|
end
|
|
87
96
|
|
|
@@ -259,11 +268,166 @@ module Samagotchi
|
|
|
259
268
|
# A call on chi's own config, hooks or guardrails is the user's to
|
|
260
269
|
# allow, never a parent agent's (Guardrails::ParentPolicy).
|
|
261
270
|
payload[:user_only] = Guardrails::ParentApprovals.protected_facts?(payload[:approval])
|
|
262
|
-
|
|
271
|
+
# System 1: the ask is judged before the user sees it, and in `on` mode
|
|
272
|
+
# one that passes every gate is allowed once without reaching them
|
|
273
|
+
# (docs/guardrails.md, "System 1").
|
|
274
|
+
judgment = system_one_judge(payload)
|
|
275
|
+
if judgment && judgment[:auto]
|
|
276
|
+
verdict.judgment = { id: judgment[:id], p: judgment[:p] }
|
|
277
|
+
verdict.settle!(:allow, decided_by: "system one")
|
|
278
|
+
verdict.scope = "once"
|
|
279
|
+
return verdict
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
answer = @ask.call(payload)
|
|
283
|
+
system_one_label(judgment, answer, payload)
|
|
284
|
+
Guardrails::Approval.settle(verdict, answer, payload[:approval][:scopes])
|
|
263
285
|
end
|
|
264
286
|
|
|
265
287
|
private
|
|
266
288
|
|
|
289
|
+
# The judgments log (SystemOne::Log), beside the approvals store; nil
|
|
290
|
+
# while there is no store dir.
|
|
291
|
+
def system_one_log
|
|
292
|
+
return nil if @approvals.nil?
|
|
293
|
+
|
|
294
|
+
@system_one_log ||= SystemOne::Log.new(dir: File.dirname(@approvals.path))
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
# System 1's judgment of an ask, written to the log before the user is
|
|
298
|
+
# asked, with the decision it led to: "auto" (it answered for the user),
|
|
299
|
+
# "asked" (on mode, the user decides) or "audit" (audit mode). A
|
|
300
|
+
# participant's review turn is never judged, and no client (mode off, no
|
|
301
|
+
# host) means no line at all. A judgment that fails is logged with p nil,
|
|
302
|
+
# so the failures are in the log too.
|
|
303
|
+
# @return [Hash, nil] {id:, p:, auto:}, nil when nothing was written
|
|
304
|
+
def system_one_judge(payload)
|
|
305
|
+
return nil if Steer.participant_author?(turn_author)
|
|
306
|
+
|
|
307
|
+
client = @system_one.call
|
|
308
|
+
log = system_one_log
|
|
309
|
+
return nil if client.nil? || log.nil?
|
|
310
|
+
|
|
311
|
+
approval = payload[:approval]
|
|
312
|
+
# The judgments log is read once for this ask: the state's examples and
|
|
313
|
+
# the record gate both use these entries. The judgment of the ask being
|
|
314
|
+
# judged is written after it, so the examples are always earlier asks.
|
|
315
|
+
entries = log.entries
|
|
316
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
317
|
+
answers = begin
|
|
318
|
+
client.ask(state: SystemOne::ApprovalQuestion.state(approval, session_title: @session_lookup.call&.title,
|
|
319
|
+
entries: entries),
|
|
320
|
+
questions: { SystemOne::ApprovalQuestion::ID => SystemOne::ApprovalQuestion.question },
|
|
321
|
+
cancel_controller: @cancel_controller.call)
|
|
322
|
+
rescue StandardError => e
|
|
323
|
+
# The client already turns every failure it knows into a nil; this is
|
|
324
|
+
# the unexpected one. Either way the ask goes to the user.
|
|
325
|
+
Samagotchi::Log.warn(:guardrails, "system_one_ask_failed", error: "#{e.class}: #{e.message}")
|
|
326
|
+
nil
|
|
327
|
+
end
|
|
328
|
+
latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
|
|
329
|
+
p = answers && answers[SystemOne::ApprovalQuestion::ID]&.noul
|
|
330
|
+
auto = system_one_auto_allow?(client, approval, p, entries)
|
|
331
|
+
decision = system_one_decision(client, auto)
|
|
332
|
+
|
|
333
|
+
id = log.judgment(session_id: @session_lookup.call&.id,
|
|
334
|
+
rule: approval[:rule], source: approval[:source], tool: approval[:tool],
|
|
335
|
+
**SystemOne::ApprovalQuestion.call_fields(approval),
|
|
336
|
+
repo_name: approval[:repo_name],
|
|
337
|
+
p: p, threshold: Config.get("guardrails.system_one.threshold"), model: client.model,
|
|
338
|
+
latency_ms: latency_ms, decision: decision, origin: "live")
|
|
339
|
+
# A judgment the log cannot hold is no judgment: the user is asked.
|
|
340
|
+
{ id: id, p: p, auto: auto && !id.nil? }
|
|
341
|
+
rescue StandardError => e
|
|
342
|
+
# The approval never fails because of System 1.
|
|
343
|
+
Samagotchi::Log.warn(:guardrails, "system_one_judgment_failed", error: "#{e.class}: #{e.message}")
|
|
344
|
+
nil
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
# What the judgment line says it did: "auto" (System 1 answered for the
|
|
348
|
+
# user), "asked" (on mode, the user decides), "audit" (nothing decided).
|
|
349
|
+
def system_one_decision(client, auto)
|
|
350
|
+
return "auto" if auto
|
|
351
|
+
|
|
352
|
+
client.mode == "on" ? "asked" : "audit"
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
# Every gate an auto-allow must pass, in order: the mode, the rule the
|
|
356
|
+
# user opted in, the never-list and chi's own files, the probability, and
|
|
357
|
+
# the rule's record. Any one that fails leaves the ask to the user. The
|
|
358
|
+
# entries are the judgment log read once for this ask.
|
|
359
|
+
# @param entries [Array<Hash>] SystemOne::Log#entries
|
|
360
|
+
def system_one_auto_allow?(client, approval, p, entries)
|
|
361
|
+
return false unless client.mode == "on"
|
|
362
|
+
|
|
363
|
+
rule = approval[:rule].to_s
|
|
364
|
+
return false if rule.empty?
|
|
365
|
+
return false unless system_one_rules.include?(rule)
|
|
366
|
+
return false if SystemOne::SelfProtect.refuse?(approval)
|
|
367
|
+
|
|
368
|
+
threshold = Config.get("guardrails.system_one.threshold").to_f
|
|
369
|
+
return false if p.nil? || p.to_f < threshold
|
|
370
|
+
|
|
371
|
+
min_record = Config.get("guardrails.system_one.min_record").to_i
|
|
372
|
+
SystemOne::Record.new(entries, rule: rule).qualifies?(threshold: threshold, min_record: min_record)
|
|
373
|
+
rescue StandardError => e
|
|
374
|
+
Samagotchi::Log.warn(:guardrails, "system_one_gate_failed", error: "#{e.class}: #{e.message}")
|
|
375
|
+
false
|
|
376
|
+
end
|
|
377
|
+
|
|
378
|
+
# guardrails.system_one.rules: a "|"-separated string or a YAML list,
|
|
379
|
+
# read with the same grammar as guardrails.small_models.
|
|
380
|
+
def system_one_rules
|
|
381
|
+
ModelMatch.parse(Config.get("guardrails.system_one.rules"))
|
|
382
|
+
rescue StandardError
|
|
383
|
+
[]
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
# The user's answer to that ask, as a label on the judgment: the scope
|
|
387
|
+
# they took, a denial, or none for an ask that was cancelled. A parent
|
|
388
|
+
# agent's answer is not the user's: its allow labels nothing, and its
|
|
389
|
+
# deny is a label only where the parent could have allowed — a parent
|
|
390
|
+
# that was not allowed to permit says nothing about what the user would
|
|
391
|
+
# have done (docs/guardrails.md, "System 1").
|
|
392
|
+
def system_one_label(judgment, answer, payload)
|
|
393
|
+
id = judgment && judgment[:id]
|
|
394
|
+
return nil if id.nil?
|
|
395
|
+
|
|
396
|
+
scopes = Array(payload[:approval][:scopes])
|
|
397
|
+
if !answer.is_a?(Samagotchi::QuestionAnswer)
|
|
398
|
+
label = "none"
|
|
399
|
+
elsif answer.parent_agent?
|
|
400
|
+
return nil if parent_allowed?(answer, scopes) || !parent_may_permit?(payload)
|
|
401
|
+
|
|
402
|
+
label = "deny"
|
|
403
|
+
else
|
|
404
|
+
index = answer.selected_indices.first
|
|
405
|
+
label = if index && index < scopes.size
|
|
406
|
+
scopes[index] == "once" ? "allow_once" : "allow_wide"
|
|
407
|
+
else
|
|
408
|
+
"deny"
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
system_one_log.label(id, label)
|
|
412
|
+
rescue StandardError => e
|
|
413
|
+
Samagotchi::Log.warn(:guardrails, "system_one_label_failed", error: "#{e.class}: #{e.message}")
|
|
414
|
+
nil
|
|
415
|
+
end
|
|
416
|
+
|
|
417
|
+
# Whether a parent's answer picked one of the offered scopes.
|
|
418
|
+
def parent_allowed?(answer, scopes)
|
|
419
|
+
index = answer.selected_indices.first
|
|
420
|
+
!index.nil? && index < scopes.size
|
|
421
|
+
end
|
|
422
|
+
|
|
423
|
+
# Whether this session's parent level lets a parent allow a call at all,
|
|
424
|
+
# and the ask is not one only the user may answer.
|
|
425
|
+
def parent_may_permit?(payload)
|
|
426
|
+
return false if payload[:user_only]
|
|
427
|
+
|
|
428
|
+
Guardrails::ParentPolicy.for_session(@session_lookup.call).allow?("permit")
|
|
429
|
+
end
|
|
430
|
+
|
|
267
431
|
# The work tree top of the session's working_directory (the folder
|
|
268
432
|
# itself outside a repo, or when it's gone); never Dir.pwd, which is
|
|
269
433
|
# the waker's cwd for a worker whose folder is gone.
|
|
@@ -51,6 +51,9 @@ module Samagotchi
|
|
|
51
51
|
paths: targets && !targets.paths.empty? ? targets.paths : nil,
|
|
52
52
|
# No command or path (an MCP tool): its arguments, as the question.
|
|
53
53
|
args: args_for_card(targets),
|
|
54
|
+
# A memory_write ask, in words: the entry, its scope, and for a
|
|
55
|
+
# removal the description its index line carries.
|
|
56
|
+
memory: memory_for_card(targets),
|
|
54
57
|
cwd: targets&.cwd,
|
|
55
58
|
repo_root: targets&.repo_root,
|
|
56
59
|
# The repository's name (a worktree's main checkout's folder).
|
|
@@ -78,6 +81,23 @@ module Samagotchi
|
|
|
78
81
|
text.empty? ? nil : text
|
|
79
82
|
end
|
|
80
83
|
|
|
84
|
+
# A memory_write ask names an entry, not a file: which one, and in which
|
|
85
|
+
# scope. For a removal the ask also carries that entry's description from
|
|
86
|
+
# its scoped index line, which is what System 1 otherwise cannot see
|
|
87
|
+
# (docs/guardrails.md, "System 1"). nil for every other tool.
|
|
88
|
+
def memory_for_card(targets)
|
|
89
|
+
entry = targets&.memory
|
|
90
|
+
return nil if entry.nil?
|
|
91
|
+
|
|
92
|
+
facts = { name: entry[:name], scope: entry[:scope] }
|
|
93
|
+
if targets.memory_remove?
|
|
94
|
+
require_relative "../memory_bundle/index_updater"
|
|
95
|
+
description = MemoryBundle::IndexUpdater.description_for(entry[:scope], entry[:name])
|
|
96
|
+
facts[:description] = description unless description.to_s.strip.empty?
|
|
97
|
+
end
|
|
98
|
+
facts
|
|
99
|
+
end
|
|
100
|
+
|
|
81
101
|
# Settle +verdict+ from open_question's result.
|
|
82
102
|
# @param answer [QuestionAnswer, Hash, String] the answer, or {error:} (anything else is a cancel)
|
|
83
103
|
# @param scopes [Array<String>] the offered scopes, in option order
|
|
@@ -74,6 +74,12 @@ module Samagotchi
|
|
|
74
74
|
attr_reader :memory_overlay
|
|
75
75
|
alias memory_overlay? memory_overlay
|
|
76
76
|
|
|
77
|
+
# @return [Hash, nil] a memory_write call's entry and its scope,
|
|
78
|
+
# {name:, scope:}: which memory the ask is about, in words rather than
|
|
79
|
+
# as the file it names (#paths); nil for another tool or a call the
|
|
80
|
+
# tool would refuse
|
|
81
|
+
attr_reader :memory
|
|
82
|
+
|
|
77
83
|
# @param call [Hash] the parsed tool call
|
|
78
84
|
# @param context [Context]
|
|
79
85
|
# @param model_key [String, nil] for a memory_write model overlay, and
|
|
@@ -114,9 +120,10 @@ module Samagotchi
|
|
|
114
120
|
end
|
|
115
121
|
memory_remove = tool == "memory_write" && call[:remove].to_s.strip.downcase == "true"
|
|
116
122
|
memory_overlay = tool == "memory_write" && !memory_remove && call[:current_model_only].to_s.strip.downcase == "true"
|
|
123
|
+
memory = tool == "memory_write" ? memory_facts(call) : nil
|
|
117
124
|
new(tool: tool, command: command, paths: paths.compact, cwd: cwd, repo_root: context.repo_root(cwd),
|
|
118
125
|
repo: context.repo(cwd), args: args, acts_as: acts_as, label: label, memory_remove: memory_remove,
|
|
119
|
-
memory_overlay: memory_overlay,
|
|
126
|
+
memory_overlay: memory_overlay, memory: memory,
|
|
120
127
|
model_name: model_name, model_key: model_key, session_root: context.repo_root(base) || base,
|
|
121
128
|
chi_dirs: -> { chi_dirs(context, base) })
|
|
122
129
|
end
|
|
@@ -224,14 +231,28 @@ module Samagotchi
|
|
|
224
231
|
nil
|
|
225
232
|
end
|
|
226
233
|
|
|
234
|
+
# The memory a memory_write call is about, in the words the ask needs:
|
|
235
|
+
# the entry and its scope, not its file; nil for a call the tool would
|
|
236
|
+
# refuse (no entry, no scope, a scope chi has no folder for).
|
|
237
|
+
def self.memory_facts(call)
|
|
238
|
+
entry = call[:path].to_s.strip
|
|
239
|
+
scope = Tools::MemoryRead.normalize_scope(call[:scope])
|
|
240
|
+
return nil if entry.empty? || scope.nil?
|
|
241
|
+
|
|
242
|
+
{ name: entry, scope: scope }
|
|
243
|
+
rescue StandardError
|
|
244
|
+
nil
|
|
245
|
+
end
|
|
246
|
+
|
|
227
247
|
# @param chi_dirs [#call, nil] → the dirs touches_chi? looks for
|
|
228
248
|
def initialize(tool:, command:, paths:, cwd:, repo_root:, repo: nil, args: nil, acts_as: nil, label: nil,
|
|
229
249
|
model_name: nil, model_key: nil, session_root: nil, chi_dirs: nil, memory_remove: false,
|
|
230
|
-
memory_overlay: false)
|
|
250
|
+
memory_overlay: false, memory: nil)
|
|
231
251
|
@acts_as = acts_as
|
|
232
252
|
@label = label
|
|
233
253
|
@memory_remove = memory_remove
|
|
234
254
|
@memory_overlay = memory_overlay
|
|
255
|
+
@memory = memory
|
|
235
256
|
@repo = repo
|
|
236
257
|
@chi_dirs = chi_dirs
|
|
237
258
|
@session_root = session_root || repo_root || cwd
|
|
@@ -19,8 +19,10 @@ module Samagotchi
|
|
|
19
19
|
# parent model would be writing a model's prompt); and the bundle's
|
|
20
20
|
# asks before answering chi's questions around chi answer (a parent
|
|
21
21
|
# allowing one would answer a grandchild's question as the user).
|
|
22
|
-
#
|
|
23
|
-
# ParentPolicy's user_only)
|
|
22
|
+
# A parent agent may never allow them (ParentApprovals.protected?,
|
|
23
|
+
# ParentPolicy's user_only); System 1 may, only if the user lists the
|
|
24
|
+
# rule by name in guardrails.system_one.rules, and never for the
|
|
25
|
+
# chi-answer/chi-broadcast ones (SystemOne::SelfProtect::NEVER_AUTO).
|
|
24
26
|
PROTECTED_RULES = %w[chi-config chi-hooks shell-touches-chi chi-context-cmd chi-broadcast prompt-memory-write
|
|
25
27
|
model-overlay-write chi-answer-piped chi-answer-http].freeze
|
|
26
28
|
SCOPES = %w[once session repo rule].freeze
|
|
@@ -33,6 +35,8 @@ module Samagotchi
|
|
|
33
35
|
attr_accessor :context, :targets
|
|
34
36
|
# The scope the user allowed an ask for ("once", "session", …).
|
|
35
37
|
attr_accessor :scope
|
|
38
|
+
# System 1's judgment that decided an ask, when it did: {id:, p:}.
|
|
39
|
+
attr_accessor :judgment
|
|
36
40
|
|
|
37
41
|
def initialize(call:)
|
|
38
42
|
@call = call
|
|
@@ -121,8 +125,11 @@ module Samagotchi
|
|
|
121
125
|
end
|
|
122
126
|
|
|
123
127
|
# For the activity entry; an allowed ask notes how ("approved (repo)",
|
|
124
|
-
# "approved earlier (session)")
|
|
128
|
+
# "approved earlier (session)"), one System 1 answered notes that and
|
|
129
|
+
# carries the judgment (docs/guardrails.md, "System 1").
|
|
125
130
|
def to_activity
|
|
131
|
+
return system_one_activity if @decided_by == "system one"
|
|
132
|
+
|
|
126
133
|
note = if allow? && @scope
|
|
127
134
|
"approved#{" earlier" if @decided_by == "approval"} (#{@scope})"
|
|
128
135
|
end
|
|
@@ -131,6 +138,19 @@ module Samagotchi
|
|
|
131
138
|
|
|
132
139
|
private
|
|
133
140
|
|
|
141
|
+
# The auto-allow's activity: who decided, for how long, and the
|
|
142
|
+
# judgment behind it (the p and the judgment's id), so a UI can show
|
|
143
|
+
# why the call was never asked about.
|
|
144
|
+
def system_one_activity
|
|
145
|
+
facts = @judgment || {}
|
|
146
|
+
p = facts[:p]
|
|
147
|
+
activity = { rule: @rule, verdict: @decision.to_s, decided_by: @decided_by, scope: @scope,
|
|
148
|
+
note: p ? "auto-allowed by System 1 (p=#{format("%.2f", p.to_f)})" : "auto-allowed by System 1" }
|
|
149
|
+
activity[:judgment_id] = facts[:id] if facts[:id]
|
|
150
|
+
activity[:p] = p if p
|
|
151
|
+
activity.compact
|
|
152
|
+
end
|
|
153
|
+
|
|
134
154
|
def decider
|
|
135
155
|
return [("rule #{@rule}" if @rule), @source].compact.join(", ") if @rule || @source
|
|
136
156
|
|
|
@@ -0,0 +1,329 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
require_relative "cli/command"
|
|
6
|
+
require_relative "cli/exit"
|
|
7
|
+
require_relative "cli/flags"
|
|
8
|
+
require_relative "config"
|
|
9
|
+
require_relative "bridge/card_store"
|
|
10
|
+
require_relative "guardrails/approvals"
|
|
11
|
+
require_relative "guardrails/parent_policy"
|
|
12
|
+
require_relative "host_registry"
|
|
13
|
+
require_relative "session"
|
|
14
|
+
require_relative "system_one"
|
|
15
|
+
|
|
16
|
+
module Samagotchi
|
|
17
|
+
# `chi guardrails`: the guardrails' own maintenance commands, on this
|
|
18
|
+
# machine only (docs/guardrails.md, "System 1"). `replay` is the record
|
|
19
|
+
# seed: System 1 judges every approval this machine's sessions already
|
|
20
|
+
# answered, and the judgment and the user's answer go to the log as they
|
|
21
|
+
# would live. A rule's record then starts with the asks the user really
|
|
22
|
+
# decided on, instead of waiting for a few days of live ones.
|
|
23
|
+
class GuardrailsCommand
|
|
24
|
+
include CLI::Command
|
|
25
|
+
|
|
26
|
+
USAGE = <<~TEXT
|
|
27
|
+
Usage: chi guardrails <subcommand> [options]
|
|
28
|
+
replay [--dry-run] [--threshold T] [--model M]
|
|
29
|
+
Judge every approval this machine's sessions have already answered
|
|
30
|
+
(their cards, $XDG_STATE_HOME/samagotchi/sessions/<id>/cards.json)
|
|
31
|
+
through System 1 and write the judgment and the user's answer to
|
|
32
|
+
guardrails/judgments.jsonl, so a rule's record starts with the asks
|
|
33
|
+
the user really decided on. Asks with no approval card, unanswered
|
|
34
|
+
ones, and question ids a previous replay already logged are skipped.
|
|
35
|
+
A parent agent's answer is not the user's: only its deny is logged,
|
|
36
|
+
and only when the parent could have allowed (the session's
|
|
37
|
+
guardrails.parent_answers allows "permit" and the ask is not
|
|
38
|
+
user-only). --threshold overrides guardrails.system_one.threshold
|
|
39
|
+
for the table; --model overrides the configured model for this run.
|
|
40
|
+
--dry-run judges and prints the table without writing a line. Needs
|
|
41
|
+
guardrails.system_one.mode audit or on and a guardrails.system_one.host
|
|
42
|
+
(exit 2 without one).
|
|
43
|
+
TEXT
|
|
44
|
+
|
|
45
|
+
SUBCOMMANDS = %w[replay].freeze
|
|
46
|
+
FLAGS = {
|
|
47
|
+
"replay" => CLI::Flags.new(help: HELP_WORDS, args: false) do |f|
|
|
48
|
+
f.switch "--dry-run"
|
|
49
|
+
f.value "--threshold"
|
|
50
|
+
f.value "--model"
|
|
51
|
+
end
|
|
52
|
+
}.freeze
|
|
53
|
+
# The labels that mean the user let the call through.
|
|
54
|
+
ALLOW_LABELS = %w[allow_once allow_wide].freeze
|
|
55
|
+
# replay's table, in column order: the rule, how many asks it had, how
|
|
56
|
+
# many the user allowed, how many they denied, how many System 1 would
|
|
57
|
+
# have auto-allowed at the threshold, the false allows among them (the
|
|
58
|
+
# model said allow, the user did not), and the asks with no judgment.
|
|
59
|
+
COLUMNS = %w[rule asks allowed denied auto false_allows no_judgment].freeze
|
|
60
|
+
# Why an ask in a cards file was not replayed, in the words the summary
|
|
61
|
+
# uses.
|
|
62
|
+
SKIP_WORDS = {
|
|
63
|
+
already_replayed: "already replayed",
|
|
64
|
+
answered_by_parent: "answered by a parent agent",
|
|
65
|
+
parent_cannot_allow: "answered by a parent that could not allow",
|
|
66
|
+
unanswered: "unanswered",
|
|
67
|
+
no_approval: "no approval card",
|
|
68
|
+
no_question_id: "no question id"
|
|
69
|
+
}.freeze
|
|
70
|
+
|
|
71
|
+
# One ask a replay judges: the card's question id, the session it came
|
|
72
|
+
# from, its approval as the card saved it, the label its answer gives,
|
|
73
|
+
# and when the user was asked.
|
|
74
|
+
Ask = Data.define(:question_id, :session_id, :approval, :label, :asked_at)
|
|
75
|
+
|
|
76
|
+
# @param argv [Array<String>] the arguments after "guardrails"
|
|
77
|
+
# @param client [SystemOne::Client, nil] the judgment client; nil: the
|
|
78
|
+
# one config.yml describes (the seam the specs pass a fake through)
|
|
79
|
+
# @param sessions_dir [String, nil] where the sessions are; nil: this
|
|
80
|
+
# machine's ($XDG_STATE_HOME/samagotchi/sessions)
|
|
81
|
+
# @param log [SystemOne::Log, nil] the judgments log; nil: the one beside
|
|
82
|
+
# the approvals store
|
|
83
|
+
def initialize(argv, stdout: $stdout, stderr: $stderr, client: nil, sessions_dir: nil, log: nil)
|
|
84
|
+
@argv = argv.dup
|
|
85
|
+
@stdout = stdout
|
|
86
|
+
@stderr = stderr
|
|
87
|
+
@client = client
|
|
88
|
+
@sessions_dir = sessions_dir
|
|
89
|
+
@log = log
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
# @return [Integer] exit status: 0 done, 2 a usage error or no System 1
|
|
93
|
+
# client to judge with
|
|
94
|
+
def run
|
|
95
|
+
@sub = @argv[0]
|
|
96
|
+
if @sub.nil? || HELP_WORDS.include?(@sub)
|
|
97
|
+
@stdout.puts(USAGE)
|
|
98
|
+
return CLI::Exit::OK
|
|
99
|
+
end
|
|
100
|
+
return usage_error("unknown subcommand #{@sub}") unless SUBCOMMANDS.include?(@sub)
|
|
101
|
+
|
|
102
|
+
parsed = parse_flags(FLAGS.fetch(@sub), @argv[1..])
|
|
103
|
+
return parsed if parsed.is_a?(Integer)
|
|
104
|
+
|
|
105
|
+
replay(parsed.options)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
private
|
|
109
|
+
|
|
110
|
+
def command_name = @sub ? "chi guardrails #{@sub}" : "chi guardrails"
|
|
111
|
+
|
|
112
|
+
def replay(options)
|
|
113
|
+
threshold = threshold(options[:threshold])
|
|
114
|
+
return usage_error(threshold) if threshold.is_a?(String)
|
|
115
|
+
|
|
116
|
+
client = @client || SystemOne::Client.from_config(host_registry: HostRegistry.new, model: options[:model])
|
|
117
|
+
if client.nil?
|
|
118
|
+
error_line("chi guardrails replay: no System 1 client — set guardrails.system_one.mode to audit or on " \
|
|
119
|
+
"and guardrails.system_one.host in config.yml")
|
|
120
|
+
return CLI::Exit::USAGE
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
sessions_dir = @sessions_dir || Session.default_state_dir
|
|
124
|
+
log = @log || SystemOne::Log.new(dir: Guardrails::Approvals.dir_for(sessions_dir))
|
|
125
|
+
replayed = File.exist?(log.path) ? log.entries.filter_map { |entry| entry["question_id"] if entry["origin"] == "replay" }.to_set : Set.new
|
|
126
|
+
@dry_run = options[:dry_run] ? true : false
|
|
127
|
+
|
|
128
|
+
asks = []
|
|
129
|
+
skipped = Hash.new(0)
|
|
130
|
+
malformed = 0
|
|
131
|
+
if Dir.exist?(sessions_dir)
|
|
132
|
+
Dir.children(sessions_dir).sort.each do |name|
|
|
133
|
+
path = File.join(sessions_dir, name, Bridge::CardStore::FILE)
|
|
134
|
+
next unless File.file?(path)
|
|
135
|
+
|
|
136
|
+
entries = sidecar(path)
|
|
137
|
+
if entries.nil?
|
|
138
|
+
malformed += 1
|
|
139
|
+
next
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
entries.each do |entry|
|
|
143
|
+
ask = ask_of(entry, name, replayed, sessions_dir)
|
|
144
|
+
next if ask.nil?
|
|
145
|
+
|
|
146
|
+
ask.is_a?(Symbol) ? skipped[ask] += 1 : asks << ask
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
stats = asks.map { |ask| judge(ask, client, log, threshold, sessions_dir) }
|
|
152
|
+
print_table(stats, threshold, client, skipped, malformed)
|
|
153
|
+
CLI::Exit::OK
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The asks of one session's cards file, or nil when it isn't one: a
|
|
157
|
+
# sidecar that doesn't parse costs one warning line, never the run.
|
|
158
|
+
def sidecar(path)
|
|
159
|
+
return nil unless File.file?(path)
|
|
160
|
+
|
|
161
|
+
data = JSON.parse(File.read(path), symbolize_names: true)
|
|
162
|
+
return nil unless data.is_a?(Hash) && data[:entries].is_a?(Array)
|
|
163
|
+
|
|
164
|
+
data[:entries]
|
|
165
|
+
rescue StandardError => e
|
|
166
|
+
error_line("chi guardrails replay: #{path}: unreadable cards (#{e.class}: #{e.message}); skipped")
|
|
167
|
+
nil
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# One card entry as an ask to judge: an Ask, a skip reason (:already_replayed,
|
|
171
|
+
# :answered_by_parent, :parent_cannot_allow, :unanswered, :no_approval,
|
|
172
|
+
# :no_question_id), or nil for an entry that is not a question at all.
|
|
173
|
+
def ask_of(entry, session_id, replayed, sessions_dir)
|
|
174
|
+
return nil unless entry.is_a?(Hash) && entry[:type] == "question"
|
|
175
|
+
|
|
176
|
+
pending = entry[:pending_question]
|
|
177
|
+
return :no_approval unless pending.is_a?(Hash) && pending[:approval].is_a?(Hash)
|
|
178
|
+
|
|
179
|
+
question_id = pending[:id].to_s
|
|
180
|
+
return :no_question_id if question_id.empty?
|
|
181
|
+
return :already_replayed if replayed.include?(question_id)
|
|
182
|
+
|
|
183
|
+
answer = entry[:answer]
|
|
184
|
+
label = if entry[:cancelled]
|
|
185
|
+
"none"
|
|
186
|
+
elsif answer.nil?
|
|
187
|
+
return :unanswered
|
|
188
|
+
elsif answer[:by] == "parent_agent"
|
|
189
|
+
# A parent's allow is not the user's decision. Its deny is a
|
|
190
|
+
# label only where the parent could have allowed: a parent
|
|
191
|
+
# that was not allowed to permit says nothing about the user
|
|
192
|
+
# (docs/guardrails.md, "System 1").
|
|
193
|
+
return :answered_by_parent unless label_for(answer, pending[:approval][:scopes]) == "deny"
|
|
194
|
+
return :parent_cannot_allow unless parent_may_permit?(pending, session_id, sessions_dir)
|
|
195
|
+
|
|
196
|
+
"deny"
|
|
197
|
+
else
|
|
198
|
+
label_for(answer, pending[:approval][:scopes])
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
Ask.new(question_id: question_id, session_id: session_id, approval: pending[:approval],
|
|
202
|
+
label: label, asked_at: pending[:created_at])
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Whether this session's parent could have allowed the ask: not a
|
|
206
|
+
# user-only ask, and a parent level that permits one. No session file:
|
|
207
|
+
# not permitted.
|
|
208
|
+
def parent_may_permit?(pending, session_id, sessions_dir)
|
|
209
|
+
return false if pending[:user_only] == true
|
|
210
|
+
|
|
211
|
+
session = session_of(session_id, sessions_dir)
|
|
212
|
+
return false if session.nil?
|
|
213
|
+
|
|
214
|
+
Guardrails::ParentPolicy.for_session(session).allow?("permit")
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# The session a card belongs to, or nil when there is no session file;
|
|
218
|
+
# read once per session for the run.
|
|
219
|
+
def session_of(session_id, sessions_dir)
|
|
220
|
+
return nil unless Session.valid_id?(session_id)
|
|
221
|
+
return nil unless File.exist?(Session.session_file(session_id, state_dir: sessions_dir))
|
|
222
|
+
|
|
223
|
+
(@sessions ||= {})[session_id] ||= begin
|
|
224
|
+
Session.load(session_id, state_dir: sessions_dir)
|
|
225
|
+
rescue StandardError
|
|
226
|
+
nil
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# What the user did, as the log's label: the scope they took, a denial
|
|
231
|
+
# when the answer is not one of the offered scopes, none for an ask that
|
|
232
|
+
# was cancelled (docs/guardrails.md, "System 1").
|
|
233
|
+
def label_for(answer, scopes)
|
|
234
|
+
index = answer[:selected_indices]&.first
|
|
235
|
+
return "deny" if index.nil?
|
|
236
|
+
|
|
237
|
+
scopes = Array(scopes)
|
|
238
|
+
if index < scopes.size
|
|
239
|
+
scopes[index] == "once" ? "allow_once" : "allow_wide"
|
|
240
|
+
else
|
|
241
|
+
"deny"
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# Judge one ask, write its two lines unless this is a dry run, and return
|
|
246
|
+
# what the table counts. A judgment that never came is written with p
|
|
247
|
+
# null, exactly as the live audit writes it.
|
|
248
|
+
def judge(ask, client, log, threshold, sessions_dir)
|
|
249
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
250
|
+
answers = begin
|
|
251
|
+
client.ask(state: SystemOne::ApprovalQuestion.state(ask.approval, session_title: session_title(ask, sessions_dir)),
|
|
252
|
+
questions: { SystemOne::ApprovalQuestion::ID => SystemOne::ApprovalQuestion.question })
|
|
253
|
+
rescue StandardError => e
|
|
254
|
+
Samagotchi::Log.warn(:guardrails, "system_one_replay_ask_failed", error: "#{e.class}: #{e.message}")
|
|
255
|
+
nil
|
|
256
|
+
end
|
|
257
|
+
latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
|
|
258
|
+
p = answers && answers[SystemOne::ApprovalQuestion::ID]&.noul
|
|
259
|
+
|
|
260
|
+
unless @dry_run
|
|
261
|
+
id = log.judgment(session_id: ask.session_id, rule: ask.approval[:rule], source: ask.approval[:source],
|
|
262
|
+
tool: ask.approval[:tool], **SystemOne::ApprovalQuestion.call_fields(ask.approval),
|
|
263
|
+
repo_name: ask.approval[:repo_name], p: p, threshold: threshold, model: client.model,
|
|
264
|
+
latency_ms: latency_ms, decision: "replay", origin: "replay",
|
|
265
|
+
question_id: ask.question_id, asked_at: ask.asked_at)
|
|
266
|
+
log.label(id, ask.label)
|
|
267
|
+
end
|
|
268
|
+
|
|
269
|
+
{ rule: ask.approval[:rule].to_s, label: ask.label, p: p, session: ask.session_id }
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# The session's title, as the ask's state carries it; nil when there is
|
|
273
|
+
# no session file (the cards alone are enough to replay).
|
|
274
|
+
def session_title(ask, sessions_dir)
|
|
275
|
+
session_of(ask.session_id, sessions_dir)&.display_title
|
|
276
|
+
end
|
|
277
|
+
|
|
278
|
+
# The threshold for this run: --threshold, or guardrails.system_one.threshold.
|
|
279
|
+
# @return [Float, String] the number, or the usage error's message
|
|
280
|
+
def threshold(value)
|
|
281
|
+
return Config.get("guardrails.system_one.threshold").to_f if value.nil?
|
|
282
|
+
|
|
283
|
+
number = Float(value.to_s.strip)
|
|
284
|
+
return "--threshold takes a number between 0 and 1, got #{value}" if number.negative? || number > 1.0
|
|
285
|
+
|
|
286
|
+
number
|
|
287
|
+
rescue ArgumentError, TypeError
|
|
288
|
+
"--threshold takes a number between 0 and 1, got #{value}"
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
def print_table(stats, threshold, client, skipped, malformed)
|
|
292
|
+
rows = stats.group_by { |row| row[:rule] }.map { |rule, asks| rule_row(rule, asks, threshold) }
|
|
293
|
+
rows.sort_by! { |rule, count, _| [-count, rule] }
|
|
294
|
+
totals = ["total", *COLUMNS[1..].each_with_index.map { |_column, offset| rows.sum { |row| row[offset + 1] } }]
|
|
295
|
+
all = rows + [totals]
|
|
296
|
+
widths = COLUMNS.each_with_index.map { |column, index| [column.size, all.map { |row| row[index].to_s.size }.max].max }
|
|
297
|
+
|
|
298
|
+
@stdout.puts(row_line(COLUMNS, widths))
|
|
299
|
+
@stdout.puts(widths.map { |width| "-" * width }.join(" "))
|
|
300
|
+
rows.each { |row| @stdout.puts(row_line(row, widths)) }
|
|
301
|
+
@stdout.puts(row_line(totals, widths))
|
|
302
|
+
|
|
303
|
+
sessions = stats.map { |row| row[:session] }.uniq.size
|
|
304
|
+
@stdout.puts("#{stats.size} asks replayed from #{sessions} session#{"s" unless sessions == 1} " \
|
|
305
|
+
"· threshold #{threshold} · model #{client.model || "the host's default"}")
|
|
306
|
+
notes = skipped.map { |reason, count| "#{count} #{SKIP_WORDS[reason] || reason}" }
|
|
307
|
+
notes << "#{malformed} unreadable cards file#{"s" unless malformed == 1}" if malformed.positive?
|
|
308
|
+
@stdout.puts("skipped: #{notes.sort.join(", ")}") unless notes.empty?
|
|
309
|
+
@stdout.puts("dry run: nothing was written") if @dry_run
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# One table row, columns left-aligned to their width.
|
|
313
|
+
def row_line(values, widths)
|
|
314
|
+
values.each_with_index.map { |value, index| value.to_s.ljust(widths[index]) }.join(" ").rstrip
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# One rule's row: asks, allowed, denied, auto-allowed at the threshold,
|
|
318
|
+
# false allows, asks with no judgment.
|
|
319
|
+
def rule_row(rule, rows, threshold)
|
|
320
|
+
allowed = rows.count { |row| ALLOW_LABELS.include?(row[:label]) }
|
|
321
|
+
denied = rows.count { |row| row[:label] == "deny" }
|
|
322
|
+
high = rows.select { |row| row[:p] && row[:p] >= threshold }
|
|
323
|
+
[rule, rows.size, allowed, denied,
|
|
324
|
+
high.count { |row| ALLOW_LABELS.include?(row[:label]) },
|
|
325
|
+
high.count { |row| row[:label] == "deny" },
|
|
326
|
+
rows.count { |row| row[:p].nil? }]
|
|
327
|
+
end
|
|
328
|
+
end
|
|
329
|
+
end
|