samagotchi 0.114.0 → 0.116.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +72 -1
  3. data/bin/chi +2 -0
  4. data/docs/cli.md +11 -3
  5. data/docs/configuration.md +7 -0
  6. data/docs/guardrails.md +188 -1
  7. data/docs/internals/web-push.md +3 -2
  8. data/docs/sessions.md +2 -2
  9. data/lib/samagotchi/bridge/turn_accumulator.rb +4 -0
  10. data/lib/samagotchi/bridge.rb +26 -0
  11. data/lib/samagotchi/bridge_client.rb +7 -0
  12. data/lib/samagotchi/bundles/system/manifest.yml +1 -1
  13. data/lib/samagotchi/config.rb +15 -0
  14. data/lib/samagotchi/engine.rb +6 -1
  15. data/lib/samagotchi/guardrail_wiring.rb +166 -2
  16. data/lib/samagotchi/guardrails/approval.rb +20 -0
  17. data/lib/samagotchi/guardrails/targets.rb +23 -2
  18. data/lib/samagotchi/guardrails/verdict.rb +23 -3
  19. data/lib/samagotchi/guardrails_command.rb +329 -0
  20. data/lib/samagotchi/memory_bundle/index_updater.rb +13 -0
  21. data/lib/samagotchi/session_access/contract.rb +1 -1
  22. data/lib/samagotchi/session_commands.rb +79 -2
  23. data/lib/samagotchi/session_metrics.rb +7 -1
  24. data/lib/samagotchi/system_one/answer.rb +78 -0
  25. data/lib/samagotchi/system_one/approval_question.rb +113 -0
  26. data/lib/samagotchi/system_one/client.rb +89 -0
  27. data/lib/samagotchi/system_one/errors.rb +14 -0
  28. data/lib/samagotchi/system_one/fake_backend.rb +45 -0
  29. data/lib/samagotchi/system_one/http_backend.rb +129 -0
  30. data/lib/samagotchi/system_one/log.rb +133 -0
  31. data/lib/samagotchi/system_one/question.rb +86 -0
  32. data/lib/samagotchi/system_one/record.rb +83 -0
  33. data/lib/samagotchi/system_one/self_protect.rb +90 -0
  34. data/lib/samagotchi/system_one/similar.rb +69 -0
  35. data/lib/samagotchi/system_one.rb +25 -0
  36. data/lib/samagotchi/terminal_ui/attached_loop.rb +15 -4
  37. data/lib/samagotchi/terminal_ui/event_renderer.rb +1 -0
  38. data/lib/samagotchi/terminal_ui/formatting.rb +14 -0
  39. data/lib/samagotchi/title_command.rb +13 -4
  40. data/lib/samagotchi/version.rb +1 -1
  41. data/lib/samagotchi/web/app.rb +1 -0
  42. data/lib/samagotchi/web/public/activity.js +10 -0
  43. data/lib/samagotchi/web/public/annotate_bar.js +22 -2
  44. data/lib/samagotchi/web/public/app.js +4 -0
  45. data/lib/samagotchi/web/public/apple-touch-icon.png +0 -0
  46. data/lib/samagotchi/web/public/composer_fold.js +8 -8
  47. data/lib/samagotchi/web/public/composer_grip.js +65 -22
  48. data/lib/samagotchi/web/public/composer_size.js +30 -2
  49. data/lib/samagotchi/web/public/data.js +10 -0
  50. data/lib/samagotchi/web/public/favicon.svg +1 -1
  51. data/lib/samagotchi/web/public/icons/chi-180.png +0 -0
  52. data/lib/samagotchi/web/public/icons/chi-192.png +0 -0
  53. data/lib/samagotchi/web/public/icons/chi-512.png +0 -0
  54. data/lib/samagotchi/web/public/index.html +7 -0
  55. data/lib/samagotchi/web/public/notify.js +4 -2
  56. data/lib/samagotchi/web/public/system_one_wrong.js +32 -0
  57. data/lib/samagotchi/web/public/timing.js +9 -5
  58. data/lib/samagotchi/web/public/turn_events.js +1 -0
  59. data/lib/samagotchi/web/public/turn_html.js +13 -2
  60. data/lib/samagotchi/web/push/attention.rb +3 -1
  61. data/lib/samagotchi/web/session_control.rb +22 -0
  62. data/lib/samagotchi/web/session_show.rb +35 -5
  63. metadata +15 -1
@@ -6,10 +6,13 @@ require_relative "config"
6
6
  require_relative "log"
7
7
  require_relative "guardrails"
8
8
  require_relative "guardrails/parent_approvals"
9
+ require_relative "guardrails/parent_policy"
10
+ require_relative "model_match"
9
11
  require_relative "memory_bundle/manifest"
10
12
  require_relative "version"
11
13
  require_relative "hooks"
12
14
  require_relative "edit_preview"
15
+ require_relative "system_one"
13
16
  require_relative "tool_activity"
14
17
  require_relative "turn_origin"
15
18
 
@@ -38,8 +41,11 @@ module Samagotchi
38
41
  # @param ask [#call] (fields) → the answer; the Engine's question flow
39
42
  # @param notify [#call] (message) → shows a guardrails line in every UI
40
43
  # (a bundle's rules kept because it needs a newer chi)
44
+ # @param system_one [#call] → SystemOne::Client or nil: the judgment client
45
+ # (nil while `guardrails.system_one.mode` is off or no host is set)
46
+ # @param cancel_controller [#call] → the running turn's CancellationController, nil
41
47
  def initialize(scratch:, hooks:, tools:, session:, model_key:, cancelled:, ask:, model_name: -> {},
42
- notify: ->(_message) {})
48
+ notify: ->(_message) {}, system_one: -> {}, cancel_controller: -> {})
43
49
  @scratch = scratch
44
50
  @hooks_lookup = hooks
45
51
  @tools_lookup = tools
@@ -49,6 +55,8 @@ module Samagotchi
49
55
  @cancelled_lookup = cancelled
50
56
  @ask = ask
51
57
  @notify = notify
58
+ @system_one = system_one
59
+ @cancel_controller = cancel_controller
52
60
  # Bundle name → the rules last loaded from it whole, kept while the
53
61
  # installed version needs a newer chi; [name, version] said so once.
54
62
  @loaded_bundle_rules = {}
@@ -82,6 +90,7 @@ module Samagotchi
82
90
  def state_dir=(state_dir)
83
91
  @approvals = Guardrails::Approvals.new(dir: Guardrails::Approvals.dir_for(state_dir))
84
92
  @approvals = Guardrails::Approvals::InMemory.new(@approvals) if @scratch
93
+ @system_one_log = nil
85
94
  @protected = nil
86
95
  end
87
96
 
@@ -259,11 +268,166 @@ module Samagotchi
259
268
  # A call on chi's own config, hooks or guardrails is the user's to
260
269
  # allow, never a parent agent's (Guardrails::ParentPolicy).
261
270
  payload[:user_only] = Guardrails::ParentApprovals.protected_facts?(payload[:approval])
262
- Guardrails::Approval.settle(verdict, @ask.call(payload), payload[:approval][:scopes])
271
+ # System 1: the ask is judged before the user sees it, and in `on` mode
272
+ # one that passes every gate is allowed once without reaching them
273
+ # (docs/guardrails.md, "System 1").
274
+ judgment = system_one_judge(payload)
275
+ if judgment && judgment[:auto]
276
+ verdict.judgment = { id: judgment[:id], p: judgment[:p] }
277
+ verdict.settle!(:allow, decided_by: "system one")
278
+ verdict.scope = "once"
279
+ return verdict
280
+ end
281
+
282
+ answer = @ask.call(payload)
283
+ system_one_label(judgment, answer, payload)
284
+ Guardrails::Approval.settle(verdict, answer, payload[:approval][:scopes])
263
285
  end
264
286
 
265
287
  private
266
288
 
289
+ # The judgments log (SystemOne::Log), beside the approvals store; nil
290
+ # while there is no store dir.
291
+ def system_one_log
292
+ return nil if @approvals.nil?
293
+
294
+ @system_one_log ||= SystemOne::Log.new(dir: File.dirname(@approvals.path))
295
+ end
296
+
297
+ # System 1's judgment of an ask, written to the log before the user is
298
+ # asked, with the decision it led to: "auto" (it answered for the user),
299
+ # "asked" (on mode, the user decides) or "audit" (audit mode). A
300
+ # participant's review turn is never judged, and no client (mode off, no
301
+ # host) means no line at all. A judgment that fails is logged with p nil,
302
+ # so the failures are in the log too.
303
+ # @return [Hash, nil] {id:, p:, auto:}, nil when nothing was written
304
+ def system_one_judge(payload)
305
+ return nil if Steer.participant_author?(turn_author)
306
+
307
+ client = @system_one.call
308
+ log = system_one_log
309
+ return nil if client.nil? || log.nil?
310
+
311
+ approval = payload[:approval]
312
+ # The judgments log is read once for this ask: the state's examples and
313
+ # the record gate both use these entries. The judgment of the ask being
314
+ # judged is written after it, so the examples are always earlier asks.
315
+ entries = log.entries
316
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
317
+ answers = begin
318
+ client.ask(state: SystemOne::ApprovalQuestion.state(approval, session_title: @session_lookup.call&.title,
319
+ entries: entries),
320
+ questions: { SystemOne::ApprovalQuestion::ID => SystemOne::ApprovalQuestion.question },
321
+ cancel_controller: @cancel_controller.call)
322
+ rescue StandardError => e
323
+ # The client already turns every failure it knows into a nil; this is
324
+ # the unexpected one. Either way the ask goes to the user.
325
+ Samagotchi::Log.warn(:guardrails, "system_one_ask_failed", error: "#{e.class}: #{e.message}")
326
+ nil
327
+ end
328
+ latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
329
+ p = answers && answers[SystemOne::ApprovalQuestion::ID]&.noul
330
+ auto = system_one_auto_allow?(client, approval, p, entries)
331
+ decision = system_one_decision(client, auto)
332
+
333
+ id = log.judgment(session_id: @session_lookup.call&.id,
334
+ rule: approval[:rule], source: approval[:source], tool: approval[:tool],
335
+ **SystemOne::ApprovalQuestion.call_fields(approval),
336
+ repo_name: approval[:repo_name],
337
+ p: p, threshold: Config.get("guardrails.system_one.threshold"), model: client.model,
338
+ latency_ms: latency_ms, decision: decision, origin: "live")
339
+ # A judgment the log cannot hold is no judgment: the user is asked.
340
+ { id: id, p: p, auto: auto && !id.nil? }
341
+ rescue StandardError => e
342
+ # The approval never fails because of System 1.
343
+ Samagotchi::Log.warn(:guardrails, "system_one_judgment_failed", error: "#{e.class}: #{e.message}")
344
+ nil
345
+ end
346
+
347
+ # What the judgment line says it did: "auto" (System 1 answered for the
348
+ # user), "asked" (on mode, the user decides), "audit" (nothing decided).
349
+ def system_one_decision(client, auto)
350
+ return "auto" if auto
351
+
352
+ client.mode == "on" ? "asked" : "audit"
353
+ end
354
+
355
+ # Every gate an auto-allow must pass, in order: the mode, the rule the
356
+ # user opted in, the never-list and chi's own files, the probability, and
357
+ # the rule's record. Any one that fails leaves the ask to the user. The
358
+ # entries are the judgment log read once for this ask.
359
+ # @param entries [Array<Hash>] SystemOne::Log#entries
360
+ def system_one_auto_allow?(client, approval, p, entries)
361
+ return false unless client.mode == "on"
362
+
363
+ rule = approval[:rule].to_s
364
+ return false if rule.empty?
365
+ return false unless system_one_rules.include?(rule)
366
+ return false if SystemOne::SelfProtect.refuse?(approval)
367
+
368
+ threshold = Config.get("guardrails.system_one.threshold").to_f
369
+ return false if p.nil? || p.to_f < threshold
370
+
371
+ min_record = Config.get("guardrails.system_one.min_record").to_i
372
+ SystemOne::Record.new(entries, rule: rule).qualifies?(threshold: threshold, min_record: min_record)
373
+ rescue StandardError => e
374
+ Samagotchi::Log.warn(:guardrails, "system_one_gate_failed", error: "#{e.class}: #{e.message}")
375
+ false
376
+ end
377
+
378
+ # guardrails.system_one.rules: a "|"-separated string or a YAML list,
379
+ # read with the same grammar as guardrails.small_models.
380
+ def system_one_rules
381
+ ModelMatch.parse(Config.get("guardrails.system_one.rules"))
382
+ rescue StandardError
383
+ []
384
+ end
385
+
386
+ # The user's answer to that ask, as a label on the judgment: the scope
387
+ # they took, a denial, or none for an ask that was cancelled. A parent
388
+ # agent's answer is not the user's: its allow labels nothing, and its
389
+ # deny is a label only where the parent could have allowed — a parent
390
+ # that was not allowed to permit says nothing about what the user would
391
+ # have done (docs/guardrails.md, "System 1").
392
+ def system_one_label(judgment, answer, payload)
393
+ id = judgment && judgment[:id]
394
+ return nil if id.nil?
395
+
396
+ scopes = Array(payload[:approval][:scopes])
397
+ if !answer.is_a?(Samagotchi::QuestionAnswer)
398
+ label = "none"
399
+ elsif answer.parent_agent?
400
+ return nil if parent_allowed?(answer, scopes) || !parent_may_permit?(payload)
401
+
402
+ label = "deny"
403
+ else
404
+ index = answer.selected_indices.first
405
+ label = if index && index < scopes.size
406
+ scopes[index] == "once" ? "allow_once" : "allow_wide"
407
+ else
408
+ "deny"
409
+ end
410
+ end
411
+ system_one_log.label(id, label)
412
+ rescue StandardError => e
413
+ Samagotchi::Log.warn(:guardrails, "system_one_label_failed", error: "#{e.class}: #{e.message}")
414
+ nil
415
+ end
416
+
417
+ # Whether a parent's answer picked one of the offered scopes.
418
+ def parent_allowed?(answer, scopes)
419
+ index = answer.selected_indices.first
420
+ !index.nil? && index < scopes.size
421
+ end
422
+
423
+ # Whether this session's parent level lets a parent allow a call at all,
424
+ # and the ask is not one only the user may answer.
425
+ def parent_may_permit?(payload)
426
+ return false if payload[:user_only]
427
+
428
+ Guardrails::ParentPolicy.for_session(@session_lookup.call).allow?("permit")
429
+ end
430
+
267
431
  # The work tree top of the session's working_directory (the folder
268
432
  # itself outside a repo, or when it's gone); never Dir.pwd, which is
269
433
  # the waker's cwd for a worker whose folder is gone.
@@ -51,6 +51,9 @@ module Samagotchi
51
51
  paths: targets && !targets.paths.empty? ? targets.paths : nil,
52
52
  # No command or path (an MCP tool): its arguments, as the question.
53
53
  args: args_for_card(targets),
54
+ # A memory_write ask, in words: the entry, its scope, and for a
55
+ # removal the description its index line carries.
56
+ memory: memory_for_card(targets),
54
57
  cwd: targets&.cwd,
55
58
  repo_root: targets&.repo_root,
56
59
  # The repository's name (a worktree's main checkout's folder).
@@ -78,6 +81,23 @@ module Samagotchi
78
81
  text.empty? ? nil : text
79
82
  end
80
83
 
84
+ # A memory_write ask names an entry, not a file: which one, and in which
85
+ # scope. For a removal the ask also carries that entry's description from
86
+ # its scoped index line, which is what System 1 otherwise cannot see
87
+ # (docs/guardrails.md, "System 1"). nil for every other tool.
88
+ def memory_for_card(targets)
89
+ entry = targets&.memory
90
+ return nil if entry.nil?
91
+
92
+ facts = { name: entry[:name], scope: entry[:scope] }
93
+ if targets.memory_remove?
94
+ require_relative "../memory_bundle/index_updater"
95
+ description = MemoryBundle::IndexUpdater.description_for(entry[:scope], entry[:name])
96
+ facts[:description] = description unless description.to_s.strip.empty?
97
+ end
98
+ facts
99
+ end
100
+
81
101
  # Settle +verdict+ from open_question's result.
82
102
  # @param answer [QuestionAnswer, Hash, String] the answer, or {error:} (anything else is a cancel)
83
103
  # @param scopes [Array<String>] the offered scopes, in option order
@@ -74,6 +74,12 @@ module Samagotchi
74
74
  attr_reader :memory_overlay
75
75
  alias memory_overlay? memory_overlay
76
76
 
77
+ # @return [Hash, nil] a memory_write call's entry and its scope,
78
+ # {name:, scope:}: which memory the ask is about, in words rather than
79
+ # as the file it names (#paths); nil for another tool or a call the
80
+ # tool would refuse
81
+ attr_reader :memory
82
+
77
83
  # @param call [Hash] the parsed tool call
78
84
  # @param context [Context]
79
85
  # @param model_key [String, nil] for a memory_write model overlay, and
@@ -114,9 +120,10 @@ module Samagotchi
114
120
  end
115
121
  memory_remove = tool == "memory_write" && call[:remove].to_s.strip.downcase == "true"
116
122
  memory_overlay = tool == "memory_write" && !memory_remove && call[:current_model_only].to_s.strip.downcase == "true"
123
+ memory = tool == "memory_write" ? memory_facts(call) : nil
117
124
  new(tool: tool, command: command, paths: paths.compact, cwd: cwd, repo_root: context.repo_root(cwd),
118
125
  repo: context.repo(cwd), args: args, acts_as: acts_as, label: label, memory_remove: memory_remove,
119
- memory_overlay: memory_overlay,
126
+ memory_overlay: memory_overlay, memory: memory,
120
127
  model_name: model_name, model_key: model_key, session_root: context.repo_root(base) || base,
121
128
  chi_dirs: -> { chi_dirs(context, base) })
122
129
  end
@@ -224,14 +231,28 @@ module Samagotchi
224
231
  nil
225
232
  end
226
233
 
234
+ # The memory a memory_write call is about, in the words the ask needs:
235
+ # the entry and its scope, not its file; nil for a call the tool would
236
+ # refuse (no entry, no scope, a scope chi has no folder for).
237
+ def self.memory_facts(call)
238
+ entry = call[:path].to_s.strip
239
+ scope = Tools::MemoryRead.normalize_scope(call[:scope])
240
+ return nil if entry.empty? || scope.nil?
241
+
242
+ { name: entry, scope: scope }
243
+ rescue StandardError
244
+ nil
245
+ end
246
+
227
247
  # @param chi_dirs [#call, nil] → the dirs touches_chi? looks for
228
248
  def initialize(tool:, command:, paths:, cwd:, repo_root:, repo: nil, args: nil, acts_as: nil, label: nil,
229
249
  model_name: nil, model_key: nil, session_root: nil, chi_dirs: nil, memory_remove: false,
230
- memory_overlay: false)
250
+ memory_overlay: false, memory: nil)
231
251
  @acts_as = acts_as
232
252
  @label = label
233
253
  @memory_remove = memory_remove
234
254
  @memory_overlay = memory_overlay
255
+ @memory = memory
235
256
  @repo = repo
236
257
  @chi_dirs = chi_dirs
237
258
  @session_root = session_root || repo_root || cwd
@@ -19,8 +19,10 @@ module Samagotchi
19
19
  # parent model would be writing a model's prompt); and the bundle's
20
20
  # asks before answering chi's questions around chi answer (a parent
21
21
  # allowing one would answer a grandchild's question as the user).
22
- # Only the user may allow them (ParentApprovals.protected?,
23
- # ParentPolicy's user_only).
22
+ # A parent agent may never allow them (ParentApprovals.protected?,
23
+ # ParentPolicy's user_only); System 1 may, only if the user lists the
24
+ # rule by name in guardrails.system_one.rules, and never for the
25
+ # chi-answer/chi-broadcast ones (SystemOne::SelfProtect::NEVER_AUTO).
24
26
  PROTECTED_RULES = %w[chi-config chi-hooks shell-touches-chi chi-context-cmd chi-broadcast prompt-memory-write
25
27
  model-overlay-write chi-answer-piped chi-answer-http].freeze
26
28
  SCOPES = %w[once session repo rule].freeze
@@ -33,6 +35,8 @@ module Samagotchi
33
35
  attr_accessor :context, :targets
34
36
  # The scope the user allowed an ask for ("once", "session", …).
35
37
  attr_accessor :scope
38
+ # System 1's judgment that decided an ask, when it did: {id:, p:}.
39
+ attr_accessor :judgment
36
40
 
37
41
  def initialize(call:)
38
42
  @call = call
@@ -121,8 +125,11 @@ module Samagotchi
121
125
  end
122
126
 
123
127
  # For the activity entry; an allowed ask notes how ("approved (repo)",
124
- # "approved earlier (session)").
128
+ # "approved earlier (session)"), one System 1 answered notes that and
129
+ # carries the judgment (docs/guardrails.md, "System 1").
125
130
  def to_activity
131
+ return system_one_activity if @decided_by == "system one"
132
+
126
133
  note = if allow? && @scope
127
134
  "approved#{" earlier" if @decided_by == "approval"} (#{@scope})"
128
135
  end
@@ -131,6 +138,19 @@ module Samagotchi
131
138
 
132
139
  private
133
140
 
141
+ # The auto-allow's activity: who decided, for how long, and the
142
+ # judgment behind it (the p and the judgment's id), so a UI can show
143
+ # why the call was never asked about.
144
+ def system_one_activity
145
+ facts = @judgment || {}
146
+ p = facts[:p]
147
+ activity = { rule: @rule, verdict: @decision.to_s, decided_by: @decided_by, scope: @scope,
148
+ note: p ? "auto-allowed by System 1 (p=#{format("%.2f", p.to_f)})" : "auto-allowed by System 1" }
149
+ activity[:judgment_id] = facts[:id] if facts[:id]
150
+ activity[:p] = p if p
151
+ activity.compact
152
+ end
153
+
134
154
  def decider
135
155
  return [("rule #{@rule}" if @rule), @source].compact.join(", ") if @rule || @source
136
156
 
@@ -0,0 +1,329 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "json"
4
+
5
+ require_relative "cli/command"
6
+ require_relative "cli/exit"
7
+ require_relative "cli/flags"
8
+ require_relative "config"
9
+ require_relative "bridge/card_store"
10
+ require_relative "guardrails/approvals"
11
+ require_relative "guardrails/parent_policy"
12
+ require_relative "host_registry"
13
+ require_relative "session"
14
+ require_relative "system_one"
15
+
16
+ module Samagotchi
17
+ # `chi guardrails`: the guardrails' own maintenance commands, on this
18
+ # machine only (docs/guardrails.md, "System 1"). `replay` is the record
19
+ # seed: System 1 judges every approval this machine's sessions already
20
+ # answered, and the judgment and the user's answer go to the log as they
21
+ # would live. A rule's record then starts with the asks the user really
22
+ # decided on, instead of waiting for a few days of live ones.
23
+ class GuardrailsCommand
24
+ include CLI::Command
25
+
26
+ USAGE = <<~TEXT
27
+ Usage: chi guardrails <subcommand> [options]
28
+ replay [--dry-run] [--threshold T] [--model M]
29
+ Judge every approval this machine's sessions have already answered
30
+ (their cards, $XDG_STATE_HOME/samagotchi/sessions/<id>/cards.json)
31
+ through System 1 and write the judgment and the user's answer to
32
+ guardrails/judgments.jsonl, so a rule's record starts with the asks
33
+ the user really decided on. Asks with no approval card, unanswered
34
+ ones, and question ids a previous replay already logged are skipped.
35
+ A parent agent's answer is not the user's: only its deny is logged,
36
+ and only when the parent could have allowed (the session's
37
+ guardrails.parent_answers allows "permit" and the ask is not
38
+ user-only). --threshold overrides guardrails.system_one.threshold
39
+ for the table; --model overrides the configured model for this run.
40
+ --dry-run judges and prints the table without writing a line. Needs
41
+ guardrails.system_one.mode audit or on and a guardrails.system_one.host
42
+ (exit 2 without one).
43
+ TEXT
44
+
45
+ SUBCOMMANDS = %w[replay].freeze
46
+ FLAGS = {
47
+ "replay" => CLI::Flags.new(help: HELP_WORDS, args: false) do |f|
48
+ f.switch "--dry-run"
49
+ f.value "--threshold"
50
+ f.value "--model"
51
+ end
52
+ }.freeze
53
+ # The labels that mean the user let the call through.
54
+ ALLOW_LABELS = %w[allow_once allow_wide].freeze
55
+ # replay's table, in column order: the rule, how many asks it had, how
56
+ # many the user allowed, how many they denied, how many System 1 would
57
+ # have auto-allowed at the threshold, the false allows among them (the
58
+ # model said allow, the user did not), and the asks with no judgment.
59
+ COLUMNS = %w[rule asks allowed denied auto false_allows no_judgment].freeze
60
+ # Why an ask in a cards file was not replayed, in the words the summary
61
+ # uses.
62
+ SKIP_WORDS = {
63
+ already_replayed: "already replayed",
64
+ answered_by_parent: "answered by a parent agent",
65
+ parent_cannot_allow: "answered by a parent that could not allow",
66
+ unanswered: "unanswered",
67
+ no_approval: "no approval card",
68
+ no_question_id: "no question id"
69
+ }.freeze
70
+
71
+ # One ask a replay judges: the card's question id, the session it came
72
+ # from, its approval as the card saved it, the label its answer gives,
73
+ # and when the user was asked.
74
+ Ask = Data.define(:question_id, :session_id, :approval, :label, :asked_at)
75
+
76
+ # @param argv [Array<String>] the arguments after "guardrails"
77
+ # @param client [SystemOne::Client, nil] the judgment client; nil: the
78
+ # one config.yml describes (the seam the specs pass a fake through)
79
+ # @param sessions_dir [String, nil] where the sessions are; nil: this
80
+ # machine's ($XDG_STATE_HOME/samagotchi/sessions)
81
+ # @param log [SystemOne::Log, nil] the judgments log; nil: the one beside
82
+ # the approvals store
83
+ def initialize(argv, stdout: $stdout, stderr: $stderr, client: nil, sessions_dir: nil, log: nil)
84
+ @argv = argv.dup
85
+ @stdout = stdout
86
+ @stderr = stderr
87
+ @client = client
88
+ @sessions_dir = sessions_dir
89
+ @log = log
90
+ end
91
+
92
+ # @return [Integer] exit status: 0 done, 2 a usage error or no System 1
93
+ # client to judge with
94
+ def run
95
+ @sub = @argv[0]
96
+ if @sub.nil? || HELP_WORDS.include?(@sub)
97
+ @stdout.puts(USAGE)
98
+ return CLI::Exit::OK
99
+ end
100
+ return usage_error("unknown subcommand #{@sub}") unless SUBCOMMANDS.include?(@sub)
101
+
102
+ parsed = parse_flags(FLAGS.fetch(@sub), @argv[1..])
103
+ return parsed if parsed.is_a?(Integer)
104
+
105
+ replay(parsed.options)
106
+ end
107
+
108
+ private
109
+
110
+ def command_name = @sub ? "chi guardrails #{@sub}" : "chi guardrails"
111
+
112
+ def replay(options)
113
+ threshold = threshold(options[:threshold])
114
+ return usage_error(threshold) if threshold.is_a?(String)
115
+
116
+ client = @client || SystemOne::Client.from_config(host_registry: HostRegistry.new, model: options[:model])
117
+ if client.nil?
118
+ error_line("chi guardrails replay: no System 1 client — set guardrails.system_one.mode to audit or on " \
119
+ "and guardrails.system_one.host in config.yml")
120
+ return CLI::Exit::USAGE
121
+ end
122
+
123
+ sessions_dir = @sessions_dir || Session.default_state_dir
124
+ log = @log || SystemOne::Log.new(dir: Guardrails::Approvals.dir_for(sessions_dir))
125
+ replayed = File.exist?(log.path) ? log.entries.filter_map { |entry| entry["question_id"] if entry["origin"] == "replay" }.to_set : Set.new
126
+ @dry_run = options[:dry_run] ? true : false
127
+
128
+ asks = []
129
+ skipped = Hash.new(0)
130
+ malformed = 0
131
+ if Dir.exist?(sessions_dir)
132
+ Dir.children(sessions_dir).sort.each do |name|
133
+ path = File.join(sessions_dir, name, Bridge::CardStore::FILE)
134
+ next unless File.file?(path)
135
+
136
+ entries = sidecar(path)
137
+ if entries.nil?
138
+ malformed += 1
139
+ next
140
+ end
141
+
142
+ entries.each do |entry|
143
+ ask = ask_of(entry, name, replayed, sessions_dir)
144
+ next if ask.nil?
145
+
146
+ ask.is_a?(Symbol) ? skipped[ask] += 1 : asks << ask
147
+ end
148
+ end
149
+ end
150
+
151
+ stats = asks.map { |ask| judge(ask, client, log, threshold, sessions_dir) }
152
+ print_table(stats, threshold, client, skipped, malformed)
153
+ CLI::Exit::OK
154
+ end
155
+
156
+ # The asks of one session's cards file, or nil when it isn't one: a
157
+ # sidecar that doesn't parse costs one warning line, never the run.
158
+ def sidecar(path)
159
+ return nil unless File.file?(path)
160
+
161
+ data = JSON.parse(File.read(path), symbolize_names: true)
162
+ return nil unless data.is_a?(Hash) && data[:entries].is_a?(Array)
163
+
164
+ data[:entries]
165
+ rescue StandardError => e
166
+ error_line("chi guardrails replay: #{path}: unreadable cards (#{e.class}: #{e.message}); skipped")
167
+ nil
168
+ end
169
+
170
+ # One card entry as an ask to judge: an Ask, a skip reason (:already_replayed,
171
+ # :answered_by_parent, :parent_cannot_allow, :unanswered, :no_approval,
172
+ # :no_question_id), or nil for an entry that is not a question at all.
173
+ def ask_of(entry, session_id, replayed, sessions_dir)
174
+ return nil unless entry.is_a?(Hash) && entry[:type] == "question"
175
+
176
+ pending = entry[:pending_question]
177
+ return :no_approval unless pending.is_a?(Hash) && pending[:approval].is_a?(Hash)
178
+
179
+ question_id = pending[:id].to_s
180
+ return :no_question_id if question_id.empty?
181
+ return :already_replayed if replayed.include?(question_id)
182
+
183
+ answer = entry[:answer]
184
+ label = if entry[:cancelled]
185
+ "none"
186
+ elsif answer.nil?
187
+ return :unanswered
188
+ elsif answer[:by] == "parent_agent"
189
+ # A parent's allow is not the user's decision. Its deny is a
190
+ # label only where the parent could have allowed: a parent
191
+ # that was not allowed to permit says nothing about the user
192
+ # (docs/guardrails.md, "System 1").
193
+ return :answered_by_parent unless label_for(answer, pending[:approval][:scopes]) == "deny"
194
+ return :parent_cannot_allow unless parent_may_permit?(pending, session_id, sessions_dir)
195
+
196
+ "deny"
197
+ else
198
+ label_for(answer, pending[:approval][:scopes])
199
+ end
200
+
201
+ Ask.new(question_id: question_id, session_id: session_id, approval: pending[:approval],
202
+ label: label, asked_at: pending[:created_at])
203
+ end
204
+
205
+ # Whether this session's parent could have allowed the ask: not a
206
+ # user-only ask, and a parent level that permits one. No session file:
207
+ # not permitted.
208
+ def parent_may_permit?(pending, session_id, sessions_dir)
209
+ return false if pending[:user_only] == true
210
+
211
+ session = session_of(session_id, sessions_dir)
212
+ return false if session.nil?
213
+
214
+ Guardrails::ParentPolicy.for_session(session).allow?("permit")
215
+ end
216
+
217
+ # The session a card belongs to, or nil when there is no session file;
218
+ # read once per session for the run.
219
+ def session_of(session_id, sessions_dir)
220
+ return nil unless Session.valid_id?(session_id)
221
+ return nil unless File.exist?(Session.session_file(session_id, state_dir: sessions_dir))
222
+
223
+ (@sessions ||= {})[session_id] ||= begin
224
+ Session.load(session_id, state_dir: sessions_dir)
225
+ rescue StandardError
226
+ nil
227
+ end
228
+ end
229
+
230
+ # What the user did, as the log's label: the scope they took, a denial
231
+ # when the answer is not one of the offered scopes, none for an ask that
232
+ # was cancelled (docs/guardrails.md, "System 1").
233
+ def label_for(answer, scopes)
234
+ index = answer[:selected_indices]&.first
235
+ return "deny" if index.nil?
236
+
237
+ scopes = Array(scopes)
238
+ if index < scopes.size
239
+ scopes[index] == "once" ? "allow_once" : "allow_wide"
240
+ else
241
+ "deny"
242
+ end
243
+ end
244
+
245
+ # Judge one ask, write its two lines unless this is a dry run, and return
246
+ # what the table counts. A judgment that never came is written with p
247
+ # null, exactly as the live audit writes it.
248
+ def judge(ask, client, log, threshold, sessions_dir)
249
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
250
+ answers = begin
251
+ client.ask(state: SystemOne::ApprovalQuestion.state(ask.approval, session_title: session_title(ask, sessions_dir)),
252
+ questions: { SystemOne::ApprovalQuestion::ID => SystemOne::ApprovalQuestion.question })
253
+ rescue StandardError => e
254
+ Samagotchi::Log.warn(:guardrails, "system_one_replay_ask_failed", error: "#{e.class}: #{e.message}")
255
+ nil
256
+ end
257
+ latency_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - started) * 1000).round
258
+ p = answers && answers[SystemOne::ApprovalQuestion::ID]&.noul
259
+
260
+ unless @dry_run
261
+ id = log.judgment(session_id: ask.session_id, rule: ask.approval[:rule], source: ask.approval[:source],
262
+ tool: ask.approval[:tool], **SystemOne::ApprovalQuestion.call_fields(ask.approval),
263
+ repo_name: ask.approval[:repo_name], p: p, threshold: threshold, model: client.model,
264
+ latency_ms: latency_ms, decision: "replay", origin: "replay",
265
+ question_id: ask.question_id, asked_at: ask.asked_at)
266
+ log.label(id, ask.label)
267
+ end
268
+
269
+ { rule: ask.approval[:rule].to_s, label: ask.label, p: p, session: ask.session_id }
270
+ end
271
+
272
+ # The session's title, as the ask's state carries it; nil when there is
273
+ # no session file (the cards alone are enough to replay).
274
+ def session_title(ask, sessions_dir)
275
+ session_of(ask.session_id, sessions_dir)&.display_title
276
+ end
277
+
278
+ # The threshold for this run: --threshold, or guardrails.system_one.threshold.
279
+ # @return [Float, String] the number, or the usage error's message
280
+ def threshold(value)
281
+ return Config.get("guardrails.system_one.threshold").to_f if value.nil?
282
+
283
+ number = Float(value.to_s.strip)
284
+ return "--threshold takes a number between 0 and 1, got #{value}" if number.negative? || number > 1.0
285
+
286
+ number
287
+ rescue ArgumentError, TypeError
288
+ "--threshold takes a number between 0 and 1, got #{value}"
289
+ end
290
+
291
+ def print_table(stats, threshold, client, skipped, malformed)
292
+ rows = stats.group_by { |row| row[:rule] }.map { |rule, asks| rule_row(rule, asks, threshold) }
293
+ rows.sort_by! { |rule, count, _| [-count, rule] }
294
+ totals = ["total", *COLUMNS[1..].each_with_index.map { |_column, offset| rows.sum { |row| row[offset + 1] } }]
295
+ all = rows + [totals]
296
+ widths = COLUMNS.each_with_index.map { |column, index| [column.size, all.map { |row| row[index].to_s.size }.max].max }
297
+
298
+ @stdout.puts(row_line(COLUMNS, widths))
299
+ @stdout.puts(widths.map { |width| "-" * width }.join(" "))
300
+ rows.each { |row| @stdout.puts(row_line(row, widths)) }
301
+ @stdout.puts(row_line(totals, widths))
302
+
303
+ sessions = stats.map { |row| row[:session] }.uniq.size
304
+ @stdout.puts("#{stats.size} asks replayed from #{sessions} session#{"s" unless sessions == 1} " \
305
+ "· threshold #{threshold} · model #{client.model || "the host's default"}")
306
+ notes = skipped.map { |reason, count| "#{count} #{SKIP_WORDS[reason] || reason}" }
307
+ notes << "#{malformed} unreadable cards file#{"s" unless malformed == 1}" if malformed.positive?
308
+ @stdout.puts("skipped: #{notes.sort.join(", ")}") unless notes.empty?
309
+ @stdout.puts("dry run: nothing was written") if @dry_run
310
+ end
311
+
312
+ # One table row, columns left-aligned to their width.
313
+ def row_line(values, widths)
314
+ values.each_with_index.map { |value, index| value.to_s.ljust(widths[index]) }.join(" ").rstrip
315
+ end
316
+
317
+ # One rule's row: asks, allowed, denied, auto-allowed at the threshold,
318
+ # false allows, asks with no judgment.
319
+ def rule_row(rule, rows, threshold)
320
+ allowed = rows.count { |row| ALLOW_LABELS.include?(row[:label]) }
321
+ denied = rows.count { |row| row[:label] == "deny" }
322
+ high = rows.select { |row| row[:p] && row[:p] >= threshold }
323
+ [rule, rows.size, allowed, denied,
324
+ high.count { |row| ALLOW_LABELS.include?(row[:label]) },
325
+ high.count { |row| row[:label] == "deny" },
326
+ rows.count { |row| row[:p].nil? }]
327
+ end
328
+ end
329
+ end