samagotchi 0.115.0 → 0.116.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +53 -1
- data/bin/chi +2 -0
- data/docs/cli.md +5 -1
- data/docs/configuration.md +7 -0
- data/docs/guardrails.md +188 -1
- data/docs/sessions.md +1 -1
- data/lib/samagotchi/bridge/turn_accumulator.rb +4 -0
- data/lib/samagotchi/bridge.rb +26 -0
- data/lib/samagotchi/bridge_client.rb +7 -0
- data/lib/samagotchi/bundles/system/manifest.yml +1 -1
- data/lib/samagotchi/config.rb +15 -0
- data/lib/samagotchi/engine.rb +6 -1
- data/lib/samagotchi/guardrail_wiring.rb +166 -2
- data/lib/samagotchi/guardrails/approval.rb +20 -0
- data/lib/samagotchi/guardrails/targets.rb +23 -2
- data/lib/samagotchi/guardrails/verdict.rb +23 -3
- data/lib/samagotchi/guardrails_command.rb +329 -0
- data/lib/samagotchi/memory_bundle/index_updater.rb +13 -0
- data/lib/samagotchi/session_access/contract.rb +1 -1
- data/lib/samagotchi/session_commands.rb +79 -2
- data/lib/samagotchi/session_metrics.rb +7 -1
- data/lib/samagotchi/system_one/answer.rb +78 -0
- data/lib/samagotchi/system_one/approval_question.rb +113 -0
- data/lib/samagotchi/system_one/client.rb +89 -0
- data/lib/samagotchi/system_one/errors.rb +14 -0
- data/lib/samagotchi/system_one/fake_backend.rb +45 -0
- data/lib/samagotchi/system_one/http_backend.rb +129 -0
- data/lib/samagotchi/system_one/log.rb +133 -0
- data/lib/samagotchi/system_one/question.rb +86 -0
- data/lib/samagotchi/system_one/record.rb +83 -0
- data/lib/samagotchi/system_one/self_protect.rb +90 -0
- data/lib/samagotchi/system_one/similar.rb +69 -0
- data/lib/samagotchi/system_one.rb +25 -0
- data/lib/samagotchi/terminal_ui/attached_loop.rb +15 -4
- data/lib/samagotchi/terminal_ui/event_renderer.rb +1 -0
- data/lib/samagotchi/terminal_ui/formatting.rb +14 -0
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +1 -0
- data/lib/samagotchi/web/public/activity.js +10 -0
- data/lib/samagotchi/web/public/app.js +3 -0
- data/lib/samagotchi/web/public/apple-touch-icon.png +0 -0
- data/lib/samagotchi/web/public/composer_fold.js +8 -8
- data/lib/samagotchi/web/public/composer_grip.js +65 -22
- data/lib/samagotchi/web/public/composer_size.js +30 -2
- data/lib/samagotchi/web/public/data.js +10 -0
- data/lib/samagotchi/web/public/favicon.svg +1 -1
- data/lib/samagotchi/web/public/icons/chi-180.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-192.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-512.png +0 -0
- data/lib/samagotchi/web/public/index.html +5 -0
- data/lib/samagotchi/web/public/system_one_wrong.js +32 -0
- data/lib/samagotchi/web/public/timing.js +9 -5
- data/lib/samagotchi/web/public/turn_events.js +1 -0
- data/lib/samagotchi/web/public/turn_html.js +13 -2
- data/lib/samagotchi/web/session_control.rb +22 -0
- data/lib/samagotchi/web/session_show.rb +35 -5
- metadata +15 -1
|
@@ -88,6 +88,19 @@ module Samagotchi
|
|
|
88
88
|
end
|
|
89
89
|
end
|
|
90
90
|
|
|
91
|
+
# +entry_name+'s description in +scope+'s index.md — the text after the
|
|
92
|
+
# " — " on its managed line — or nil when the index has no line for it
|
|
93
|
+
# or that line carries no description. One small file read.
|
|
94
|
+
def self.description_for(scope, entry_name)
|
|
95
|
+
index_path = index_path_for(scope)
|
|
96
|
+
return nil unless index_path && File.exist?(index_path)
|
|
97
|
+
|
|
98
|
+
match = File.read(index_path, encoding: "UTF-8").match(managed_pattern(entry_name))
|
|
99
|
+
match ? extract_description(match[0].chomp) : nil
|
|
100
|
+
rescue StandardError
|
|
101
|
+
nil
|
|
102
|
+
end
|
|
103
|
+
|
|
91
104
|
# The bundle a managed line names ("· from <bundle>", before any
|
|
92
105
|
# " — description"), or nil.
|
|
93
106
|
def self.extract_source(line)
|
|
@@ -26,7 +26,7 @@ module Samagotchi
|
|
|
26
26
|
# bin/chi subcommands that act on this machine: never sessions. chi
|
|
27
27
|
# remote (this machine's list of remotes) runs here under a remote too
|
|
28
28
|
# (Dispatch); the others are refused there.
|
|
29
|
-
MACHINE = %w[bootstrap update models desktop memory bundle remote].freeze
|
|
29
|
+
MACHINE = %w[bootstrap update models desktop memory bundle guardrails remote].freeze
|
|
30
30
|
|
|
31
31
|
# "command", "command sub" or either with a flag (its long name) →
|
|
32
32
|
# SUPPORTED or LOCAL_ONLY. A LOCAL_ONLY command or subcommand lists
|
|
@@ -19,6 +19,8 @@ require_relative "reasoning_history_command"
|
|
|
19
19
|
require_relative "working_set_command"
|
|
20
20
|
require_relative "recap_command"
|
|
21
21
|
require_relative "guardrails/parent_policy"
|
|
22
|
+
require_relative "system_one/log"
|
|
23
|
+
require_relative "system_one/record"
|
|
22
24
|
|
|
23
25
|
module Samagotchi
|
|
24
26
|
# The session commands a REPL and a session worker both run: /model,
|
|
@@ -91,7 +93,7 @@ module Samagotchi
|
|
|
91
93
|
registry.register(MODELS_COMMAND, "list the hosts' models (/models <text> filters)", anytime: true) do |text|
|
|
92
94
|
reply(models_listing(text.delete_prefix(MODELS_COMMAND).strip))
|
|
93
95
|
end
|
|
94
|
-
registry.register(GUARDRAILS_COMMAND, "list
|
|
96
|
+
registry.register(GUARDRAILS_COMMAND, "list guardrails, revoke an approval or flag a System 1 auto-allow",
|
|
95
97
|
mid_turn: show_mid_turn(GUARDRAILS_COMMAND)) do |text|
|
|
96
98
|
guardrails(text.delete_prefix(GUARDRAILS_COMMAND).strip)
|
|
97
99
|
end
|
|
@@ -418,11 +420,85 @@ module Samagotchi
|
|
|
418
420
|
|
|
419
421
|
def guardrails(args)
|
|
420
422
|
return guardrails_revoke(args.delete_prefix("revoke").strip) if args.start_with?("revoke")
|
|
421
|
-
return
|
|
423
|
+
return guardrails_wrong(args.sub(/\Awrong\b/, "").strip) if args.match?(/\Awrong(?:\s|\z)/)
|
|
424
|
+
return reply("usage: /guardrails [revoke N | wrong [N]]", status: :error) unless args.empty?
|
|
422
425
|
|
|
423
426
|
reply(guardrails_listing)
|
|
424
427
|
end
|
|
425
428
|
|
|
429
|
+
def guardrails_wrong(arg)
|
|
430
|
+
unless arg.empty? || arg.match?(/\A[1-9]\d*\z/)
|
|
431
|
+
return reply("usage: /guardrails wrong [N] (N from /guardrails)", status: :error)
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
index = arg.empty? ? 1 : arg.to_i
|
|
435
|
+
entry = system_one_auto_allows[index - 1]
|
|
436
|
+
return reply("no unlabelled System 1 auto-allows in this session (see /guardrails)") unless entry
|
|
437
|
+
unless system_one_log.label(entry["id"], "should_have_asked")
|
|
438
|
+
return reply("could not label System 1 auto-allow #{index} (see the guardrails log)", status: :error)
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
call = shorten_call(entry["call"].to_s.gsub(/\s+/, " "))
|
|
442
|
+
probability = entry["p"].nil? ? "unknown" : format("%.2f", entry["p"].to_f)
|
|
443
|
+
reply("labelled auto-allow #{index}: #{entry["rule"]}: #{call} (p=#{probability}); " \
|
|
444
|
+
"the rule will ask again until its record is rebuilt")
|
|
445
|
+
end
|
|
446
|
+
|
|
447
|
+
def system_one_log
|
|
448
|
+
SystemOne::Log.new(dir: File.dirname(@engine.guardrail_approvals.path))
|
|
449
|
+
end
|
|
450
|
+
|
|
451
|
+
def system_one_auto_allows(entries = system_one_entries)
|
|
452
|
+
session_id = @engine.session&.id
|
|
453
|
+
return [] unless session_id
|
|
454
|
+
|
|
455
|
+
entries.select do |entry|
|
|
456
|
+
entry["session_id"] == session_id && entry["decision"] == "auto" && entry["answer"].nil?
|
|
457
|
+
end.last(20).reverse
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
def system_one_entries
|
|
461
|
+
log = system_one_log
|
|
462
|
+
File.file?(log.path) ? log.entries : []
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
def system_one_listing(lines)
|
|
466
|
+
entries = system_one_entries
|
|
467
|
+
mode = Config.get("guardrails.system_one.mode").to_s
|
|
468
|
+
return if mode == "off" && entries.empty?
|
|
469
|
+
|
|
470
|
+
lines << "System 1 (#{mode}):"
|
|
471
|
+
auto_allows = system_one_auto_allows(entries)
|
|
472
|
+
lines << " auto-allows (this session, newest first; /guardrails wrong N):"
|
|
473
|
+
if auto_allows.empty?
|
|
474
|
+
lines << " (none)"
|
|
475
|
+
else
|
|
476
|
+
auto_allows.each_with_index do |entry, index|
|
|
477
|
+
call = shorten_call(entry["call"].to_s.gsub(/\s+/, " "))
|
|
478
|
+
probability = entry["p"].nil? ? "unknown" : format("%.2f", entry["p"].to_f)
|
|
479
|
+
lines << " #{index + 1}. #{entry["rule"]}: #{call} (p=#{probability})"
|
|
480
|
+
end
|
|
481
|
+
end
|
|
482
|
+
|
|
483
|
+
rules = entries.filter_map { |entry| entry["rule"] }.uniq.sort
|
|
484
|
+
lines << " records:"
|
|
485
|
+
if rules.empty?
|
|
486
|
+
lines << " (none)"
|
|
487
|
+
else
|
|
488
|
+
rules.each do |rule|
|
|
489
|
+
summary = SystemOne::Record.new(entries, rule: rule).summary
|
|
490
|
+
counts = summary.reject { |key, _| key == :rule }.map { |key, value| "#{key}=#{value}" }.join(", ")
|
|
491
|
+
lines << " #{rule}: #{counts}"
|
|
492
|
+
end
|
|
493
|
+
end
|
|
494
|
+
end
|
|
495
|
+
|
|
496
|
+
CALL_SHOWN = 100
|
|
497
|
+
|
|
498
|
+
def shorten_call(call)
|
|
499
|
+
call.length > CALL_SHOWN ? "#{call[0, CALL_SHOWN - 1]}…" : call
|
|
500
|
+
end
|
|
501
|
+
|
|
426
502
|
def guardrails_listing
|
|
427
503
|
rules = @engine.guardrail_rules
|
|
428
504
|
approvals = @engine.guardrail_approvals.entries
|
|
@@ -467,6 +543,7 @@ module Samagotchi
|
|
|
467
543
|
lines << " (none)" if approvals.empty?
|
|
468
544
|
approvals.each_with_index { |entry, idx| lines << " #{idx + 1}. #{approval_line(entry)}" }
|
|
469
545
|
lines << "revoke one with /guardrails revoke N" unless approvals.empty?
|
|
546
|
+
system_one_listing(lines)
|
|
470
547
|
lines.join("\n")
|
|
471
548
|
end
|
|
472
549
|
|
|
@@ -1106,6 +1106,12 @@ module Samagotchi
|
|
|
1106
1106
|
active[:via] = via.to_s if via
|
|
1107
1107
|
# A grep that found nothing (ok): the reloaded web row says "no match".
|
|
1108
1108
|
active[:no_match] = true if event.dig(:activity, :no_match)
|
|
1109
|
+
# Allowed once by System 1 instead of the user: the reloaded web row
|
|
1110
|
+
# says so and can flag the judgment it came from.
|
|
1111
|
+
guardrail = event.dig(:activity, :guardrail)
|
|
1112
|
+
if guardrail.is_a?(Hash) && guardrail[:decided_by] == "system one"
|
|
1113
|
+
active[:guardrail] = { rule: guardrail[:rule], p: guardrail[:p], judgment_id: guardrail[:judgment_id] }.compact
|
|
1114
|
+
end
|
|
1109
1115
|
finish_tool_record(active, status.empty? ? "ok" : status, waited_ms: event[:waited_ms].to_i)
|
|
1110
1116
|
end
|
|
1111
1117
|
end
|
|
@@ -1113,7 +1119,7 @@ module Samagotchi
|
|
|
1113
1119
|
# Caller holds the mutex.
|
|
1114
1120
|
def finish_tool_record(active, status, waited_ms: 0)
|
|
1115
1121
|
record = active.slice(
|
|
1116
|
-
:id, :turn_id, :iteration, :call_index, :tool, :called_as, :via, :no_match, :started_at
|
|
1122
|
+
:id, :turn_id, :iteration, :call_index, :tool, :called_as, :via, :no_match, :guardrail, :started_at
|
|
1117
1123
|
).merge(
|
|
1118
1124
|
status: status,
|
|
1119
1125
|
completed_at: now.iso8601(3),
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "question"
|
|
4
|
+
|
|
5
|
+
module Samagotchi
|
|
6
|
+
module SystemOne
|
|
7
|
+
# One judgment the System One API answered: for an :noul question,
|
|
8
|
+
# `noul` is the probability of "yes"; a :choice answer carries `choice`
|
|
9
|
+
# (the criterion chosen) and `confidence`; a :score answer carries
|
|
10
|
+
# `score` (the level's index) and `confidence`. `probabilities` is what
|
|
11
|
+
# the model put on each option, when it says.
|
|
12
|
+
Answer = Data.define(:id, :type, :noul, :choice, :score, :confidence, :probabilities) do
|
|
13
|
+
def initialize(id:, type:, noul: nil, choice: nil, score: nil, confidence: nil, probabilities: nil) = super
|
|
14
|
+
|
|
15
|
+
# The response's answers entry as the wire sends it, or nil when it is
|
|
16
|
+
# not usable: no type, a type the contract doesn't have, a noul
|
|
17
|
+
# outside 0..1, a score that isn't a whole number, … A nil means "no
|
|
18
|
+
# judgment for this question", never a guess.
|
|
19
|
+
def self.from_wire(id, raw)
|
|
20
|
+
return nil unless raw.is_a?(Hash)
|
|
21
|
+
|
|
22
|
+
type = raw["type"].to_s.strip.downcase.to_sym
|
|
23
|
+
return nil unless QUESTION_TYPES.include?(type)
|
|
24
|
+
|
|
25
|
+
confidence = probability(raw["confidence"])
|
|
26
|
+
odds = probabilities(raw["probabilities"])
|
|
27
|
+
# A field the server sent but can't read is as unusable as a missing answer.
|
|
28
|
+
return nil if bad_field(raw, "confidence", confidence) || bad_field(raw, "probabilities", odds)
|
|
29
|
+
|
|
30
|
+
answer = { id: id, type: type, confidence: confidence, probabilities: odds }
|
|
31
|
+
|
|
32
|
+
case type
|
|
33
|
+
when :noul
|
|
34
|
+
noul = probability(raw["noul"])
|
|
35
|
+
noul.nil? ? nil : new(**answer, noul: noul)
|
|
36
|
+
when :choice
|
|
37
|
+
choice = raw["choice"].to_s.strip
|
|
38
|
+
choice.empty? ? nil : new(**answer, choice: choice)
|
|
39
|
+
else
|
|
40
|
+
score = raw["score"]
|
|
41
|
+
score = score.to_i if score.is_a?(Float) && score.to_i == score
|
|
42
|
+
score.is_a?(Integer) && score >= 0 ? new(**answer, score: score) : nil
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# A value the wire sent that we could not parse.
|
|
47
|
+
private_class_method def self.bad_field(raw, name, parsed)
|
|
48
|
+
raw.key?(name) && !raw[name].nil? && parsed.nil?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# A probability as written: a number in 0..1, else nil.
|
|
52
|
+
private_class_method def self.probability(value)
|
|
53
|
+
return nil unless value.is_a?(Numeric)
|
|
54
|
+
|
|
55
|
+
value = value.to_f
|
|
56
|
+
return nil if value.nan? || value.infinite? || value.negative? || value > 1.0
|
|
57
|
+
|
|
58
|
+
value
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# The probabilities as sent: a map of option => number, or a list of
|
|
62
|
+
# numbers (the options in order). Anything else is malformed.
|
|
63
|
+
private_class_method def self.probabilities(value)
|
|
64
|
+
if value.is_a?(Hash)
|
|
65
|
+
pairs = value.map { |name, weight| [name.to_s, probability(weight)] }
|
|
66
|
+
return nil if pairs.any? { |_name, weight| weight.nil? }
|
|
67
|
+
|
|
68
|
+
pairs.to_h
|
|
69
|
+
elsif value.is_a?(Array)
|
|
70
|
+
weights = value.map { |weight| probability(weight) }
|
|
71
|
+
return nil if weights.any?(&:nil?)
|
|
72
|
+
|
|
73
|
+
weights
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
require_relative "question"
|
|
6
|
+
require_relative "similar"
|
|
7
|
+
|
|
8
|
+
module Samagotchi
|
|
9
|
+
module SystemOne
|
|
10
|
+
# The one question an approval asks System 1: "would this user allow it
|
|
11
|
+
# without hesitation?" (docs/guardrails.md, "System 1"). The instruction
|
|
12
|
+
# is a constant in code, reviewed like a prompt; the state is the ask's
|
|
13
|
+
# own facts — the rule that asked, why, the call, and where it runs — plus
|
|
14
|
+
# the memory an ask names and the earlier asks it resembles.
|
|
15
|
+
module ApprovalQuestion
|
|
16
|
+
ID = "allow"
|
|
17
|
+
# What the judgment is asked, in the model's terms. Frozen: a change
|
|
18
|
+
# here is a change in behaviour, not a wording tweak.
|
|
19
|
+
INSTRUCTIONS = "A coding agent wants to run this tool call, and a guardrail rule asked its user to approve it. " \
|
|
20
|
+
"The state names the rule, why it asked, the call and where it runs. Would the user approve it " \
|
|
21
|
+
"without hesitation? Users usually approve routine reads, edits of the agent's own notes and " \
|
|
22
|
+
"memory, and pushes of verified work; they usually refuse discarding uncommitted work, " \
|
|
23
|
+
"interactive rebases, deleting outside the project, and anything irreversible they did not ask " \
|
|
24
|
+
"for. The state may also list a few of the user's earlier decisions on calls like this one, " \
|
|
25
|
+
"most alike first, each with what they answered."
|
|
26
|
+
# How much of a command, a diff or a path list the state carries.
|
|
27
|
+
TEXT_CHARS = 2000
|
|
28
|
+
# How much of the call the judgment log keeps; its digest is of the whole.
|
|
29
|
+
CALL_CHARS = 300
|
|
30
|
+
|
|
31
|
+
# @return [Question] the noul question for id "allow"
|
|
32
|
+
def self.question
|
|
33
|
+
# The type as a string: a symbol here reads as an event type to
|
|
34
|
+
# spec/web/participant_view_spec.
|
|
35
|
+
Question.new(id: ID, type: "noul", instructions: INSTRUCTIONS)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @param approval [Hash] payload[:approval]: the ask the user is shown
|
|
39
|
+
# @param session_title [String, nil] the session's title, when it has one
|
|
40
|
+
# @param entries [Array<Hash>] SystemOne::Log#entries: the judgments the
|
|
41
|
+
# user has answered so far, read once per ask and shared with the
|
|
42
|
+
# record gate (GuardrailWiring#system_one_judge)
|
|
43
|
+
# @return [Hash] the state, with the fields the ask didn't give left out
|
|
44
|
+
def self.state(approval, session_title: nil, entries: nil)
|
|
45
|
+
{
|
|
46
|
+
"rule" => approval[:rule],
|
|
47
|
+
"source" => approval[:source],
|
|
48
|
+
"reason" => approval[:reason],
|
|
49
|
+
"tool" => approval[:tool],
|
|
50
|
+
"command" => cut(approval[:command]),
|
|
51
|
+
"paths" => Array(approval[:paths]).map { |path| cut(path) },
|
|
52
|
+
"change" => change_text(approval[:preview]),
|
|
53
|
+
"cwd" => approval[:cwd],
|
|
54
|
+
"repo_name" => approval[:repo_name],
|
|
55
|
+
"branch" => approval[:branch],
|
|
56
|
+
"session" => session_title,
|
|
57
|
+
# Which memory entry the ask is about, when it is a memory ask.
|
|
58
|
+
"memory" => memory_facts(approval[:memory]),
|
|
59
|
+
# The asks this one looks like, and what the user did about them.
|
|
60
|
+
"examples" => Similar.select(entries: entries, rule: approval[:rule], repo_name: approval[:repo_name],
|
|
61
|
+
call_text: call_text(approval))
|
|
62
|
+
}.reject do |_key, value|
|
|
63
|
+
value.nil? || (value.is_a?(String) && value.strip.empty?) ||
|
|
64
|
+
((value.is_a?(Array) || value.is_a?(Hash)) && value.empty?)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# What the call is, whole: the command, the paths, or a plugin tool's
|
|
69
|
+
# arguments.
|
|
70
|
+
def self.call_text(approval)
|
|
71
|
+
command = approval[:command]
|
|
72
|
+
return command.to_s if command && !command.to_s.empty?
|
|
73
|
+
|
|
74
|
+
paths = Array(approval[:paths])
|
|
75
|
+
return paths.join("\n") unless paths.empty?
|
|
76
|
+
|
|
77
|
+
approval[:args].to_s
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# The judgment log's two fields for the call: what it was, cut to
|
|
81
|
+
# CALL_CHARS, and the SHA256 of the whole of it.
|
|
82
|
+
def self.call_fields(approval)
|
|
83
|
+
call = call_text(approval)
|
|
84
|
+
{ "call" => cut(call, CALL_CHARS), "call_digest" => Digest::SHA256.hexdigest(call) }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# The memory an ask is about, as the approval card built it: entry name,
|
|
88
|
+
# scope, and (for a removal) the description its index line carries.
|
|
89
|
+
private_class_method def self.memory_facts(memory)
|
|
90
|
+
return nil unless memory.is_a?(Hash) && !memory.empty?
|
|
91
|
+
|
|
92
|
+
memory.transform_keys(&:to_s).reject { |_key, value| value.to_s.strip.empty? }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
private_class_method def self.change_text(preview)
|
|
96
|
+
return nil unless preview.is_a?(Hash)
|
|
97
|
+
|
|
98
|
+
text = preview[:text] || preview["text"]
|
|
99
|
+
return cut(text) if text && !text.to_s.strip.empty?
|
|
100
|
+
|
|
101
|
+
note = preview[:error] || preview["error"] || preview[:skipped] || preview["skipped"]
|
|
102
|
+
return note.to_s unless note.to_s.strip.empty?
|
|
103
|
+
|
|
104
|
+
"+#{(preview[:added] || preview["added"]).to_i} −#{(preview[:removed] || preview["removed"]).to_i}"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
private_class_method def self.cut(text, chars = TEXT_CHARS)
|
|
108
|
+
text = text.to_s
|
|
109
|
+
text.length > chars ? "#{text[0, chars - 1]}…" : text
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../config"
|
|
4
|
+
require_relative "../log"
|
|
5
|
+
require_relative "answer"
|
|
6
|
+
require_relative "errors"
|
|
7
|
+
require_relative "http_backend"
|
|
8
|
+
require_relative "question"
|
|
9
|
+
|
|
10
|
+
module Samagotchi
|
|
11
|
+
module SystemOne
|
|
12
|
+
# The one way anything asks System 1 a question: #ask answers a set of
|
|
13
|
+
# questions about a state, or says nothing. A backend that is unreachable,
|
|
14
|
+
# slow, refusing or nonsensical is a nil — no judgment, one log line — and
|
|
15
|
+
# the caller asks the user as it would have anyway.
|
|
16
|
+
#
|
|
17
|
+
# client = SystemOne::Client.from_config(host_registry: registry)
|
|
18
|
+
# client&.ask(state: "git push origin main", questions: {"allow" => question})
|
|
19
|
+
class Client
|
|
20
|
+
MODES = %w[off audit on].freeze
|
|
21
|
+
DEFAULT_TIMEOUT_MS = 2000
|
|
22
|
+
|
|
23
|
+
attr_reader :mode, :host, :model, :backend
|
|
24
|
+
|
|
25
|
+
def initialize(backend:, mode: "off", host: nil, model: nil)
|
|
26
|
+
@backend = backend
|
|
27
|
+
@mode = mode.to_s.strip.downcase
|
|
28
|
+
@host = host
|
|
29
|
+
@model = model
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# nil when judgment is off (the default) or no host is named, so the
|
|
33
|
+
# caller needs no other check. The settings are read every call: a
|
|
34
|
+
# config.yml change takes effect on the next ask.
|
|
35
|
+
# @param model [String, nil] a model for this call only (chi guardrails
|
|
36
|
+
# replay --model), else the configured one
|
|
37
|
+
def self.from_config(host_registry:, env: ENV, model: nil)
|
|
38
|
+
mode = Config.get("guardrails.system_one.mode").to_s.strip.downcase
|
|
39
|
+
return nil unless MODES.include?(mode) && mode != "off"
|
|
40
|
+
|
|
41
|
+
host = Config.get("guardrails.system_one.host").to_s.strip
|
|
42
|
+
return nil if host.empty?
|
|
43
|
+
|
|
44
|
+
model = (model.nil? ? Config.get("guardrails.system_one.model") : model).to_s.strip
|
|
45
|
+
timeout_ms = Config.get("guardrails.system_one.timeout_ms").to_i
|
|
46
|
+
timeout_ms = DEFAULT_TIMEOUT_MS unless timeout_ms.positive?
|
|
47
|
+
|
|
48
|
+
new(backend: HttpBackend.new(host_registry: host_registry, host: host, model: model.empty? ? nil : model,
|
|
49
|
+
timeout_ms: timeout_ms, env: env),
|
|
50
|
+
mode: mode, host: host, model: model.empty? ? nil : model)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# @param state [String, Hash, Array] what the question is about, as the
|
|
54
|
+
# contract takes it
|
|
55
|
+
# @param questions [Hash{String => Question|Hash}] the questions, by id
|
|
56
|
+
# @return [Hash{String => Answer}, nil] one Answer per question, or nil
|
|
57
|
+
# when there is no judgment at all
|
|
58
|
+
def ask(state:, questions:, cancel_controller: nil)
|
|
59
|
+
questions = SystemOne.normalize_questions(questions)
|
|
60
|
+
return nil if questions.empty?
|
|
61
|
+
|
|
62
|
+
answers = begin
|
|
63
|
+
@backend.ask(state: state, questions: questions, cancel_controller: cancel_controller)
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
return no_judgment("#{e.class}: #{e.message}", error: e.class.name)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
unless answers.is_a?(Hash)
|
|
69
|
+
return no_judgment("the backend returned #{answers.inspect}")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
answers = questions.to_h { |question| [question.id, answers[question.id]] }
|
|
73
|
+
missing = questions.reject { |question| answers[question.id].is_a?(Answer) }
|
|
74
|
+
return no_judgment("no answer for #{missing.map(&:id).join(", ")}") unless missing.empty?
|
|
75
|
+
|
|
76
|
+
answers
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# One warn, one nil: the ask goes to the user.
|
|
82
|
+
def no_judgment(reason, **fields)
|
|
83
|
+
# Qualified: inside SystemOne, the bare Log is now SystemOne::Log.
|
|
84
|
+
Samagotchi::Log.warn(:guardrails, "system_one_failed", echo: "[samagotchi:guardrails] system one: #{reason}", host: @host, **fields)
|
|
85
|
+
nil
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Samagotchi
|
|
4
|
+
module SystemOne
|
|
5
|
+
# A judgment that did not happen: the backend could not reach the host,
|
|
6
|
+
# the host refused the request, or its answer was not usable. Client#ask
|
|
7
|
+
# turns any of these into nil (no judgment), so the caller asks the user.
|
|
8
|
+
class BackendError < StandardError; end
|
|
9
|
+
|
|
10
|
+
# The named host is not a System One host: it isn't in hosts:, its api
|
|
11
|
+
# is not openai, or it names no model.
|
|
12
|
+
class UnsupportedHost < BackendError; end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
|
|
5
|
+
module Samagotchi
|
|
6
|
+
module SystemOne
|
|
7
|
+
# The backend the specs use: answers from a block, from a fixed hash, or
|
|
8
|
+
# an error every time. It never reaches the network.
|
|
9
|
+
#
|
|
10
|
+
# FakeBackend.new { |state, questions| {questions.first.id => answer} }
|
|
11
|
+
# FakeBackend.new("git-push" => Answer.from_wire("git-push", {"type" => "noul", "noul" => 0.97}))
|
|
12
|
+
# FakeBackend.new(failing: true)
|
|
13
|
+
class FakeBackend
|
|
14
|
+
attr_reader :calls
|
|
15
|
+
|
|
16
|
+
# @param answers [Hash{String => Answer}, nil] the same answers for
|
|
17
|
+
# every call
|
|
18
|
+
# @param failing [Boolean] true: every #ask raises
|
|
19
|
+
# @param error [String] what a failing backend says
|
|
20
|
+
# @param judgments [#call] (state, questions) => {id => Answer}
|
|
21
|
+
def initialize(answers = nil, failing: false, error: "fake System One backend is unavailable", &judgments)
|
|
22
|
+
@answers = answers
|
|
23
|
+
@judgments = judgments
|
|
24
|
+
@failing = failing
|
|
25
|
+
@error = error
|
|
26
|
+
@calls = 0
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def ask(state:, questions:, cancel_controller: nil)
|
|
30
|
+
@calls += 1
|
|
31
|
+
raise BackendError, @error if @failing
|
|
32
|
+
|
|
33
|
+
judged = @judgments&.call(state, questions)
|
|
34
|
+
return judged unless judged.nil?
|
|
35
|
+
|
|
36
|
+
ids = Array(questions).map { |question| question.respond_to?(:id) ? question.id : question.to_s }
|
|
37
|
+
(@answers || {}).select { |id, _answer| ids.include?(id) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def failing? = @failing
|
|
41
|
+
|
|
42
|
+
def inspect = "#<#{self.class.name} calls=#{@calls}#{" failing" if @failing}>"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "uri"
|
|
5
|
+
require_relative "../config"
|
|
6
|
+
require_relative "../llm/api_key"
|
|
7
|
+
require_relative "../llm/http"
|
|
8
|
+
require_relative "answer"
|
|
9
|
+
require_relative "question"
|
|
10
|
+
require_relative "errors"
|
|
11
|
+
|
|
12
|
+
module Samagotchi
|
|
13
|
+
module SystemOne
|
|
14
|
+
# The TypeSafe System One judgment API on a host from config.yml's
|
|
15
|
+
# `hosts:`: POST <openai_base_url>/systemone (that base already ends in
|
|
16
|
+
# /v1), one request per #ask, no retries — a judgment that doesn't
|
|
17
|
+
# arrive in time is no judgment, and the ask goes to the user.
|
|
18
|
+
#
|
|
19
|
+
# Works for Splash (a local `api: openai` host serving /v1/systemone)
|
|
20
|
+
# and for the hosted Jev API the same way; a host whose api isn't
|
|
21
|
+
# openai is refused.
|
|
22
|
+
class HttpBackend
|
|
23
|
+
DEFAULT_TIMEOUT_MS = 2000
|
|
24
|
+
|
|
25
|
+
# @param host_registry [HostRegistry] where the host is looked up
|
|
26
|
+
# @param host [String] the hosts: entry name
|
|
27
|
+
# @param model [String, nil] the judgment model; nil: the host's first
|
|
28
|
+
# model (hosts.<name>.models), and no judgment when it has none
|
|
29
|
+
# @param timeout_ms [Integer] the whole request's limit, in ms
|
|
30
|
+
# @param env [Hash] where the host's api_key_env: key is read
|
|
31
|
+
def initialize(host_registry:, host:, model: nil, timeout_ms: DEFAULT_TIMEOUT_MS, env: ENV)
|
|
32
|
+
@host_registry = host_registry
|
|
33
|
+
@host = host.to_s.strip.downcase
|
|
34
|
+
@model = model.nil? ? nil : model.to_s.strip
|
|
35
|
+
@timeout_ms = timeout_ms.to_i
|
|
36
|
+
@env = env
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @return [Hash{String => Answer}] one Answer per question, nil for a
|
|
40
|
+
# question the response answered badly
|
|
41
|
+
# @raise [UnsupportedHost] the host is unknown, not an openai host, or
|
|
42
|
+
# names no model
|
|
43
|
+
# @raise [BackendError] the request failed, the status isn't 200, or
|
|
44
|
+
# the body isn't a usable response
|
|
45
|
+
def ask(state:, questions:, cancel_controller: nil)
|
|
46
|
+
entry = @host_registry.find_entry(@host)
|
|
47
|
+
raise UnsupportedHost, "host #{@host.inspect} is not in hosts:" unless entry
|
|
48
|
+
unless entry.api&.to_sym == :openai
|
|
49
|
+
raise UnsupportedHost, "host #{@host} has api: #{entry.api || "llama_cpp"}; System One speaks the OpenAI-compatible API"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
model = @model || host_model(entry)
|
|
53
|
+
raise UnsupportedHost, "host #{@host} names no model for System One" if model.nil? || model.empty?
|
|
54
|
+
|
|
55
|
+
url = "#{entry.openai_base_url.to_s.chomp("/")}/systemone"
|
|
56
|
+
questions = SystemOne.normalize_questions(questions)
|
|
57
|
+
body = { model: model, state: state, questions: questions.to_h { |question| [question.id, question.to_wire] } }
|
|
58
|
+
request = json_post(url, body)
|
|
59
|
+
response = http(entry).fetch(URI(url), request, retries: false, check_status: false, cancel_controller: cancel_controller,
|
|
60
|
+
log_fields: { purpose: "system_one", model: model })
|
|
61
|
+
|
|
62
|
+
status = response.code.to_i
|
|
63
|
+
unless status == 200
|
|
64
|
+
raise BackendError, "#{@host}: #{status} #{error_detail(response.body)}"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
answers = answers_of(response.body)
|
|
68
|
+
questions.to_h { |question| [question.id, Answer.from_wire(question.id, answers[question.id])] }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# The host's first model: whatever hosts.<name>.models holds, in the
|
|
74
|
+
# shape the registry has it in (HostModel by id, or as written).
|
|
75
|
+
def host_model(entry)
|
|
76
|
+
models = entry.models
|
|
77
|
+
return nil if models.nil? || models.empty?
|
|
78
|
+
|
|
79
|
+
first = models.is_a?(Hash) ? models.values.first : models.first
|
|
80
|
+
first.respond_to?(:id) ? first.id : first.to_s
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# One attempt, closed on the timeout: a judgment is worth at most
|
|
84
|
+
# timeout_ms, not a retry chain.
|
|
85
|
+
def http(entry)
|
|
86
|
+
seconds = @timeout_ms.positive? ? @timeout_ms / 1000.0 : DEFAULT_TIMEOUT_MS / 1000.0
|
|
87
|
+
open_timeout = [seconds, Config.positive_seconds("server.open_timeout")].min
|
|
88
|
+
LLM::HTTP.new(label: entry.name, open_timeout: open_timeout, read_timeout: seconds,
|
|
89
|
+
retry_policy: LLM::HTTP::RetryPolicy.none,
|
|
90
|
+
api_key: LLM::ApiKey.for(entry.api_key_env, host: entry.name, env: @env, scheme: entry.auth_scheme))
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# The POST the adapters build: JSON body, Content-Type, and the host's
|
|
94
|
+
# key on the header its auth: names (LLM::HTTP adds it).
|
|
95
|
+
def json_post(url, body)
|
|
96
|
+
Net::HTTP::Post.new(URI(url)).tap do |request|
|
|
97
|
+
request["Content-Type"] = "application/json"
|
|
98
|
+
request.body = JSON.generate(body)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def answers_of(body)
|
|
103
|
+
data = JSON.parse(body.to_s)
|
|
104
|
+
raise BackendError, "the response is not JSON: #{body.to_s[0, 200]}" unless data.is_a?(Hash)
|
|
105
|
+
|
|
106
|
+
answers = data["answers"]
|
|
107
|
+
raise BackendError, "the response has no answers map" unless answers.is_a?(Hash)
|
|
108
|
+
|
|
109
|
+
answers
|
|
110
|
+
rescue JSON::ParserError => e
|
|
111
|
+
raise BackendError, "the response is not JSON: #{e.message[0, 200]}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# A 422's detail array, or the body's start, for the log line.
|
|
115
|
+
def error_detail(body)
|
|
116
|
+
data = JSON.parse(body.to_s)
|
|
117
|
+
detail = data["detail"]
|
|
118
|
+
text = if detail.is_a?(Array)
|
|
119
|
+
detail.map { |item| item.is_a?(Hash) ? item["msg"] || item.to_s : item.to_s }.join("; ")
|
|
120
|
+
else
|
|
121
|
+
detail.to_s
|
|
122
|
+
end
|
|
123
|
+
text.empty? ? body.to_s[0, 200] : text[0, 200]
|
|
124
|
+
rescue StandardError
|
|
125
|
+
body.to_s[0, 200]
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|