samagotchi 0.115.0 → 0.117.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +76 -1
- data/bin/chi +2 -0
- data/docs/cli.md +33 -2
- data/docs/configuration.md +13 -0
- data/docs/guardrails.md +188 -1
- data/docs/internals/background-tasks.md +1 -1
- data/docs/sessions.md +3 -3
- data/lib/samagotchi/bridge/turn_accumulator.rb +4 -0
- data/lib/samagotchi/bridge.rb +114 -14
- data/lib/samagotchi/bridge_client.rb +20 -1
- data/lib/samagotchi/bundles/system/manifest.yml +1 -1
- data/lib/samagotchi/config.rb +34 -3
- data/lib/samagotchi/engine.rb +6 -1
- data/lib/samagotchi/guardrail_wiring.rb +166 -2
- data/lib/samagotchi/guardrails/approval.rb +20 -0
- data/lib/samagotchi/guardrails/targets.rb +23 -2
- data/lib/samagotchi/guardrails/verdict.rb +23 -3
- data/lib/samagotchi/guardrails_command.rb +329 -0
- data/lib/samagotchi/hold_stop.rb +89 -0
- data/lib/samagotchi/memory_bundle/index_updater.rb +13 -0
- data/lib/samagotchi/prompt_warmup.rb +8 -3
- data/lib/samagotchi/session_access/contract.rb +1 -1
- data/lib/samagotchi/session_access/web_bridge_client.rb +9 -4
- data/lib/samagotchi/session_commands.rb +79 -2
- data/lib/samagotchi/session_manager.rb +3 -2
- data/lib/samagotchi/session_metrics.rb +7 -1
- data/lib/samagotchi/system_one/answer.rb +78 -0
- data/lib/samagotchi/system_one/approval_question.rb +113 -0
- data/lib/samagotchi/system_one/client.rb +89 -0
- data/lib/samagotchi/system_one/errors.rb +14 -0
- data/lib/samagotchi/system_one/fake_backend.rb +45 -0
- data/lib/samagotchi/system_one/http_backend.rb +129 -0
- data/lib/samagotchi/system_one/log.rb +133 -0
- data/lib/samagotchi/system_one/question.rb +86 -0
- data/lib/samagotchi/system_one/record.rb +83 -0
- data/lib/samagotchi/system_one/self_protect.rb +90 -0
- data/lib/samagotchi/system_one/similar.rb +69 -0
- data/lib/samagotchi/system_one.rb +25 -0
- data/lib/samagotchi/task_watch.rb +23 -5
- data/lib/samagotchi/terminal_ui/attached_loop.rb +145 -12
- data/lib/samagotchi/terminal_ui/event_renderer.rb +1 -0
- data/lib/samagotchi/terminal_ui/formatting.rb +14 -0
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +3 -0
- data/lib/samagotchi/web/public/activity.js +10 -0
- data/lib/samagotchi/web/public/app.js +20 -1
- data/lib/samagotchi/web/public/apple-touch-icon.png +0 -0
- data/lib/samagotchi/web/public/composer_fold.js +8 -8
- data/lib/samagotchi/web/public/composer_grip.js +65 -22
- data/lib/samagotchi/web/public/composer_size.js +40 -5
- data/lib/samagotchi/web/public/data.js +24 -0
- data/lib/samagotchi/web/public/favicon.svg +1 -1
- data/lib/samagotchi/web/public/icons/chi-180.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-192.png +0 -0
- data/lib/samagotchi/web/public/icons/chi-512.png +0 -0
- data/lib/samagotchi/web/public/index.html +21 -0
- data/lib/samagotchi/web/public/session_actions.js +39 -6
- data/lib/samagotchi/web/public/stop_sheet.js +126 -0
- data/lib/samagotchi/web/public/system_one_wrong.js +32 -0
- data/lib/samagotchi/web/public/timing.js +9 -5
- data/lib/samagotchi/web/public/turn_events.js +1 -0
- data/lib/samagotchi/web/public/turn_html.js +13 -2
- data/lib/samagotchi/web/session_access_routes.rb +9 -6
- data/lib/samagotchi/web/session_control.rb +62 -3
- data/lib/samagotchi/web/session_show.rb +35 -5
- data/lib/samagotchi/worker.rb +57 -5
- data/lib/samagotchi/worker_idle_exit.rb +111 -9
- metadata +17 -1
|
@@ -1141,11 +1141,12 @@ module Samagotchi
|
|
|
1141
1141
|
# there is no live SSE or in-process cancel/answer).
|
|
1142
1142
|
# @param on_input [#call, nil] called after the Bridge queues a turn
|
|
1143
1143
|
def self.start_bridge(engine:, state_dir:, session_id:, on_input: nil, on_command: nil, on_exit_request: nil,
|
|
1144
|
-
exit_discards: nil)
|
|
1144
|
+
exit_discards: nil, exit_stopped: nil, on_holds: nil, on_holds_stop: nil)
|
|
1145
1145
|
require_relative "bridge"
|
|
1146
1146
|
Samagotchi::Bridge.new(
|
|
1147
1147
|
engine: engine, state_dir: state_dir, session_id: session_id, input_format: SessionInbox::INPUT_FORMAT,
|
|
1148
|
-
on_input: on_input, on_command: on_command, on_exit_request: on_exit_request, exit_discards: exit_discards
|
|
1148
|
+
on_input: on_input, on_command: on_command, on_exit_request: on_exit_request, exit_discards: exit_discards,
|
|
1149
|
+
exit_stopped: exit_stopped, on_holds: on_holds, on_holds_stop: on_holds_stop
|
|
1149
1150
|
).start
|
|
1150
1151
|
rescue StandardError => e
|
|
1151
1152
|
Log.error(:bridge, "start_failed", echo: "Bridge: failed to start for session #{session_id}: #{e.class}: #{e.message}", sid: session_id, error: e.class.name)
|
|
@@ -1106,6 +1106,12 @@ module Samagotchi
|
|
|
1106
1106
|
active[:via] = via.to_s if via
|
|
1107
1107
|
# A grep that found nothing (ok): the reloaded web row says "no match".
|
|
1108
1108
|
active[:no_match] = true if event.dig(:activity, :no_match)
|
|
1109
|
+
# Allowed once by System 1 instead of the user: the reloaded web row
|
|
1110
|
+
# says so and can flag the judgment it came from.
|
|
1111
|
+
guardrail = event.dig(:activity, :guardrail)
|
|
1112
|
+
if guardrail.is_a?(Hash) && guardrail[:decided_by] == "system one"
|
|
1113
|
+
active[:guardrail] = { rule: guardrail[:rule], p: guardrail[:p], judgment_id: guardrail[:judgment_id] }.compact
|
|
1114
|
+
end
|
|
1109
1115
|
finish_tool_record(active, status.empty? ? "ok" : status, waited_ms: event[:waited_ms].to_i)
|
|
1110
1116
|
end
|
|
1111
1117
|
end
|
|
@@ -1113,7 +1119,7 @@ module Samagotchi
|
|
|
1113
1119
|
# Caller holds the mutex.
|
|
1114
1120
|
def finish_tool_record(active, status, waited_ms: 0)
|
|
1115
1121
|
record = active.slice(
|
|
1116
|
-
:id, :turn_id, :iteration, :call_index, :tool, :called_as, :via, :no_match, :started_at
|
|
1122
|
+
:id, :turn_id, :iteration, :call_index, :tool, :called_as, :via, :no_match, :guardrail, :started_at
|
|
1117
1123
|
).merge(
|
|
1118
1124
|
status: status,
|
|
1119
1125
|
completed_at: now.iso8601(3),
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "question"
|
|
4
|
+
|
|
5
|
+
module Samagotchi
|
|
6
|
+
module SystemOne
|
|
7
|
+
# One judgment the System One API answered: for an :noul question,
|
|
8
|
+
# `noul` is the probability of "yes"; a :choice answer carries `choice`
|
|
9
|
+
# (the criterion chosen) and `confidence`; a :score answer carries
|
|
10
|
+
# `score` (the level's index) and `confidence`. `probabilities` is what
|
|
11
|
+
# the model put on each option, when it says.
|
|
12
|
+
Answer = Data.define(:id, :type, :noul, :choice, :score, :confidence, :probabilities) do
|
|
13
|
+
def initialize(id:, type:, noul: nil, choice: nil, score: nil, confidence: nil, probabilities: nil) = super
|
|
14
|
+
|
|
15
|
+
# The response's answers entry as the wire sends it, or nil when it is
|
|
16
|
+
# not usable: no type, a type the contract doesn't have, a noul
|
|
17
|
+
# outside 0..1, a score that isn't a whole number, … A nil means "no
|
|
18
|
+
# judgment for this question", never a guess.
|
|
19
|
+
def self.from_wire(id, raw)
|
|
20
|
+
return nil unless raw.is_a?(Hash)
|
|
21
|
+
|
|
22
|
+
type = raw["type"].to_s.strip.downcase.to_sym
|
|
23
|
+
return nil unless QUESTION_TYPES.include?(type)
|
|
24
|
+
|
|
25
|
+
confidence = probability(raw["confidence"])
|
|
26
|
+
odds = probabilities(raw["probabilities"])
|
|
27
|
+
# A field the server sent but can't read is as unusable as a missing answer.
|
|
28
|
+
return nil if bad_field(raw, "confidence", confidence) || bad_field(raw, "probabilities", odds)
|
|
29
|
+
|
|
30
|
+
answer = { id: id, type: type, confidence: confidence, probabilities: odds }
|
|
31
|
+
|
|
32
|
+
case type
|
|
33
|
+
when :noul
|
|
34
|
+
noul = probability(raw["noul"])
|
|
35
|
+
noul.nil? ? nil : new(**answer, noul: noul)
|
|
36
|
+
when :choice
|
|
37
|
+
choice = raw["choice"].to_s.strip
|
|
38
|
+
choice.empty? ? nil : new(**answer, choice: choice)
|
|
39
|
+
else
|
|
40
|
+
score = raw["score"]
|
|
41
|
+
score = score.to_i if score.is_a?(Float) && score.to_i == score
|
|
42
|
+
score.is_a?(Integer) && score >= 0 ? new(**answer, score: score) : nil
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# A value the wire sent that we could not parse.
|
|
47
|
+
private_class_method def self.bad_field(raw, name, parsed)
|
|
48
|
+
raw.key?(name) && !raw[name].nil? && parsed.nil?
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# A probability as written: a number in 0..1, else nil.
|
|
52
|
+
private_class_method def self.probability(value)
|
|
53
|
+
return nil unless value.is_a?(Numeric)
|
|
54
|
+
|
|
55
|
+
value = value.to_f
|
|
56
|
+
return nil if value.nan? || value.infinite? || value.negative? || value > 1.0
|
|
57
|
+
|
|
58
|
+
value
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# The probabilities as sent: a map of option => number, or a list of
|
|
62
|
+
# numbers (the options in order). Anything else is malformed.
|
|
63
|
+
private_class_method def self.probabilities(value)
|
|
64
|
+
if value.is_a?(Hash)
|
|
65
|
+
pairs = value.map { |name, weight| [name.to_s, probability(weight)] }
|
|
66
|
+
return nil if pairs.any? { |_name, weight| weight.nil? }
|
|
67
|
+
|
|
68
|
+
pairs.to_h
|
|
69
|
+
elsif value.is_a?(Array)
|
|
70
|
+
weights = value.map { |weight| probability(weight) }
|
|
71
|
+
return nil if weights.any?(&:nil?)
|
|
72
|
+
|
|
73
|
+
weights
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
|
|
5
|
+
require_relative "question"
|
|
6
|
+
require_relative "similar"
|
|
7
|
+
|
|
8
|
+
module Samagotchi
|
|
9
|
+
module SystemOne
|
|
10
|
+
# The one question an approval asks System 1: "would this user allow it
|
|
11
|
+
# without hesitation?" (docs/guardrails.md, "System 1"). The instruction
|
|
12
|
+
# is a constant in code, reviewed like a prompt; the state is the ask's
|
|
13
|
+
# own facts — the rule that asked, why, the call, and where it runs — plus
|
|
14
|
+
# the memory an ask names and the earlier asks it resembles.
|
|
15
|
+
module ApprovalQuestion
|
|
16
|
+
ID = "allow"
|
|
17
|
+
# What the judgment is asked, in the model's terms. Frozen: a change
|
|
18
|
+
# here is a change in behaviour, not a wording tweak.
|
|
19
|
+
INSTRUCTIONS = "A coding agent wants to run this tool call, and a guardrail rule asked its user to approve it. " \
|
|
20
|
+
"The state names the rule, why it asked, the call and where it runs. Would the user approve it " \
|
|
21
|
+
"without hesitation? Users usually approve routine reads, edits of the agent's own notes and " \
|
|
22
|
+
"memory, and pushes of verified work; they usually refuse discarding uncommitted work, " \
|
|
23
|
+
"interactive rebases, deleting outside the project, and anything irreversible they did not ask " \
|
|
24
|
+
"for. The state may also list a few of the user's earlier decisions on calls like this one, " \
|
|
25
|
+
"most alike first, each with what they answered."
|
|
26
|
+
# How much of a command, a diff or a path list the state carries.
|
|
27
|
+
TEXT_CHARS = 2000
|
|
28
|
+
# How much of the call the judgment log keeps; its digest is of the whole.
|
|
29
|
+
CALL_CHARS = 300
|
|
30
|
+
|
|
31
|
+
# @return [Question] the noul question for id "allow"
|
|
32
|
+
def self.question
|
|
33
|
+
# The type as a string: a symbol here reads as an event type to
|
|
34
|
+
# spec/web/participant_view_spec.
|
|
35
|
+
Question.new(id: ID, type: "noul", instructions: INSTRUCTIONS)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# @param approval [Hash] payload[:approval]: the ask the user is shown
|
|
39
|
+
# @param session_title [String, nil] the session's title, when it has one
|
|
40
|
+
# @param entries [Array<Hash>] SystemOne::Log#entries: the judgments the
|
|
41
|
+
# user has answered so far, read once per ask and shared with the
|
|
42
|
+
# record gate (GuardrailWiring#system_one_judge)
|
|
43
|
+
# @return [Hash] the state, with the fields the ask didn't give left out
|
|
44
|
+
def self.state(approval, session_title: nil, entries: nil)
|
|
45
|
+
{
|
|
46
|
+
"rule" => approval[:rule],
|
|
47
|
+
"source" => approval[:source],
|
|
48
|
+
"reason" => approval[:reason],
|
|
49
|
+
"tool" => approval[:tool],
|
|
50
|
+
"command" => cut(approval[:command]),
|
|
51
|
+
"paths" => Array(approval[:paths]).map { |path| cut(path) },
|
|
52
|
+
"change" => change_text(approval[:preview]),
|
|
53
|
+
"cwd" => approval[:cwd],
|
|
54
|
+
"repo_name" => approval[:repo_name],
|
|
55
|
+
"branch" => approval[:branch],
|
|
56
|
+
"session" => session_title,
|
|
57
|
+
# Which memory entry the ask is about, when it is a memory ask.
|
|
58
|
+
"memory" => memory_facts(approval[:memory]),
|
|
59
|
+
# The asks this one looks like, and what the user did about them.
|
|
60
|
+
"examples" => Similar.select(entries: entries, rule: approval[:rule], repo_name: approval[:repo_name],
|
|
61
|
+
call_text: call_text(approval))
|
|
62
|
+
}.reject do |_key, value|
|
|
63
|
+
value.nil? || (value.is_a?(String) && value.strip.empty?) ||
|
|
64
|
+
((value.is_a?(Array) || value.is_a?(Hash)) && value.empty?)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# What the call is, whole: the command, the paths, or a plugin tool's
|
|
69
|
+
# arguments.
|
|
70
|
+
def self.call_text(approval)
|
|
71
|
+
command = approval[:command]
|
|
72
|
+
return command.to_s if command && !command.to_s.empty?
|
|
73
|
+
|
|
74
|
+
paths = Array(approval[:paths])
|
|
75
|
+
return paths.join("\n") unless paths.empty?
|
|
76
|
+
|
|
77
|
+
approval[:args].to_s
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# The judgment log's two fields for the call: what it was, cut to
|
|
81
|
+
# CALL_CHARS, and the SHA256 of the whole of it.
|
|
82
|
+
def self.call_fields(approval)
|
|
83
|
+
call = call_text(approval)
|
|
84
|
+
{ "call" => cut(call, CALL_CHARS), "call_digest" => Digest::SHA256.hexdigest(call) }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# The memory an ask is about, as the approval card built it: entry name,
|
|
88
|
+
# scope, and (for a removal) the description its index line carries.
|
|
89
|
+
private_class_method def self.memory_facts(memory)
|
|
90
|
+
return nil unless memory.is_a?(Hash) && !memory.empty?
|
|
91
|
+
|
|
92
|
+
memory.transform_keys(&:to_s).reject { |_key, value| value.to_s.strip.empty? }
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
private_class_method def self.change_text(preview)
|
|
96
|
+
return nil unless preview.is_a?(Hash)
|
|
97
|
+
|
|
98
|
+
text = preview[:text] || preview["text"]
|
|
99
|
+
return cut(text) if text && !text.to_s.strip.empty?
|
|
100
|
+
|
|
101
|
+
note = preview[:error] || preview["error"] || preview[:skipped] || preview["skipped"]
|
|
102
|
+
return note.to_s unless note.to_s.strip.empty?
|
|
103
|
+
|
|
104
|
+
"+#{(preview[:added] || preview["added"]).to_i} −#{(preview[:removed] || preview["removed"]).to_i}"
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
private_class_method def self.cut(text, chars = TEXT_CHARS)
|
|
108
|
+
text = text.to_s
|
|
109
|
+
text.length > chars ? "#{text[0, chars - 1]}…" : text
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../config"
|
|
4
|
+
require_relative "../log"
|
|
5
|
+
require_relative "answer"
|
|
6
|
+
require_relative "errors"
|
|
7
|
+
require_relative "http_backend"
|
|
8
|
+
require_relative "question"
|
|
9
|
+
|
|
10
|
+
module Samagotchi
|
|
11
|
+
module SystemOne
|
|
12
|
+
# The one way anything asks System 1 a question: #ask answers a set of
|
|
13
|
+
# questions about a state, or says nothing. A backend that is unreachable,
|
|
14
|
+
# slow, refusing or nonsensical is a nil — no judgment, one log line — and
|
|
15
|
+
# the caller asks the user as it would have anyway.
|
|
16
|
+
#
|
|
17
|
+
# client = SystemOne::Client.from_config(host_registry: registry)
|
|
18
|
+
# client&.ask(state: "git push origin main", questions: {"allow" => question})
|
|
19
|
+
class Client
|
|
20
|
+
MODES = %w[off audit on].freeze
|
|
21
|
+
DEFAULT_TIMEOUT_MS = 2000
|
|
22
|
+
|
|
23
|
+
attr_reader :mode, :host, :model, :backend
|
|
24
|
+
|
|
25
|
+
def initialize(backend:, mode: "off", host: nil, model: nil)
|
|
26
|
+
@backend = backend
|
|
27
|
+
@mode = mode.to_s.strip.downcase
|
|
28
|
+
@host = host
|
|
29
|
+
@model = model
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# nil when judgment is off (the default) or no host is named, so the
|
|
33
|
+
# caller needs no other check. The settings are read every call: a
|
|
34
|
+
# config.yml change takes effect on the next ask.
|
|
35
|
+
# @param model [String, nil] a model for this call only (chi guardrails
|
|
36
|
+
# replay --model), else the configured one
|
|
37
|
+
def self.from_config(host_registry:, env: ENV, model: nil)
|
|
38
|
+
mode = Config.get("guardrails.system_one.mode").to_s.strip.downcase
|
|
39
|
+
return nil unless MODES.include?(mode) && mode != "off"
|
|
40
|
+
|
|
41
|
+
host = Config.get("guardrails.system_one.host").to_s.strip
|
|
42
|
+
return nil if host.empty?
|
|
43
|
+
|
|
44
|
+
model = (model.nil? ? Config.get("guardrails.system_one.model") : model).to_s.strip
|
|
45
|
+
timeout_ms = Config.get("guardrails.system_one.timeout_ms").to_i
|
|
46
|
+
timeout_ms = DEFAULT_TIMEOUT_MS unless timeout_ms.positive?
|
|
47
|
+
|
|
48
|
+
new(backend: HttpBackend.new(host_registry: host_registry, host: host, model: model.empty? ? nil : model,
|
|
49
|
+
timeout_ms: timeout_ms, env: env),
|
|
50
|
+
mode: mode, host: host, model: model.empty? ? nil : model)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# @param state [String, Hash, Array] what the question is about, as the
|
|
54
|
+
# contract takes it
|
|
55
|
+
# @param questions [Hash{String => Question|Hash}] the questions, by id
|
|
56
|
+
# @return [Hash{String => Answer}, nil] one Answer per question, or nil
|
|
57
|
+
# when there is no judgment at all
|
|
58
|
+
def ask(state:, questions:, cancel_controller: nil)
|
|
59
|
+
questions = SystemOne.normalize_questions(questions)
|
|
60
|
+
return nil if questions.empty?
|
|
61
|
+
|
|
62
|
+
answers = begin
|
|
63
|
+
@backend.ask(state: state, questions: questions, cancel_controller: cancel_controller)
|
|
64
|
+
rescue StandardError => e
|
|
65
|
+
return no_judgment("#{e.class}: #{e.message}", error: e.class.name)
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
unless answers.is_a?(Hash)
|
|
69
|
+
return no_judgment("the backend returned #{answers.inspect}")
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
answers = questions.to_h { |question| [question.id, answers[question.id]] }
|
|
73
|
+
missing = questions.reject { |question| answers[question.id].is_a?(Answer) }
|
|
74
|
+
return no_judgment("no answer for #{missing.map(&:id).join(", ")}") unless missing.empty?
|
|
75
|
+
|
|
76
|
+
answers
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
private
|
|
80
|
+
|
|
81
|
+
# One warn, one nil: the ask goes to the user.
|
|
82
|
+
def no_judgment(reason, **fields)
|
|
83
|
+
# Qualified: inside SystemOne, the bare Log is now SystemOne::Log.
|
|
84
|
+
Samagotchi::Log.warn(:guardrails, "system_one_failed", echo: "[samagotchi:guardrails] system one: #{reason}", host: @host, **fields)
|
|
85
|
+
nil
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Samagotchi
|
|
4
|
+
module SystemOne
|
|
5
|
+
# A judgment that did not happen: the backend could not reach the host,
|
|
6
|
+
# the host refused the request, or its answer was not usable. Client#ask
|
|
7
|
+
# turns any of these into nil (no judgment), so the caller asks the user.
|
|
8
|
+
class BackendError < StandardError; end
|
|
9
|
+
|
|
10
|
+
# The named host is not a System One host: it isn't in hosts:, its api
|
|
11
|
+
# is not openai, or it names no model.
|
|
12
|
+
class UnsupportedHost < BackendError; end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
|
|
5
|
+
module Samagotchi
|
|
6
|
+
module SystemOne
|
|
7
|
+
# The backend the specs use: answers from a block, from a fixed hash, or
|
|
8
|
+
# an error every time. It never reaches the network.
|
|
9
|
+
#
|
|
10
|
+
# FakeBackend.new { |state, questions| {questions.first.id => answer} }
|
|
11
|
+
# FakeBackend.new("git-push" => Answer.from_wire("git-push", {"type" => "noul", "noul" => 0.97}))
|
|
12
|
+
# FakeBackend.new(failing: true)
|
|
13
|
+
class FakeBackend
|
|
14
|
+
attr_reader :calls
|
|
15
|
+
|
|
16
|
+
# @param answers [Hash{String => Answer}, nil] the same answers for
|
|
17
|
+
# every call
|
|
18
|
+
# @param failing [Boolean] true: every #ask raises
|
|
19
|
+
# @param error [String] what a failing backend says
|
|
20
|
+
# @param judgments [#call] (state, questions) => {id => Answer}
|
|
21
|
+
def initialize(answers = nil, failing: false, error: "fake System One backend is unavailable", &judgments)
|
|
22
|
+
@answers = answers
|
|
23
|
+
@judgments = judgments
|
|
24
|
+
@failing = failing
|
|
25
|
+
@error = error
|
|
26
|
+
@calls = 0
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def ask(state:, questions:, cancel_controller: nil)
|
|
30
|
+
@calls += 1
|
|
31
|
+
raise BackendError, @error if @failing
|
|
32
|
+
|
|
33
|
+
judged = @judgments&.call(state, questions)
|
|
34
|
+
return judged unless judged.nil?
|
|
35
|
+
|
|
36
|
+
ids = Array(questions).map { |question| question.respond_to?(:id) ? question.id : question.to_s }
|
|
37
|
+
(@answers || {}).select { |id, _answer| ids.include?(id) }
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def failing? = @failing
|
|
41
|
+
|
|
42
|
+
def inspect = "#<#{self.class.name} calls=#{@calls}#{" failing" if @failing}>"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "uri"
|
|
5
|
+
require_relative "../config"
|
|
6
|
+
require_relative "../llm/api_key"
|
|
7
|
+
require_relative "../llm/http"
|
|
8
|
+
require_relative "answer"
|
|
9
|
+
require_relative "question"
|
|
10
|
+
require_relative "errors"
|
|
11
|
+
|
|
12
|
+
module Samagotchi
|
|
13
|
+
module SystemOne
|
|
14
|
+
# The TypeSafe System One judgment API on a host from config.yml's
|
|
15
|
+
# `hosts:`: POST <openai_base_url>/systemone (that base already ends in
|
|
16
|
+
# /v1), one request per #ask, no retries — a judgment that doesn't
|
|
17
|
+
# arrive in time is no judgment, and the ask goes to the user.
|
|
18
|
+
#
|
|
19
|
+
# Works for Splash (a local `api: openai` host serving /v1/systemone)
|
|
20
|
+
# and for the hosted Jev API the same way; a host whose api isn't
|
|
21
|
+
# openai is refused.
|
|
22
|
+
class HttpBackend
|
|
23
|
+
DEFAULT_TIMEOUT_MS = 2000
|
|
24
|
+
|
|
25
|
+
# @param host_registry [HostRegistry] where the host is looked up
|
|
26
|
+
# @param host [String] the hosts: entry name
|
|
27
|
+
# @param model [String, nil] the judgment model; nil: the host's first
|
|
28
|
+
# model (hosts.<name>.models), and no judgment when it has none
|
|
29
|
+
# @param timeout_ms [Integer] the whole request's limit, in ms
|
|
30
|
+
# @param env [Hash] where the host's api_key_env: key is read
|
|
31
|
+
def initialize(host_registry:, host:, model: nil, timeout_ms: DEFAULT_TIMEOUT_MS, env: ENV)
|
|
32
|
+
@host_registry = host_registry
|
|
33
|
+
@host = host.to_s.strip.downcase
|
|
34
|
+
@model = model.nil? ? nil : model.to_s.strip
|
|
35
|
+
@timeout_ms = timeout_ms.to_i
|
|
36
|
+
@env = env
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# @return [Hash{String => Answer}] one Answer per question, nil for a
|
|
40
|
+
# question the response answered badly
|
|
41
|
+
# @raise [UnsupportedHost] the host is unknown, not an openai host, or
|
|
42
|
+
# names no model
|
|
43
|
+
# @raise [BackendError] the request failed, the status isn't 200, or
|
|
44
|
+
# the body isn't a usable response
|
|
45
|
+
def ask(state:, questions:, cancel_controller: nil)
|
|
46
|
+
entry = @host_registry.find_entry(@host)
|
|
47
|
+
raise UnsupportedHost, "host #{@host.inspect} is not in hosts:" unless entry
|
|
48
|
+
unless entry.api&.to_sym == :openai
|
|
49
|
+
raise UnsupportedHost, "host #{@host} has api: #{entry.api || "llama_cpp"}; System One speaks the OpenAI-compatible API"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
model = @model || host_model(entry)
|
|
53
|
+
raise UnsupportedHost, "host #{@host} names no model for System One" if model.nil? || model.empty?
|
|
54
|
+
|
|
55
|
+
url = "#{entry.openai_base_url.to_s.chomp("/")}/systemone"
|
|
56
|
+
questions = SystemOne.normalize_questions(questions)
|
|
57
|
+
body = { model: model, state: state, questions: questions.to_h { |question| [question.id, question.to_wire] } }
|
|
58
|
+
request = json_post(url, body)
|
|
59
|
+
response = http(entry).fetch(URI(url), request, retries: false, check_status: false, cancel_controller: cancel_controller,
|
|
60
|
+
log_fields: { purpose: "system_one", model: model })
|
|
61
|
+
|
|
62
|
+
status = response.code.to_i
|
|
63
|
+
unless status == 200
|
|
64
|
+
raise BackendError, "#{@host}: #{status} #{error_detail(response.body)}"
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
answers = answers_of(response.body)
|
|
68
|
+
questions.to_h { |question| [question.id, Answer.from_wire(question.id, answers[question.id])] }
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
# The host's first model: whatever hosts.<name>.models holds, in the
|
|
74
|
+
# shape the registry has it in (HostModel by id, or as written).
|
|
75
|
+
def host_model(entry)
|
|
76
|
+
models = entry.models
|
|
77
|
+
return nil if models.nil? || models.empty?
|
|
78
|
+
|
|
79
|
+
first = models.is_a?(Hash) ? models.values.first : models.first
|
|
80
|
+
first.respond_to?(:id) ? first.id : first.to_s
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# One attempt, closed on the timeout: a judgment is worth at most
|
|
84
|
+
# timeout_ms, not a retry chain.
|
|
85
|
+
def http(entry)
|
|
86
|
+
seconds = @timeout_ms.positive? ? @timeout_ms / 1000.0 : DEFAULT_TIMEOUT_MS / 1000.0
|
|
87
|
+
open_timeout = [seconds, Config.positive_seconds("server.open_timeout")].min
|
|
88
|
+
LLM::HTTP.new(label: entry.name, open_timeout: open_timeout, read_timeout: seconds,
|
|
89
|
+
retry_policy: LLM::HTTP::RetryPolicy.none,
|
|
90
|
+
api_key: LLM::ApiKey.for(entry.api_key_env, host: entry.name, env: @env, scheme: entry.auth_scheme))
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# The POST the adapters build: JSON body, Content-Type, and the host's
|
|
94
|
+
# key on the header its auth: names (LLM::HTTP adds it).
|
|
95
|
+
def json_post(url, body)
|
|
96
|
+
Net::HTTP::Post.new(URI(url)).tap do |request|
|
|
97
|
+
request["Content-Type"] = "application/json"
|
|
98
|
+
request.body = JSON.generate(body)
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
def answers_of(body)
|
|
103
|
+
data = JSON.parse(body.to_s)
|
|
104
|
+
raise BackendError, "the response is not JSON: #{body.to_s[0, 200]}" unless data.is_a?(Hash)
|
|
105
|
+
|
|
106
|
+
answers = data["answers"]
|
|
107
|
+
raise BackendError, "the response has no answers map" unless answers.is_a?(Hash)
|
|
108
|
+
|
|
109
|
+
answers
|
|
110
|
+
rescue JSON::ParserError => e
|
|
111
|
+
raise BackendError, "the response is not JSON: #{e.message[0, 200]}"
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# A 422's detail array, or the body's start, for the log line.
|
|
115
|
+
def error_detail(body)
|
|
116
|
+
data = JSON.parse(body.to_s)
|
|
117
|
+
detail = data["detail"]
|
|
118
|
+
text = if detail.is_a?(Array)
|
|
119
|
+
detail.map { |item| item.is_a?(Hash) ? item["msg"] || item.to_s : item.to_s }.join("; ")
|
|
120
|
+
else
|
|
121
|
+
detail.to_s
|
|
122
|
+
end
|
|
123
|
+
text.empty? ? body.to_s[0, 200] : text[0, 200]
|
|
124
|
+
rescue StandardError
|
|
125
|
+
body.to_s[0, 200]
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
require "time"
|
|
7
|
+
|
|
8
|
+
require_relative "../log"
|
|
9
|
+
|
|
10
|
+
module Samagotchi
|
|
11
|
+
module SystemOne
|
|
12
|
+
# The judgments log: one JSON line per System 1 judgment, and a later
|
|
13
|
+
# line for the user's answer to the ask it judged (docs/guardrails.md,
|
|
14
|
+
# "System 1"). It sits beside the approvals store,
|
|
15
|
+
# `$XDG_STATE_HOME/samagotchi/guardrails/judgments.jsonl`, and is written
|
|
16
|
+
# append-only under an exclusive flock: several workers judge at once,
|
|
17
|
+
# and a line is never rewritten to add its label. Past 5 MB the file is
|
|
18
|
+
# rewritten, under the same lock, keeping the newest half of its lines.
|
|
19
|
+
#
|
|
20
|
+
# Nothing here can break an approval: every write that fails is one
|
|
21
|
+
# Log.warn and a nil, and the ask reaches the user as it would have.
|
|
22
|
+
class Log
|
|
23
|
+
FILE = "judgments.jsonl"
|
|
24
|
+
# Above this the file is rewritten keeping the newest half of the lines.
|
|
25
|
+
MAX_BYTES = 5 * 1024 * 1024
|
|
26
|
+
MODE = 0o600
|
|
27
|
+
# What a label's answer can be (P2a writes the first four;
|
|
28
|
+
# "should_have_asked" comes with `/guardrails wrong`).
|
|
29
|
+
ANSWERS = %w[allow_once allow_wide deny none should_have_asked].freeze
|
|
30
|
+
|
|
31
|
+
attr_reader :dir, :path
|
|
32
|
+
|
|
33
|
+
# @param dir [String] the guardrails store dir (Guardrails::Approvals.dir_for)
|
|
34
|
+
# @param warn [#call] (reason) — the record for a failed write
|
|
35
|
+
def initialize(dir:, warn: nil)
|
|
36
|
+
@dir = dir
|
|
37
|
+
@path = File.join(dir, FILE)
|
|
38
|
+
@warn = warn || ->(reason) { Samagotchi::Log.warn(:guardrails, "system_one_log_failed", echo: "[samagotchi:guardrails] #{reason}") }
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @param fields [Hash] the judgment's own fields (rule, tool, p, …)
|
|
42
|
+
# @return [String, nil] the judgment's id, nil when nothing was written
|
|
43
|
+
def judgment(fields)
|
|
44
|
+
id = SecureRandom.uuid
|
|
45
|
+
append({ "kind" => "judgment", "id" => id, "at" => Time.now.utc.iso8601 }.merge(fields)) ? id : nil
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# @param id [String] a judgment's id
|
|
49
|
+
# @param answer [String] one of ANSWERS
|
|
50
|
+
# @return [Boolean] whether the line was written
|
|
51
|
+
def label(id, answer)
|
|
52
|
+
return false if id.to_s.strip.empty?
|
|
53
|
+
|
|
54
|
+
append({ "kind" => "label", "id" => id.to_s, "at" => Time.now.utc.iso8601, "answer" => answer.to_s })
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# The judgments, oldest first, each joined with its LAST label
|
|
58
|
+
# (answer: nil when it has none). Lines that don't parse are skipped;
|
|
59
|
+
# labels of a judgment that isn't there are dropped.
|
|
60
|
+
# @return [Array<Hash>]
|
|
61
|
+
def entries
|
|
62
|
+
judgments = {}
|
|
63
|
+
labels = {}
|
|
64
|
+
with_lock(File::LOCK_SH) do |file|
|
|
65
|
+
file.each_line do |line|
|
|
66
|
+
data = parse(line)
|
|
67
|
+
next if data.nil?
|
|
68
|
+
|
|
69
|
+
case data["kind"]
|
|
70
|
+
when "judgment" then judgments[data["id"]] = data
|
|
71
|
+
when "label" then labels[data["id"]] = data["answer"] if data.key?("id")
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
judgments.map { |id, judgment| judgment.merge("answer" => labels[id]) }
|
|
76
|
+
rescue StandardError => e
|
|
77
|
+
warn_about(e)
|
|
78
|
+
[]
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
private
|
|
82
|
+
|
|
83
|
+
def append(line)
|
|
84
|
+
with_lock(File::LOCK_EX) do |file|
|
|
85
|
+
# The handle is read-write (not O_APPEND) so #trim can rewrite the
|
|
86
|
+
# file: write at the end, under the lock nobody else holds.
|
|
87
|
+
file.seek(file.size)
|
|
88
|
+
file.puts(JSON.generate(line))
|
|
89
|
+
trim(file)
|
|
90
|
+
end
|
|
91
|
+
true
|
|
92
|
+
rescue StandardError => e
|
|
93
|
+
warn_about(e)
|
|
94
|
+
false
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
# @param mode [Integer] File::LOCK_EX for a write, LOCK_SH to read
|
|
98
|
+
def with_lock(mode)
|
|
99
|
+
FileUtils.mkdir_p(@dir)
|
|
100
|
+
File.open(@path, File::RDWR | File::CREAT, MODE) do |file|
|
|
101
|
+
file.flock(mode)
|
|
102
|
+
yield file
|
|
103
|
+
end
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Past MAX_BYTES, keep the newest half of the lines (whole, so a
|
|
107
|
+
# judgment and its labels stay together as far as the cut reaches).
|
|
108
|
+
def trim(file)
|
|
109
|
+
file.flush
|
|
110
|
+
return if file.size <= MAX_BYTES
|
|
111
|
+
|
|
112
|
+
file.rewind
|
|
113
|
+
lines = file.readlines
|
|
114
|
+
keep = lines.last([lines.size / 2, 1].max).join
|
|
115
|
+
file.rewind
|
|
116
|
+
file.write(keep)
|
|
117
|
+
file.flush
|
|
118
|
+
file.truncate(keep.bytesize)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def parse(line)
|
|
122
|
+
data = JSON.parse(line)
|
|
123
|
+
data.is_a?(Hash) ? data : nil
|
|
124
|
+
rescue JSON::ParserError
|
|
125
|
+
nil
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def warn_about(error)
|
|
129
|
+
@warn.call("system one log: #{path}: #{error.class}: #{error.message}")
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|