samagotchi 0.141.0 → 0.142.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +46 -1
- data/README.md +1 -1
- data/bin/chi +2 -0
- data/docs/architecture.md +12 -283
- data/docs/bundle-authoring.md +1 -0
- data/docs/cli.md +15 -7
- data/docs/configuration.md +57 -18
- data/docs/context.md +2 -2
- data/docs/guardrails.md +20 -11
- data/docs/hooks.md +9 -7
- data/docs/internals/attached-context.md +226 -0
- data/docs/internals/background-tasks.md +1 -1
- data/docs/internals/bridge.md +380 -0
- data/docs/internals/bundles.md +1 -0
- data/docs/internals/context-telemetry.md +1 -1
- data/docs/internals/engine.md +24 -18
- data/docs/internals/participants.md +13 -12
- data/docs/internals/remote-sessions.md +1 -1
- data/docs/internals/{tool-guardrails.md → tool-output-limits.md} +21 -2
- data/docs/memory.md +31 -1
- data/docs/plugins.md +122 -1
- data/docs/sessions.md +50 -26
- data/docs/system-one.md +122 -0
- data/docs/testing.md +3 -1
- data/lib/samagotchi/bridge.rb +2 -0
- data/lib/samagotchi/bundles/check-in/manifest.yml +6 -3
- data/lib/samagotchi/bundles/check-in/plugin.rb +101 -4
- data/lib/samagotchi/bundles/core/manifest.yml +2 -2
- data/lib/samagotchi/bundles/system/config_modification_protocol.md +2 -2
- data/lib/samagotchi/bundles/system/manifest.yml +4 -4
- data/lib/samagotchi/bundles/system/memory_guide.md +1 -1
- data/lib/samagotchi/bundles/system/self_map.md +1 -1
- data/lib/samagotchi/config.rb +74 -14
- data/lib/samagotchi/context_candidate_choice.rb +8 -0
- data/lib/samagotchi/context_candidates_log.rb +25 -1
- data/lib/samagotchi/context_judge.rb +6 -2
- data/lib/samagotchi/engine.rb +68 -7
- data/lib/samagotchi/extension_load.rb +3 -2
- data/lib/samagotchi/guardrail_wiring.rb +36 -31
- data/lib/samagotchi/guardrails/chi_dirs.rb +7 -4
- data/lib/samagotchi/guardrails/path_resolve.rb +1 -1
- data/lib/samagotchi/guardrails/protected_paths.rb +7 -4
- data/lib/samagotchi/guardrails_command.rb +3 -3
- data/lib/samagotchi/judgments_command.rb +155 -0
- data/lib/samagotchi/kernel_loop.rb +13 -1
- data/lib/samagotchi/llm/chat_loop.rb +74 -10
- data/lib/samagotchi/llm/errors.rb +7 -5
- data/lib/samagotchi/llm/stray_tool_call.rb +61 -0
- data/lib/samagotchi/log_line.rb +1 -1
- data/lib/samagotchi/memory_bundle/entry_size.rb +100 -0
- data/lib/samagotchi/plugin/api.rb +33 -2
- data/lib/samagotchi/plugin/context.rb +49 -1
- data/lib/samagotchi/plugin/judgments.rb +72 -0
- data/lib/samagotchi/plugin/loader.rb +3 -2
- data/lib/samagotchi/run_options.rb +1 -0
- data/lib/samagotchi/self_report.rb +36 -0
- data/lib/samagotchi/session_access/contract.rb +1 -1
- data/lib/samagotchi/session_commands.rb +3 -0
- data/lib/samagotchi/session_manager.rb +13 -2
- data/lib/samagotchi/session_retention.rb +10 -0
- data/lib/samagotchi/side_judgments.rb +12 -1
- data/lib/samagotchi/system_one/answer.rb +63 -11
- data/lib/samagotchi/system_one/approval_label.rb +56 -0
- data/lib/samagotchi/system_one/approval_question.rb +2 -0
- data/lib/samagotchi/system_one/client.rb +43 -17
- data/lib/samagotchi/system_one/fake_backend.rb +1 -1
- data/lib/samagotchi/system_one/http_backend.rb +12 -4
- data/lib/samagotchi/system_one/judgment_settings.rb +61 -0
- data/lib/samagotchi/system_one/judgments.rb +260 -0
- data/lib/samagotchi/system_one/log.rb +4 -2
- data/lib/samagotchi/system_one/recording.rb +117 -0
- data/lib/samagotchi/system_one/runs.rb +179 -0
- data/lib/samagotchi/system_one/runs_store.rb +230 -0
- data/lib/samagotchi/system_one.rb +9 -0
- data/lib/samagotchi/testing/bundle.rb +9 -2
- data/lib/samagotchi/testing/host.rb +2 -1
- data/lib/samagotchi/testing/judgments.rb +185 -0
- data/lib/samagotchi/testing/recorder.rb +5 -0
- data/lib/samagotchi/testing/recording_api.rb +1 -1
- data/lib/samagotchi/tool_output_cut.rb +26 -6
- data/lib/samagotchi/tool_response.rb +1 -1
- data/lib/samagotchi/tool_runner.rb +63 -4
- data/lib/samagotchi/toolkit_suggest.rb +11 -2
- data/lib/samagotchi/tools/memory.rb +28 -0
- data/lib/samagotchi/turn_note.rb +9 -0
- data/lib/samagotchi/version.rb +1 -1
- metadata +16 -2
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "json"
|
|
5
|
+
require "time"
|
|
6
|
+
|
|
7
|
+
module Samagotchi
|
|
8
|
+
module SystemOne
|
|
9
|
+
# Runs on disk: <state>/system_one/runs/<judgment>/YYYY-MM-DD.jsonl (UTC,
|
|
10
|
+
# 0600), one append under flock per line, read by copying the lines out
|
|
11
|
+
# and parsing outside the lock; each judgment's questions once per digest
|
|
12
|
+
# pair in versions.jsonl beside them. Nothing is rewritten but for a
|
|
13
|
+
# session's purge.
|
|
14
|
+
class RunsStore
|
|
15
|
+
DIR = File.join("system_one", "runs")
|
|
16
|
+
# A judgment's name: safe as a folder name, dot-separated words.
|
|
17
|
+
NAME = /\A[a-z0-9][a-z0-9_-]*(?:\.[a-z0-9][a-z0-9_-]*)*\z/
|
|
18
|
+
VERSIONS = "versions.jsonl"
|
|
19
|
+
DAY_FILE = /\A\d{4}-\d{2}-\d{2}\.jsonl\z/
|
|
20
|
+
FILE_MODE = 0o600
|
|
21
|
+
DIR_MODE = 0o700
|
|
22
|
+
# The longest line: a longer state is saved cut, marked "truncated".
|
|
23
|
+
MAX_LINE_BYTES = 16 * 1024
|
|
24
|
+
|
|
25
|
+
attr_reader :dir
|
|
26
|
+
|
|
27
|
+
# @param root [String] the state dir the sessions folder sits in
|
|
28
|
+
def initialize(root)
|
|
29
|
+
@dir = File.join(root, DIR)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# @param line [Hash] with its "at"
|
|
33
|
+
# @return [Boolean] true: raises what the filesystem raises
|
|
34
|
+
def append(judgment, line)
|
|
35
|
+
write(File.join(judgment_dir(judgment), "#{day_of(line)}.jsonl"), fit(line))
|
|
36
|
+
true
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# +line+ ({digests…, "questions"}) unless the file has its digest pair.
|
|
40
|
+
# @return [Boolean] whether it was new
|
|
41
|
+
def version(judgment, line)
|
|
42
|
+
path = File.join(judgment_dir(judgment), VERSIONS)
|
|
43
|
+
with_file(path) do |file|
|
|
44
|
+
known = file.each_line.any? do |text|
|
|
45
|
+
data = parse(text)
|
|
46
|
+
data && data["instructions_digest"] == line["instructions_digest"] && data["criteria_digest"] == line["criteria_digest"]
|
|
47
|
+
end
|
|
48
|
+
next false if known
|
|
49
|
+
|
|
50
|
+
file.seek(0, IO::SEEK_END)
|
|
51
|
+
file.write("#{JSON.generate(line)}\n")
|
|
52
|
+
true
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The judgments with a folder, sorted; a folder that isn't one is left
|
|
57
|
+
# alone, and in no one's way.
|
|
58
|
+
# @return [Array<String>]
|
|
59
|
+
def names
|
|
60
|
+
return [] unless Dir.exist?(@dir)
|
|
61
|
+
|
|
62
|
+
Dir.children(@dir).select { |name| name.match?(NAME) && File.directory?(File.join(@dir, name)) }.sort
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Every parseable line, oldest day first.
|
|
66
|
+
# @param since [String, nil] the first day (YYYY-MM-DD) wanted
|
|
67
|
+
# @return [Array<Hash>]
|
|
68
|
+
def lines(judgment, since: nil)
|
|
69
|
+
day_files(judgment).flat_map do |path|
|
|
70
|
+
next [] if since && File.basename(path, ".jsonl") < since
|
|
71
|
+
|
|
72
|
+
read(path).each_line.filter_map { |text| parse(text) }
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# The questions saved, one line per digest pair.
|
|
77
|
+
# @return [Array<Hash>]
|
|
78
|
+
def versions(judgment)
|
|
79
|
+
path = File.join(judgment_dir(judgment), VERSIONS)
|
|
80
|
+
File.exist?(path) ? read(path).each_line.filter_map { |text| parse(text) } : []
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @return [Integer] the judgment's bytes on disk
|
|
84
|
+
def size(judgment)
|
|
85
|
+
Dir.children(judgment_dir(judgment)).sum { |name| File.size(File.join(judgment_dir(judgment), name)) }
|
|
86
|
+
rescue SystemCallError
|
|
87
|
+
0
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# Day files past +days+ (0: none), then the oldest until the total is
|
|
91
|
+
# under +max_bytes+, today's stay. A judgment with no day file left
|
|
92
|
+
# loses its versions and folder.
|
|
93
|
+
# @return [Integer] the day files deleted
|
|
94
|
+
def sweep(days:, max_bytes:, now: Time.now.utc)
|
|
95
|
+
return 0 unless Dir.exist?(@dir)
|
|
96
|
+
|
|
97
|
+
today = now.utc.strftime("%F")
|
|
98
|
+
cutoff = (now.utc - (days * 86_400)).strftime("%F") if days.positive?
|
|
99
|
+
files = names.flat_map { |name| day_files(name) }.sort_by { |path| File.basename(path) }
|
|
100
|
+
expired, kept = files.partition { |path| cutoff && File.basename(path, ".jsonl") < cutoff }
|
|
101
|
+
total = names.sum { |name| size(name) } - expired.sum { |path| File.size(path) }
|
|
102
|
+
over = kept.take_while do |path|
|
|
103
|
+
next false if total <= max_bytes || File.basename(path, ".jsonl") >= today
|
|
104
|
+
|
|
105
|
+
total -= File.size(path)
|
|
106
|
+
true
|
|
107
|
+
end
|
|
108
|
+
(expired + over).each { |path| FileUtils.rm_f(path) }
|
|
109
|
+
names.each { |name| remove_if_empty(name) }
|
|
110
|
+
expired.size + over.size
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# One session's runs and the later lines on them, rewritten out of the
|
|
114
|
+
# day files under the lock.
|
|
115
|
+
# @return [Integer] the runs purged
|
|
116
|
+
def purge_session(session_id)
|
|
117
|
+
needle = session_id.to_s
|
|
118
|
+
return 0 if needle.empty?
|
|
119
|
+
|
|
120
|
+
names.sum do |name|
|
|
121
|
+
files = day_files(name).select { |path| read(path).include?(needle) }
|
|
122
|
+
ids = files.flat_map { |path| session_run_ids(path, needle) }
|
|
123
|
+
next 0 if ids.empty?
|
|
124
|
+
|
|
125
|
+
known = ids.to_h { |id| [id, true] }
|
|
126
|
+
day_files(name).each { |path| rewrite_without(path, known) }
|
|
127
|
+
remove_if_empty(name)
|
|
128
|
+
ids.size
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
private
|
|
133
|
+
|
|
134
|
+
def judgment_dir(judgment)
|
|
135
|
+
name = judgment.to_s
|
|
136
|
+
raise ArgumentError, "not a judgment's name: #{name.inspect}" unless name.match?(NAME)
|
|
137
|
+
|
|
138
|
+
File.join(@dir, name)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def day_files(judgment)
|
|
142
|
+
dir = judgment_dir(judgment)
|
|
143
|
+
return [] unless Dir.exist?(dir)
|
|
144
|
+
|
|
145
|
+
Dir.children(dir).grep(DAY_FILE).sort.map { |name| File.join(dir, name) }
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def day_of(line) = line.fetch("at")[0, 10]
|
|
149
|
+
|
|
150
|
+
# The line as written: past the cap its state is cut, as a string, as
|
|
151
|
+
# little as fits (escaping makes a byte cost more than one in the line:
|
|
152
|
+
# the most that fits is searched for) and marked; a line that doesn't
|
|
153
|
+
# fit with no state at all keeps its facts only.
|
|
154
|
+
def fit(line)
|
|
155
|
+
text = JSON.generate(line)
|
|
156
|
+
return text if text.bytesize <= MAX_LINE_BYTES
|
|
157
|
+
|
|
158
|
+
line = line.merge("truncated" => true)
|
|
159
|
+
state = line["state"].is_a?(String) ? line["state"] : JSON.generate(line["state"])
|
|
160
|
+
cut = ->(bytes) { JSON.generate(line.merge("state" => state.byteslice(0, bytes).scrub(""))) }
|
|
161
|
+
return JSON.generate(line.except("state", "answers", "failure_text")) if cut.call(0).bytesize > MAX_LINE_BYTES
|
|
162
|
+
|
|
163
|
+
bytes = (0..[state.bytesize, MAX_LINE_BYTES].min).bsearch { |size| cut.call(size + 1).bytesize > MAX_LINE_BYTES }
|
|
164
|
+
cut.call(bytes || [state.bytesize, MAX_LINE_BYTES].min)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
def write(path, text)
|
|
168
|
+
FileUtils.mkdir_p(File.dirname(path), mode: DIR_MODE)
|
|
169
|
+
File.open(path, File::WRONLY | File::APPEND | File::CREAT, FILE_MODE) do |file|
|
|
170
|
+
file.flock(File::LOCK_EX)
|
|
171
|
+
file.write("#{text}\n")
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
def with_file(path)
|
|
176
|
+
FileUtils.mkdir_p(File.dirname(path), mode: DIR_MODE)
|
|
177
|
+
File.open(path, File::RDWR | File::CREAT, FILE_MODE) do |file|
|
|
178
|
+
file.flock(File::LOCK_EX)
|
|
179
|
+
yield file
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# The file's text, copied out under a shared lock.
|
|
184
|
+
def read(path)
|
|
185
|
+
File.open(path, File::RDONLY) do |file|
|
|
186
|
+
file.flock(File::LOCK_SH)
|
|
187
|
+
file.read
|
|
188
|
+
end
|
|
189
|
+
rescue SystemCallError
|
|
190
|
+
""
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def parse(text)
|
|
194
|
+
data = JSON.parse(text)
|
|
195
|
+
data.is_a?(Hash) ? data : nil
|
|
196
|
+
rescue JSON::ParserError
|
|
197
|
+
nil
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def session_run_ids(path, session_id)
|
|
201
|
+
read(path).each_line.filter_map do |text|
|
|
202
|
+
next unless text.include?(session_id)
|
|
203
|
+
|
|
204
|
+
data = parse(text)
|
|
205
|
+
data["id"] if data && data["kind"] == "run" && data["session_id"] == session_id
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def rewrite_without(path, ids)
|
|
210
|
+
with_file(path) do |file|
|
|
211
|
+
kept = file.each_line.reject do |text|
|
|
212
|
+
data = parse(text)
|
|
213
|
+
data && (ids[data["id"]] || ids[data["run_id"]])
|
|
214
|
+
end
|
|
215
|
+
file.rewind
|
|
216
|
+
file.write(kept.join)
|
|
217
|
+
file.flush
|
|
218
|
+
file.truncate(file.pos)
|
|
219
|
+
end
|
|
220
|
+
FileUtils.rm_f(path) if File.empty?(path)
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
def remove_if_empty(name)
|
|
224
|
+
return unless day_files(name).empty?
|
|
225
|
+
|
|
226
|
+
FileUtils.rm_rf(judgment_dir(name))
|
|
227
|
+
end
|
|
228
|
+
end
|
|
229
|
+
end
|
|
230
|
+
end
|
|
@@ -6,11 +6,16 @@ require_relative "system_one/answer"
|
|
|
6
6
|
require_relative "system_one/fake_backend"
|
|
7
7
|
require_relative "system_one/http_backend"
|
|
8
8
|
require_relative "system_one/client"
|
|
9
|
+
require_relative "system_one/runs"
|
|
10
|
+
require_relative "system_one/recording"
|
|
11
|
+
require_relative "system_one/judgment_settings"
|
|
12
|
+
require_relative "system_one/judgments"
|
|
9
13
|
require_relative "system_one/log"
|
|
10
14
|
require_relative "system_one/record"
|
|
11
15
|
require_relative "system_one/similar"
|
|
12
16
|
require_relative "system_one/self_protect"
|
|
13
17
|
require_relative "system_one/approval_question"
|
|
18
|
+
require_relative "system_one/approval_label"
|
|
14
19
|
|
|
15
20
|
module Samagotchi
|
|
16
21
|
# System 1: the cheap judgment model that will, one phase at a time,
|
|
@@ -20,6 +25,10 @@ module Samagotchi
|
|
|
20
25
|
# that passes every gate allows the call once, without asking
|
|
21
26
|
# (SystemOne::Record, SystemOne::SelfProtect). The line a UI shows for an
|
|
22
27
|
# auto-allow is P3b (docs/guardrails.md, "System 1").
|
|
28
|
+
# Every core ask goes through SystemOne::Recording, the seam that saves it
|
|
29
|
+
# as a run, with labels and what came of it (SystemOne::Runs, RunsStore;
|
|
30
|
+
# docs/system-one.md). Bundles ask through SystemOne::Judgments, on the same
|
|
31
|
+
# Client and Recording (chi.judgment, ctx.judge).
|
|
23
32
|
module SystemOne
|
|
24
33
|
end
|
|
25
34
|
end
|
|
@@ -101,7 +101,8 @@ module Samagotchi
|
|
|
101
101
|
Plugin::Registries.new(
|
|
102
102
|
commands: SessionCommands.register_builtins(Commands::Registry.new), tools: Tools::Builtins.registry,
|
|
103
103
|
services: Plugin::Services.new, stage_tools: ->(*) {}, toolkits: [], tool_index: Tools::ToolIndex.new,
|
|
104
|
-
peer_sources: PeerSources.new, prompt_sections: Plugin::PromptSections.new,
|
|
104
|
+
peer_sources: PeerSources.new, prompt_sections: Plugin::PromptSections.new,
|
|
105
|
+
judgments: SystemOne::Judgments.new, **fields
|
|
105
106
|
)
|
|
106
107
|
end
|
|
107
108
|
end
|
|
@@ -375,6 +376,11 @@ module Samagotchi
|
|
|
375
376
|
# Queue ask_model answers: a String, an exception, or a callable.
|
|
376
377
|
def model_answers = @recorder.model_answers
|
|
377
378
|
|
|
379
|
+
# ctx.judge and ctx.judge_later's System 1, on a FakeBackend: what it
|
|
380
|
+
# answers, and the asks, runs and labels it saw (Testing::Judgments).
|
|
381
|
+
# @return [Judgments]
|
|
382
|
+
def judgments = @recorder.judgments.scoped(@name)
|
|
383
|
+
|
|
378
384
|
# Queue ask_user answers ({selected:, freeform:, selected_indices:},
|
|
379
385
|
# or a callable given the HookQuestion).
|
|
380
386
|
def answer(*answers)
|
|
@@ -425,7 +431,8 @@ module Samagotchi
|
|
|
425
431
|
@inits << Init.new(label: label, plugin_label: plugin_label, provides_tools: provides_tools, quiet: quiet,
|
|
426
432
|
timeout: timeout, failed: failed, block: block)
|
|
427
433
|
},
|
|
428
|
-
toolkits: toolkits
|
|
434
|
+
toolkits: toolkits,
|
|
435
|
+
judgments: @recorder.judgments.layer
|
|
429
436
|
)
|
|
430
437
|
end
|
|
431
438
|
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../llm/errors"
|
|
4
|
+
require_relative "../side_judgments"
|
|
5
|
+
require_relative "../system_one/answer"
|
|
6
|
+
require_relative "../system_one/client"
|
|
7
|
+
require_relative "../system_one/errors"
|
|
8
|
+
require_relative "../system_one/fake_backend"
|
|
9
|
+
require_relative "../system_one/judgments"
|
|
10
|
+
|
|
11
|
+
module Samagotchi
|
|
12
|
+
module Testing
|
|
13
|
+
# A ctx.judge or ctx.judge_later that reached System 1. +cancel+ is the
|
|
14
|
+
# controller it would stop with (nil for a judge_later's).
|
|
15
|
+
JudgmentAsk = Data.define(:name, :state, :questions, :cancel)
|
|
16
|
+
|
|
17
|
+
# What bundle.judgments is: chi's real SystemOne::Judgments, Client,
|
|
18
|
+
# Recording and Runs, with a FakeBackend per judgment and the runs in
|
|
19
|
+
# memory. So a judgment's off, skip, busy and failure paths are chi's own.
|
|
20
|
+
# Each judgment is on until the test says otherwise (chi's are off until
|
|
21
|
+
# the user sets a mode, which isn't the bundle's to test).
|
|
22
|
+
#
|
|
23
|
+
# bundle.judgments.answer("progress", noul: 0.9)
|
|
24
|
+
# bundle.fire(:after_turn)
|
|
25
|
+
# bundle.judgments.wait # judge_later's thread, and its block
|
|
26
|
+
# bundle.judgments.asks.last.state
|
|
27
|
+
# bundle.judgments.runs("progress").last["answers"]
|
|
28
|
+
# bundle.judgments.labels("progress").map { _1["value"] }
|
|
29
|
+
class Judgments
|
|
30
|
+
# The backend behind one judgment: records the ask, stops if it's cancelled.
|
|
31
|
+
class Backend < SystemOne::FakeBackend
|
|
32
|
+
def initialize(owner, name)
|
|
33
|
+
@owner = owner
|
|
34
|
+
@name = name
|
|
35
|
+
super() { |state, questions| owner.answers_for(name, state, questions) }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def ask(state:, questions:, cancel_controller: nil, observer: nil)
|
|
39
|
+
@owner.asked(JudgmentAsk.new(name: @name, state: state, questions: questions, cancel: cancel_controller))
|
|
40
|
+
raise LLM::RequestCancelled, :manual if cancel_controller&.cancelled?
|
|
41
|
+
|
|
42
|
+
super
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# @return [SystemOne::Judgments] what the Host and registries hold
|
|
47
|
+
attr_reader :layer
|
|
48
|
+
|
|
49
|
+
# @param recorder [Recorder] the session's values: session_id, scratch,
|
|
50
|
+
# model_name and turn_author are read at each ask
|
|
51
|
+
def initialize(recorder)
|
|
52
|
+
@recorder = recorder
|
|
53
|
+
@scope = nil
|
|
54
|
+
@mutex = Mutex.new
|
|
55
|
+
@answers = Hash.new { |hash, name| hash[name] = {} }
|
|
56
|
+
@failing = {}
|
|
57
|
+
@modes = {}
|
|
58
|
+
@no_host = {}
|
|
59
|
+
@shared_host_busy = false
|
|
60
|
+
@asks = []
|
|
61
|
+
@store = SystemOne::Runs::MemoryStore.new
|
|
62
|
+
@side = SideJudgments.new
|
|
63
|
+
@layer = build_layer
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Which bundle's short names these are (Bundle#judgments).
|
|
67
|
+
# @return [Judgments] self
|
|
68
|
+
def scoped(bundle)
|
|
69
|
+
@scope = bundle
|
|
70
|
+
self
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# What the judgment answers from now on, until the next #answer: the
|
|
74
|
+
# Answer's fields (noul:, choice:, score:, confidence:), or a block given
|
|
75
|
+
# the state and returning them, which may also block, to hold a
|
|
76
|
+
# judge_later in flight. A question with no answer set is no judgment.
|
|
77
|
+
# @param question [String, nil] nil: every question that has none of its own
|
|
78
|
+
# @return [Judgments] self
|
|
79
|
+
def answer(name, question: nil, **fields, &block)
|
|
80
|
+
@mutex.synchronize do
|
|
81
|
+
@answers[full(name)][question&.to_s || :any] = block || fields
|
|
82
|
+
@failing.delete(full(name))
|
|
83
|
+
end
|
|
84
|
+
self
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# The backend fails from now on, until the next #answer: a failed run.
|
|
88
|
+
# @return [Judgments] self
|
|
89
|
+
def fail(name, error: "fake System 1 is unavailable")
|
|
90
|
+
@mutex.synchronize { @failing[full(name)] = error }
|
|
91
|
+
self
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# The judgment's mode: "off", "audit" or "on" (the default here).
|
|
95
|
+
# @return [Judgments] self
|
|
96
|
+
def mode(name, word)
|
|
97
|
+
@mutex.synchronize { @modes[full(name)] = word.to_s }
|
|
98
|
+
self
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# No System 1 host is configured for the judgment.
|
|
102
|
+
# @return [Judgments] self
|
|
103
|
+
def no_host(name)
|
|
104
|
+
@mutex.synchronize { @no_host[full(name)] = true }
|
|
105
|
+
self
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# Whether a turn runs on the System 1 host (shared_host: :skip skips then).
|
|
109
|
+
# @return [Judgments] self
|
|
110
|
+
def shared_host_busy(busy = true)
|
|
111
|
+
@shared_host_busy = busy
|
|
112
|
+
self
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Wait for judge_later's asks and blocks, and for the blocks of those
|
|
116
|
+
# that asked nothing.
|
|
117
|
+
# @return [Judgments] self
|
|
118
|
+
def wait(seconds = 5)
|
|
119
|
+
@side.join(seconds)
|
|
120
|
+
self
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Every ask that reached System 1, in order.
|
|
124
|
+
# @return [Array<JudgmentAsk>]
|
|
125
|
+
def asks = @mutex.synchronize { @asks.dup }
|
|
126
|
+
|
|
127
|
+
# The runs saved, as chi saves them (docs/system-one.md).
|
|
128
|
+
# @return [Array<Hash>]
|
|
129
|
+
def runs(name) = lines(name, "run")
|
|
130
|
+
|
|
131
|
+
# The label lines saved.
|
|
132
|
+
# @return [Array<Hash>]
|
|
133
|
+
def labels(name) = lines(name, "label")
|
|
134
|
+
|
|
135
|
+
# The acted lines saved.
|
|
136
|
+
# @return [Array<Hash>]
|
|
137
|
+
def acted_lines(name) = lines(name, "acted")
|
|
138
|
+
|
|
139
|
+
# @api private
|
|
140
|
+
def asked(ask)
|
|
141
|
+
@mutex.synchronize { @asks << ask }
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# @api private: Backend's answers, one per question that has one.
|
|
145
|
+
def answers_for(name, state, questions)
|
|
146
|
+
answers, error = @mutex.synchronize { [@answers[name].dup, @failing[name]] }
|
|
147
|
+
raise SystemOne::BackendError, error if error
|
|
148
|
+
|
|
149
|
+
questions.to_h do |question|
|
|
150
|
+
entry = answers.fetch(question.id) { answers[:any] }
|
|
151
|
+
entry = entry.call(state) if entry.respond_to?(:call)
|
|
152
|
+
[question.id, entry && SystemOne::Answer.new(id: question.id, type: question.type, **entry)]
|
|
153
|
+
end.compact
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
private
|
|
157
|
+
|
|
158
|
+
def lines(name, kind) = @store.lines[full(name)].select { |line| line["kind"] == kind }
|
|
159
|
+
|
|
160
|
+
def full(name)
|
|
161
|
+
name = name.to_s
|
|
162
|
+
@scope && !@layer.declared?(name) ? "#{@scope}.#{name}" : name
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def build_layer
|
|
166
|
+
SystemOne::Judgments.new(
|
|
167
|
+
settings: ->(name) { @mutex.synchronize { SystemOne::JudgmentSettings.new(mode: @modes.fetch(name, "on")) } },
|
|
168
|
+
client_for: lambda { |name, settings|
|
|
169
|
+
next nil if @mutex.synchronize { @no_host[name] }
|
|
170
|
+
|
|
171
|
+
SystemOne::Client.new(backend: Backend.new(self, name), mode: settings.mode, host: "fake", model: "fake", tag: :plugins)
|
|
172
|
+
},
|
|
173
|
+
runs: -> { SystemOne::Runs.new(store: @recorder[:scratch] ? nil : @store) },
|
|
174
|
+
where: lambda {
|
|
175
|
+
id = @recorder[:session_id]
|
|
176
|
+
id && SystemOne::Recording::Context.new(session_id: id, session_model: @recorder[:model_name])
|
|
177
|
+
},
|
|
178
|
+
side: @side,
|
|
179
|
+
shared_host_busy: ->(_host) { @shared_host_busy },
|
|
180
|
+
participant: -> { !@recorder[:turn_author].nil? }
|
|
181
|
+
)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
@@ -4,6 +4,7 @@ require_relative "../plugin/card"
|
|
|
4
4
|
require_relative "../plugin/context"
|
|
5
5
|
require_relative "../guardrails/parent_policy"
|
|
6
6
|
require_relative "../hook_question"
|
|
7
|
+
require_relative "judgments"
|
|
7
8
|
|
|
8
9
|
module Samagotchi
|
|
9
10
|
module Testing
|
|
@@ -83,6 +84,10 @@ module Samagotchi
|
|
|
83
84
|
|
|
84
85
|
def [](key) = @mutex.synchronize { @values.fetch(key) }
|
|
85
86
|
|
|
87
|
+
# ctx.judge and ctx.judge_later's System 1 (Testing::Judgments).
|
|
88
|
+
# @return [Judgments]
|
|
89
|
+
def judgments = @mutex.synchronize { @judgments ||= Judgments.new(self) }
|
|
90
|
+
|
|
86
91
|
# Only stop_turn's, or only stop_generation's.
|
|
87
92
|
def turn_stops = stops.select { |stop| stop.kind == :turn }
|
|
88
93
|
def generation_stops = stops.select { |stop| stop.kind == :generation }
|
|
@@ -11,7 +11,7 @@ module Samagotchi
|
|
|
11
11
|
# The real Plugin::Api, also recording each declaration in order
|
|
12
12
|
# (#declarations), for a spec of what #register declares as written.
|
|
13
13
|
class RecordingApi < Plugin::Api
|
|
14
|
-
DECLARATIONS = %i[command annotation tool on service init toolkit tool_source peer_source prompt_section].freeze
|
|
14
|
+
DECLARATIONS = %i[command annotation tool on service init toolkit tool_source peer_source prompt_section judgment].freeze
|
|
15
15
|
|
|
16
16
|
# @return [Array<Declaration>]
|
|
17
17
|
def declarations = (@declarations ||= [])
|
|
@@ -9,7 +9,9 @@ module Samagotchi
|
|
|
9
9
|
#
|
|
10
10
|
# :head keeps the first chars and ends with head_line ("[cut: N of M
|
|
11
11
|
# chars; read it in parts]"): read (which has line ranges) and every tool
|
|
12
|
-
# with no policy of its own.
|
|
12
|
+
# with no policy of its own. memory_read's cut (a memory.read_max_chars
|
|
13
|
+
# cap) ends with memory_line instead, which says where the rest is
|
|
14
|
+
# (the memory's .md file, read with start_line): memory_read has no parts.
|
|
13
15
|
#
|
|
14
16
|
# :head_tail_file (execute) keeps a short head and a long tail, whole
|
|
15
17
|
# lines, with one gap line between them, and saves the whole output
|
|
@@ -95,9 +97,11 @@ module Samagotchi
|
|
|
95
97
|
# +save+ returned (nil when it wasn't called)
|
|
96
98
|
# @param ref [String, nil] :head_tail_ref: the file the whole text is in
|
|
97
99
|
# @param ref_stderr [String, nil] :head_tail_ref: its stderr's file, if apart
|
|
98
|
-
|
|
100
|
+
# @param line [#call, nil] (kept, total) → the cut line of a :head cut
|
|
101
|
+
# (head_line when nil): memory_read's names the memory file
|
|
102
|
+
def apply(output, cap:, tool:, save: nil, ref: nil, ref_stderr: nil, head_chars: HEAD_CHARS, line: nil)
|
|
99
103
|
kind = policy(tool)
|
|
100
|
-
return [head(output, cap), nil] if kind == :head || cap < MIN_TAIL * 3
|
|
104
|
+
return [head(output, cap, line: line), nil] if kind == :head || cap < MIN_TAIL * 3
|
|
101
105
|
|
|
102
106
|
if kind == :head_tail_ref
|
|
103
107
|
text = head_tail(output, cap: cap, blocks: [Block.new(from: 0, length: output.length)], head_chars: head_chars,
|
|
@@ -137,10 +141,26 @@ module Samagotchi
|
|
|
137
141
|
# +output+ cut to +cap+ chars, the cut line included: the model knows
|
|
138
142
|
# what it didn't get. A cap smaller than the line keeps +cap+ chars and
|
|
139
143
|
# the line goes past it.
|
|
140
|
-
|
|
141
|
-
|
|
144
|
+
# @param line [#call, nil] (kept, total) → the cut line, head_line when nil
|
|
145
|
+
def head(output, cap, line: nil)
|
|
146
|
+
line ||= method(:head_line)
|
|
147
|
+
keep = cap - line.call(cap, output.length).length
|
|
142
148
|
keep = cap unless keep.positive?
|
|
143
|
-
"#{output[0, keep]}#{
|
|
149
|
+
"#{output[0, keep]}#{line.call(keep, output.length)}"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# memory_read's cut line: the rest is in the memory's file, which read
|
|
153
|
+
# takes with start_line. +where+ names the memory dir(s) searched
|
|
154
|
+
# ("the memory .md file" for one name, "the .md files" for a list or the
|
|
155
|
+
# indexes).
|
|
156
|
+
# @param dirs [Array<String>] the memories dirs searched
|
|
157
|
+
# @param single [Boolean] one memory name was read
|
|
158
|
+
# @return [#call] a +line+ for head/apply
|
|
159
|
+
def memory_line(dirs, single:)
|
|
160
|
+
what = single ? "the memory's .md file" : "the memories' .md files"
|
|
161
|
+
lambda do |kept, total|
|
|
162
|
+
"\n[cut: #{kept} of #{total} chars; the rest is in #{what} in #{dirs.join(" or ")}: read it with start_line]"
|
|
163
|
+
end
|
|
144
164
|
end
|
|
145
165
|
|
|
146
166
|
def head_line(kept, total)
|
|
@@ -6,7 +6,7 @@ module Samagotchi
|
|
|
6
6
|
# what the model gets back is per format: the native loop answers a batch
|
|
7
7
|
# with ONE tool_response (#joined), the chat loop each call with its own,
|
|
8
8
|
# paired by tool_call_id (#single). Both feed the capped outputs
|
|
9
|
-
# (max_tool_output_chars).
|
|
9
|
+
# (max_tool_output_chars; memory_read's is memory.read_max_chars).
|
|
10
10
|
module ToolResponse
|
|
11
11
|
SEPARATOR = "\n\n---\n\n"
|
|
12
12
|
# Saved with a result for the web's reload (a plugin tool's params line,
|