samagotchi 0.79.0 → 0.81.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +59 -1
- data/README.md +7 -5
- data/docs/architecture.md +9 -1
- data/docs/cli.md +32 -7
- data/docs/configuration.md +10 -4
- data/docs/guardrails.md +26 -3
- data/docs/internals/anthropic.md +3 -1
- data/docs/internals/bundles.md +4 -0
- data/docs/internals/engine.md +13 -3
- data/docs/plugins.md +149 -3
- data/docs/releasing.md +2 -2
- data/docs/sessions.md +9 -1
- data/docs/testing.md +51 -2
- data/lib/samagotchi/bootstrap/config_writer.rb +13 -3
- data/lib/samagotchi/bootstrap/probe.rb +153 -31
- data/lib/samagotchi/bootstrap_command.rb +93 -16
- data/lib/samagotchi/bridge.rb +1 -1
- data/lib/samagotchi/bundles/guardrails/guardrails/rules.yml +4 -4
- data/lib/samagotchi/bundles/guardrails/manifest.yml +1 -1
- data/lib/samagotchi/bundles/slow-commands/manifest.yml +2 -2
- data/lib/samagotchi/bundles/slow-commands/plugin.rb +2 -1
- data/lib/samagotchi/bundles/system/manifest.yml +2 -2
- data/lib/samagotchi/bundles/system/self_map.md +1 -1
- data/lib/samagotchi/continue_offer.rb +5 -5
- data/lib/samagotchi/engine.rb +68 -39
- data/lib/samagotchi/guardrails/approval.rb +6 -4
- data/lib/samagotchi/guardrails/data_readers.rb +214 -0
- data/lib/samagotchi/guardrails/gate.rb +7 -3
- data/lib/samagotchi/guardrails/rules.rb +4 -2
- data/lib/samagotchi/guardrails/shell_lex.rb +166 -28
- data/lib/samagotchi/guardrails/targets.rb +14 -0
- data/lib/samagotchi/hooks/bundle_loader.rb +31 -16
- data/lib/samagotchi/hooks/registry.rb +44 -3
- data/lib/samagotchi/host_config.rb +8 -8
- data/lib/samagotchi/host_registry.rb +12 -4
- data/lib/samagotchi/idle_client.rb +41 -11
- data/lib/samagotchi/idle_recap.rb +6 -1
- data/lib/samagotchi/idle_target.rb +12 -4
- data/lib/samagotchi/llm/adapter_common.rb +4 -2
- data/lib/samagotchi/llm/anthropic_binding_recovery.rb +3 -0
- data/lib/samagotchi/llm/anthropic_messages.rb +2 -1
- data/lib/samagotchi/llm/http.rb +12 -7
- data/lib/samagotchi/llm/usage.rb +9 -3
- data/lib/samagotchi/model_price.rb +16 -0
- data/lib/samagotchi/plugin/card.rb +44 -0
- data/lib/samagotchi/plugin/context.rb +61 -9
- data/lib/samagotchi/plugin/http.rb +154 -0
- data/lib/samagotchi/plugin/http_transport.rb +77 -0
- data/lib/samagotchi/plugin/loader.rb +29 -10
- data/lib/samagotchi/plugin/service.rb +24 -1
- data/lib/samagotchi/question_answer.rb +29 -0
- data/lib/samagotchi/question_desk.rb +20 -19
- data/lib/samagotchi/recap_setup.rb +2 -1
- data/lib/samagotchi/relay_verifier.rb +1 -1
- data/lib/samagotchi/session_metrics/side_totals.rb +110 -0
- data/lib/samagotchi/session_metrics.rb +46 -12
- data/lib/samagotchi/session_observer.rb +3 -0
- data/lib/samagotchi/side_usage.rb +38 -0
- data/lib/samagotchi/terminal_ui/formatting.rb +3 -0
- data/lib/samagotchi/terminal_ui/model_segments_text.rb +27 -3
- data/lib/samagotchi/testing/boundary.rb +156 -0
- data/lib/samagotchi/testing/bundle.rb +349 -0
- data/lib/samagotchi/testing/fake_executable.rb +42 -0
- data/lib/samagotchi/testing/fake_http.rb +137 -0
- data/lib/samagotchi/testing/helpers.rb +56 -0
- data/lib/samagotchi/testing/host.rb +76 -0
- data/lib/samagotchi/testing/minitest.rb +31 -0
- data/lib/samagotchi/testing/recorder.rb +141 -0
- data/lib/samagotchi/testing/recording_api.rb +27 -0
- data/lib/samagotchi/testing/rspec.rb +31 -0
- data/lib/samagotchi/testing/sandbox.rb +165 -0
- data/lib/samagotchi/testing/sessions.rb +77 -0
- data/lib/samagotchi/testing/verdict.rb +28 -0
- data/lib/samagotchi/testing/waiting.rb +36 -0
- data/lib/samagotchi/testing.rb +26 -0
- data/lib/samagotchi/tools/delegate_relay.rb +7 -6
- data/lib/samagotchi/tools/tool_index.rb +3 -0
- data/lib/samagotchi/turn_state.rb +28 -1
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +4 -1
- data/lib/samagotchi/web/public/app.js +47 -23
- data/lib/samagotchi/web/public/ctx.js +32 -6
- data/lib/samagotchi/web/public/images.js +33 -0
- data/lib/samagotchi/web/public/index.html +71 -38
- data/lib/samagotchi/web/public/info_bar_model.js +1 -1
- data/lib/samagotchi/web/public/level_chip.js +11 -11
- data/lib/samagotchi/web/public/lists_view.js +20 -15
- data/lib/samagotchi/web/public/llm_context_chip.js +15 -15
- data/lib/samagotchi/web/public/model_picker.js +1 -1
- data/lib/samagotchi/web/public/scope.js +3 -3
- data/lib/samagotchi/web/public/stream_handlers.js +7 -0
- data/lib/samagotchi/web/public/strip.js +35 -9
- data/lib/samagotchi/web/public/strip_grip.js +76 -0
- data/lib/samagotchi/web/public/thinking_chip.js +5 -5
- data/lib/samagotchi/web/public/timing.js +6 -1
- data/lib/samagotchi/web/session_summary.rb +2 -0
- metadata +24 -1
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../plugin/loader"
|
|
4
|
+
require_relative "../plugin/tool_spec"
|
|
5
|
+
require_relative "../hooks"
|
|
6
|
+
require_relative "../commands/registry"
|
|
7
|
+
require_relative "../session_commands"
|
|
8
|
+
require_relative "../tools/builtins"
|
|
9
|
+
require_relative "../tools/tool_index"
|
|
10
|
+
require_relative "../peer_sources"
|
|
11
|
+
require_relative "../toolkit"
|
|
12
|
+
require_relative "../guardrails/gate"
|
|
13
|
+
require_relative "../memory_bundle/manifest"
|
|
14
|
+
require_relative "sandbox"
|
|
15
|
+
require_relative "recorder"
|
|
16
|
+
require_relative "host"
|
|
17
|
+
require_relative "sessions"
|
|
18
|
+
require_relative "fake_http"
|
|
19
|
+
require_relative "verdict"
|
|
20
|
+
require_relative "recording_api"
|
|
21
|
+
|
|
22
|
+
module Samagotchi
|
|
23
|
+
module Testing
|
|
24
|
+
# A hook (a plugin's chi.on block or a hook file) raised while the kit
|
|
25
|
+
# fired it: chi would log it and go on, a test fails with it.
|
|
26
|
+
class HookFailed < StandardError
|
|
27
|
+
# @return [Hooks::Failure]
|
|
28
|
+
attr_reader :failure
|
|
29
|
+
|
|
30
|
+
def initialize(failure)
|
|
31
|
+
@failure = failure
|
|
32
|
+
error = failure.error
|
|
33
|
+
super("#{failure.hook} raised on #{failure.event}: #{error.class}: #{error.message}")
|
|
34
|
+
set_backtrace(error.backtrace) if error.backtrace
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def cause = @failure.error
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# A chi.init the plugin registered: its arguments and block.
|
|
41
|
+
Init = Data.define(:label, :plugin_label, :provides_tools, :quiet, :timeout, :failed, :block)
|
|
42
|
+
# A chi.replace_tools set, staged as chi stages it for the turn thread.
|
|
43
|
+
StagedTools = Data.define(:specs, :context)
|
|
44
|
+
|
|
45
|
+
# One bundle loaded for a test through chi's own seams: its plugin
|
|
46
|
+
# through Plugin::Loader.register_file (Plugin::Api, a real
|
|
47
|
+
# Plugin::Context) and its hook files through
|
|
48
|
+
# Hooks::BundleLoader.load_hook_file, into real registries (commands
|
|
49
|
+
# and tools with chi's built-ins in them, so a name clash fails as in
|
|
50
|
+
# chi). Only the edges are recorded: the Host and hook Runtime
|
|
51
|
+
# (Recorder), ctx.sessions (Testing::Sessions), chi.init tasks and
|
|
52
|
+
# staged tool sets. No sha256 or requires_chi check: it loads the
|
|
53
|
+
# source as it is.
|
|
54
|
+
#
|
|
55
|
+
# bundle = Samagotchi::Testing::Bundle.load("path/to/bundle", settings: { "every" => 3 })
|
|
56
|
+
# bundle.fire(:after_turn, status: "completed")
|
|
57
|
+
# bundle.command("/checkin stop")
|
|
58
|
+
# bundle.tool_call({ name: "execute", content: "ls" }) # → Verdict
|
|
59
|
+
# bundle.notices, bundle.cards, bundle.steers …
|
|
60
|
+
# bundle.shutdown
|
|
61
|
+
#
|
|
62
|
+
# A hook that raises while #fire, #tool_call, #command or #tool runs
|
|
63
|
+
# raises HookFailed after the call (chi would log it and go on);
|
|
64
|
+
# #lenient collects them in #errors instead.
|
|
65
|
+
class Bundle
|
|
66
|
+
SHIPPED_DIR = File.expand_path("../bundles", __dir__)
|
|
67
|
+
|
|
68
|
+
class << self
|
|
69
|
+
# @param dir [String] the bundle's folder (its manifest.yml)
|
|
70
|
+
# @param settings [Hash] config.yml `bundles: <name>:`
|
|
71
|
+
# @param host [Hash] session values (Recorder::DEFAULTS' keys)
|
|
72
|
+
# @param sandbox [Sandbox, nil] default Sandbox.current, else one of
|
|
73
|
+
# its own, left at #shutdown
|
|
74
|
+
# @return [Bundle]
|
|
75
|
+
def load(dir, settings: {}, host: {}, sandbox: nil)
|
|
76
|
+
new(dir, settings: settings, host: host, sandbox: sandbox).tap(&:load!)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# A bundle chi ships (lib/samagotchi/bundles/<name>), as #load.
|
|
80
|
+
def shipped(name, **) = load(File.join(SHIPPED_DIR, name.to_s), **)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @return [String] the bundle's name (manifest.yml)
|
|
84
|
+
attr_reader :name
|
|
85
|
+
# @return [MemoryBundle::Manifest]
|
|
86
|
+
attr_reader :manifest
|
|
87
|
+
# @return [Plugin::Registries]
|
|
88
|
+
attr_reader :registries
|
|
89
|
+
# @return [Recorder]
|
|
90
|
+
attr_reader :recorder
|
|
91
|
+
# @return [Sessions] ctx.sessions
|
|
92
|
+
attr_reader :sessions
|
|
93
|
+
# @return [FakeHttp] what answers ctx.http (stub, requests, unmatched)
|
|
94
|
+
attr_reader :http
|
|
95
|
+
# @return [Sandbox]
|
|
96
|
+
attr_reader :sandbox
|
|
97
|
+
# @return [Plugin::Api, nil] the plugin's committed Api (nil without one)
|
|
98
|
+
attr_reader :api
|
|
99
|
+
# @return [Object, nil] the plugin's instance (the object #register ran
|
|
100
|
+
# on), for specs of its own methods
|
|
101
|
+
attr_reader :plugin
|
|
102
|
+
# @return [Module, nil] the module the plugin was evaluated in
|
|
103
|
+
# (bundle.namespace::ThinkingWatch)
|
|
104
|
+
attr_reader :namespace
|
|
105
|
+
# @return [Module] the module the hook files were evaluated in
|
|
106
|
+
attr_reader :hooks_namespace
|
|
107
|
+
# @return [Array<Init>]
|
|
108
|
+
attr_reader :inits
|
|
109
|
+
# @return [Array<StagedTools>]
|
|
110
|
+
attr_reader :staged_tools
|
|
111
|
+
# @return [Array<Hooks::Failure>] what raised under #lenient
|
|
112
|
+
attr_reader :errors
|
|
113
|
+
# @return [Integer] how often chi was told the tools changed
|
|
114
|
+
attr_reader :tools_changes
|
|
115
|
+
|
|
116
|
+
def initialize(dir, settings: {}, host: {}, sandbox: nil)
|
|
117
|
+
@dir = File.expand_path(dir)
|
|
118
|
+
@manifest = MemoryBundle::Manifest.read(dir: @dir)
|
|
119
|
+
@name = @manifest.name
|
|
120
|
+
@settings = stringify(settings || {})
|
|
121
|
+
@own_sandbox = sandbox.nil? && Sandbox.current.nil?
|
|
122
|
+
@sandbox = sandbox || Sandbox.current || Sandbox.enter
|
|
123
|
+
@recorder = Recorder.new(cwd: @sandbox.work_dir, **host)
|
|
124
|
+
@sessions = Sessions.new
|
|
125
|
+
@http = FakeHttp.new
|
|
126
|
+
@inits = []
|
|
127
|
+
@staged_tools = []
|
|
128
|
+
@errors = []
|
|
129
|
+
@lenient = false
|
|
130
|
+
@tools_changes = 0
|
|
131
|
+
@contexts = {}
|
|
132
|
+
@registries = build_registries
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# @api private (Bundle.load)
|
|
136
|
+
def load!
|
|
137
|
+
if (plugin = @manifest.plugin)
|
|
138
|
+
@namespace = Plugin::Loader.fresh_namespace(@name)
|
|
139
|
+
@api = Plugin::Loader.register_file(@name, File.join(@dir, plugin[:file]), @registries, settings: @settings,
|
|
140
|
+
namespace: @namespace,
|
|
141
|
+
api_class: RecordingApi) do |built|
|
|
142
|
+
@plugin = built
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
@hooks_namespace = Hooks::BundleLoader.namespace_for(@name).const_set(:"KitHooks#{object_id}", Module.new)
|
|
146
|
+
@manifest.hooks.each do |basename, hook|
|
|
147
|
+
next if hook.event.empty?
|
|
148
|
+
|
|
149
|
+
Hooks::BundleLoader.load_hook_file(@name, basename, File.join(@dir, "hooks", basename), hook, hooks,
|
|
150
|
+
settings: @settings, namespace: @hooks_namespace)
|
|
151
|
+
end
|
|
152
|
+
self
|
|
153
|
+
rescue Exception # rubocop:disable Lint/RescueException -- leave the sandbox it entered, whatever failed
|
|
154
|
+
shutdown
|
|
155
|
+
raise
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# ── Firing ──────────────────────────────────────────────────────────
|
|
159
|
+
|
|
160
|
+
# Fire hook +event+ through the registry, as chi's fire sites do: the
|
|
161
|
+
# event is {type: event, **fields} (a fire site's own keys, such as
|
|
162
|
+
# after_turn's present:/refs:, come from +fields+), with the runtime
|
|
163
|
+
# helpers added. Returns the event after the hooks.
|
|
164
|
+
# @return [Hash]
|
|
165
|
+
def fire(event, **fields)
|
|
166
|
+
payload = { type: event.to_sym, **fields }
|
|
167
|
+
failing { hooks.fire(event.to_sym, payload) }
|
|
168
|
+
payload
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
# One tool call through a real Guardrails::Gate: the before_tool_call
|
|
172
|
+
# hooks vote on a Testing::Verdict (pass +verdict+ to start from one,
|
|
173
|
+
# another voter's deny in it), with the context and targets the Gate
|
|
174
|
+
# computes for the call (cwd: the session's by default).
|
|
175
|
+
# @param call [Hash] the parsed call ({name:, content:/path:/args:})
|
|
176
|
+
# @param context [Guardrails::Context, nil]
|
|
177
|
+
# @return [Verdict]
|
|
178
|
+
def tool_call(call, params: nil, cwd: nil, context: nil, verdict: nil, iteration: 1)
|
|
179
|
+
context ||= Guardrails::Context.new(cwd: cwd || @recorder[:cwd], session_id: @recorder[:session_id],
|
|
180
|
+
interface: @recorder[:frontend])
|
|
181
|
+
gate = Guardrails::Gate.new(-> { hooks }, context_lookup: -> { context }, tools_lookup: -> { tools },
|
|
182
|
+
checks_lookup: -> { [] })
|
|
183
|
+
verdict ||= Verdict.new(call: call)
|
|
184
|
+
params ||= "#{call[:name]} #{call[:content] || call[:path]}".strip
|
|
185
|
+
failing { gate.evaluate(call, iteration: iteration, params: params, verdict: verdict) }
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Run a command line ("/name args") as chi does: the first matching
|
|
189
|
+
# command, given the text after its name. Returns the handler's value
|
|
190
|
+
# (a raise propagates: chi would show it as an error result).
|
|
191
|
+
def command(line)
|
|
192
|
+
entry = commands.lookup(line)
|
|
193
|
+
raise ArgumentError, "no command matches #{line.inspect}" unless entry
|
|
194
|
+
raise ArgumentError, "#{entry.name} is chi's own command, not the bundle's" if entry.source == "core"
|
|
195
|
+
|
|
196
|
+
failing { entry.handler.call(line.to_s.strip.delete_prefix(entry.name).strip) }
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
# Call a registered tool as chi's dispatch does: its handler gets the
|
|
200
|
+
# call ({name:, args:}), types the args by the tool's schema
|
|
201
|
+
# (ToolSpec.args_of) and returns the text the model reads.
|
|
202
|
+
# @return [String]
|
|
203
|
+
def tool(name, args = {})
|
|
204
|
+
entry = tools[name.to_s]
|
|
205
|
+
raise ArgumentError, "no tool #{name}" unless entry
|
|
206
|
+
|
|
207
|
+
failing { entry.handler.call({ name: name.to_s, args: stringify(args) }, nil) }
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Run the chi.init blocks in order, as chi's init tasks do.
|
|
211
|
+
# @return [Array] their results
|
|
212
|
+
def run_inits = failing { @inits.map { |init| init.block.call } }
|
|
213
|
+
|
|
214
|
+
# Apply the staged chi.replace_tools sets, as chi does before a turn
|
|
215
|
+
# (ToolSpec.apply). @return [Boolean] whether the tools changed
|
|
216
|
+
def apply_staged_tools!
|
|
217
|
+
sets = @staged_tools.dup
|
|
218
|
+
@staged_tools.clear
|
|
219
|
+
sets.map { |set| Plugin::ToolSpec.apply(tools, @name, set.specs, set.context)[:changed] }.any?
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Collect what raises in the block's fires in #errors instead of
|
|
223
|
+
# raising HookFailed.
|
|
224
|
+
def lenient
|
|
225
|
+
was = @lenient
|
|
226
|
+
@lenient = true
|
|
227
|
+
yield
|
|
228
|
+
ensure
|
|
229
|
+
@lenient = was
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
# ── What it registered ─────────────────────────────────────────────
|
|
233
|
+
|
|
234
|
+
# @return [Commands::Registry] chi's built-ins and the bundle's
|
|
235
|
+
def commands = @registries.commands
|
|
236
|
+
# @return [Tools::Registry] chi's built-ins and the bundle's
|
|
237
|
+
def tools = @registries.tools
|
|
238
|
+
# @return [Hooks::Registry]
|
|
239
|
+
def hooks = @registries.hooks
|
|
240
|
+
# @return [Array<Toolkit::Kit>]
|
|
241
|
+
def toolkits = @registries.toolkits
|
|
242
|
+
# @return [Tools::ToolIndex]
|
|
243
|
+
def tool_index = @registries.tool_index
|
|
244
|
+
# @return [PeerSources]
|
|
245
|
+
def peer_sources = @registries.peer_sources
|
|
246
|
+
|
|
247
|
+
# The bundle's own command entries.
|
|
248
|
+
def bundle_commands = commands.entries.select { |entry| entry.source == @name }
|
|
249
|
+
# The bundle's own tool entries.
|
|
250
|
+
def bundle_tools = tools.by_source(@name)
|
|
251
|
+
# Its hooks for +event+ as [bundle, hook_name, priority], fire order.
|
|
252
|
+
def hook_names(event) = hooks.bundle_hook_names(event.to_sym)
|
|
253
|
+
# The annotations its commands carry.
|
|
254
|
+
def annotations = bundle_commands.flat_map(&:annotations)
|
|
255
|
+
|
|
256
|
+
# The chi.* calls #register made, in order, as made (RecordingApi).
|
|
257
|
+
# @return [Array<Declaration>]
|
|
258
|
+
def declarations = @api ? @api.declarations : []
|
|
259
|
+
|
|
260
|
+
# The plugin's ctx (a real Plugin::Context; nil without a plugin).
|
|
261
|
+
def ctx = @contexts.values.first
|
|
262
|
+
|
|
263
|
+
# ── Recorders ──────────────────────────────────────────────────────
|
|
264
|
+
|
|
265
|
+
def notices = @recorder.notices
|
|
266
|
+
def cards = @recorder.cards
|
|
267
|
+
def steers = @recorder.steers
|
|
268
|
+
# Every stop_turn and stop_generation, in order (Stop#kind).
|
|
269
|
+
def stops = @recorder.stops
|
|
270
|
+
def turn_stops = @recorder.turn_stops
|
|
271
|
+
def generation_stops = @recorder.generation_stops
|
|
272
|
+
def questions = @recorder.questions
|
|
273
|
+
def model_requests = @recorder.model_requests
|
|
274
|
+
def toolkit_uses = @recorder.toolkit_uses
|
|
275
|
+
# Queue ask_model answers: a String, an exception, or a callable.
|
|
276
|
+
def model_answers = @recorder.model_answers
|
|
277
|
+
|
|
278
|
+
# Queue ask_user answers ({selected:, freeform:, selected_indices:},
|
|
279
|
+
# or a callable given the Question).
|
|
280
|
+
def answer(*answers)
|
|
281
|
+
@recorder.answers.concat(answers)
|
|
282
|
+
self
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# Change the session's values (cwd:, messages:, running: …).
|
|
286
|
+
def set(**values)
|
|
287
|
+
@recorder.set(**values)
|
|
288
|
+
self
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# Stop the plugin's services; leave the sandbox it entered.
|
|
292
|
+
def shutdown
|
|
293
|
+
@registries&.services&.stop_all
|
|
294
|
+
return unless @own_sandbox
|
|
295
|
+
|
|
296
|
+
@own_sandbox = false
|
|
297
|
+
@sandbox.leave
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
private
|
|
301
|
+
|
|
302
|
+
def build_registries
|
|
303
|
+
builtin_tools = Tools::Builtins.registry
|
|
304
|
+
toolkits = []
|
|
305
|
+
host = Host.build(@recorder, toolkits: -> { Toolkit::BUILTIN_KITS.keys + toolkits.map(&:name) },
|
|
306
|
+
tool_names: -> { @registries.tools.names })
|
|
307
|
+
Plugin::Registries.new(
|
|
308
|
+
commands: SessionCommands.register_builtins(Commands::Registry.new),
|
|
309
|
+
tools: builtin_tools,
|
|
310
|
+
hooks: Hooks::Registry.new.tap { |registry| registry.runtime = Runtime.build(@recorder) },
|
|
311
|
+
context_for: lambda { |bundle, settings, label|
|
|
312
|
+
@contexts[label] = Plugin::Context.new(bundle: bundle, label: label, settings: settings, host: host,
|
|
313
|
+
env: ENV, sessions: @sessions, http_transport: @http)
|
|
314
|
+
},
|
|
315
|
+
tools_changed: -> { @tools_changes += 1 },
|
|
316
|
+
services: Plugin::Services.new,
|
|
317
|
+
stage_tools: ->(_bundle, specs, context) { @staged_tools << StagedTools.new(specs: specs, context: context) },
|
|
318
|
+
init: lambda { |_bundle, label, plugin_label, provides_tools:, quiet:, timeout:, failed: nil, &block|
|
|
319
|
+
@inits << Init.new(label: label, plugin_label: plugin_label, provides_tools: provides_tools, quiet: quiet,
|
|
320
|
+
timeout: timeout, failed: failed, block: block)
|
|
321
|
+
},
|
|
322
|
+
toolkits: toolkits,
|
|
323
|
+
tool_index: Tools::ToolIndex.new,
|
|
324
|
+
peer_sources: PeerSources.new
|
|
325
|
+
)
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
# Run the block catching the hooks that raise in it.
|
|
329
|
+
def failing(&)
|
|
330
|
+
failures = []
|
|
331
|
+
result = Hooks.capture_failures(failures, &)
|
|
332
|
+
if @lenient
|
|
333
|
+
@errors.concat(failures)
|
|
334
|
+
elsif (first = failures.first)
|
|
335
|
+
raise HookFailed, first
|
|
336
|
+
end
|
|
337
|
+
result
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
def stringify(value)
|
|
341
|
+
case value
|
|
342
|
+
when Hash then value.to_h { |key, item| [key.to_s, stringify(item)] }
|
|
343
|
+
when Array then value.map { |item| stringify(item) }
|
|
344
|
+
else value
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
end
|
|
348
|
+
end
|
|
349
|
+
end
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "tmpdir"
|
|
5
|
+
|
|
6
|
+
module Samagotchi
|
|
7
|
+
module Testing
|
|
8
|
+
# Fake commands for a test's PATH (Sandbox#fake_executable), cheap to
|
|
9
|
+
# exec: macOS checks a new executable file on its first exec (about
|
|
10
|
+
# 0.5–1 s, once per file). A fake is a symlink to one launcher, made
|
|
11
|
+
# once per process, that runs the fake's body (<name>.sh beside the
|
|
12
|
+
# link) with /bin/sh: writing a new body costs nothing.
|
|
13
|
+
module FakeExecutable
|
|
14
|
+
LAUNCHER = "#!/bin/sh\nexec /bin/sh \"$0.sh\" \"$@\"\n"
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
# Writes +dir+/+name+, which runs +body+ (sh) with its arguments;
|
|
19
|
+
# again for the same name replaces the body.
|
|
20
|
+
# @return [String] the command's path
|
|
21
|
+
def write(dir, name, body)
|
|
22
|
+
FileUtils.mkdir_p(dir)
|
|
23
|
+
path = File.join(dir, name)
|
|
24
|
+
File.write("#{path}.sh", body.end_with?("\n") ? body : "#{body}\n")
|
|
25
|
+
FileUtils.ln_sf(launcher, path)
|
|
26
|
+
path
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def launcher
|
|
30
|
+
@launcher ||= begin
|
|
31
|
+
dir = Dir.mktmpdir("chi-fake-exec")
|
|
32
|
+
owner = Process.pid
|
|
33
|
+
at_exit { FileUtils.rm_rf(dir) if Process.pid == owner }
|
|
34
|
+
File.join(dir, "launcher").tap do |path|
|
|
35
|
+
File.write(path, LAUNCHER)
|
|
36
|
+
File.chmod(0o755, path)
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../plugin/http"
|
|
5
|
+
|
|
6
|
+
module Samagotchi
|
|
7
|
+
module Testing
|
|
8
|
+
# A request no stub answered (FakeHttp). Not an HttpError: a plugin's
|
|
9
|
+
# `rescue Plugin::HttpError` doesn't swallow it, and the test fails at
|
|
10
|
+
# teardown even when the plugin rescued it (FakeHttp#unmatched).
|
|
11
|
+
class UnstubbedRequest < StandardError; end
|
|
12
|
+
|
|
13
|
+
# ctx.http's wire in a bundle's test (docs/plugins.md, "Testing a
|
|
14
|
+
# bundle"): requests are answered by stubs, never by the network.
|
|
15
|
+
#
|
|
16
|
+
# bundle.http.stub(:post, "http://127.0.0.1:9000/classify", json: { label: "safe" })
|
|
17
|
+
# bundle.http.stub(:get, %r{/health\z}, status: 503, body: "down")
|
|
18
|
+
# bundle.http.stub(:post, "http://127.0.0.1:9000/classify") { |req| { json: { label: req.json["text"] } } }
|
|
19
|
+
# bundle.http.stub(:get, "http://127.0.0.1:9000/x", error: :refused) # also :timeout
|
|
20
|
+
# bundle.http.requests # what the plugin sent, in order
|
|
21
|
+
# bundle.http.unmatched # what no stub answered
|
|
22
|
+
#
|
|
23
|
+
# A String matches the full URL exactly (query included), a Regexp is
|
|
24
|
+
# matched against it; the method too. Later stubs win. A stub answers
|
|
25
|
+
# every matching call: a sequence is a block that counts.
|
|
26
|
+
class FakeHttp
|
|
27
|
+
# One request the plugin made (headers as the plugin gave them).
|
|
28
|
+
# `method` shadows Object#method on purpose: it is the plugin's word (request.method == "POST").
|
|
29
|
+
Request = Data.define(:method, :url, :headers, :body) do # rubocop:disable Lint/DataDefineOverride
|
|
30
|
+
# The body parsed as JSON.
|
|
31
|
+
def json = JSON.parse(body.to_s)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
Stub = Data.define(:verb, :matcher, :answer) do
|
|
35
|
+
def match?(request)
|
|
36
|
+
request.method == verb &&
|
|
37
|
+
(matcher.is_a?(Regexp) ? matcher.match?(request.url) : matcher == request.url)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
private_constant :Stub
|
|
41
|
+
|
|
42
|
+
# What error: names, and the words the real transport's message says.
|
|
43
|
+
ERRORS = { refused: "connection refused", timeout: "timed out" }.freeze
|
|
44
|
+
|
|
45
|
+
# The failure message for requests no stub answered (the adapters'
|
|
46
|
+
# teardown check).
|
|
47
|
+
# @param requests [Array<Request>] FakeHttp#unmatched, of every bundle
|
|
48
|
+
def self.unmatched_message(requests)
|
|
49
|
+
lines = requests.map { |request| " #{request.method} #{request.url}" }
|
|
50
|
+
"ctx.http made #{requests.size} request#{"s" unless requests.size == 1} no stub answered " \
|
|
51
|
+
"(bundle.http.stub):\n#{lines.join("\n")}"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def initialize
|
|
55
|
+
@mutex = Mutex.new
|
|
56
|
+
@stubs = []
|
|
57
|
+
@requests = []
|
|
58
|
+
@unmatched = []
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Answer +method+ requests to +url+.
|
|
62
|
+
# @param method [Symbol, String] :get, :post, :put, :patch, :delete, :head
|
|
63
|
+
# @param url [String, Regexp]
|
|
64
|
+
# @param status [Integer]
|
|
65
|
+
# @param headers [Hash]
|
|
66
|
+
# @param body [String, nil]
|
|
67
|
+
# @param json [Object, nil] a JSON body (with its Content-Type)
|
|
68
|
+
# @param error [Symbol, nil] :refused or :timeout: the request fails
|
|
69
|
+
# with the HttpError the real transport raises
|
|
70
|
+
# @yield [Request] to decide per request: returns a Hash with the same
|
|
71
|
+
# keywords (status:, headers:, body:, json:, error:) or a
|
|
72
|
+
# Plugin::Http::Response
|
|
73
|
+
# @return [self]
|
|
74
|
+
def stub(method, url, status: 200, headers: {}, body: nil, json: nil, error: nil, &block)
|
|
75
|
+
verb = method.to_s.upcase
|
|
76
|
+
raise ArgumentError, "unknown HTTP method #{method.inspect}" unless Plugin::Http::METHODS.include?(verb)
|
|
77
|
+
raise ArgumentError, "the URL is a String or a Regexp" unless url.is_a?(String) || url.is_a?(Regexp)
|
|
78
|
+
raise ArgumentError, "unknown error #{error.inspect} (#{ERRORS.keys.join(", ")})" if error && !ERRORS.key?(error)
|
|
79
|
+
|
|
80
|
+
answer = block || { status: status, headers: headers, body: body, json: json, error: error }
|
|
81
|
+
@mutex.synchronize { @stubs << Stub.new(verb: verb, matcher: url, answer: answer) }
|
|
82
|
+
self
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# @return [Array<Request>] every request the plugin sent, in order
|
|
86
|
+
def requests = @mutex.synchronize { @requests.dup }
|
|
87
|
+
|
|
88
|
+
# @return [Array<Request>] the requests no stub answered (the test
|
|
89
|
+
# fails at teardown when there are any)
|
|
90
|
+
def unmatched = @mutex.synchronize { @unmatched.dup }
|
|
91
|
+
|
|
92
|
+
# The transport contract (Plugin::Http): the +cancel+ it is given is
|
|
93
|
+
# the only cancel it checks.
|
|
94
|
+
# @raise [UnstubbedRequest, Plugin::HttpError, Plugin::HttpCancelled]
|
|
95
|
+
def call(method:, uri:, headers:, body:, timeout:, cancel:)
|
|
96
|
+
raise Plugin::HttpCancelled, "#{method} #{where(uri)}: cancelled" if cancel&.cancelled?
|
|
97
|
+
|
|
98
|
+
request = Request.new(method: method, url: uri.to_s, headers: headers.dup.freeze, body: body)
|
|
99
|
+
stub = @mutex.synchronize do
|
|
100
|
+
@requests << request
|
|
101
|
+
@stubs.reverse_each.find { |candidate| candidate.match?(request) }.tap { |found| @unmatched << request unless found }
|
|
102
|
+
end
|
|
103
|
+
raise UnstubbedRequest, unstubbed_message(request) unless stub
|
|
104
|
+
|
|
105
|
+
respond(request, uri, stub.answer.respond_to?(:call) ? stub.answer.call(request) : stub.answer)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
private
|
|
109
|
+
|
|
110
|
+
def respond(request, uri, answer)
|
|
111
|
+
return answer if answer.is_a?(Plugin::Http::Response)
|
|
112
|
+
raise ArgumentError, "a stub's block returns a Hash or a Plugin::Http::Response" unless answer.is_a?(Hash)
|
|
113
|
+
|
|
114
|
+
if (error = answer[:error])
|
|
115
|
+
raise Plugin::HttpError, "#{request.method} #{where(uri)}: #{ERRORS.fetch(error) { raise ArgumentError, "unknown error #{error.inspect}" }}"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
headers = (answer[:headers] || {}).to_h { |name, value| [name.to_s.downcase, value.to_s] }
|
|
119
|
+
body = answer[:body].to_s
|
|
120
|
+
unless answer[:json].nil?
|
|
121
|
+
body = JSON.generate(answer[:json])
|
|
122
|
+
headers = { "content-type" => "application/json" }.merge(headers)
|
|
123
|
+
end
|
|
124
|
+
body = "" if request.method == "HEAD"
|
|
125
|
+
Plugin::Http::Response.new(status: Integer(answer[:status] || 200), headers: headers.freeze, body: body.dup)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def where(uri) = "#{uri.host}:#{uri.port}"
|
|
129
|
+
|
|
130
|
+
def unstubbed_message(request)
|
|
131
|
+
stubs = @mutex.synchronize { @stubs.map { |stub| "#{stub.verb} #{stub.matcher.inspect}" } }
|
|
132
|
+
"no stub for #{request.method} #{request.url}; stubbed: #{stubs.empty? ? "none" : stubs.join(", ")} " \
|
|
133
|
+
"(bundle.http.stub(:#{request.method.downcase}, ...))"
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "bundle"
|
|
4
|
+
require_relative "fake_http"
|
|
5
|
+
require_relative "sandbox"
|
|
6
|
+
require_relative "waiting"
|
|
7
|
+
|
|
8
|
+
module Samagotchi
|
|
9
|
+
module Testing
|
|
10
|
+
# For any test framework: load bundles that #shutdown_bundles stops,
|
|
11
|
+
# in the test's sandbox. The RSpec and Minitest adapters call
|
|
12
|
+
# #enter_sandbox before each test and #leave_sandbox after it.
|
|
13
|
+
module Helpers
|
|
14
|
+
include Waiting
|
|
15
|
+
|
|
16
|
+
# The bundle in +dir+ (Bundle.load), shut down after the test.
|
|
17
|
+
def load_bundle(dir, **) = remember_bundle(Bundle.load(dir, sandbox: sandbox, **))
|
|
18
|
+
|
|
19
|
+
# A shipped bundle by name (Bundle.shipped), shut down after the test.
|
|
20
|
+
def shipped_bundle(name, **) = remember_bundle(Bundle.shipped(name, sandbox: sandbox, **))
|
|
21
|
+
|
|
22
|
+
# This test's sandbox (entered on first use when no adapter did).
|
|
23
|
+
# @return [Sandbox]
|
|
24
|
+
def sandbox = (@samagotchi_sandbox ||= Sandbox.enter)
|
|
25
|
+
|
|
26
|
+
def enter_sandbox = sandbox
|
|
27
|
+
|
|
28
|
+
# Shut the loaded bundles down and leave the sandbox, then say what
|
|
29
|
+
# no stub answered: the adapters fail the test with it (only a test
|
|
30
|
+
# that passed: a failure of its own is not hidden). The requests are
|
|
31
|
+
# collected before the shutdown, and nothing is raised before the
|
|
32
|
+
# cleanup is done.
|
|
33
|
+
# @return [Array<FakeHttp::Request>] the ctx.http requests no stub
|
|
34
|
+
# answered, of every bundle the test loaded
|
|
35
|
+
def leave_sandbox
|
|
36
|
+
unmatched = (@samagotchi_bundles || []).flat_map { |bundle| bundle.http.unmatched }
|
|
37
|
+
shutdown_bundles
|
|
38
|
+
@samagotchi_sandbox&.leave
|
|
39
|
+
@samagotchi_sandbox = nil
|
|
40
|
+
unmatched
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def shutdown_bundles
|
|
44
|
+
(@samagotchi_bundles || []).reverse_each(&:shutdown)
|
|
45
|
+
@samagotchi_bundles = []
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def remember_bundle(bundle)
|
|
51
|
+
(@samagotchi_bundles ||= []) << bundle
|
|
52
|
+
bundle
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../cancellation_controller"
|
|
4
|
+
require_relative "../plugin/context"
|
|
5
|
+
require_relative "recorder"
|
|
6
|
+
|
|
7
|
+
module Samagotchi
|
|
8
|
+
module Testing
|
|
9
|
+
# The Engine side of a bundle's Context, recorded: a real Plugin::Host
|
|
10
|
+
# with every member set, its values read from the Recorder at each call
|
|
11
|
+
# and its actions landing in it.
|
|
12
|
+
module Host
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# @param recorder [Recorder]
|
|
16
|
+
# @param toolkits [#call] → the toolkit names ctx.use_toolkit accepts
|
|
17
|
+
# @param tool_names [#call] → the declared tools when none is set
|
|
18
|
+
# @return [Plugin::Host]
|
|
19
|
+
def build(recorder, toolkits: -> { [] }, tool_names: -> { [] })
|
|
20
|
+
# ctx.http's default cancel: a cancelled controller while the test
|
|
21
|
+
# says the turn is (cancelled: true), none otherwise.
|
|
22
|
+
stopped = CancellationController.new.tap { |controller| controller.cancel!(:test) }
|
|
23
|
+
Plugin::Host.new(
|
|
24
|
+
session_id: -> { recorder[:session_id] },
|
|
25
|
+
cwd: -> { recorder[:cwd] },
|
|
26
|
+
messages: -> { recorder[:messages] },
|
|
27
|
+
messages_partial: -> { recorder[:messages_partial] },
|
|
28
|
+
notify: ->(text, level, label, fallback_for: nil) { recorder.notify(text, level, label, fallback_for) },
|
|
29
|
+
ask_user: lambda { |question:, options:, header:, allow_freeform:, hook:|
|
|
30
|
+
recorder.ask_user(question: question, options: options, header: header, allow_freeform: allow_freeform, hook: hook)
|
|
31
|
+
},
|
|
32
|
+
cancelled: -> { recorder[:cancelled] },
|
|
33
|
+
card: ->(**card) { recorder.card(**card) },
|
|
34
|
+
ask_model: lambda { |request, timeout:, max_tokens:, cancel_controller:, model: nil, source: nil|
|
|
35
|
+
recorder.ask_model(request, timeout: timeout, max_tokens: max_tokens, cancel_controller: cancel_controller,
|
|
36
|
+
model: model, source: source)
|
|
37
|
+
},
|
|
38
|
+
model_name: -> { recorder[:model_name] },
|
|
39
|
+
model_key: -> { recorder[:model_key] },
|
|
40
|
+
state_dir: -> { recorder[:state_dir] || Session.default_state_dir },
|
|
41
|
+
scratch: -> { recorder[:scratch] },
|
|
42
|
+
setup: -> { recorder[:setup] },
|
|
43
|
+
prompt_files: -> { recorder[:prompt_files] },
|
|
44
|
+
steer: ->(text, label) { recorder.steer(text, label) },
|
|
45
|
+
stop_turn: ->(reason, label) { recorder.stop(:turn, reason, label) },
|
|
46
|
+
stop_generation: ->(reason, label) { recorder.stop(:generation, reason, label) },
|
|
47
|
+
frontend: -> { recorder[:frontend] },
|
|
48
|
+
toolkit: -> { recorder[:toolkit] },
|
|
49
|
+
declared_tools: -> { recorder[:declared_tools] || tool_names.call },
|
|
50
|
+
use_toolkit: ->(name) { recorder.use_toolkit(name, toolkits.call) },
|
|
51
|
+
cancel_controller: ->(handler: false) { recorder[:cancelled] ? stopped : nil }
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The hook runtime (Hooks::Runtime: event[:notify] and the other
|
|
57
|
+
# helpers), recorded in the same Recorder as the Host.
|
|
58
|
+
module Runtime
|
|
59
|
+
module_function
|
|
60
|
+
|
|
61
|
+
# @param recorder [Recorder]
|
|
62
|
+
# @return [Hooks::Runtime]
|
|
63
|
+
def build(recorder)
|
|
64
|
+
Hooks::Runtime.new(
|
|
65
|
+
notify: ->(text:, level:, hook:, fallback_for: nil) { recorder.notify(text, level, hook, fallback_for) },
|
|
66
|
+
ask_user: lambda { |question:, options:, header:, allow_freeform:, hook:|
|
|
67
|
+
recorder.ask_user(question: question, options: options, header: header, allow_freeform: allow_freeform, hook: hook)
|
|
68
|
+
},
|
|
69
|
+
stop_turn: ->(reason:, hook:) { recorder.stop(:turn, reason, hook) },
|
|
70
|
+
steer: ->(text:, hook:) { recorder.steer(text, hook) },
|
|
71
|
+
stop_generation: ->(reason:, hook:) { recorder.stop(:generation, reason, hook) }
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|