samagotchi 0.79.0 → 0.80.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +30 -1
- data/README.md +7 -5
- data/docs/cli.md +24 -7
- data/docs/configuration.md +7 -4
- data/docs/internals/anthropic.md +3 -1
- data/docs/internals/bundles.md +4 -0
- data/docs/internals/engine.md +13 -3
- data/docs/plugins.md +144 -3
- data/docs/releasing.md +2 -2
- data/docs/sessions.md +1 -1
- data/docs/testing.md +32 -2
- data/lib/samagotchi/bootstrap/config_writer.rb +13 -3
- data/lib/samagotchi/bootstrap/probe.rb +153 -31
- data/lib/samagotchi/bootstrap_command.rb +92 -15
- data/lib/samagotchi/bridge.rb +1 -1
- data/lib/samagotchi/bundles/system/manifest.yml +2 -2
- data/lib/samagotchi/bundles/system/self_map.md +1 -1
- data/lib/samagotchi/continue_offer.rb +5 -5
- data/lib/samagotchi/engine.rb +40 -37
- data/lib/samagotchi/guardrails/approval.rb +6 -4
- data/lib/samagotchi/guardrails/gate.rb +7 -3
- data/lib/samagotchi/hooks/bundle_loader.rb +31 -16
- data/lib/samagotchi/hooks/registry.rb +44 -3
- data/lib/samagotchi/host_registry.rb +12 -4
- data/lib/samagotchi/llm/adapter_common.rb +4 -2
- data/lib/samagotchi/llm/anthropic_binding_recovery.rb +3 -0
- data/lib/samagotchi/llm/anthropic_messages.rb +2 -1
- data/lib/samagotchi/llm/http.rb +12 -7
- data/lib/samagotchi/plugin/card.rb +44 -0
- data/lib/samagotchi/plugin/context.rb +53 -8
- data/lib/samagotchi/plugin/http.rb +154 -0
- data/lib/samagotchi/plugin/http_transport.rb +77 -0
- data/lib/samagotchi/plugin/loader.rb +29 -10
- data/lib/samagotchi/question_answer.rb +29 -0
- data/lib/samagotchi/question_desk.rb +20 -19
- data/lib/samagotchi/relay_verifier.rb +1 -1
- data/lib/samagotchi/testing/boundary.rb +156 -0
- data/lib/samagotchi/testing/bundle.rb +349 -0
- data/lib/samagotchi/testing/fake_executable.rb +42 -0
- data/lib/samagotchi/testing/fake_http.rb +137 -0
- data/lib/samagotchi/testing/helpers.rb +56 -0
- data/lib/samagotchi/testing/host.rb +76 -0
- data/lib/samagotchi/testing/minitest.rb +31 -0
- data/lib/samagotchi/testing/recorder.rb +138 -0
- data/lib/samagotchi/testing/recording_api.rb +27 -0
- data/lib/samagotchi/testing/rspec.rb +31 -0
- data/lib/samagotchi/testing/sandbox.rb +163 -0
- data/lib/samagotchi/testing/sessions.rb +77 -0
- data/lib/samagotchi/testing/verdict.rb +28 -0
- data/lib/samagotchi/testing/waiting.rb +36 -0
- data/lib/samagotchi/testing.rb +26 -0
- data/lib/samagotchi/tools/delegate_relay.rb +7 -6
- data/lib/samagotchi/tools/tool_index.rb +3 -0
- data/lib/samagotchi/turn_state.rb +28 -1
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/public/app.js +47 -23
- data/lib/samagotchi/web/public/images.js +33 -0
- data/lib/samagotchi/web/public/index.html +71 -38
- data/lib/samagotchi/web/public/level_chip.js +11 -11
- data/lib/samagotchi/web/public/lists_view.js +19 -14
- data/lib/samagotchi/web/public/llm_context_chip.js +15 -15
- data/lib/samagotchi/web/public/model_picker.js +1 -1
- data/lib/samagotchi/web/public/scope.js +3 -3
- data/lib/samagotchi/web/public/strip.js +35 -9
- data/lib/samagotchi/web/public/strip_grip.js +76 -0
- data/lib/samagotchi/web/public/thinking_chip.js +5 -5
- metadata +21 -1
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../plugin/http"
|
|
5
|
+
|
|
6
|
+
module Samagotchi
|
|
7
|
+
module Testing
|
|
8
|
+
# A request no stub answered (FakeHttp). Not an HttpError: a plugin's
|
|
9
|
+
# `rescue Plugin::HttpError` doesn't swallow it, and the test fails at
|
|
10
|
+
# teardown even when the plugin rescued it (FakeHttp#unmatched).
|
|
11
|
+
class UnstubbedRequest < StandardError; end
|
|
12
|
+
|
|
13
|
+
# ctx.http's wire in a bundle's test (docs/plugins.md, "Testing a
|
|
14
|
+
# bundle"): requests are answered by stubs, never by the network.
|
|
15
|
+
#
|
|
16
|
+
# bundle.http.stub(:post, "http://127.0.0.1:9000/classify", json: { label: "safe" })
|
|
17
|
+
# bundle.http.stub(:get, %r{/health\z}, status: 503, body: "down")
|
|
18
|
+
# bundle.http.stub(:post, "http://127.0.0.1:9000/classify") { |req| { json: { label: req.json["text"] } } }
|
|
19
|
+
# bundle.http.stub(:get, "http://127.0.0.1:9000/x", error: :refused) # also :timeout
|
|
20
|
+
# bundle.http.requests # what the plugin sent, in order
|
|
21
|
+
# bundle.http.unmatched # what no stub answered
|
|
22
|
+
#
|
|
23
|
+
# A String matches the full URL exactly (query included), a Regexp is
|
|
24
|
+
# matched against it; the method too. Later stubs win. A stub answers
|
|
25
|
+
# every matching call: a sequence is a block that counts.
|
|
26
|
+
class FakeHttp
|
|
27
|
+
# One request the plugin made (headers as the plugin gave them).
|
|
28
|
+
# `method` shadows Object#method on purpose: it is the plugin's word (request.method == "POST").
|
|
29
|
+
Request = Data.define(:method, :url, :headers, :body) do # rubocop:disable Lint/DataDefineOverride
|
|
30
|
+
# The body parsed as JSON.
|
|
31
|
+
def json = JSON.parse(body.to_s)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
Stub = Data.define(:verb, :matcher, :answer) do
|
|
35
|
+
def match?(request)
|
|
36
|
+
request.method == verb &&
|
|
37
|
+
(matcher.is_a?(Regexp) ? matcher.match?(request.url) : matcher == request.url)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
private_constant :Stub
|
|
41
|
+
|
|
42
|
+
# What error: names, and the words the real transport's message says.
|
|
43
|
+
ERRORS = { refused: "connection refused", timeout: "timed out" }.freeze
|
|
44
|
+
|
|
45
|
+
# The failure message for requests no stub answered (the adapters'
|
|
46
|
+
# teardown check).
|
|
47
|
+
# @param requests [Array<Request>] FakeHttp#unmatched, of every bundle
|
|
48
|
+
def self.unmatched_message(requests)
|
|
49
|
+
lines = requests.map { |request| " #{request.method} #{request.url}" }
|
|
50
|
+
"ctx.http made #{requests.size} request#{"s" unless requests.size == 1} no stub answered " \
|
|
51
|
+
"(bundle.http.stub):\n#{lines.join("\n")}"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def initialize
|
|
55
|
+
@mutex = Mutex.new
|
|
56
|
+
@stubs = []
|
|
57
|
+
@requests = []
|
|
58
|
+
@unmatched = []
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# Answer +method+ requests to +url+.
|
|
62
|
+
# @param method [Symbol, String] :get, :post, :put, :patch, :delete, :head
|
|
63
|
+
# @param url [String, Regexp]
|
|
64
|
+
# @param status [Integer]
|
|
65
|
+
# @param headers [Hash]
|
|
66
|
+
# @param body [String, nil]
|
|
67
|
+
# @param json [Object, nil] a JSON body (with its Content-Type)
|
|
68
|
+
# @param error [Symbol, nil] :refused or :timeout: the request fails
|
|
69
|
+
# with the HttpError the real transport raises
|
|
70
|
+
# @yield [Request] to decide per request: returns a Hash with the same
|
|
71
|
+
# keywords (status:, headers:, body:, json:, error:) or a
|
|
72
|
+
# Plugin::Http::Response
|
|
73
|
+
# @return [self]
|
|
74
|
+
def stub(method, url, status: 200, headers: {}, body: nil, json: nil, error: nil, &block)
|
|
75
|
+
verb = method.to_s.upcase
|
|
76
|
+
raise ArgumentError, "unknown HTTP method #{method.inspect}" unless Plugin::Http::METHODS.include?(verb)
|
|
77
|
+
raise ArgumentError, "the URL is a String or a Regexp" unless url.is_a?(String) || url.is_a?(Regexp)
|
|
78
|
+
raise ArgumentError, "unknown error #{error.inspect} (#{ERRORS.keys.join(", ")})" if error && !ERRORS.key?(error)
|
|
79
|
+
|
|
80
|
+
answer = block || { status: status, headers: headers, body: body, json: json, error: error }
|
|
81
|
+
@mutex.synchronize { @stubs << Stub.new(verb: verb, matcher: url, answer: answer) }
|
|
82
|
+
self
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# @return [Array<Request>] every request the plugin sent, in order
|
|
86
|
+
def requests = @mutex.synchronize { @requests.dup }
|
|
87
|
+
|
|
88
|
+
# @return [Array<Request>] the requests no stub answered (the test
|
|
89
|
+
# fails at teardown when there are any)
|
|
90
|
+
def unmatched = @mutex.synchronize { @unmatched.dup }
|
|
91
|
+
|
|
92
|
+
# The transport contract (Plugin::Http): the +cancel+ it is given is
|
|
93
|
+
# the only cancel it checks.
|
|
94
|
+
# @raise [UnstubbedRequest, Plugin::HttpError, Plugin::HttpCancelled]
|
|
95
|
+
def call(method:, uri:, headers:, body:, timeout:, cancel:)
|
|
96
|
+
raise Plugin::HttpCancelled, "#{method} #{where(uri)}: cancelled" if cancel&.cancelled?
|
|
97
|
+
|
|
98
|
+
request = Request.new(method: method, url: uri.to_s, headers: headers.dup.freeze, body: body)
|
|
99
|
+
stub = @mutex.synchronize do
|
|
100
|
+
@requests << request
|
|
101
|
+
@stubs.reverse_each.find { |candidate| candidate.match?(request) }.tap { |found| @unmatched << request unless found }
|
|
102
|
+
end
|
|
103
|
+
raise UnstubbedRequest, unstubbed_message(request) unless stub
|
|
104
|
+
|
|
105
|
+
respond(request, uri, stub.answer.respond_to?(:call) ? stub.answer.call(request) : stub.answer)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
private
|
|
109
|
+
|
|
110
|
+
def respond(request, uri, answer)
|
|
111
|
+
return answer if answer.is_a?(Plugin::Http::Response)
|
|
112
|
+
raise ArgumentError, "a stub's block returns a Hash or a Plugin::Http::Response" unless answer.is_a?(Hash)
|
|
113
|
+
|
|
114
|
+
if (error = answer[:error])
|
|
115
|
+
raise Plugin::HttpError, "#{request.method} #{where(uri)}: #{ERRORS.fetch(error) { raise ArgumentError, "unknown error #{error.inspect}" }}"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
headers = (answer[:headers] || {}).to_h { |name, value| [name.to_s.downcase, value.to_s] }
|
|
119
|
+
body = answer[:body].to_s
|
|
120
|
+
unless answer[:json].nil?
|
|
121
|
+
body = JSON.generate(answer[:json])
|
|
122
|
+
headers = { "content-type" => "application/json" }.merge(headers)
|
|
123
|
+
end
|
|
124
|
+
body = "" if request.method == "HEAD"
|
|
125
|
+
Plugin::Http::Response.new(status: Integer(answer[:status] || 200), headers: headers.freeze, body: body.dup)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def where(uri) = "#{uri.host}:#{uri.port}"
|
|
129
|
+
|
|
130
|
+
def unstubbed_message(request)
|
|
131
|
+
stubs = @mutex.synchronize { @stubs.map { |stub| "#{stub.verb} #{stub.matcher.inspect}" } }
|
|
132
|
+
"no stub for #{request.method} #{request.url}; stubbed: #{stubs.empty? ? "none" : stubs.join(", ")} " \
|
|
133
|
+
"(bundle.http.stub(:#{request.method.downcase}, ...))"
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "bundle"
|
|
4
|
+
require_relative "fake_http"
|
|
5
|
+
require_relative "sandbox"
|
|
6
|
+
require_relative "waiting"
|
|
7
|
+
|
|
8
|
+
module Samagotchi
|
|
9
|
+
module Testing
|
|
10
|
+
# For any test framework: load bundles that #shutdown_bundles stops,
|
|
11
|
+
# in the test's sandbox. The RSpec and Minitest adapters call
|
|
12
|
+
# #enter_sandbox before each test and #leave_sandbox after it.
|
|
13
|
+
module Helpers
|
|
14
|
+
include Waiting
|
|
15
|
+
|
|
16
|
+
# The bundle in +dir+ (Bundle.load), shut down after the test.
|
|
17
|
+
def load_bundle(dir, **) = remember_bundle(Bundle.load(dir, sandbox: sandbox, **))
|
|
18
|
+
|
|
19
|
+
# A shipped bundle by name (Bundle.shipped), shut down after the test.
|
|
20
|
+
def shipped_bundle(name, **) = remember_bundle(Bundle.shipped(name, sandbox: sandbox, **))
|
|
21
|
+
|
|
22
|
+
# This test's sandbox (entered on first use when no adapter did).
|
|
23
|
+
# @return [Sandbox]
|
|
24
|
+
def sandbox = (@samagotchi_sandbox ||= Sandbox.enter)
|
|
25
|
+
|
|
26
|
+
def enter_sandbox = sandbox
|
|
27
|
+
|
|
28
|
+
# Shut the loaded bundles down and leave the sandbox, then say what
|
|
29
|
+
# no stub answered: the adapters fail the test with it (only a test
|
|
30
|
+
# that passed: a failure of its own is not hidden). The requests are
|
|
31
|
+
# collected before the shutdown, and nothing is raised before the
|
|
32
|
+
# cleanup is done.
|
|
33
|
+
# @return [Array<FakeHttp::Request>] the ctx.http requests no stub
|
|
34
|
+
# answered, of every bundle the test loaded
|
|
35
|
+
def leave_sandbox
|
|
36
|
+
unmatched = (@samagotchi_bundles || []).flat_map { |bundle| bundle.http.unmatched }
|
|
37
|
+
shutdown_bundles
|
|
38
|
+
@samagotchi_sandbox&.leave
|
|
39
|
+
@samagotchi_sandbox = nil
|
|
40
|
+
unmatched
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def shutdown_bundles
|
|
44
|
+
(@samagotchi_bundles || []).reverse_each(&:shutdown)
|
|
45
|
+
@samagotchi_bundles = []
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def remember_bundle(bundle)
|
|
51
|
+
(@samagotchi_bundles ||= []) << bundle
|
|
52
|
+
bundle
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../cancellation_controller"
|
|
4
|
+
require_relative "../plugin/context"
|
|
5
|
+
require_relative "recorder"
|
|
6
|
+
|
|
7
|
+
module Samagotchi
|
|
8
|
+
module Testing
|
|
9
|
+
# The Engine side of a bundle's Context, recorded: a real Plugin::Host
|
|
10
|
+
# with every member set, its values read from the Recorder at each call
|
|
11
|
+
# and its actions landing in it.
|
|
12
|
+
module Host
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
# @param recorder [Recorder]
|
|
16
|
+
# @param toolkits [#call] → the toolkit names ctx.use_toolkit accepts
|
|
17
|
+
# @param tool_names [#call] → the declared tools when none is set
|
|
18
|
+
# @return [Plugin::Host]
|
|
19
|
+
def build(recorder, toolkits: -> { [] }, tool_names: -> { [] })
|
|
20
|
+
# ctx.http's default cancel: a cancelled controller while the test
|
|
21
|
+
# says the turn is (cancelled: true), none otherwise.
|
|
22
|
+
stopped = CancellationController.new.tap { |controller| controller.cancel!(:test) }
|
|
23
|
+
Plugin::Host.new(
|
|
24
|
+
session_id: -> { recorder[:session_id] },
|
|
25
|
+
cwd: -> { recorder[:cwd] },
|
|
26
|
+
messages: -> { recorder[:messages] },
|
|
27
|
+
messages_partial: -> { recorder[:messages_partial] },
|
|
28
|
+
notify: ->(text, level, label, fallback_for: nil) { recorder.notify(text, level, label, fallback_for) },
|
|
29
|
+
ask_user: lambda { |question:, options:, header:, allow_freeform:, hook:|
|
|
30
|
+
recorder.ask_user(question: question, options: options, header: header, allow_freeform: allow_freeform, hook: hook)
|
|
31
|
+
},
|
|
32
|
+
cancelled: -> { recorder[:cancelled] },
|
|
33
|
+
card: ->(**card) { recorder.card(**card) },
|
|
34
|
+
ask_model: lambda { |request, timeout:, max_tokens:, cancel_controller:, model: nil|
|
|
35
|
+
recorder.ask_model(request, timeout: timeout, max_tokens: max_tokens, cancel_controller: cancel_controller,
|
|
36
|
+
model: model)
|
|
37
|
+
},
|
|
38
|
+
model_name: -> { recorder[:model_name] },
|
|
39
|
+
model_key: -> { recorder[:model_key] },
|
|
40
|
+
state_dir: -> { recorder[:state_dir] || Session.default_state_dir },
|
|
41
|
+
scratch: -> { recorder[:scratch] },
|
|
42
|
+
setup: -> { recorder[:setup] },
|
|
43
|
+
prompt_files: -> { recorder[:prompt_files] },
|
|
44
|
+
steer: ->(text, label) { recorder.steer(text, label) },
|
|
45
|
+
stop_turn: ->(reason, label) { recorder.stop(:turn, reason, label) },
|
|
46
|
+
stop_generation: ->(reason, label) { recorder.stop(:generation, reason, label) },
|
|
47
|
+
frontend: -> { recorder[:frontend] },
|
|
48
|
+
toolkit: -> { recorder[:toolkit] },
|
|
49
|
+
declared_tools: -> { recorder[:declared_tools] || tool_names.call },
|
|
50
|
+
use_toolkit: ->(name) { recorder.use_toolkit(name, toolkits.call) },
|
|
51
|
+
cancel_controller: ->(handler: false) { recorder[:cancelled] ? stopped : nil }
|
|
52
|
+
)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The hook runtime (Hooks::Runtime: event[:notify] and the other
|
|
57
|
+
# helpers), recorded in the same Recorder as the Host.
|
|
58
|
+
module Runtime
|
|
59
|
+
module_function
|
|
60
|
+
|
|
61
|
+
# @param recorder [Recorder]
|
|
62
|
+
# @return [Hooks::Runtime]
|
|
63
|
+
def build(recorder)
|
|
64
|
+
Hooks::Runtime.new(
|
|
65
|
+
notify: ->(text:, level:, hook:, fallback_for: nil) { recorder.notify(text, level, hook, fallback_for) },
|
|
66
|
+
ask_user: lambda { |question:, options:, header:, allow_freeform:, hook:|
|
|
67
|
+
recorder.ask_user(question: question, options: options, header: header, allow_freeform: allow_freeform, hook: hook)
|
|
68
|
+
},
|
|
69
|
+
stop_turn: ->(reason:, hook:) { recorder.stop(:turn, reason, hook) },
|
|
70
|
+
steer: ->(text:, hook:) { recorder.steer(text, hook) },
|
|
71
|
+
stop_generation: ->(reason:, hook:) { recorder.stop(:generation, reason, hook) }
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "minitest"
|
|
4
|
+
require_relative "../testing"
|
|
5
|
+
|
|
6
|
+
module Samagotchi
|
|
7
|
+
module Testing
|
|
8
|
+
# The bundle test kit in Minitest (EXPERIMENTAL, docs/plugins.md): a
|
|
9
|
+
# test case with Helpers (load_bundle, shipped_bundle, sandbox), each
|
|
10
|
+
# test in its own sandbox, its bundles shut down after it.
|
|
11
|
+
#
|
|
12
|
+
# class MyBundleTest < Samagotchi::Testing::Case
|
|
13
|
+
# def test_denies = assert load_bundle(__dir__ + "/..").tool_call({ name: "execute", content: "rm -rf /" }).deny?
|
|
14
|
+
# end
|
|
15
|
+
class Case < Minitest::Test
|
|
16
|
+
include Helpers
|
|
17
|
+
|
|
18
|
+
def before_setup
|
|
19
|
+
super
|
|
20
|
+
enter_sandbox
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def after_teardown
|
|
24
|
+
unmatched = leave_sandbox
|
|
25
|
+
super
|
|
26
|
+
# After the cleanup, and only over a test that passed.
|
|
27
|
+
flunk(FakeHttp.unmatched_message(unmatched)) if !unmatched.empty? && passed?
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../plugin/card"
|
|
4
|
+
require_relative "../plugin/context"
|
|
5
|
+
|
|
6
|
+
module Samagotchi
|
|
7
|
+
module Testing
|
|
8
|
+
# A line to the user (ctx.notify, event[:notify]); +hook+ is the
|
|
9
|
+
# plugin's or hook file's label ("plugin.rb (bundle x)").
|
|
10
|
+
Notice = Data.define(:text, :level, :hook, :fallback_for)
|
|
11
|
+
# Text put into the running turn (ctx.steer, event[:steer]).
|
|
12
|
+
Steer = Data.define(:text, :hook)
|
|
13
|
+
# A stop_turn or stop_generation (+kind+ :turn or :generation).
|
|
14
|
+
Stop = Data.define(:kind, :reason, :hook)
|
|
15
|
+
# An ask_user question.
|
|
16
|
+
Question = Data.define(:question, :options, :header, :allow_freeform, :hook)
|
|
17
|
+
# An ask_model request: the request Context built (SideQuestion), its
|
|
18
|
+
# limits, cancel controller and the model: it named (nil: the session's;
|
|
19
|
+
# the kit doesn't check it against any hosts).
|
|
20
|
+
ModelRequest = Data.define(:request, :timeout, :max_tokens, :cancel, :model)
|
|
21
|
+
|
|
22
|
+
# What a bundle did at chi's edges, for a test to read: the Engine side
|
|
23
|
+
# of its Context (Testing::Host) and the hook runtime (Testing::Runtime)
|
|
24
|
+
# land here, file hooks and plugin hooks in the same lists. It also
|
|
25
|
+
# holds what those edges answer: queued ask_user / ask_model answers and
|
|
26
|
+
# the session's values (#values), read at each call.
|
|
27
|
+
class Recorder
|
|
28
|
+
# The session values a Host reads when the test sets none.
|
|
29
|
+
DEFAULTS = {
|
|
30
|
+
session_id: "test-session", cwd: nil, messages: [], messages_partial: false, model_name: "test:model",
|
|
31
|
+
model_key: "model", toolkit: "full", declared_tools: nil, frontend: :worker, scratch: false, setup: nil,
|
|
32
|
+
prompt_files: [], cancelled: false, state_dir: nil, running: true, toolkit_staged: true
|
|
33
|
+
}.freeze
|
|
34
|
+
|
|
35
|
+
attr_reader :notices, :cards, :steers, :stops, :questions, :model_requests, :toolkit_uses, :answers,
|
|
36
|
+
:model_answers, :values
|
|
37
|
+
|
|
38
|
+
def initialize(**values)
|
|
39
|
+
unknown = values.keys - DEFAULTS.keys
|
|
40
|
+
raise ArgumentError, "unknown host values: #{unknown.join(", ")}" unless unknown.empty?
|
|
41
|
+
|
|
42
|
+
@values = DEFAULTS.merge(values)
|
|
43
|
+
@notices = []
|
|
44
|
+
@cards = []
|
|
45
|
+
@steers = []
|
|
46
|
+
@stops = []
|
|
47
|
+
@questions = []
|
|
48
|
+
@model_requests = []
|
|
49
|
+
@toolkit_uses = []
|
|
50
|
+
@answers = []
|
|
51
|
+
@model_answers = []
|
|
52
|
+
@mutex = Mutex.new
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Change session values (cwd:, messages:, running:, cancelled: …).
|
|
56
|
+
def set(**values)
|
|
57
|
+
unknown = values.keys - DEFAULTS.keys
|
|
58
|
+
raise ArgumentError, "unknown host values: #{unknown.join(", ")}" unless unknown.empty?
|
|
59
|
+
|
|
60
|
+
@mutex.synchronize { @values.merge!(values) }
|
|
61
|
+
self
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def [](key) = @mutex.synchronize { @values.fetch(key) }
|
|
65
|
+
|
|
66
|
+
# Only stop_turn's, or only stop_generation's.
|
|
67
|
+
def turn_stops = stops.select { |stop| stop.kind == :turn }
|
|
68
|
+
def generation_stops = stops.select { |stop| stop.kind == :generation }
|
|
69
|
+
|
|
70
|
+
def notify(text, level, hook, fallback_for)
|
|
71
|
+
record(@notices, Notice.new(text: text.to_s, level: level, hook: hook, fallback_for: fallback_for))
|
|
72
|
+
nil
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Checked as chi's cards are (Plugin::Card.build); the id it shows.
|
|
76
|
+
def card(**fields)
|
|
77
|
+
card = Plugin::Card.build(**fields).freeze
|
|
78
|
+
record(@cards, card)
|
|
79
|
+
card[:id]
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def steer(text, hook)
|
|
83
|
+
return false unless self[:running]
|
|
84
|
+
|
|
85
|
+
record(@steers, Steer.new(text: text.to_s, hook: hook))
|
|
86
|
+
true
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def stop(kind, reason, hook)
|
|
90
|
+
return false unless self[:running]
|
|
91
|
+
|
|
92
|
+
record(@stops, Stop.new(kind: kind, reason: reason.to_s, hook: hook))
|
|
93
|
+
true
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The next queued answer (#answers), nil with none (no one to ask).
|
|
97
|
+
def ask_user(question:, options:, header:, allow_freeform:, hook:)
|
|
98
|
+
record(@questions, Question.new(question: question, options: options, header: header,
|
|
99
|
+
allow_freeform: allow_freeform, hook: hook))
|
|
100
|
+
answer = @mutex.synchronize { @answers.shift }
|
|
101
|
+
answer.respond_to?(:call) ? answer.call(@questions.last) : answer
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The next queued model answer (#model_answers): a String, an
|
|
105
|
+
# exception to raise, or a callable given the ModelRequest. None left:
|
|
106
|
+
# Plugin::ModelError, as an unreachable model.
|
|
107
|
+
def ask_model(request, timeout:, max_tokens:, cancel_controller:, model: nil)
|
|
108
|
+
entry = ModelRequest.new(request: request, timeout: timeout, max_tokens: max_tokens, cancel: cancel_controller,
|
|
109
|
+
model: model)
|
|
110
|
+
record(@model_requests, entry)
|
|
111
|
+
answer = @mutex.synchronize { @model_answers.empty? ? :none : @model_answers.shift }
|
|
112
|
+
raise Plugin::ModelError, "no model answer queued (bundle.model_answers << \"text\")" if answer == :none
|
|
113
|
+
raise answer if answer.is_a?(Exception) || (answer.is_a?(Class) && answer <= Exception)
|
|
114
|
+
|
|
115
|
+
answer.respond_to?(:call) ? answer.call(entry) : answer
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# ctx.use_toolkit: +known+ the toolkits chi and the plugins declare.
|
|
119
|
+
# Staged (toolkit_staged:, true by default), the session is on it at
|
|
120
|
+
# once, as chi saves it on the session (ToolkitBinding#use).
|
|
121
|
+
def use_toolkit(name, known)
|
|
122
|
+
raise ArgumentError, "no toolkit named #{name}" unless known.include?(name)
|
|
123
|
+
|
|
124
|
+
record(@toolkit_uses, name)
|
|
125
|
+
return false unless self[:toolkit_staged]
|
|
126
|
+
|
|
127
|
+
set(toolkit: name)
|
|
128
|
+
true
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
private
|
|
132
|
+
|
|
133
|
+
def record(list, item)
|
|
134
|
+
@mutex.synchronize { list << item }
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
138
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../plugin/api"
|
|
4
|
+
|
|
5
|
+
module Samagotchi
|
|
6
|
+
module Testing
|
|
7
|
+
# One chi.* call a plugin's #register made, as made: its arguments
|
|
8
|
+
# before Plugin::Api checks and normalizes them (params: as given).
|
|
9
|
+
Declaration = Data.define(:name, :args, :options)
|
|
10
|
+
|
|
11
|
+
# The real Plugin::Api, also recording each declaration in order
|
|
12
|
+
# (#declarations), for a spec of what #register declares as written.
|
|
13
|
+
class RecordingApi < Plugin::Api
|
|
14
|
+
DECLARATIONS = %i[command annotation tool on service init toolkit tool_source peer_source].freeze
|
|
15
|
+
|
|
16
|
+
# @return [Array<Declaration>]
|
|
17
|
+
def declarations = (@declarations ||= [])
|
|
18
|
+
|
|
19
|
+
DECLARATIONS.each do |name|
|
|
20
|
+
define_method(name) do |*args, **options, &block|
|
|
21
|
+
declarations << Declaration.new(name: name, args: args.freeze, options: options.freeze)
|
|
22
|
+
super(*args, **options, &block)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rspec/core"
|
|
4
|
+
require_relative "../testing"
|
|
5
|
+
|
|
6
|
+
# The bundle test kit in RSpec (EXPERIMENTAL, docs/plugins.md): example groups
|
|
7
|
+
# tagged :bundle get Samagotchi::Testing::Helpers (load_bundle,
|
|
8
|
+
# shipped_bundle, sandbox), and each of their examples runs in its own
|
|
9
|
+
# sandbox, its bundles shut down after it. A spec under a bundle's own spec/
|
|
10
|
+
# folder (bundles/<name>/spec/) is tagged :bundle by itself; elsewhere tag the
|
|
11
|
+
# group:
|
|
12
|
+
#
|
|
13
|
+
# RSpec.describe "my bundle", :bundle do
|
|
14
|
+
# it("says hi") { expect(load_bundle(BUNDLE_DIR).command("/hi")).to eq("hi") }
|
|
15
|
+
# end
|
|
16
|
+
RSpec.configure do |config|
|
|
17
|
+
config.define_derived_metadata(file_path: %r{(?:\A|/)bundles/[^/]+/spec/}) { |metadata| metadata[:bundle] = true }
|
|
18
|
+
config.include Samagotchi::Testing::Helpers, :bundle
|
|
19
|
+
config.around(:each, :bundle) do |example|
|
|
20
|
+
enter_sandbox
|
|
21
|
+
example.run
|
|
22
|
+
ensure
|
|
23
|
+
unmatched = leave_sandbox
|
|
24
|
+
# A request no stub answered fails an example that passed, after its
|
|
25
|
+
# cleanup (a plugin that rescued the UnstubbedRequest can't hide it); an
|
|
26
|
+
# example that failed keeps its own failure.
|
|
27
|
+
unless unmatched.empty? || example.exception
|
|
28
|
+
raise Samagotchi::Testing::UnstubbedRequest, Samagotchi::Testing::FakeHttp.unmatched_message(unmatched)
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "tmpdir"
|
|
5
|
+
require "yaml"
|
|
6
|
+
require_relative "../log"
|
|
7
|
+
require_relative "../paths"
|
|
8
|
+
require_relative "fake_executable"
|
|
9
|
+
|
|
10
|
+
module Samagotchi
|
|
11
|
+
module Testing
|
|
12
|
+
# The environment a bundle test runs in: temp config and state homes
|
|
13
|
+
# (config.yml, memories, ctx.data_dir, the debug log), HOME and the peer
|
|
14
|
+
# CLIs' folders, git without the user's config, no Keychain, none of
|
|
15
|
+
# the user's SAMAGOTCHI_* settings, a bin/ first on PATH for fake
|
|
16
|
+
# commands. ENV is put back on #leave.
|
|
17
|
+
#
|
|
18
|
+
# Samagotchi::Testing::Sandbox.with { |sandbox| sandbox.config("bundles" => {…}) … }
|
|
19
|
+
# sandbox = Samagotchi::Testing::Sandbox.enter # … sandbox.leave
|
|
20
|
+
class Sandbox
|
|
21
|
+
# The environment switches a run of the test keeps (as the suite does).
|
|
22
|
+
KEPT = %w[SAMAGOTCHI_LOG_STRICT].freeze
|
|
23
|
+
|
|
24
|
+
class << self
|
|
25
|
+
# The sandbox entered last and not left yet (Bundle.load uses it).
|
|
26
|
+
# @return [Sandbox, nil]
|
|
27
|
+
attr_accessor :current
|
|
28
|
+
|
|
29
|
+
# @return [Sandbox] entered
|
|
30
|
+
def enter = new.tap(&:enter)
|
|
31
|
+
|
|
32
|
+
# Run the block in a new sandbox, left after it.
|
|
33
|
+
# @yieldparam sandbox [Sandbox]
|
|
34
|
+
def with
|
|
35
|
+
sandbox = enter
|
|
36
|
+
yield sandbox
|
|
37
|
+
ensure
|
|
38
|
+
sandbox&.leave
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# @api private
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# @return [String] the temp folder holding everything
|
|
45
|
+
attr_reader :root
|
|
46
|
+
|
|
47
|
+
def initialize
|
|
48
|
+
@root = nil
|
|
49
|
+
@saved = nil
|
|
50
|
+
@outer = nil
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Point ENV at fresh temp folders (and remember the previous ENV).
|
|
54
|
+
# @return [self]
|
|
55
|
+
def enter
|
|
56
|
+
raise ArgumentError, "the sandbox was entered already" if @root
|
|
57
|
+
|
|
58
|
+
@root = File.realpath(Dir.mktmpdir("chi-bundle-test-"))
|
|
59
|
+
@saved = ENV.to_h
|
|
60
|
+
@outer = Sandbox.current
|
|
61
|
+
ENV.each_key.select { |key| key.start_with?("SAMAGOTCHI_") && !KEPT.include?(key) }.each { |key| ENV.delete(key) }
|
|
62
|
+
%w[config state home work bin].each { |dir| FileUtils.mkdir_p(File.join(@root, dir)) }
|
|
63
|
+
File.write(File.join(@root, "gitconfig"), "")
|
|
64
|
+
ENV.update(
|
|
65
|
+
"HOME" => File.join(@root, "home"),
|
|
66
|
+
"XDG_CONFIG_HOME" => File.join(@root, "config"),
|
|
67
|
+
"XDG_STATE_HOME" => File.join(@root, "state"),
|
|
68
|
+
"CLAUDE_CONFIG_DIR" => File.join(@root, "home", ".claude"),
|
|
69
|
+
"CODEX_HOME" => File.join(@root, "home", ".codex"),
|
|
70
|
+
"GIT_CONFIG_GLOBAL" => File.join(@root, "gitconfig"),
|
|
71
|
+
"GIT_CONFIG_NOSYSTEM" => "1",
|
|
72
|
+
"SAMAGOTCHI_KEYCHAIN" => "off",
|
|
73
|
+
"SAMAGOTCHI_ENV" => "test",
|
|
74
|
+
"SAMAGOTCHI_RECAP_ENABLED" => "false",
|
|
75
|
+
"SAMAGOTCHI_CACHE_WARMUP" => "off",
|
|
76
|
+
"PATH" => [bin_dir, ENV.fetch("PATH", nil)].compact.join(File::PATH_SEPARATOR)
|
|
77
|
+
)
|
|
78
|
+
# The log resolves its file from the (now temp) state home.
|
|
79
|
+
Log.reset!
|
|
80
|
+
Sandbox.current = self
|
|
81
|
+
self
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Put ENV back as it was and remove the temp folders.
|
|
85
|
+
def leave
|
|
86
|
+
return unless @root
|
|
87
|
+
|
|
88
|
+
ENV.replace(@saved)
|
|
89
|
+
Log.reset!
|
|
90
|
+
FileUtils.rm_rf(@root)
|
|
91
|
+
Sandbox.current = @outer
|
|
92
|
+
@root = @saved = @outer = nil
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Write config.yml (what Config.get reads): a Hash or YAML text.
|
|
96
|
+
# @return [String] its path
|
|
97
|
+
def config(data)
|
|
98
|
+
path = File.join(Paths.config_home, Paths::APP_DIR, "config.yml")
|
|
99
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
100
|
+
File.write(path, data.is_a?(String) ? data : YAML.dump(stringify(data)))
|
|
101
|
+
path
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# @return [String] $XDG_STATE_HOME/samagotchi (ctx.data_dir's parent)
|
|
105
|
+
def state_dir = Paths.state_dir
|
|
106
|
+
|
|
107
|
+
# A memory scope's folder ("system", or "project" for the checkout
|
|
108
|
+
# at +cwd+), as chi's memory tools and bundles find it; created.
|
|
109
|
+
# @return [String]
|
|
110
|
+
def memory_dir(scope = "system", cwd: Dir.pwd)
|
|
111
|
+
require_relative "../memory_paths"
|
|
112
|
+
MemoryPaths.scope_dir!(scope, cwd: cwd).tap { |dir| FileUtils.mkdir_p(dir) }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# Write a memory as memory_write does (the file and its index line,
|
|
116
|
+
# with +description+), for the checkout at +cwd+ (the project scope's;
|
|
117
|
+
# default the process's own); +source+ marks the line as a bundle's
|
|
118
|
+
# ("· from <source>"), as an install writes it.
|
|
119
|
+
# @return [String] the memory's path
|
|
120
|
+
# @raise [ArgumentError] what memory_write refused
|
|
121
|
+
def write_memory(name, content, scope: "system", description: nil, source: nil, cwd: nil)
|
|
122
|
+
require_relative "../tools/memory"
|
|
123
|
+
require_relative "../memory_bundle/index_updater"
|
|
124
|
+
in_dir(cwd) do
|
|
125
|
+
result = Tools::MemoryWrite.call(content, path: name, scope: scope, description: description)
|
|
126
|
+
raise ArgumentError, result if result.start_with?("Error")
|
|
127
|
+
|
|
128
|
+
MemoryBundle::IndexUpdater.update_index(scope, name, content.bytesize, description, source: source) if source
|
|
129
|
+
end
|
|
130
|
+
File.join(memory_dir(scope, cwd: cwd || Dir.pwd), "#{name}.md")
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
# @return [String] a temp folder for the session's cwd
|
|
134
|
+
def work_dir = File.join(@root, "work")
|
|
135
|
+
|
|
136
|
+
# @return [String] the folder first on PATH (#fake_executable's)
|
|
137
|
+
def bin_dir = File.join(@root, "bin")
|
|
138
|
+
|
|
139
|
+
# A command +name+ first on PATH that runs +body+ (sh) with its
|
|
140
|
+
# arguments; again for the same name replaces the body.
|
|
141
|
+
# @return [String] its path
|
|
142
|
+
def fake_executable(name, body) = FakeExecutable.write(bin_dir, name, body)
|
|
143
|
+
|
|
144
|
+
private
|
|
145
|
+
|
|
146
|
+
# Run the block with +dir+ as the process's cwd (the memory tools
|
|
147
|
+
# read the project from it); nil: as it is.
|
|
148
|
+
def in_dir(dir, &)
|
|
149
|
+
return yield if dir.nil? || File.expand_path(dir) == Dir.pwd
|
|
150
|
+
|
|
151
|
+
Dir.chdir(dir, &)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def stringify(value)
|
|
155
|
+
case value
|
|
156
|
+
when Hash then value.to_h { |key, item| [key.to_s, stringify(item)] }
|
|
157
|
+
when Array then value.map { |item| stringify(item) }
|
|
158
|
+
else value
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|