samagotchi 0.121.0 → 0.121.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +82 -1
- data/bin/chi +3 -2
- data/docs/architecture.md +36 -11
- data/docs/bundle-authoring.md +2 -1
- data/docs/cli.md +10 -4
- data/docs/configuration.md +7 -3
- data/docs/guardrails.md +48 -19
- data/docs/internals/anthropic.md +1 -1
- data/docs/internals/background-tasks.md +1 -1
- data/docs/internals/participants.md +3 -1
- data/docs/internals/remote-sessions.md +15 -11
- data/docs/internals/web-push.md +22 -11
- data/docs/plugins.md +6 -2
- data/docs/sessions.md +15 -0
- data/docs/testing.md +41 -1
- data/lib/samagotchi/answer_command.rb +1 -1
- data/lib/samagotchi/bridge.rb +8 -8
- data/lib/samagotchi/bridge_client/event_stream.rb +11 -27
- data/lib/samagotchi/bridge_client/stream_open.rb +79 -0
- data/lib/samagotchi/bridge_client/stream_request.rb +8 -2
- data/lib/samagotchi/bridge_client.rb +52 -65
- data/lib/samagotchi/bridge_secret.rb +9 -7
- data/lib/samagotchi/bundle_command/check.rb +27 -2
- data/lib/samagotchi/bundles/claude-peer/manifest.yml +2 -2
- data/lib/samagotchi/bundles/claude-peer/scripts/pair +20 -6
- data/lib/samagotchi/bundles/codex-peer/manifest.yml +2 -2
- data/lib/samagotchi/bundles/codex-peer/scripts/pair +14 -3
- data/lib/samagotchi/bundles/github-pr/manifest.yml +2 -2
- data/lib/samagotchi/bundles/github-pr/plugin.rb +2 -6
- data/lib/samagotchi/bundles/system/manifest.yml +1 -1
- data/lib/samagotchi/child_move.rb +5 -7
- data/lib/samagotchi/children_status.rb +2 -1
- data/lib/samagotchi/guardrail_wiring.rb +19 -2
- data/lib/samagotchi/guardrails/parent_approvals.rb +9 -0
- data/lib/samagotchi/guardrails_command.rb +41 -6
- data/lib/samagotchi/image_store.rb +6 -3
- data/lib/samagotchi/interrupted_turn.rb +6 -3
- data/lib/samagotchi/live_versions.rb +6 -7
- data/lib/samagotchi/memory_bundle/profile.rb +1 -1
- data/lib/samagotchi/memory_bundle/uninstaller.rb +6 -2
- data/lib/samagotchi/plugin/api.rb +8 -5
- data/lib/samagotchi/plugin/sessions.rb +13 -4
- data/lib/samagotchi/push_command.rb +1 -1
- data/lib/samagotchi/relay_verifier.rb +2 -3
- data/lib/samagotchi/relay_watcher.rb +2 -2
- data/lib/samagotchi/remote_attach_command.rb +1 -1
- data/lib/samagotchi/reply_wait/poll.rb +8 -2
- data/lib/samagotchi/reply_wait.rb +4 -3
- data/lib/samagotchi/review_command.rb +1 -1
- data/lib/samagotchi/run_options.rb +1 -0
- data/lib/samagotchi/session.rb +4 -0
- data/lib/samagotchi/session_access/local.rb +18 -8
- data/lib/samagotchi/session_access/remote.rb +5 -0
- data/lib/samagotchi/session_access/web_bridge_client.rb +31 -19
- data/lib/samagotchi/session_access.rb +21 -9
- data/lib/samagotchi/session_continue.rb +2 -3
- data/lib/samagotchi/session_manager.rb +118 -28
- data/lib/samagotchi/sessions_command.rb +10 -5
- data/lib/samagotchi/system_one/client.rb +6 -1
- data/lib/samagotchi/system_one/fake_backend.rb +0 -2
- data/lib/samagotchi/system_one/log.rb +21 -6
- data/lib/samagotchi/system_one/self_protect.rb +39 -43
- data/lib/samagotchi/terminal_ui/attach_launcher.rb +8 -3
- data/lib/samagotchi/terminal_ui/attached_loop.rb +4 -2
- data/lib/samagotchi/terminal_ui/local_link.rb +28 -19
- data/lib/samagotchi/toolkit.rb +68 -2
- data/lib/samagotchi/tools/delegate.rb +1 -1
- data/lib/samagotchi/tools/delegate_relay.rb +9 -7
- data/lib/samagotchi/turn_checkpoint.rb +15 -9
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/web/app.rb +123 -65
- data/lib/samagotchi/web/participant_access.rb +26 -7
- data/lib/samagotchi/web/participant_review.rb +2 -1
- data/lib/samagotchi/web/participant_view.rb +7 -3
- data/lib/samagotchi/web/participants_admin.rb +13 -0
- data/lib/samagotchi/web/public/app.js +1 -1
- data/lib/samagotchi/web/public/notifications.js +21 -1
- data/lib/samagotchi/web/public/participant.html +3 -2
- data/lib/samagotchi/web/public/participant.js +23 -9
- data/lib/samagotchi/web/public/participants_panel.js +46 -11
- data/lib/samagotchi/web/public/push.js +4 -3
- data/lib/samagotchi/web/public/push_devices.js +17 -3
- data/lib/samagotchi/web/public/review_history.js +14 -2
- data/lib/samagotchi/web/public/review_start.js +1 -3
- data/lib/samagotchi/web/public/session_actions.js +14 -2
- data/lib/samagotchi/web/public/storage.js +16 -0
- data/lib/samagotchi/web/push/client.rb +5 -0
- data/lib/samagotchi/web/push/store.rb +16 -3
- data/lib/samagotchi/web/push_routes.rb +3 -2
- data/lib/samagotchi/web/session_access_routes.rb +25 -17
- data/lib/samagotchi/web/session_create.rb +4 -4
- data/lib/samagotchi/web/session_hub.rb +6 -4
- data/lib/samagotchi/web/session_summary.rb +8 -16
- data/lib/samagotchi/worker.rb +1 -1
- data/lib/samagotchi/worker_link.rb +213 -0
- data/lib/samagotchi/worker_sidecar.rb +15 -37
- metadata +3 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative "
|
|
3
|
+
require_relative "worker_link"
|
|
4
4
|
require_relative "log"
|
|
5
5
|
|
|
6
6
|
module Samagotchi
|
|
@@ -20,7 +20,7 @@ module Samagotchi
|
|
|
20
20
|
# @param live [#call] whether the parent's worker is up
|
|
21
21
|
# @return [Thread]
|
|
22
22
|
def start(engine:, question_id:, relay_id:, parent_dir:, interval: INTERVAL,
|
|
23
|
-
live: -> {
|
|
23
|
+
live: -> { WorkerLink.resolve(parent_dir, unlink: false).live? })
|
|
24
24
|
Thread.new do
|
|
25
25
|
loop do
|
|
26
26
|
sleep(interval)
|
|
@@ -48,7 +48,7 @@ module Samagotchi
|
|
|
48
48
|
ignored(id) if existing
|
|
49
49
|
launch(id, existing: existing)
|
|
50
50
|
rescue TerminalUI::AttachLauncher::Error, SessionAccess::RemoteFailed, SessionAccess::Refused,
|
|
51
|
-
SessionManager::OwnedByTUI, ModelProfile::MissingModel, ArgumentError => e
|
|
51
|
+
SessionManager::OwnedByTUI, ModelProfile::MissingModel, Toolkit::Refused, ArgumentError => e
|
|
52
52
|
e.is_a?(SessionAccess::BadRequest) ? usage(e.message) : failed(e.message)
|
|
53
53
|
rescue Interrupt
|
|
54
54
|
CLI::Exit::INTERRUPTED
|
|
@@ -5,6 +5,7 @@ require_relative "../session_inbox"
|
|
|
5
5
|
require_relative "../turn_checkpoint"
|
|
6
6
|
require_relative "../continue_offer"
|
|
7
7
|
require_relative "../steer"
|
|
8
|
+
require_relative "../worker_link"
|
|
8
9
|
|
|
9
10
|
module Samagotchi
|
|
10
11
|
module ReplyWait
|
|
@@ -164,11 +165,16 @@ module Samagotchi
|
|
|
164
165
|
pending = session.pending_question
|
|
165
166
|
return nil unless pending && !@baseline&.question?(pending.id) &&
|
|
166
167
|
(@turn.nil? || ReplyWait.own_question?(session, @turn, @claimed)) &&
|
|
167
|
-
session.waiting_question(live:
|
|
168
|
+
session.waiting_question(live: worker_owned?)
|
|
168
169
|
|
|
169
170
|
Result.new(status: :waiting_for_answer, question: pending, session: session)
|
|
170
171
|
end
|
|
171
172
|
|
|
173
|
+
# A worker holds the session's lock (a chi REPL's question is its own).
|
|
174
|
+
def worker_owned?
|
|
175
|
+
WorkerLink.resolve(Session.session_dir(@id, state_dir: @state_dir), unlink: false).worker_owned?
|
|
176
|
+
end
|
|
177
|
+
|
|
172
178
|
# A participant's turn its worker holds while the owner's Continue
|
|
173
179
|
# offer stands (Worker#run_input_files): it waits for the owner's
|
|
174
180
|
# answer to that question, whenever it was asked (the baseline's
|
|
@@ -176,7 +182,7 @@ module Samagotchi
|
|
|
176
182
|
def held(session)
|
|
177
183
|
pending = session.pending_question
|
|
178
184
|
return nil unless @participant_input && !@claimed && pending&.kind.to_s == ContinueOffer::KIND &&
|
|
179
|
-
session.waiting_question(live:
|
|
185
|
+
session.waiting_question(live: worker_owned?)
|
|
180
186
|
|
|
181
187
|
Result.new(status: :waiting_for_answer, question: pending, session: session)
|
|
182
188
|
end
|
|
@@ -4,7 +4,7 @@ require_relative "session"
|
|
|
4
4
|
require_relative "turn_note"
|
|
5
5
|
require_relative "session_inbox"
|
|
6
6
|
require_relative "turn_checkpoint"
|
|
7
|
-
require_relative "
|
|
7
|
+
require_relative "worker_link"
|
|
8
8
|
require_relative "reply_wait/poll"
|
|
9
9
|
|
|
10
10
|
module Samagotchi
|
|
@@ -214,9 +214,10 @@ module Samagotchi
|
|
|
214
214
|
running || queued || ended
|
|
215
215
|
end
|
|
216
216
|
|
|
217
|
-
# Whether the session's worker
|
|
217
|
+
# Whether the session's live worker names TURN_IDS_FEATURE.
|
|
218
218
|
def turn_ids?(session_dir)
|
|
219
|
-
|
|
219
|
+
link = WorkerLink.resolve(session_dir, unlink: false)
|
|
220
|
+
link.live? && link.features.include?(TURN_IDS_FEATURE)
|
|
220
221
|
end
|
|
221
222
|
|
|
222
223
|
# A pending question is a turn: wait's (its enqueued_id) only when that
|
|
@@ -61,7 +61,7 @@ module Samagotchi
|
|
|
61
61
|
start(parsed.options, parsed.args)
|
|
62
62
|
rescue ReviewStart::Refused, SessionAccess::BadRequest => e
|
|
63
63
|
refused(e.message)
|
|
64
|
-
rescue ModelProfile::MissingModel => e
|
|
64
|
+
rescue ModelProfile::MissingModel, Toolkit::Refused => e
|
|
65
65
|
refused("could not start: #{e.message}")
|
|
66
66
|
rescue ArgumentError => e
|
|
67
67
|
refused(e.is_a?(Session::AmbiguousId) ? e.message : e.message.sub(/\ASession not found: /, "no session "))
|
|
@@ -165,6 +165,7 @@ module Samagotchi
|
|
|
165
165
|
chi review FILE... [--dir DIR] | --from ID a review session to share (chi share)
|
|
166
166
|
chi memory <ls|rm> [--scope project|system] the memories; rm moves one to the bundle trash
|
|
167
167
|
chi bundle <install|upgrade|uninstall|status|diff|list|build|trash|new|test|check|try>
|
|
168
|
+
chi guardrails replay [--dry-run] [--threshold T] [--model M] seed System 1's record from answered approvals
|
|
168
169
|
chi desktop <install|upgrade|uninstall|status> macOS "Send to chi" helper
|
|
169
170
|
chi models [--format json] [TEXT] the models --model takes, from every host
|
|
170
171
|
chi remote <add|ls|rm> other machines' chi web, for chi --remote NAME
|
data/lib/samagotchi/session.rb
CHANGED
|
@@ -427,10 +427,14 @@ module Samagotchi
|
|
|
427
427
|
# seed); [] by default
|
|
428
428
|
# @param setup [SessionSetup] the session's own settings (none by default)
|
|
429
429
|
# @param now [Time] its created_at and updated_at (a spec's clock)
|
|
430
|
+
# @raise [Toolkit::Refused] its toolkit (setup's, else the configured
|
|
431
|
+
# one SessionManager.spawn_session gives it) builds on a config kit
|
|
432
|
+
# with an invalid groups: (Toolkit.check_new!)
|
|
430
433
|
def self.new_session(mode:, model_name:, working_directory:, test_run: nil,
|
|
431
434
|
preloaded_memory_names: [], muted_memory_names: [], parent_id: nil, messages: [],
|
|
432
435
|
scratch: false, model_typed: nil, delegate: false, setup: SessionSetup.new, continues: nil,
|
|
433
436
|
review: nil, now: Time.now)
|
|
437
|
+
Toolkit.check_new!(setup.toolkit || Toolkit.configured(delegate: delegate))
|
|
434
438
|
now = now.iso8601(3)
|
|
435
439
|
resolved_test = if test_run.nil?
|
|
436
440
|
test_session_env?
|
|
@@ -14,6 +14,7 @@ require_relative "../context_sources"
|
|
|
14
14
|
require_relative "../send_command/busy"
|
|
15
15
|
require_relative "../chi_invocation"
|
|
16
16
|
require_relative "../worker_sidecar"
|
|
17
|
+
require_relative "../worker_link"
|
|
17
18
|
require_relative "../review_start"
|
|
18
19
|
require_relative "../launch_mode"
|
|
19
20
|
require_relative "../participants"
|
|
@@ -95,13 +96,22 @@ module Samagotchi
|
|
|
95
96
|
session = show(id)
|
|
96
97
|
raise Refused.new(LaunchMode::SCRATCH_LEFTOVER, code: "scratch") if session.scratch
|
|
97
98
|
|
|
98
|
-
|
|
99
|
-
|
|
99
|
+
link = WorkerLink.resolve(session_dir(id), unlink: true)
|
|
100
|
+
case link
|
|
101
|
+
when WorkerLink::Live then return Woken.new(woke: false, sidecar: link.sidecar)
|
|
102
|
+
when WorkerLink::Skew then raise Refused.new(link.message(id), code: "worker_newer")
|
|
103
|
+
when WorkerLink::Starting
|
|
104
|
+
# A worker that never publishes its Bridge (a failed start, a
|
|
105
|
+
# crash's teardown) lets go of the lock: then resume, not 504.
|
|
106
|
+
found = WorkerLink.await(id, session_dir: session_dir(id), timeout: wait,
|
|
107
|
+
give_up: ->(link) { !link.worker_owned? })
|
|
108
|
+
return Woken.new(woke: false, sidecar: found.live? ? found.sidecar : nil) if found.worker_owned?
|
|
109
|
+
end
|
|
100
110
|
|
|
101
111
|
OwnerLock.await_release(session_dir(id))
|
|
102
112
|
SessionManager.resume_session(id, **dir)
|
|
103
|
-
found =
|
|
104
|
-
Woken.new(woke: true, sidecar: found
|
|
113
|
+
found = WorkerLink.await(id, session_dir: session_dir(id), timeout: wait)
|
|
114
|
+
Woken.new(woke: true, sidecar: found.live? ? found.sidecar : nil)
|
|
105
115
|
end
|
|
106
116
|
|
|
107
117
|
# The session's worker now, never started (chi web's GET …/worker).
|
|
@@ -109,8 +119,8 @@ module Samagotchi
|
|
|
109
119
|
# @raise [ArgumentError] no such session
|
|
110
120
|
def worker(id)
|
|
111
121
|
show(id)
|
|
112
|
-
WorkerState.new(
|
|
113
|
-
stopped: Session.stopped_marker?(id, state_dir: state_dir!)
|
|
122
|
+
WorkerState.new(link: WorkerLink.resolve(session_dir(id), unlink: true),
|
|
123
|
+
stopped: Session.stopped_marker?(id, state_dir: state_dir!))
|
|
114
124
|
end
|
|
115
125
|
|
|
116
126
|
# --- create and send
|
|
@@ -148,7 +158,7 @@ module Samagotchi
|
|
|
148
158
|
# Whether a just-created session's worker came up (its Bridge
|
|
149
159
|
# answers) within +timeout+ seconds.
|
|
150
160
|
def await_worker(id, timeout:)
|
|
151
|
-
|
|
161
|
+
WorkerLink.await(id, session_dir: session_dir(id), timeout: timeout).live?
|
|
152
162
|
end
|
|
153
163
|
|
|
154
164
|
# Copies of +refs+ (ImageStore refs stored in +from+) in the
|
|
@@ -479,7 +489,7 @@ module Samagotchi
|
|
|
479
489
|
# The block's post to the session's live worker, again on a 408 (the
|
|
480
490
|
# worker didn't take it in time).
|
|
481
491
|
def on_worker(id)
|
|
482
|
-
client =
|
|
492
|
+
client = WorkerLink.client(id, session_dir: session_dir(id)) or return nil
|
|
483
493
|
reply = nil
|
|
484
494
|
2.times do
|
|
485
495
|
reply = yield client
|
|
@@ -12,6 +12,7 @@ require_relative "../bridge_client"
|
|
|
12
12
|
require_relative "../session_continue"
|
|
13
13
|
require_relative "../guardrails/parent_policy"
|
|
14
14
|
require_relative "../model_profile"
|
|
15
|
+
require_relative "../web/grants"
|
|
15
16
|
require_relative "http_transport"
|
|
16
17
|
require_relative "../chi_invocation"
|
|
17
18
|
|
|
@@ -679,6 +680,10 @@ module Samagotchi
|
|
|
679
680
|
# No model configured there, or --model's host unknown there: what
|
|
680
681
|
# spawn_session raises here (chi send: could not start a session).
|
|
681
682
|
when "invalid_model" then raise ModelProfile::MissingModel, message.to_s
|
|
683
|
+
# A toolkit whose config kit's groups: is invalid there: the same.
|
|
684
|
+
when "toolkit_refused" then raise Toolkit::Refused, message.to_s
|
|
685
|
+
# The grants store doesn't read there: chi share says what it says here.
|
|
686
|
+
when "grants_unwritable" then raise Web::Grants::Unwritable, message.to_s
|
|
682
687
|
when /\Ainvalid_/, "missing_fields", "bad_images"
|
|
683
688
|
# invalid_participant: chi share's name, kind or ttl, an archived session
|
|
684
689
|
raise BadRequest.new(message.to_s, code: error)
|
|
@@ -6,6 +6,7 @@ require_relative "../session_access"
|
|
|
6
6
|
require_relative "../bridge_client"
|
|
7
7
|
require_relative "../delivery"
|
|
8
8
|
require_relative "../log"
|
|
9
|
+
require_relative "../worker_link"
|
|
9
10
|
require_relative "http_transport"
|
|
10
11
|
|
|
11
12
|
module Samagotchi
|
|
@@ -39,9 +40,6 @@ module Samagotchi
|
|
|
39
40
|
# A read a join waits on (a turn's saved records, the children
|
|
40
41
|
# counts): a slow chi web can't hold the join up for long (L9).
|
|
41
42
|
JOIN_READ_TIMEOUT = 5
|
|
42
|
-
# chi web answers no live worker for the session (Mapping).
|
|
43
|
-
NO_WORKER = "no live worker for session %s on %s"
|
|
44
|
-
|
|
45
43
|
# chi web's answers as the session's Bridge would have given them.
|
|
46
44
|
module Mapping
|
|
47
45
|
module_function
|
|
@@ -60,7 +58,7 @@ module Samagotchi
|
|
|
60
58
|
def response(route, reply, session_id:, remote:)
|
|
61
59
|
json = reply.json.is_a?(Hash) ? reply.json : {}
|
|
62
60
|
case [reply.status, json["error"]]
|
|
63
|
-
in [503, "not_live"] | [404, "not_found"] then raise Unreachable,
|
|
61
|
+
in [503, "not_live"] | [404, "not_found"] then raise Unreachable, "#{WorkerLink.not_live(session_id)} on #{remote}"
|
|
64
62
|
in [504, _] then bridge(408, error: "deadline_passed", detail: json["detail"])
|
|
65
63
|
in [501, _] if route == :command && json["command"]
|
|
66
64
|
bridge(400, error: "unknown_command", detail: json["detail"], command: json["command"])
|
|
@@ -69,6 +67,8 @@ module Samagotchi
|
|
|
69
67
|
end
|
|
70
68
|
end
|
|
71
69
|
|
|
70
|
+
def stream(sock, status, headers) = BridgeClient::StreamOpen.classify(sock, status, headers)
|
|
71
|
+
|
|
72
72
|
def bridge(status, **body) = BridgeClient::Response.new(status: status, body: JSON.generate(body.compact))
|
|
73
73
|
end
|
|
74
74
|
|
|
@@ -198,24 +198,35 @@ module Samagotchi
|
|
|
198
198
|
idle_timeout: STREAM_IDLE_TIMEOUT, &on_event).start
|
|
199
199
|
end
|
|
200
200
|
|
|
201
|
-
# GET /api/sessions/:id/stream through StreamRequest,
|
|
202
|
-
#
|
|
203
|
-
#
|
|
204
|
-
#
|
|
205
|
-
#
|
|
206
|
-
|
|
201
|
+
# GET /api/sessions/:id/stream through StreamRequest, classifying chi
|
|
202
|
+
# web's stream answers as Bridge outcomes.
|
|
203
|
+
# @param header_timeout [Float] see BridgeClient::STREAM_HEADER_TIMEOUT
|
|
204
|
+
# @yield [sock] called after response headers arrive, before classifying a
|
|
205
|
+
# refusal; EventStream uses it to publish the socket for #close
|
|
206
|
+
# @return [BridgeClient::StreamOpen::Opened, BridgeClient::StreamOpen::Refused,
|
|
207
|
+
# BridgeClient::StreamOpen::Unreachable]
|
|
208
|
+
def open_stream(query: "", last_event_id: nil, header_timeout: BridgeClient::STREAM_HEADER_TIMEOUT, attempts: 1)
|
|
207
209
|
uri = URI(@remote.entry.url)
|
|
208
210
|
return refused("tls_unsupported") if uri.scheme == "https"
|
|
209
211
|
|
|
210
212
|
token = @remote.entry.token
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
return
|
|
213
|
+
connection = BridgeClient::StreamRequest.open(host: uri.hostname, port: uri.port,
|
|
214
|
+
path: "/api/sessions/#{@session_id}/stream#{query}",
|
|
215
|
+
last_event_id: last_event_id, timeout: header_timeout,
|
|
216
|
+
headers: token ? { "Authorization" => "Bearer #{token}" } : {})
|
|
217
|
+
return BridgeClient::StreamOpen::Unreachable.new(attempts: 1) unless connection
|
|
218
|
+
|
|
219
|
+
sock, status, headers = connection
|
|
220
|
+
unless !block_given? || yield(sock)
|
|
221
|
+
sock.close rescue nil
|
|
222
|
+
return BridgeClient::StreamOpen::Unreachable.new(attempts: 1)
|
|
223
|
+
end
|
|
224
|
+
return refused("chunked_unsupported", sock) if headers["transfer-encoding"].to_s.downcase.include?("chunked")
|
|
216
225
|
|
|
217
|
-
|
|
218
|
-
|
|
226
|
+
Mapping.stream(sock, status, headers)
|
|
227
|
+
rescue SystemCallError, SocketError, IOError
|
|
228
|
+
sock&.close rescue nil
|
|
229
|
+
BridgeClient::StreamOpen::Unreachable.new(attempts: 1)
|
|
219
230
|
end
|
|
220
231
|
|
|
221
232
|
private
|
|
@@ -234,9 +245,10 @@ module Samagotchi
|
|
|
234
245
|
raise Unreachable, e.message
|
|
235
246
|
end
|
|
236
247
|
|
|
237
|
-
def refused(reason)
|
|
248
|
+
def refused(reason, sock = nil)
|
|
249
|
+
sock&.close rescue nil
|
|
238
250
|
Log.warn(:attached, "stream_refused", reason: reason, remote: @remote.name)
|
|
239
|
-
|
|
251
|
+
BridgeClient::StreamOpen::Unreachable.new(attempts: 1)
|
|
240
252
|
end
|
|
241
253
|
end
|
|
242
254
|
end
|
|
@@ -179,15 +179,27 @@ module Samagotchi
|
|
|
179
179
|
end
|
|
180
180
|
|
|
181
181
|
# A session's worker as an attach finds it (Local#worker, chi web's GET
|
|
182
|
-
# …/worker): +
|
|
183
|
-
#
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
#
|
|
190
|
-
def
|
|
182
|
+
# …/worker): +link+ the one WorkerLink outcome every field reads, +stopped+
|
|
183
|
+
# its stop marker.
|
|
184
|
+
WorkerState = Data.define(:link, :stopped) do
|
|
185
|
+
# @return [WorkerSidecar, nil] the live worker's (port, version,
|
|
186
|
+
# features, epoch; nil: none answers)
|
|
187
|
+
def sidecar = link.live? ? link.sidecar : nil
|
|
188
|
+
|
|
189
|
+
# @return [OwnerLock::Owner, nil] a worker, a chi REPL, nil
|
|
190
|
+
def owner = link.owner
|
|
191
|
+
|
|
192
|
+
def live? = link.live?
|
|
193
|
+
|
|
194
|
+
# The owner is a worker leaving (writing its recap on leave): a wake
|
|
195
|
+
# waits for it (Local#wake).
|
|
196
|
+
def leaving? = link.is_a?(WorkerLink::Leaving)
|
|
197
|
+
|
|
198
|
+
# A worker holds the lock past WorkerLink::START_LIMIT with no Bridge.
|
|
199
|
+
def stuck? = link.is_a?(WorkerLink::Starting) && link.stuck
|
|
200
|
+
|
|
201
|
+
# A worker whose sidecar speaks a newer protocol than this chi.
|
|
202
|
+
def skew? = link.is_a?(WorkerLink::Skew)
|
|
191
203
|
end
|
|
192
204
|
|
|
193
205
|
# What a wake for an attach did (Local#wake, chi web's POST …/wake):
|
|
@@ -9,6 +9,7 @@ require_relative "child_move"
|
|
|
9
9
|
require_relative "archive_store"
|
|
10
10
|
require_relative "recap_store"
|
|
11
11
|
require_relative "bridge_client"
|
|
12
|
+
require_relative "worker_link"
|
|
12
13
|
require_relative "model_profile"
|
|
13
14
|
require_relative "log"
|
|
14
15
|
require_relative "session_manager"
|
|
@@ -303,10 +304,8 @@ module Samagotchi
|
|
|
303
304
|
# answers at once. Best effort: the continue goes on without. A review
|
|
304
305
|
# fork (ReviewStart) asks the same, with nothing archived after.
|
|
305
306
|
def self.recap_before_archive(id, state_dir, wait:)
|
|
306
|
-
return unless SessionManager.worker_live?(id, state_dir: state_dir)
|
|
307
|
-
|
|
308
307
|
dir = Session.session_dir(id, state_dir: state_dir)
|
|
309
|
-
client =
|
|
308
|
+
client = WorkerLink.client(id, session_dir: dir) or return
|
|
310
309
|
before = RecapStore.read(dir)
|
|
311
310
|
reply = client.request_recap
|
|
312
311
|
asked = reply.status == 200 && reply.json.is_a?(Hash) ? reply.json["request"] : nil
|
|
@@ -25,6 +25,7 @@ require_relative "turn_note"
|
|
|
25
25
|
require_relative "owner_lock"
|
|
26
26
|
require_relative "bridge_client"
|
|
27
27
|
require_relative "worker_sidecar"
|
|
28
|
+
require_relative "worker_link"
|
|
28
29
|
require_relative "worker_signals"
|
|
29
30
|
require_relative "log"
|
|
30
31
|
require_relative "log_path"
|
|
@@ -40,6 +41,7 @@ require_relative "plugin_session_state"
|
|
|
40
41
|
require_relative "context_sources"
|
|
41
42
|
require_relative "session_chain"
|
|
42
43
|
require_relative "child_move"
|
|
44
|
+
require_relative "tools/task_runtime"
|
|
43
45
|
|
|
44
46
|
module Samagotchi
|
|
45
47
|
# SessionManager coordinates background session processes.
|
|
@@ -587,7 +589,7 @@ module Samagotchi
|
|
|
587
589
|
|
|
588
590
|
# How long after a spawn a resume trusts that worker to take the
|
|
589
591
|
# session, while it runs (and the session wasn't stopped since).
|
|
590
|
-
WORKER_START_GRACE =
|
|
592
|
+
WORKER_START_GRACE = WorkerLink::START_LIMIT
|
|
591
593
|
|
|
592
594
|
@spawns = {}
|
|
593
595
|
@spawns_mutex = Mutex.new
|
|
@@ -626,15 +628,17 @@ module Samagotchi
|
|
|
626
628
|
# the next prompt starts one on the newest chi anyway), :unsupported (a
|
|
627
629
|
# worker from before restarts, running +version+: stop it instead), a
|
|
628
630
|
# WorkerIdleExit#hold_for_restart hold (:turn_running, :reminders, ...),
|
|
629
|
-
#
|
|
631
|
+
# :fallback_held (disk-visible work blocks an unresponsive-worker restart),
|
|
632
|
+
# or :failed (the worker answered something else or didn't release).
|
|
630
633
|
class RestartRefused < StandardError
|
|
631
|
-
attr_reader :session_id, :reason, :version
|
|
634
|
+
attr_reader :session_id, :reason, :version, :holds
|
|
632
635
|
|
|
633
|
-
def initialize(session_id, reason, version: nil)
|
|
636
|
+
def initialize(session_id, reason, version: nil, holds: [])
|
|
634
637
|
@session_id = session_id
|
|
635
638
|
@reason = reason.to_sym
|
|
636
639
|
@version = version
|
|
637
|
-
|
|
640
|
+
@holds = Array(holds).map(&:to_sym).freeze
|
|
641
|
+
super(RestartRefused.words(session_id, @reason, version, @holds))
|
|
638
642
|
end
|
|
639
643
|
|
|
640
644
|
HOLDS = {
|
|
@@ -650,13 +654,17 @@ module Samagotchi
|
|
|
650
654
|
}.freeze
|
|
651
655
|
|
|
652
656
|
# What the CLI and the web say.
|
|
653
|
-
def self.words(session_id, reason, version = nil)
|
|
657
|
+
def self.words(session_id, reason, version = nil, holds = [])
|
|
654
658
|
case reason
|
|
655
659
|
when :not_running
|
|
656
660
|
"session #{session_id} has no running worker; the next prompt starts one on the newest chi"
|
|
657
661
|
when :unsupported
|
|
658
662
|
"session #{session_id}'s worker runs chi #{version || "(older)"}, which can't restart: " \
|
|
659
663
|
"chi sessions stop #{session_id}, then any prompt starts it on the newest chi"
|
|
664
|
+
when :fallback_held
|
|
665
|
+
why = holds.map { |hold| HOLDS.fetch(hold, hold.to_s.tr("_", " ")) }.join("; ")
|
|
666
|
+
"session #{session_id} has disk-visible work that a restart might lose#{": #{why}" unless why.empty?}; " \
|
|
667
|
+
"run `chi sessions stop #{session_id}`, then any prompt starts it on the newest chi"
|
|
660
668
|
when :failed then "session #{session_id}'s worker didn't take the restart"
|
|
661
669
|
else "not now: #{HOLDS.fetch(reason, reason.to_s.tr("_", " "))}"
|
|
662
670
|
end
|
|
@@ -668,12 +676,14 @@ module Samagotchi
|
|
|
668
676
|
Restarted = Struct.new(:session_id, :from_version, :version, keyword_init: true)
|
|
669
677
|
|
|
670
678
|
RESTART_WAIT = 10.0
|
|
679
|
+
# A TERMed worker writes its recap on leave (an LLM call) while it still
|
|
680
|
+
# holds the lock: STOP_WAIT plus the recap's own wait (SessionContinue::RECAP_WAIT).
|
|
681
|
+
FALLBACK_STOP_WAIT = 40
|
|
671
682
|
|
|
672
683
|
# Hand a session to a new worker on the newest chi installed: ask its
|
|
673
684
|
# worker (POST /exit restart: true), which leaves only when nothing would
|
|
674
|
-
# be lost and starts its successor
|
|
675
|
-
#
|
|
676
|
-
# terminals reconnect to it.
|
|
685
|
+
# be lost and starts its successor. If the worker cannot be asked, stop
|
|
686
|
+
# it without a stop marker and resume only when disk shows no work to lose.
|
|
677
687
|
# @return [Restarted]
|
|
678
688
|
# @raise [OwnedByTUI] a chi REPL owns the session
|
|
679
689
|
# @raise [RestartRefused]
|
|
@@ -683,23 +693,109 @@ module Samagotchi
|
|
|
683
693
|
Session.load(session_id, state_dir: sd) # ArgumentError for none
|
|
684
694
|
owner = refuse_tui!(session_id, state_dir: sd)
|
|
685
695
|
dir = Session.session_dir(session_id, state_dir: sd)
|
|
686
|
-
|
|
687
|
-
|
|
696
|
+
link = WorkerLink.resolve(dir, unlink: false, owner: owner)
|
|
697
|
+
if link.is_a?(WorkerLink::Skew)
|
|
698
|
+
return restart_fallback(session_id, owner: owner, sidecar: link.sidecar, state_dir: sd, wait: wait, why: :skew)
|
|
699
|
+
end
|
|
700
|
+
|
|
701
|
+
if link.is_a?(WorkerLink::Starting)
|
|
702
|
+
return restart_fallback(session_id, owner: owner, sidecar: nil, state_dir: sd, wait: wait, why: :starting_stuck) if link.stuck
|
|
703
|
+
|
|
704
|
+
raise RestartRefused.new(session_id, :starting)
|
|
705
|
+
end
|
|
706
|
+
raise RestartRefused.new(session_id, :not_running) unless link.live?
|
|
707
|
+
|
|
708
|
+
sidecar = link.sidecar
|
|
688
709
|
unless sidecar.features.include?("restart")
|
|
689
710
|
raise RestartRefused.new(session_id, :unsupported, version: sidecar.version)
|
|
690
711
|
end
|
|
691
712
|
|
|
692
|
-
|
|
713
|
+
begin
|
|
714
|
+
reply = BridgeClient.for_sidecar(session_id, sidecar).request_exit(client_id: client_id, restart: true)
|
|
715
|
+
rescue SystemCallError, IOError => e
|
|
716
|
+
return restart_fallback(session_id, owner: owner, sidecar: sidecar, state_dir: sd, wait: wait, why: :unreachable,
|
|
717
|
+
error: e)
|
|
718
|
+
end
|
|
693
719
|
raise RestartRefused.new(session_id, reply.json&.fetch("reason", nil) || :failed) if reply.status == 409
|
|
720
|
+
if reply.secret_refused?
|
|
721
|
+
return restart_fallback(session_id, owner: owner, sidecar: sidecar, state_dir: sd, wait: wait, why: :bridge_secret)
|
|
722
|
+
end
|
|
694
723
|
raise RestartRefused.new(session_id, :failed) unless reply.status == 200
|
|
695
724
|
|
|
696
725
|
Log.info(:worker, "restart", sid: session_id, from: sidecar.version, by: client_id)
|
|
697
|
-
fresh =
|
|
698
|
-
|
|
699
|
-
|
|
726
|
+
fresh = WorkerLink.await(session_id, session_dir: dir, timeout: wait, unlink: false, interval: 0.1,
|
|
727
|
+
until: ->(live) { live.epoch != sidecar.epoch || live.started_at != sidecar.started_at })
|
|
728
|
+
# Timed out: the last outcome, the old worker's Live too, is no successor.
|
|
729
|
+
changed = fresh.live? && (fresh.epoch != sidecar.epoch || fresh.started_at != sidecar.started_at)
|
|
730
|
+
version = fresh.version if changed
|
|
731
|
+
Restarted.new(session_id: session_id, from_version: sidecar.version, version: version)
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
def self.restart_fallback(session_id, owner:, sidecar:, state_dir:, wait:, why:, error: nil)
|
|
735
|
+
Log.info(:worker, "restart_fallback", sid: session_id, why: why, error: error&.class&.name, msg: error&.message)
|
|
736
|
+
dir = Session.session_dir(session_id, state_dir: state_dir)
|
|
737
|
+
session = Session.load(session_id, state_dir: state_dir)
|
|
738
|
+
holds = restart_fallback_holds(session, dir)
|
|
739
|
+
raise RestartRefused.new(session_id, :fallback_held, holds: holds) unless holds.empty?
|
|
740
|
+
raise RestartRefused.new(session_id, :failed, version: sidecar&.version) unless (owner&.pid || 0).to_i.positive?
|
|
741
|
+
|
|
742
|
+
# The ask may have taken READ_TIMEOUT: TERM only the owner read at the
|
|
743
|
+
# start; a successor that took the lock meanwhile is awaited, not killed.
|
|
744
|
+
current = session_owner(session_id, state_dir: state_dir)
|
|
745
|
+
if current.nil?
|
|
746
|
+
resume_session(session_id, state_dir: state_dir)
|
|
747
|
+
elsif current.pid == owner.pid && current.started_at == owner.started_at
|
|
748
|
+
begin
|
|
749
|
+
Process.kill("TERM", owner.pid)
|
|
750
|
+
rescue Errno::ESRCH
|
|
751
|
+
nil
|
|
752
|
+
rescue SystemCallError => e
|
|
753
|
+
Log.warn(:worker, "restart_fallback_failed", sid: session_id, why: why, error: e.class.name, msg: e.message)
|
|
754
|
+
raise RestartRefused.new(session_id, :failed, version: sidecar&.version)
|
|
755
|
+
end
|
|
756
|
+
unless wait_for_owner_release(session_id, timeout: FALLBACK_STOP_WAIT, state_dir: state_dir)
|
|
757
|
+
raise RestartRefused.new(session_id, :failed, version: sidecar&.version)
|
|
758
|
+
end
|
|
759
|
+
|
|
760
|
+
resume_session(session_id, state_dir: state_dir)
|
|
761
|
+
end
|
|
762
|
+
previous_epoch = sidecar&.epoch
|
|
763
|
+
previous_started_at = sidecar&.started_at
|
|
764
|
+
fresh = WorkerLink.await(session_id, session_dir: dir, timeout: wait, unlink: false, interval: 0.1,
|
|
765
|
+
until: ->(live) { live.epoch != previous_epoch || live.started_at != previous_started_at })
|
|
766
|
+
changed = fresh.live? && (fresh.epoch != previous_epoch || fresh.started_at != previous_started_at)
|
|
767
|
+
Restarted.new(session_id: session_id, from_version: sidecar&.version, version: (fresh.version if changed))
|
|
768
|
+
end
|
|
769
|
+
private_class_method :restart_fallback
|
|
770
|
+
|
|
771
|
+
def self.restart_fallback_holds(session, session_dir)
|
|
772
|
+
holds = []
|
|
773
|
+
holds << :turn_running if session.status == Session::STATUS_RUNNING
|
|
774
|
+
holds << :question_pending if session.pending_question
|
|
775
|
+
holds << :input_queued unless SessionInbox.find_new_input_files(session_dir).empty?
|
|
776
|
+
holds << :background_tasks if running_created_task?(session)
|
|
777
|
+
holds
|
|
778
|
+
end
|
|
779
|
+
private_class_method :restart_fallback_holds
|
|
780
|
+
|
|
781
|
+
def self.running_created_task?(session)
|
|
782
|
+
root = session.working_directory.to_s
|
|
783
|
+
return false if root.empty?
|
|
784
|
+
|
|
785
|
+
Tools::TaskRuntime.created_ids_in(session.messages).any? do |id|
|
|
786
|
+
next false unless Tools::TaskRuntime::TASK_ID.match?(id)
|
|
787
|
+
|
|
788
|
+
path = File.join(root, Tools::TaskRuntime::TASKS_DIR, id, Tools::TaskRuntime::METADATA_FILENAME)
|
|
789
|
+
next false unless File.file?(path)
|
|
790
|
+
|
|
791
|
+
Tools::TaskRecord.from_file(JSON.parse(File.read(path)))&.running?
|
|
792
|
+
rescue JSON::ParserError, SystemCallError => e
|
|
793
|
+
Log.warn(:worker, "restart_task_check_failed", sid: session.id, task_id: id,
|
|
794
|
+
error: e.class.name, msg: e.message)
|
|
795
|
+
true
|
|
700
796
|
end
|
|
701
|
-
Restarted.new(session_id: session_id, from_version: sidecar.version, version: fresh&.version)
|
|
702
797
|
end
|
|
798
|
+
private_class_method :running_created_task?
|
|
703
799
|
|
|
704
800
|
# Stop a session by sending TERM to its process.
|
|
705
801
|
# With +wait+ (seconds), also wait for the owner to let go of the session,
|
|
@@ -741,7 +837,7 @@ module Samagotchi
|
|
|
741
837
|
private_class_method def self.cancel_running_turn(session_id, state_dir:)
|
|
742
838
|
return unless Session.load(session_id, state_dir: state_dir).status == Session::STATUS_RUNNING
|
|
743
839
|
|
|
744
|
-
client =
|
|
840
|
+
client = WorkerLink.client(session_id, session_dir: Session.session_dir(session_id, state_dir: state_dir))
|
|
745
841
|
return unless client && client.cancel(reason: STOP_CANCEL_REASON).status == 202
|
|
746
842
|
|
|
747
843
|
BridgeClient.poll(STOP_CANCEL_WAIT) do
|
|
@@ -757,7 +853,7 @@ module Samagotchi
|
|
|
757
853
|
# session keeps. "pid" is the pid file workers wrote until 2026-10-01:
|
|
758
854
|
# nothing writes or reads it now, but a folder from before still has one
|
|
759
855
|
# and is still empty.
|
|
760
|
-
EMPTY_SKELETON_FILES = ["pid", "owner.lock",
|
|
856
|
+
EMPTY_SKELETON_FILES = ["pid", "owner.lock", WorkerLink::FILE, AnalyticsFile::NAME].freeze
|
|
761
857
|
EMPTY_DIRS = [SessionInbox::INPUT_DIR, SessionInbox::NOTES_DIR, "images"].freeze
|
|
762
858
|
EMPTY_SKELETON_DIRS = (EMPTY_DIRS + [SessionInbox::OUTPUT_DIR]).freeze
|
|
763
859
|
|
|
@@ -1252,7 +1348,10 @@ module Samagotchi
|
|
|
1252
1348
|
def self.deliver_turn(session_id, prompt:, client_id: nil, images: [], delivery: nil, no_interrupt: false,
|
|
1253
1349
|
author: nil, pending_limit: nil, state_dir: nil, manager: self, bridge: nil)
|
|
1254
1350
|
session_dir = Session.session_dir(session_id, state_dir: state_dir || Session.default_state_dir)
|
|
1255
|
-
bridge ||=
|
|
1351
|
+
bridge ||= lambda do
|
|
1352
|
+
link = WorkerLink.await(session_id, session_dir: session_dir, timeout: TURN_BRIDGE_WAIT)
|
|
1353
|
+
link.live? ? link.client(session_id) : nil
|
|
1354
|
+
end
|
|
1256
1355
|
manager.resume_session(session_id, state_dir: state_dir)
|
|
1257
1356
|
if (client = bridge.call)
|
|
1258
1357
|
# A worker from before queueing (no "delivery" feature) ignores the
|
|
@@ -1404,15 +1503,6 @@ module Samagotchi
|
|
|
1404
1503
|
OwnerLock.owner(Session.session_dir(session_id, state_dir: state_dir || Session.default_state_dir))
|
|
1405
1504
|
end
|
|
1406
1505
|
|
|
1407
|
-
# A worker owns the session now: the one rule for whether a saved
|
|
1408
|
-
# pending question waits for anyone (Session#waiting_question's live:).
|
|
1409
|
-
# A chi REPL's question is its own (nothing else can answer it), and one
|
|
1410
|
-
# a dead worker saved waits for no one.
|
|
1411
|
-
# @return [Boolean]
|
|
1412
|
-
def self.worker_live?(session_id, state_dir: nil)
|
|
1413
|
-
!!session_owner(session_id, state_dir: state_dir)&.worker?
|
|
1414
|
-
end
|
|
1415
|
-
|
|
1416
1506
|
# The session's owner (#session_owner), unless it is the interactive
|
|
1417
1507
|
# TUI, which nothing else may resume, stop or delete.
|
|
1418
1508
|
# @return [OwnerLock::Owner, nil]
|