samagotchi 0.155.0 → 0.157.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +57 -1
- data/bin/chi +2 -1
- data/docs/architecture.md +22 -13
- data/docs/bundles.md +3 -1
- data/docs/cli.md +9 -8
- data/docs/configuration.md +36 -4
- data/docs/guardrails.md +9 -5
- data/docs/hooks.md +11 -5
- data/docs/internals/anthropic.md +1 -1
- data/docs/internals/attached-context.md +3 -2
- data/docs/internals/background-tasks.md +1 -1
- data/docs/internals/bridge.md +83 -6
- data/docs/internals/bundles.md +28 -18
- data/docs/internals/context-telemetry.md +4 -3
- data/docs/internals/engine.md +641 -937
- data/docs/internals/llm-context-forget.md +8 -3
- data/docs/internals/objectives.md +2 -2
- data/docs/internals/participants.md +3 -2
- data/docs/internals/prompt-caching.md +4 -1
- data/docs/internals/remote-sessions.md +8 -3
- data/docs/internals/web-frontend.md +3 -2
- data/docs/internals/working-set.md +2 -2
- data/docs/memory.md +13 -2
- data/docs/plugins.md +35 -2
- data/docs/sessions.md +11 -11
- data/docs/sub-agent.md +17 -2
- data/docs/system-one.md +3 -2
- data/docs/testing.md +53 -2
- data/lib/samagotchi/answer_command.rb +2 -1
- data/lib/samagotchi/archive_store.rb +4 -3
- data/lib/samagotchi/bootstrap/config_writer.rb +1 -1
- data/lib/samagotchi/bootstrap_command.rb +3 -2
- data/lib/samagotchi/bridge/card_places.rb +10 -4
- data/lib/samagotchi/bridge/card_store.rb +3 -0
- data/lib/samagotchi/bridge/turn_accumulator.rb +109 -9
- data/lib/samagotchi/bridge/turn_drawings.rb +265 -0
- data/lib/samagotchi/bridge.rb +31 -6
- data/lib/samagotchi/broadcast/recipients.rb +8 -8
- data/lib/samagotchi/broadcast/triage_model.rb +1 -1
- data/lib/samagotchi/broadcast_command.rb +3 -3
- data/lib/samagotchi/bundle_command/check.rb +18 -0
- data/lib/samagotchi/bundle_command/try/sandbox.rb +2 -2
- data/lib/samagotchi/bundle_command/try/trace.rb +1 -1
- data/lib/samagotchi/bundle_command.rb +5 -2
- data/lib/samagotchi/bundles/known-names/manifest.yml +4 -7
- data/lib/samagotchi/bundles/known-names/{hooks/known_names.rb → plugin.rb} +17 -0
- data/lib/samagotchi/bundles/source-links/manifest.yml +4 -7
- data/lib/samagotchi/bundles/source-links/{hooks/source_links.rb → plugin.rb} +23 -4
- data/lib/samagotchi/bundles/system/config_modification_protocol.md +2 -2
- data/lib/samagotchi/bundles/system/manifest.yml +3 -3
- data/lib/samagotchi/bundles/system/memory_guide.md +1 -1
- data/lib/samagotchi/child_move.rb +13 -57
- data/lib/samagotchi/child_reports.rb +2 -2
- data/lib/samagotchi/children_status.rb +15 -21
- data/lib/samagotchi/cli/parent_wait.rb +9 -23
- data/lib/samagotchi/config.rb +20 -112
- data/lib/samagotchi/context_command.rb +1 -2
- data/lib/samagotchi/context_sources.rb +5 -4
- data/lib/samagotchi/context_window.rb +9 -12
- data/lib/samagotchi/desktop/macos.rb +1 -1
- data/lib/samagotchi/effective_model.rb +57 -22
- data/lib/samagotchi/engine/context_judge.rb +19 -11
- data/lib/samagotchi/engine/messages.rb +1 -1
- data/lib/samagotchi/engine/models.rb +5 -2
- data/lib/samagotchi/engine/objectives.rb +29 -11
- data/lib/samagotchi/engine/plugins.rb +1 -1
- data/lib/samagotchi/engine/recap.rb +8 -4
- data/lib/samagotchi/engine/side_requests.rb +21 -11
- data/lib/samagotchi/engine/toolkit.rb +10 -9
- data/lib/samagotchi/engine/turn_end.rb +1 -1
- data/lib/samagotchi/engine/turn_helpers.rb +7 -7
- data/lib/samagotchi/engine/verify.rb +1 -1
- data/lib/samagotchi/engine.rb +43 -12
- data/lib/samagotchi/events.rb +3 -2
- data/lib/samagotchi/extension_load.rb +6 -3
- data/lib/samagotchi/guardrails/targets.rb +24 -9
- data/lib/samagotchi/guardrails_command.rb +1 -1
- data/lib/samagotchi/host_address.rb +52 -0
- data/lib/samagotchi/host_config.rb +72 -1
- data/lib/samagotchi/host_model.rb +2 -6
- data/lib/samagotchi/host_registry.rb +58 -46
- data/lib/samagotchi/kernel_loop.rb +53 -124
- data/lib/samagotchi/llm/anthropic_wire.rb +2 -2
- data/lib/samagotchi/llm/chat_loop.rb +40 -21
- data/lib/samagotchi/llm/native_backend.rb +1 -1
- data/lib/samagotchi/llm_context_apply.rb +54 -1
- data/lib/samagotchi/llm_context_notice.rb +1 -1
- data/lib/samagotchi/llm_context_strategy.rb +29 -67
- data/lib/samagotchi/llm_context_view.rb +9 -0
- data/lib/samagotchi/memory_bundle/asset_installer.rb +71 -76
- data/lib/samagotchi/memory_bundle/builder.rb +2 -1
- data/lib/samagotchi/memory_bundle/file_actions.rb +193 -0
- data/lib/samagotchi/memory_bundle/file_digest.rb +13 -0
- data/lib/samagotchi/memory_bundle/file_state.rb +152 -0
- data/lib/samagotchi/memory_bundle/installed_copy.rb +2 -1
- data/lib/samagotchi/memory_bundle/installed_manifest.rb +8 -1
- data/lib/samagotchi/memory_bundle/installer.rb +185 -169
- data/lib/samagotchi/memory_bundle/manifest.rb +14 -4
- data/lib/samagotchi/memory_bundle/merger.rb +69 -65
- data/lib/samagotchi/memory_bundle/plugin_ref.rb +6 -5
- data/lib/samagotchi/memory_bundle/profile.rb +5 -5
- data/lib/samagotchi/memory_bundle/provenance.rb +15 -1
- data/lib/samagotchi/memory_bundle/remerge.rb +25 -12
- data/lib/samagotchi/memory_bundle/shipped_update.rb +51 -6
- data/lib/samagotchi/memory_bundle/status.rb +14 -20
- data/lib/samagotchi/memory_bundle/system_bundle.rb +24 -6
- data/lib/samagotchi/memory_bundle/trash.rb +23 -8
- data/lib/samagotchi/memory_bundle/uninstaller.rb +16 -11
- data/lib/samagotchi/memory_usage.rb +1 -1
- data/lib/samagotchi/model_lookup.rb +75 -0
- data/lib/samagotchi/model_price.rb +4 -10
- data/lib/samagotchi/model_profile.rb +7 -6
- data/lib/samagotchi/model_settings.rb +37 -2
- data/lib/samagotchi/models_command.rb +1 -1
- data/lib/samagotchi/note_command.rb +2 -3
- data/lib/samagotchi/objectives_audit.rb +4 -3
- data/lib/samagotchi/participant_mode.rb +2 -1
- data/lib/samagotchi/participants.rb +2 -2
- data/lib/samagotchi/plugin/api.rb +41 -6
- data/lib/samagotchi/plugin/context.rb +1 -1
- data/lib/samagotchi/plugin/loader.rb +33 -10
- data/lib/samagotchi/plugin/sessions.rb +3 -6
- data/lib/samagotchi/plugin/tool_spec.rb +5 -3
- data/lib/samagotchi/project_env.rb +1 -1
- data/lib/samagotchi/reasoning_history.rb +14 -20
- data/lib/samagotchi/recap_place.rb +27 -0
- data/lib/samagotchi/reminder_carry.rb +1 -1
- data/lib/samagotchi/reply_wait/liveness.rb +38 -0
- data/lib/samagotchi/reply_wait/poll.rb +44 -15
- data/lib/samagotchi/reply_wait/target.rb +113 -0
- data/lib/samagotchi/reply_wait.rb +22 -2
- data/lib/samagotchi/reprefill_meter.rb +45 -0
- data/lib/samagotchi/review_command.rb +1 -1
- data/lib/samagotchi/sampling_settings.rb +9 -16
- data/lib/samagotchi/self_report.rb +36 -37
- data/lib/samagotchi/send_command.rb +17 -23
- data/lib/samagotchi/session.rb +370 -445
- data/lib/samagotchi/session_access/local.rb +12 -11
- data/lib/samagotchi/session_access/remote.rb +22 -6
- data/lib/samagotchi/session_archive_command.rb +2 -2
- data/lib/samagotchi/session_chain.rb +7 -6
- data/lib/samagotchi/session_continue.rb +6 -5
- data/lib/samagotchi/session_delete_command.rb +1 -1
- data/lib/samagotchi/session_field.rb +133 -0
- data/lib/samagotchi/session_inbox.rb +49 -1
- data/lib/samagotchi/session_manager/delivery.rb +254 -0
- data/lib/samagotchi/session_manager/emptiness.rb +139 -0
- data/lib/samagotchi/session_manager/lifecycle.rb +274 -0
- data/lib/samagotchi/session_manager/listing.rb +118 -0
- data/lib/samagotchi/session_manager/removal.rb +190 -0
- data/lib/samagotchi/session_manager/spawn.rb +242 -0
- data/lib/samagotchi/session_manager/worker_command.rb +157 -0
- data/lib/samagotchi/session_manager.rb +28 -1541
- data/lib/samagotchi/session_metrics.rb +8 -5
- data/lib/samagotchi/session_paths.rb +57 -0
- data/lib/samagotchi/session_retention.rb +2 -2
- data/lib/samagotchi/session_row.rb +200 -0
- data/lib/samagotchi/session_setup.rb +15 -12
- data/lib/samagotchi/sessions_command.rb +11 -129
- data/lib/samagotchi/share_command.rb +1 -1
- data/lib/samagotchi/starting_marker.rb +65 -0
- data/lib/samagotchi/steer_cut_after.rb +7 -14
- data/lib/samagotchi/system_one/recorder.rb +49 -0
- data/lib/samagotchi/terminal_ui/attached_loop.rb +6 -26
- data/lib/samagotchi/terminal_ui/exit_lines.rb +1 -1
- data/lib/samagotchi/terminal_ui/join_replay.rb +86 -8
- data/lib/samagotchi/terminal_ui/local_link.rb +2 -2
- data/lib/samagotchi/terminal_ui.rb +5 -7
- data/lib/samagotchi/testing/install.rb +2 -1
- data/lib/samagotchi/testing/sessions.rb +1 -1
- data/lib/samagotchi/thinking.rb +16 -21
- data/lib/samagotchi/token_usage.rb +1 -1
- data/lib/samagotchi/tool_activity.rb +10 -4
- data/lib/samagotchi/tool_declarations.rb +2 -2
- data/lib/samagotchi/tool_output_cut.rb +17 -0
- data/lib/samagotchi/tool_response.rb +5 -3
- data/lib/samagotchi/tool_runner.rb +25 -5
- data/lib/samagotchi/toolkit_command.rb +1 -1
- data/lib/samagotchi/tools/delegate.rb +55 -45
- data/lib/samagotchi/tools/delegate_relay.rb +5 -5
- data/lib/samagotchi/tools/delegate_result.rb +5 -5
- data/lib/samagotchi/tools/delegate_wait.rb +41 -17
- data/lib/samagotchi/tools/index_hint.rb +14 -0
- data/lib/samagotchi/tools/list_sessions.rb +7 -7
- data/lib/samagotchi/tools/registry.rb +28 -11
- data/lib/samagotchi/tools/send_note.rb +4 -6
- data/lib/samagotchi/tools/tool_index.rb +52 -51
- data/lib/samagotchi/tools/tool_surface.rb +22 -8
- data/lib/samagotchi/turn_context.rb +2 -1
- data/lib/samagotchi/turn_stages/context_judge.rb +38 -36
- data/lib/samagotchi/turn_stages/objectives.rb +56 -52
- data/lib/samagotchi/turn_stages/toolkit_suggestion.rb +13 -17
- data/lib/samagotchi/turn_stages/verify_anchors.rb +9 -10
- data/lib/samagotchi/turn_stages/warmup.rb +14 -15
- data/lib/samagotchi/update_command.rb +11 -7
- data/lib/samagotchi/verify_fresh.rb +7 -11
- data/lib/samagotchi/version.rb +1 -1
- data/lib/samagotchi/vision_support.rb +11 -14
- data/lib/samagotchi/web/app.rb +45 -28
- data/lib/samagotchi/web/message_display.rb +57 -7
- data/lib/samagotchi/web/participant_access.rb +3 -3
- data/lib/samagotchi/web/public/activity.js +16 -7
- data/lib/samagotchi/web/public/app.js +29 -8
- data/lib/samagotchi/web/public/bubbles.js +8 -3
- data/lib/samagotchi/web/public/cards_place.js +15 -0
- data/lib/samagotchi/web/public/commands.js +5 -1
- data/lib/samagotchi/web/public/data.js +4 -2
- data/lib/samagotchi/web/public/history_shape.js +220 -0
- data/lib/samagotchi/web/public/recap_stamp.js +17 -0
- data/lib/samagotchi/web/public/reload_check.js +71 -0
- data/lib/samagotchi/web/public/stage_model.js +7 -5
- data/lib/samagotchi/web/public/stage_view.js +7 -3
- data/lib/samagotchi/web/public/stream_handlers.js +8 -2
- data/lib/samagotchi/web/public/timing.js +43 -2
- data/lib/samagotchi/web/public/turn_block.js +215 -30
- data/lib/samagotchi/web/public/turn_drawing.js +43 -0
- data/lib/samagotchi/web/public/turn_events.js +10 -1
- data/lib/samagotchi/web/public/turn_pieces.js +29 -0
- data/lib/samagotchi/web/reload_check_route.rb +95 -0
- data/lib/samagotchi/web/server.rb +4 -2
- data/lib/samagotchi/web/session_access_routes.rb +6 -6
- data/lib/samagotchi/web/session_control.rb +4 -4
- data/lib/samagotchi/web/session_create.rb +3 -3
- data/lib/samagotchi/web/session_hub.rb +9 -17
- data/lib/samagotchi/web/session_show.rb +67 -5
- data/lib/samagotchi/web/session_summary.rb +39 -28
- data/lib/samagotchi/worker.rb +83 -32
- data/lib/samagotchi/worker_boot.rb +184 -0
- data/lib/samagotchi/worker_link.rb +3 -0
- data/lib/samagotchi/worker_signals.rb +1 -1
- data/lib/samagotchi/worker_task_wakes.rb +2 -2
- metadata +32 -4
- data/lib/samagotchi/send_command/wait_target.rb +0 -85
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Samagotchi
|
|
4
|
+
class SessionManager
|
|
5
|
+
# How long a turn waits for the Bridge of a worker a resume just spawned.
|
|
6
|
+
TURN_BRIDGE_WAIT = 5.0
|
|
7
|
+
|
|
8
|
+
# What #deliver_turn did with a message.
|
|
9
|
+
# +status+ :accepted; :refused (the Bridge refused it, or a queue would
|
|
10
|
+
# reach a worker that doesn't queue); :timeout (the Bridge took it and
|
|
11
|
+
# never answered); :failed (the input file couldn't be written).
|
|
12
|
+
# +via+ :bridge (the worker's Bridge answered) or :file (the input-file
|
|
13
|
+
# fallback); nil when nothing went in.
|
|
14
|
+
# +ack+ the Bridge's reply body, or the file's {"status", "enqueued_id",
|
|
15
|
+
# "session_id"}: string keys either way, {} for none.
|
|
16
|
+
# +code+ a refusal's HTTP status.
|
|
17
|
+
DeliverResult = Data.define(:status, :via, :ack, :code) do
|
|
18
|
+
def initialize(status:, via: nil, ack: {}, code: nil) = super(status:, via:, ack: ack || {}, code:)
|
|
19
|
+
|
|
20
|
+
def accepted? = status == :accepted
|
|
21
|
+
|
|
22
|
+
# Sent as an input file (no Bridge): its turn and outcome carry the
|
|
23
|
+
# file's enqueued_id, which a wait follows (chi send --wait).
|
|
24
|
+
def file? = via == :file
|
|
25
|
+
|
|
26
|
+
# @return [String, nil] the id the message's turn will carry
|
|
27
|
+
def enqueued_id
|
|
28
|
+
id = ack["enqueued_id"].to_s
|
|
29
|
+
id.empty? ? nil : id
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# Whether the worker took the line as a session command (a POST /turn
|
|
33
|
+
# answered with its command_id).
|
|
34
|
+
def command? = !ack["command_id"].to_s.empty?
|
|
35
|
+
|
|
36
|
+
def error = ack["error"]
|
|
37
|
+
|
|
38
|
+
def detail = ack["detail"]
|
|
39
|
+
|
|
40
|
+
# What the worker applied (a Bridge ack only): the delivery asked for,
|
|
41
|
+
# or next_step for a --queue that found no turn running; nil from the
|
|
42
|
+
# input file or a worker older than delivery.
|
|
43
|
+
def delivery = ack["delivery"]
|
|
44
|
+
|
|
45
|
+
# A --cut's outcome ("now", "waits", "off"), nil when nothing was cut.
|
|
46
|
+
def cut = ack["cut"]
|
|
47
|
+
|
|
48
|
+
# The steer cut-after seconds, with a cut that waits only.
|
|
49
|
+
def cut_after = ack["cut_after"]
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The unclaimed input files +author+ wrote.
|
|
53
|
+
PARTICIPANT_QUEUE_LOCK = "participants.lock"
|
|
54
|
+
|
|
55
|
+
# Delivery: a user message into a session, the way a UI sends one (its
|
|
56
|
+
# Bridge, else the input file). Calls SpawnPart's resume_session and the
|
|
57
|
+
# facade's session_owner through +manager+ (this class by default).
|
|
58
|
+
# Meant to be loaded through samagotchi/session_manager.
|
|
59
|
+
module DeliveryPart
|
|
60
|
+
# Write a user turn into a session's input directory via the same file IPC
|
|
61
|
+
# the worker polls. Reused by the bridge's POST surface so a turn is
|
|
62
|
+
# fire-and-forget and never calls run_turn across the thread/process
|
|
63
|
+
# boundary. The file is JSON carrying the sender's ids.
|
|
64
|
+
# @param client_id [String, nil] the sending UI
|
|
65
|
+
# @param enqueued_id [String, nil] the id its ACK / :turn_enqueued carry
|
|
66
|
+
# @param delivery [String, nil] the wire value: "next_step" (default),
|
|
67
|
+
# "cut" or "queue"
|
|
68
|
+
# @param images [Array<Hash>] image refs ({file:, name:}) in the
|
|
69
|
+
# session's images/
|
|
70
|
+
# @return [String, false] the input file's path, or false.
|
|
71
|
+
def write_turn_input(session_id, prompt:, client_id: nil, enqueued_id: nil, no_interrupt: false, state_dir: nil,
|
|
72
|
+
images: [], delivery: nil, author: nil, anchors: nil)
|
|
73
|
+
session_dir = Session.session_dir(session_id, state_dir: state_dir || Session.default_state_dir)
|
|
74
|
+
SessionInbox.write_input(session_dir, prompt: prompt, client_id: client_id, enqueued_id: enqueued_id,
|
|
75
|
+
no_interrupt: no_interrupt, images: images, delivery: delivery, author: author,
|
|
76
|
+
anchors: anchors)
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Hand a user message to the session the way a UI does, for the web
|
|
80
|
+
# composer and `chi send` alike: wakes the worker when none runs, posts
|
|
81
|
+
# through its Bridge so every live UI sees :turn_enqueued, and falls back
|
|
82
|
+
# to the input file when the Bridge is gone (a worker closing it on idle
|
|
83
|
+
# exit). Race-safe against a TUI taking the session, or the worker
|
|
84
|
+
# exiting, between the resume and the write.
|
|
85
|
+
# @param images [Array<Hash>] refs ({file:, name:}) already in images/
|
|
86
|
+
# @param delivery [String, nil] what the message asks the running turn
|
|
87
|
+
# for: "next_step" (default), "cut" or "queue"
|
|
88
|
+
# @param no_interrupt [Boolean] the turn runs with the raised iteration
|
|
89
|
+
# limit (an attached TUI's --no-interrupt), through the Bridge or the
|
|
90
|
+
# file; sent only when true, so older callees see no new field
|
|
91
|
+
# @param author [String, nil] who wrote it when it isn't the session's
|
|
92
|
+
# owner (Web::Principal#author: chi web stamps it, never the body);
|
|
93
|
+
# to the Bridge or the file only when given, so an owner's turn is
|
|
94
|
+
# sent and saved as before
|
|
95
|
+
# @param pending_limit [Integer, nil] with +author+: how many of their
|
|
96
|
+
# turns may be running or waiting (the Bridge counts them; with no
|
|
97
|
+
# worker, their waiting files): :refused 429 too_many_turns past it
|
|
98
|
+
# @param anchors [TurnAnchors, Hash, nil] a Verify turn's claim, a Revise
|
|
99
|
+
# turn's draft: each refused (409, its UNAVAILABLE) by a worker without
|
|
100
|
+
# its feature
|
|
101
|
+
# @param manager [#resume_session, #write_turn_input] this class, or a
|
|
102
|
+
# stand-in (the web's specs); #session_owner is optional
|
|
103
|
+
# @param bridge [#call, nil] returns the BridgeClient or nil; defaults to
|
|
104
|
+
# the session's sidecar, waiting TURN_BRIDGE_WAIT for a new worker's
|
|
105
|
+
# @return [DeliverResult] :accepted (via the Bridge or a file),
|
|
106
|
+
# :refused when the Bridge refused the request (any 4xx with an
|
|
107
|
+
# error: bad images, a delivery it doesn't run) or a queue would reach
|
|
108
|
+
# a worker that doesn't queue (no file then: it would steer), :timeout
|
|
109
|
+
# when it took the request but never answered (no file then), or
|
|
110
|
+
# :failed when the input file couldn't be written
|
|
111
|
+
# @raise [OwnedByTUI] a one-turn chi run owns the session
|
|
112
|
+
# @raise [ArgumentError] no such session
|
|
113
|
+
def deliver_turn(session_id, prompt:, client_id: nil, images: [], delivery: nil, no_interrupt: false,
|
|
114
|
+
author: nil, pending_limit: nil, anchors: nil, state_dir: nil, manager: self, bridge: nil)
|
|
115
|
+
anchors = TurnAnchors.from(anchors)
|
|
116
|
+
session_dir = Session.session_dir(session_id, state_dir: state_dir || Session.default_state_dir)
|
|
117
|
+
bridge ||= lambda do
|
|
118
|
+
link = WorkerLink.await(session_id, session_dir: session_dir, timeout: TURN_BRIDGE_WAIT)
|
|
119
|
+
link.live? ? link.client(session_id) : nil
|
|
120
|
+
end
|
|
121
|
+
manager.resume_session(session_id, state_dir: state_dir)
|
|
122
|
+
if (client = bridge.call)
|
|
123
|
+
if (missing = anchors.present.find { |anchor| !worker_has?(client, anchor.class::FEATURE) })
|
|
124
|
+
return DeliverResult.new(status: :refused, code: 409, ack: missing.class::UNAVAILABLE)
|
|
125
|
+
end
|
|
126
|
+
# A worker from before queueing (no "delivery" feature) ignores the
|
|
127
|
+
# field or refuses it; either way a file would steer the running turn.
|
|
128
|
+
return queue_unavailable if Delivery.queue?(delivery) && !worker_has?(client, Delivery::QUEUE_FEATURE)
|
|
129
|
+
# A participant's turn only to a worker with the review boundary.
|
|
130
|
+
return review_unavailable if Steer.participant_author?(author) && !worker_has?(client, Delivery::REVIEW_FEATURE)
|
|
131
|
+
|
|
132
|
+
begin
|
|
133
|
+
options = { prompt: prompt, client_id: client_id }
|
|
134
|
+
options[:images] = images unless images.empty?
|
|
135
|
+
# An old worker ignores the field: it cuts anyway (its own default).
|
|
136
|
+
options[:delivery] = delivery unless Delivery.next_step?(delivery)
|
|
137
|
+
options[:no_interrupt] = true if no_interrupt
|
|
138
|
+
options[:author] = author if author
|
|
139
|
+
options[:pending_limit] = pending_limit if author && pending_limit
|
|
140
|
+
options[:anchors] = anchors unless anchors.empty?
|
|
141
|
+
reply = client.post_turn(**options)
|
|
142
|
+
ack = reply.json
|
|
143
|
+
return DeliverResult.new(status: :accepted, via: :bridge, ack: ack) if reply.status == 202 && ack.is_a?(Hash)
|
|
144
|
+
# Read after its deadline and dropped: a file would run it after all.
|
|
145
|
+
return turn_timeout if ack.is_a?(Hash) && ack["error"] == "deadline_passed"
|
|
146
|
+
# Any other 4xx with an error is a refusal (bad images, a delivery
|
|
147
|
+
# it doesn't run, a client bug): a file would bypass it, and a
|
|
148
|
+
# refused queue would steer. A 413 (a body over the Bridge's
|
|
149
|
+
# limit, a big piped chi send) and a 5xx keep the file fallback
|
|
150
|
+
# below, which carries the delivery (a queue too).
|
|
151
|
+
if (400..499).cover?(reply.status) && reply.status != 413 && ack.is_a?(Hash) && ack["error"]
|
|
152
|
+
return DeliverResult.new(status: :refused, via: :bridge, code: reply.status, ack: ack)
|
|
153
|
+
end
|
|
154
|
+
rescue Errno::ETIMEDOUT
|
|
155
|
+
# The Bridge drops a turn it reads after the request's deadline
|
|
156
|
+
# (BridgeClient#post_turn), so a worker that wakes later won't run
|
|
157
|
+
# it; a file would.
|
|
158
|
+
return turn_timeout
|
|
159
|
+
rescue SystemCallError, IOError
|
|
160
|
+
nil # the worker closed its Bridge on the way out: queue the file
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
# A participant's file only for a worker started after it: a worker
|
|
164
|
+
# still holding the session with its Bridge gone (exiting) may be an
|
|
165
|
+
# older one, which would run it without the review boundary.
|
|
166
|
+
# (A one-turn chi run holding it is refused as for anyone, below.)
|
|
167
|
+
if Steer.participant_author?(author) && (holder = delivery_owner(manager, session_id, state_dir)) && !holder.tui?
|
|
168
|
+
return DeliverResult.new(status: :timeout, via: :file,
|
|
169
|
+
ack: { "error" => "worker_timeout",
|
|
170
|
+
"detail" => "the session's worker is exiting, so the message was not sent; send it again" })
|
|
171
|
+
end
|
|
172
|
+
enqueued_id = SecureRandom.uuid
|
|
173
|
+
input = { prompt: prompt, client_id: client_id, enqueued_id: enqueued_id, state_dir: state_dir }
|
|
174
|
+
input[:images] = images unless images.empty?
|
|
175
|
+
input[:delivery] = delivery unless Delivery.next_step?(delivery)
|
|
176
|
+
input[:no_interrupt] = true if no_interrupt
|
|
177
|
+
input[:author] = author if author
|
|
178
|
+
input[:anchors] = anchors unless anchors.empty?
|
|
179
|
+
path = participant_queue_lock(session_dir, author && pending_limit) do
|
|
180
|
+
# No worker: the participant's waiting files are all their turns,
|
|
181
|
+
# counted and written under one lock (the Bridge counts under its).
|
|
182
|
+
next :too_many if author && pending_limit && pending_files(session_dir, author) >= pending_limit
|
|
183
|
+
|
|
184
|
+
manager.write_turn_input(session_id, **input)
|
|
185
|
+
end
|
|
186
|
+
if path == :too_many
|
|
187
|
+
return DeliverResult.new(status: :refused, code: 429,
|
|
188
|
+
ack: { "error" => "too_many_turns", "detail" => ParticipantMessages::TOO_MANY_TURNS })
|
|
189
|
+
end
|
|
190
|
+
return DeliverResult.new(status: :failed) unless path
|
|
191
|
+
|
|
192
|
+
owner = delivery_owner(manager, session_id, state_dir)
|
|
193
|
+
# A TUI that took the session between the resume and the write never
|
|
194
|
+
# reads input files; a later worker would replay this one.
|
|
195
|
+
if owner&.tui?
|
|
196
|
+
FileUtils.rm_f(path) if path.is_a?(String)
|
|
197
|
+
raise OwnedByTUI, session_id
|
|
198
|
+
end
|
|
199
|
+
# A worker that idle-exited since the resume never reads it either:
|
|
200
|
+
# wake a new one. (The exiting worker also looks for input it left;
|
|
201
|
+
# one the first resume spawned that is still starting is left be.)
|
|
202
|
+
manager.resume_session(session_id, state_dir: state_dir) if owner.nil?
|
|
203
|
+
DeliverResult.new(status: :accepted, via: :file,
|
|
204
|
+
ack: { "status" => "accepted", "enqueued_id" => enqueued_id, "session_id" => session_id })
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Whether the worker +client+ talks to names +feature+: the features of
|
|
208
|
+
# the sidecar the client was made from (the Live link WorkerLink.await
|
|
209
|
+
# found), not a second read of bridge.json.
|
|
210
|
+
private def worker_has?(client, feature) = client.features.include?(feature)
|
|
211
|
+
|
|
212
|
+
# The block under the session's participant queue lock (a file lock:
|
|
213
|
+
# chi web's threads, and another chi web), when +locked+; else as is.
|
|
214
|
+
private def participant_queue_lock(session_dir, locked)
|
|
215
|
+
return yield unless locked
|
|
216
|
+
|
|
217
|
+
FileUtils.mkdir_p(session_dir)
|
|
218
|
+
File.open(File.join(session_dir, PARTICIPANT_QUEUE_LOCK), File::RDWR | File::CREAT, 0o600) do |lock|
|
|
219
|
+
lock.flock(File::LOCK_EX)
|
|
220
|
+
yield
|
|
221
|
+
end
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
private def pending_files(session_dir, author)
|
|
225
|
+
SessionInbox.find_new_input_files(session_dir).count do |file|
|
|
226
|
+
data = JSON.parse(File.read(file))
|
|
227
|
+
data.is_a?(Hash) && data["author"] == author
|
|
228
|
+
rescue JSON::ParserError, SystemCallError
|
|
229
|
+
false
|
|
230
|
+
end
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
private def queue_unavailable
|
|
234
|
+
DeliverResult.new(status: :refused, code: 409, ack: Delivery::QUEUE_UNAVAILABLE)
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
private def review_unavailable
|
|
238
|
+
DeliverResult.new(status: :refused, code: 409, ack: Delivery::REVIEW_UNAVAILABLE)
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
private def turn_timeout
|
|
242
|
+
DeliverResult.new(status: :timeout, via: :bridge,
|
|
243
|
+
ack: { "error" => "worker_timeout",
|
|
244
|
+
"detail" => "the session's worker did not answer, so the message was not sent" })
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
private def delivery_owner(manager, session_id, state_dir)
|
|
248
|
+
Degrade.call(:worker, "owner_unreadable", sid: session_id) { manager.session_owner(session_id, state_dir: state_dir) }
|
|
249
|
+
end
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
extend DeliveryPart
|
|
253
|
+
end
|
|
254
|
+
end
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../analytics_file"
|
|
4
|
+
require_relative "../session_inbox"
|
|
5
|
+
require_relative "../worker_link"
|
|
6
|
+
|
|
7
|
+
module Samagotchi
|
|
8
|
+
class SessionManager
|
|
9
|
+
# What a session's directory holds before anything happened in it. Any
|
|
10
|
+
# other entry (or a file in one of the EMPTY_DIRS) is something the
|
|
11
|
+
# session keeps. "pid" is the pid file workers wrote until 2026-10-01:
|
|
12
|
+
# nothing writes or reads it now, but a folder from before still has one
|
|
13
|
+
# and is still empty. A turn's drawing (Bridge::TurnDrawings::FILE)
|
|
14
|
+
# exists only after a turn that kept messages: never a skeleton file.
|
|
15
|
+
EMPTY_SKELETON_FILES = ["pid", "owner.lock", WorkerLink::FILE, AnalyticsFile::NAME].freeze
|
|
16
|
+
EMPTY_DIRS = [SessionInbox::INPUT_DIR, SessionInbox::NOTES_DIR, "images"].freeze
|
|
17
|
+
EMPTY_SKELETON_DIRS = (EMPTY_DIRS + [SessionInbox::OUTPUT_DIR]).freeze
|
|
18
|
+
|
|
19
|
+
# Emptiness: whether nothing happened in a session (pure predicates over
|
|
20
|
+
# its file and directory) and whether one being left is deleted, asked by
|
|
21
|
+
# WorkerCommandPart and the retention sweep. Calls no other part. Meant
|
|
22
|
+
# to be loaded through samagotchi/session_manager.
|
|
23
|
+
module EmptinessPart
|
|
24
|
+
# session.keep_empty off: sessions left with nothing in them are deleted
|
|
25
|
+
# (the worker as it exits, the retention sweep).
|
|
26
|
+
def discard_empty?
|
|
27
|
+
Degrade.call(:config, "keep_empty_unreadable", fallback: false) { Samagotchi::Config.get("session.keep_empty") != true }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Whether a session being left (or found left) is deleted: session.keep_empty
|
|
31
|
+
# is off and nothing happened in it (#empty_session?). The worker as it
|
|
32
|
+
# leaves and the retention sweep both ask this.
|
|
33
|
+
# @param default_model [String, nil] what a new session starts on
|
|
34
|
+
# @param used_memory_names [Array<String>] memory its owner's Engine used
|
|
35
|
+
# before any save
|
|
36
|
+
def discardable?(session_id, default_model:, state_dir: nil, used_memory_names: [])
|
|
37
|
+
return false unless discard_empty? && Array(used_memory_names).empty?
|
|
38
|
+
|
|
39
|
+
empty_session?(session_id, state_dir: state_dir || Session.default_state_dir, default_model: default_model)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
# A session nothing happened in: no conversation, no turn tried (a failed
|
|
43
|
+
# one leaves no messages but a last_prompt and analytics turns), nothing
|
|
44
|
+
# queued or attached, and the model and mode a new session gets. A
|
|
45
|
+
# session prepared for later (/model, --model, a note, an image, memory)
|
|
46
|
+
# is not empty. Anything unreadable or unknown counts as not empty.
|
|
47
|
+
# @param default_model [String, nil] what a new session starts on
|
|
48
|
+
def empty_session?(session_id, state_dir: nil, default_model: nil)
|
|
49
|
+
sd = state_dir || Session.default_state_dir
|
|
50
|
+
session = Session.load(session_id, state_dir: sd)
|
|
51
|
+
return false unless no_conversation?(session.messages) && session.pending_question.nil? && session.used_memory_names.empty?
|
|
52
|
+
return false unless session.last_prompt.to_s.strip.empty? && session.first_preview.to_s.strip.empty?
|
|
53
|
+
return false unless session.mode.to_s == "assist" && !default_model.nil? && same_model?(session.model_name, default_model)
|
|
54
|
+
|
|
55
|
+
empty_session_dir?(Session.session_dir(session_id, state_dir: sd))
|
|
56
|
+
rescue ArgumentError, SystemCallError, JSON::ParserError
|
|
57
|
+
false
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Whether two model names resolve to the same ref (ModelRef#ref): a
|
|
61
|
+
# session stores the resolved ref, the default may be an alias.
|
|
62
|
+
def same_model?(one, other)
|
|
63
|
+
return true if one.to_s == other.to_s
|
|
64
|
+
return false if one.nil? || other.nil?
|
|
65
|
+
|
|
66
|
+
Degrade.call(:config, "model_ref_unresolved", fallback: false) do
|
|
67
|
+
HostConfig.model_ref(one).ref == HostConfig.model_ref(other).ref
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Only the system prompt (an Engine seeds one, a saved session keeps
|
|
72
|
+
# it); a context note is a system message too, but with its kind.
|
|
73
|
+
# A turn note (the tail system message after a failed, cancelled or empty
|
|
74
|
+
# turn) is not a conversation either: a session whose only turn failed
|
|
75
|
+
# before any answer is still empty.
|
|
76
|
+
def no_conversation?(messages)
|
|
77
|
+
Array(messages).all? do |msg|
|
|
78
|
+
(msg[:role] || msg["role"]).to_s == "system" &&
|
|
79
|
+
((msg[:kind] || msg["kind"]).nil? || TurnNote.note?(msg))
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# @return [Boolean] whether a session's directory holds only the skeleton
|
|
84
|
+
def empty_session_dir?(dir)
|
|
85
|
+
return true unless Dir.exist?(dir)
|
|
86
|
+
|
|
87
|
+
Dir.children(dir).all? do |name|
|
|
88
|
+
path = File.join(dir, name)
|
|
89
|
+
if EMPTY_SKELETON_DIRS.include?(name)
|
|
90
|
+
File.directory?(path) && (!EMPTY_DIRS.include?(name) || Dir.empty?(path))
|
|
91
|
+
elsif name == AnalyticsFile::NAME
|
|
92
|
+
analytics_empty?(path)
|
|
93
|
+
# Cards and notices from before any turn (Bridge::CardStore).
|
|
94
|
+
elsif name == "cards.json"
|
|
95
|
+
cards_empty?(path)
|
|
96
|
+
else
|
|
97
|
+
EMPTY_SKELETON_FILES.include?(name)
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# A folder's analytics.json is empty only when it is a JSON object with
|
|
103
|
+
# an empty turn_records array. Anything unreadable, not JSON, not an
|
|
104
|
+
# object, the wrong type, or turn_records the wrong shape reads as
|
|
105
|
+
# "not empty": a broken analytics keeps the folder instead of crashing
|
|
106
|
+
# the retention sweep (or of a wrong-shape one reading as empty).
|
|
107
|
+
def analytics_empty?(path)
|
|
108
|
+
data = read_json(path)
|
|
109
|
+
turns = data.is_a?(Hash) ? data["turn_records"] : nil
|
|
110
|
+
turns.is_a?(Array) && turns.empty?
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# A folder's cards.json is empty only when it is a JSON object with no
|
|
114
|
+
# turn in it: an empty turn_log, or an older file's (Bridge::CardStore
|
|
115
|
+
# version 1) turns an integer zero. Anything unreadable, not JSON, not an
|
|
116
|
+
# object, the wrong type, or neither of those reads as "not empty": a
|
|
117
|
+
# broken or wrong-shape cards file keeps the folder instead of raising
|
|
118
|
+
# (Array#/Hash#to_i) or reading as empty.
|
|
119
|
+
def cards_empty?(path)
|
|
120
|
+
data = read_json(path)
|
|
121
|
+
return false unless data.is_a?(Hash)
|
|
122
|
+
|
|
123
|
+
turns = data["turns"]
|
|
124
|
+
data["turn_log"] == [] || (turns.is_a?(Integer) && turns.zero?)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# A session-dir JSON file's contents, or nil when it is missing,
|
|
128
|
+
# unreadable, not JSON or not an object (mirrors AnalyticsFile#read): a
|
|
129
|
+
# broken file reads as none, so its folder is kept, not deleted.
|
|
130
|
+
def read_json(path)
|
|
131
|
+
JSON.parse(File.read(path))
|
|
132
|
+
rescue JSON::ParserError, SystemCallError
|
|
133
|
+
nil
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
extend EmptinessPart
|
|
138
|
+
end
|
|
139
|
+
end
|
|
@@ -0,0 +1,274 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../worker_idle_exit"
|
|
4
|
+
|
|
5
|
+
module Samagotchi
|
|
6
|
+
class SessionManager
|
|
7
|
+
# A restart that can't happen now. reason: :not_running (no live worker:
|
|
8
|
+
# the next prompt starts one on the newest chi anyway), :unsupported (a
|
|
9
|
+
# worker from before restarts, running +version+: stop it instead), a
|
|
10
|
+
# WorkerIdleExit#hold_for_restart hold (:turn_running, :reminders, ...),
|
|
11
|
+
# :fallback_held (disk-visible work blocks an unresponsive-worker restart),
|
|
12
|
+
# :stopped (its worker, +pid+, is SIGSTOPped: neither the ask nor a TERM
|
|
13
|
+
# reaches it until it is continued), or :failed (the worker answered something else or didn't release).
|
|
14
|
+
class RestartRefused < StandardError
|
|
15
|
+
attr_reader :session_id, :reason, :version, :holds, :pid
|
|
16
|
+
|
|
17
|
+
def initialize(session_id, reason, version: nil, holds: [], pid: nil, via: ChiInvocation::HERE)
|
|
18
|
+
@session_id = session_id
|
|
19
|
+
@reason = reason.to_sym
|
|
20
|
+
@version = version
|
|
21
|
+
@holds = Array(holds).map(&:to_sym).freeze
|
|
22
|
+
@pid = pid
|
|
23
|
+
super(RestartRefused.words(session_id, @reason, version, @holds, pid: pid, via: via))
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
HOLDS = {
|
|
27
|
+
starting: "its worker is still starting",
|
|
28
|
+
turn_running: "a turn is running",
|
|
29
|
+
input_queued: "a prompt is queued",
|
|
30
|
+
continue_offered: "the step-limit question waits for an answer",
|
|
31
|
+
question_pending: "a question or approval waits for an answer",
|
|
32
|
+
# Only a worker from before ReminderCarry still holds a restart for its reminders.
|
|
33
|
+
reminders: "it has reminders, which its worker's older chi can't carry over a restart",
|
|
34
|
+
command_running: "a /btw (or another command) is still running",
|
|
35
|
+
relays_open: "a delegate's approval relayed here is open or just answered",
|
|
36
|
+
background_tasks: "background tasks it started are still running",
|
|
37
|
+
scratch: "a chi scratch session's worker ends with its terminal"
|
|
38
|
+
}.merge(WorkerIdleExit::LEAVING_WORDS).freeze
|
|
39
|
+
|
|
40
|
+
# What the CLI and the web say.
|
|
41
|
+
def self.words(session_id, reason, version = nil, holds = [], pid: nil, via: ChiInvocation::HERE)
|
|
42
|
+
stop = via.command("sessions stop #{session_id}")
|
|
43
|
+
case reason
|
|
44
|
+
when :stopped
|
|
45
|
+
where = via.remote? ? " on #{via.remote}" : ""
|
|
46
|
+
"session #{session_id}'s worker#{" (pid #{pid})" if pid} is stopped (SIGSTOP or Ctrl-Z) and can't take " \
|
|
47
|
+
"a restart: continue it#{" with `kill -CONT #{pid}`" if pid}#{where}, then restart again"
|
|
48
|
+
when :not_running
|
|
49
|
+
"session #{session_id} has no running worker; the next prompt starts one on the newest chi"
|
|
50
|
+
when :unsupported
|
|
51
|
+
"session #{session_id}'s worker runs chi #{version || "(older)"}, which can't restart: " \
|
|
52
|
+
"#{stop}, then any prompt starts it on the newest chi"
|
|
53
|
+
when :fallback_held
|
|
54
|
+
why = holds.map { |hold| HOLDS.fetch(hold, hold.to_s.tr("_", " ")) }.join("; ")
|
|
55
|
+
"session #{session_id} has disk-visible work that a restart might lose#{": #{why}" unless why.empty?}; " \
|
|
56
|
+
"run `#{stop}`, then any prompt starts it on the newest chi"
|
|
57
|
+
when :failed then "session #{session_id}'s worker didn't take the restart"
|
|
58
|
+
else "not now: #{HOLDS.fetch(reason, reason.to_s.tr("_", " "))}"
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# What a restart did: the worker's version before and after (nil when
|
|
64
|
+
# the new worker hadn't published its Bridge within the wait).
|
|
65
|
+
Restarted = Struct.new(:session_id, :from_version, :version, keyword_init: true)
|
|
66
|
+
|
|
67
|
+
RESTART_WAIT = 10.0
|
|
68
|
+
# A TERMed worker writes its recap on leave (an LLM call) while it still
|
|
69
|
+
# holds the lock: STOP_WAIT plus the recap's own wait (SessionContinue::RECAP_WAIT).
|
|
70
|
+
FALLBACK_STOP_WAIT = 40
|
|
71
|
+
|
|
72
|
+
STOP_CANCEL_WAIT = 3.0
|
|
73
|
+
# How long a stop from the CLI (chi sessions stop, delete, a continue's
|
|
74
|
+
# orphan) waits for the worker to let go of the session, here or
|
|
75
|
+
# through chi web (POST …/stop's wait, capped at it).
|
|
76
|
+
STOP_WAIT = 10
|
|
77
|
+
# The cancel reason of a turn a session stop cancels (Bridge::CANCEL_REASONS):
|
|
78
|
+
# not the user's Stop, which only ends the turn ("user").
|
|
79
|
+
STOP_CANCEL_REASON = "session_stopped"
|
|
80
|
+
|
|
81
|
+
# Lifecycle: stop a session's worker (TERM, then CONT) and restart it on
|
|
82
|
+
# the newest chi (its Bridge's POST /exit, else the fallback). Calls
|
|
83
|
+
# SpawnPart's resume_session and the facade's ownership checks. Meant to
|
|
84
|
+
# be loaded through samagotchi/session_manager.
|
|
85
|
+
module LifecyclePart
|
|
86
|
+
# Hand a session to a new worker on the newest chi installed: ask its
|
|
87
|
+
# worker (POST /exit restart: true), which leaves only when nothing would
|
|
88
|
+
# be lost and starts its successor. If the worker cannot be asked, stop
|
|
89
|
+
# it without a stop marker and resume only when disk shows no work to lose.
|
|
90
|
+
# @return [Restarted]
|
|
91
|
+
# @raise [OwnedByTUI] a one-turn chi run owns the session
|
|
92
|
+
# @raise [RestartRefused]
|
|
93
|
+
# @raise [ArgumentError] no such session
|
|
94
|
+
def restart_session(session_id, state_dir: nil, wait: RESTART_WAIT, client_id: ClientId::CLI_RESTART)
|
|
95
|
+
sd = state_dir || Session.default_state_dir
|
|
96
|
+
Session.load(session_id, state_dir: sd) # ArgumentError for none
|
|
97
|
+
owner = refuse_tui!(session_id, state_dir: sd)
|
|
98
|
+
# A stopped worker answers neither the ask (READ_TIMEOUT) nor the
|
|
99
|
+
# fallback's TERM (FALLBACK_STOP_WAIT): say so now, not after ~70 s.
|
|
100
|
+
raise RestartRefused.new(session_id, :stopped, pid: owner.pid) if owner&.worker? && ProcessGroup.stopped?(owner.pid)
|
|
101
|
+
|
|
102
|
+
dir = Session.session_dir(session_id, state_dir: sd)
|
|
103
|
+
link = WorkerLink.resolve(dir, unlink: false, owner: owner)
|
|
104
|
+
if link.is_a?(WorkerLink::Skew)
|
|
105
|
+
return restart_fallback(session_id, owner: owner, sidecar: link.sidecar, state_dir: sd, wait: wait, why: :skew)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
if link.is_a?(WorkerLink::Starting)
|
|
109
|
+
return restart_fallback(session_id, owner: owner, sidecar: nil, state_dir: sd, wait: wait, why: :starting_stuck) if link.stuck
|
|
110
|
+
|
|
111
|
+
raise RestartRefused.new(session_id, :starting)
|
|
112
|
+
end
|
|
113
|
+
raise RestartRefused.new(session_id, :not_running) unless link.live?
|
|
114
|
+
|
|
115
|
+
sidecar = link.sidecar
|
|
116
|
+
unless sidecar.features.include?("restart")
|
|
117
|
+
raise RestartRefused.new(session_id, :unsupported, version: sidecar.version)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
begin
|
|
121
|
+
reply = BridgeClient.for_sidecar(session_id, sidecar).request_exit(client_id: client_id, restart: true)
|
|
122
|
+
rescue SystemCallError, IOError => e
|
|
123
|
+
return restart_fallback(session_id, owner: owner, sidecar: sidecar, state_dir: sd, wait: wait, why: :unreachable,
|
|
124
|
+
error: e)
|
|
125
|
+
end
|
|
126
|
+
raise RestartRefused.new(session_id, reply.json&.fetch("reason", nil) || :failed) if reply.status == 409
|
|
127
|
+
if reply.secret_refused?
|
|
128
|
+
return restart_fallback(session_id, owner: owner, sidecar: sidecar, state_dir: sd, wait: wait, why: :bridge_secret)
|
|
129
|
+
end
|
|
130
|
+
raise RestartRefused.new(session_id, :failed) unless reply.status == 200
|
|
131
|
+
|
|
132
|
+
Log.info(:worker, "restart", sid: session_id, from: sidecar.version, by: client_id)
|
|
133
|
+
fresh = WorkerLink.await(session_id, session_dir: dir, timeout: wait, unlink: false, interval: 0.1,
|
|
134
|
+
until: ->(live) { live.epoch != sidecar.epoch || live.started_at != sidecar.started_at })
|
|
135
|
+
# Timed out: the last outcome, the old worker's Live too, is no successor.
|
|
136
|
+
changed = fresh.live? && (fresh.epoch != sidecar.epoch || fresh.started_at != sidecar.started_at)
|
|
137
|
+
version = fresh.version if changed
|
|
138
|
+
Restarted.new(session_id: session_id, from_version: sidecar.version, version: version)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
private def restart_fallback(session_id, owner:, sidecar:, state_dir:, wait:, why:, error: nil)
|
|
142
|
+
Log.info(:worker, "restart_fallback", sid: session_id, why: why, error: error&.class&.name, msg: error&.message)
|
|
143
|
+
dir = Session.session_dir(session_id, state_dir: state_dir)
|
|
144
|
+
session = Session.load(session_id, state_dir: state_dir)
|
|
145
|
+
holds = restart_fallback_holds(session, dir)
|
|
146
|
+
raise RestartRefused.new(session_id, :fallback_held, holds: holds) unless holds.empty?
|
|
147
|
+
raise RestartRefused.new(session_id, :failed, version: sidecar&.version) unless (owner&.pid || 0).to_i.positive?
|
|
148
|
+
|
|
149
|
+
# The ask may have taken READ_TIMEOUT: TERM only the owner read at the
|
|
150
|
+
# start; a successor that took the lock meanwhile is awaited, not killed.
|
|
151
|
+
current = session_owner(session_id, state_dir: state_dir)
|
|
152
|
+
if current.nil?
|
|
153
|
+
resume_session(session_id, state_dir: state_dir)
|
|
154
|
+
elsif current.pid == owner.pid && current.started_at == owner.started_at
|
|
155
|
+
begin
|
|
156
|
+
Process.kill("TERM", owner.pid)
|
|
157
|
+
rescue Errno::ESRCH
|
|
158
|
+
nil
|
|
159
|
+
rescue SystemCallError => e
|
|
160
|
+
Log.warn(:worker, "restart_fallback_failed", sid: session_id, why: why, error: e.class.name, msg: e.message)
|
|
161
|
+
raise RestartRefused.new(session_id, :failed, version: sidecar&.version)
|
|
162
|
+
end
|
|
163
|
+
unless wait_for_owner_release(session_id, timeout: FALLBACK_STOP_WAIT, state_dir: state_dir)
|
|
164
|
+
raise RestartRefused.new(session_id, :failed, version: sidecar&.version)
|
|
165
|
+
end
|
|
166
|
+
|
|
167
|
+
resume_session(session_id, state_dir: state_dir)
|
|
168
|
+
end
|
|
169
|
+
previous_epoch = sidecar&.epoch
|
|
170
|
+
previous_started_at = sidecar&.started_at
|
|
171
|
+
fresh = WorkerLink.await(session_id, session_dir: dir, timeout: wait, unlink: false, interval: 0.1,
|
|
172
|
+
until: ->(live) { live.epoch != previous_epoch || live.started_at != previous_started_at })
|
|
173
|
+
changed = fresh.live? && (fresh.epoch != previous_epoch || fresh.started_at != previous_started_at)
|
|
174
|
+
Restarted.new(session_id: session_id, from_version: sidecar&.version, version: (fresh.version if changed))
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
private def restart_fallback_holds(session, session_dir)
|
|
178
|
+
holds = []
|
|
179
|
+
holds << :turn_running if session.status == Session::STATUS_RUNNING
|
|
180
|
+
holds << :question_pending if session.pending_question
|
|
181
|
+
holds << :input_queued unless SessionInbox.find_new_input_files(session_dir).empty?
|
|
182
|
+
holds << :background_tasks if running_created_task?(session)
|
|
183
|
+
holds
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
private def running_created_task?(session)
|
|
187
|
+
root = session.working_directory.to_s
|
|
188
|
+
return false if root.empty?
|
|
189
|
+
|
|
190
|
+
Tools::TaskRuntime.created_ids_in(session.messages).any? do |id|
|
|
191
|
+
next false unless Tools::TaskRuntime::TASK_ID.match?(id)
|
|
192
|
+
|
|
193
|
+
path = File.join(root, Tools::TaskRuntime::TASKS_DIR, id, Tools::TaskRuntime::METADATA_FILENAME)
|
|
194
|
+
next false unless File.file?(path)
|
|
195
|
+
|
|
196
|
+
Tools::TaskRecord.from_file(JSON.parse(File.read(path)))&.running?
|
|
197
|
+
rescue JSON::ParserError, SystemCallError => e
|
|
198
|
+
Log.warn(:worker, "restart_task_check_failed", sid: session.id, task_id: id,
|
|
199
|
+
error: e.class.name, msg: e.message)
|
|
200
|
+
true
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Stop a session by sending TERM (then CONT, for a stopped one) to its process.
|
|
205
|
+
# With +wait+ (seconds), also wait for the owner to let go of the session,
|
|
206
|
+
# so a resume right after spawns a fresh worker instead of finding the
|
|
207
|
+
# dying one.
|
|
208
|
+
# A turn still running is canceled first (STOP_CANCEL_WAIT), as the
|
|
209
|
+
# web's Cancel does, with reason STOP_CANCEL_REASON: its worker saves it. The TERM stops (and saves)
|
|
210
|
+
# what still runs (WorkerSignals), with a short grace after the marker.
|
|
211
|
+
# @return [Boolean, nil] with +wait+: whether the owner was gone in time
|
|
212
|
+
# @raise [OwnedByTUI] when the interactive TUI owns the session
|
|
213
|
+
def stop_session(session_id, state_dir: nil, wait: nil)
|
|
214
|
+
sd = state_dir || Session.default_state_dir
|
|
215
|
+
owner = refuse_tui!(session_id, state_dir: sd)
|
|
216
|
+
|
|
217
|
+
cancel_running_turn(session_id, state_dir: sd)
|
|
218
|
+
# Mark first: a worker that has not taken the lock yet sees it and exits.
|
|
219
|
+
Session.mark_stopped(session_id, state_dir: sd)
|
|
220
|
+
# A stop drops the reminders: a restart's hand-off no successor took
|
|
221
|
+
# yet must not bring them back on the next resume.
|
|
222
|
+
ReminderCarry.remove(Session.session_dir(session_id, state_dir: sd))
|
|
223
|
+
pid = owner&.pid.to_i
|
|
224
|
+
begin
|
|
225
|
+
Process.kill("TERM", pid) if pid && pid > 0
|
|
226
|
+
rescue Errno::ESRCH
|
|
227
|
+
nil # already exited
|
|
228
|
+
end
|
|
229
|
+
continue_stopped(pid)
|
|
230
|
+
wait_for_owner_release(session_id, timeout: wait, state_dir: sd) if wait
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
# A stopped worker (SIGSTOP, Ctrl-Z) runs its TERM handler only once
|
|
234
|
+
# continued: CONT it after the TERM (as ProcessGroup.stop does), so it
|
|
235
|
+
# leaves now rather than outliving the stop's wait. Best effort: one the
|
|
236
|
+
# TERM ended may be gone already.
|
|
237
|
+
private def continue_stopped(pid)
|
|
238
|
+
Process.kill("CONT", pid) if pid && pid > 0
|
|
239
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
240
|
+
nil
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Cancel the session's running turn over its Bridge and wait (up to
|
|
244
|
+
# STOP_CANCEL_WAIT) for the worker to save it. Best effort: without a
|
|
245
|
+
# live Bridge, or when it doesn't answer, the stop goes on as before.
|
|
246
|
+
private def cancel_running_turn(session_id, state_dir:)
|
|
247
|
+
return unless Session.load(session_id, state_dir: state_dir).status == Session::STATUS_RUNNING
|
|
248
|
+
|
|
249
|
+
client = WorkerLink.client(session_id, session_dir: Session.session_dir(session_id, state_dir: state_dir))
|
|
250
|
+
return unless client && client.cancel(reason: STOP_CANCEL_REASON).status == 202
|
|
251
|
+
|
|
252
|
+
BridgeClient.poll(STOP_CANCEL_WAIT) do
|
|
253
|
+
Session.load(session_id, state_dir: state_dir).status != Session::STATUS_RUNNING
|
|
254
|
+
end
|
|
255
|
+
rescue StandardError => e
|
|
256
|
+
Log.info(:worker, "stop_cancel_failed", sid: session_id, error: e.class.name, msg: e.message)
|
|
257
|
+
nil
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
# @return [Boolean] whether the session had no owner within +timeout+ seconds
|
|
261
|
+
def wait_for_owner_release(session_id, timeout:, state_dir:)
|
|
262
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout.to_f
|
|
263
|
+
while session_owner(session_id, state_dir: state_dir)
|
|
264
|
+
return false if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
265
|
+
|
|
266
|
+
sleep(OwnerLock::RETRY_INTERVAL)
|
|
267
|
+
end
|
|
268
|
+
true
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
extend LifecyclePart
|
|
273
|
+
end
|
|
274
|
+
end
|