ace-overseer 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +26 -0
- data/README.md +2 -2
- data/docs/getting-started.md +1 -1
- data/docs/usage.md +47 -25
- data/handbook/workflow-instructions/overseer/bootstrap-delegation.wf.md +258 -40
- data/handbook/workflow-instructions/overseer/lab-start.wf.md +60 -24
- data/handbook/workflow-instructions/overseer/queue.wf.md +33 -13
- data/handbook/workflow-instructions/overseer/start.wf.md +198 -21
- data/lib/ace/overseer/atoms/observation_sources.rb +17 -0
- data/lib/ace/overseer/atoms/report_interpretation.rb +28 -14
- data/lib/ace/overseer/atoms/start_contract.rb +20 -0
- data/lib/ace/overseer/cli/commands/handoff.rb +4 -0
- data/lib/ace/overseer/cli/commands/process_reports.rb +4 -2
- data/lib/ace/overseer/cli/commands/start.rb +37 -5
- data/lib/ace/overseer/cli/commands/work_on.rb +6 -2
- data/lib/ace/overseer/cli.rb +2 -0
- data/lib/ace/overseer/molecules/coworker_launcher.rb +189 -0
- data/lib/ace/overseer/molecules/event_observation.rb +54 -19
- data/lib/ace/overseer/molecules/eyes_evidence.rb +103 -0
- data/lib/ace/overseer/molecules/eyes_readiness.rb +21 -0
- data/lib/ace/overseer/molecules/eyes_session_evidence.rb +132 -0
- data/lib/ace/overseer/molecules/hitl_inbox.rb +14 -5
- data/lib/ace/overseer/molecules/local_sources.rb +83 -19
- data/lib/ace/overseer/molecules/project_bundle.rb +50 -6
- data/lib/ace/overseer/molecules/project_observations.rb +82 -0
- data/lib/ace/overseer/molecules/project_sessions.rb +99 -0
- data/lib/ace/overseer/molecules/remote_sources.rb +46 -16
- data/lib/ace/overseer/molecules/report_completion_check.rb +52 -0
- data/lib/ace/overseer/molecules/session_environment.rb +126 -0
- data/lib/ace/overseer/organisms/handoff_orchestrator.rb +26 -15
- data/lib/ace/overseer/organisms/report_processor.rb +56 -11
- data/lib/ace/overseer/organisms/start_orchestrator.rb +107 -52
- data/lib/ace/overseer/organisms/work_on_orchestrator.rb +43 -1
- data/lib/ace/overseer/version.rb +1 -1
- data/lib/ace/overseer.rb +3 -1
- metadata +65 -2
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "ace/herdr"
|
|
5
|
+
require "ace/herdr/molecules/bounded_observation"
|
|
6
|
+
require "ace/runtime/molecules/process_identity"
|
|
7
|
+
require_relative "eyes_evidence"
|
|
8
|
+
|
|
9
|
+
module Ace
|
|
10
|
+
module Overseer
|
|
11
|
+
module Molecules
|
|
12
|
+
# Reconstruct Pi's current persisted branch and bind its eyes evidence
|
|
13
|
+
# to the live Herdr pane and process, without persisting another copy.
|
|
14
|
+
class EyesSessionEvidence
|
|
15
|
+
MAX_BYTES = 64 * 1024 * 1024
|
|
16
|
+
MAX_ENTRIES = 200_000
|
|
17
|
+
SESSION_PATH = %r{\A/[^\0]*_([0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12})\.jsonl\z}
|
|
18
|
+
|
|
19
|
+
# Preserve exact kernel birth identity on Linux and Darwin. A stale
|
|
20
|
+
# PID or zombie never proves observation readiness.
|
|
21
|
+
class NativeIdentity
|
|
22
|
+
def initialize(birth: Ace::Runtime::Molecules::ProcessIdentity.new)
|
|
23
|
+
@birth = birth
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def capture(pid)
|
|
27
|
+
return nil unless pid.is_a?(Integer) && pid.positive?
|
|
28
|
+
return @birth.capture(pid) if RUBY_PLATFORM.include?("darwin")
|
|
29
|
+
return nil unless RUBY_PLATFORM.include?("linux")
|
|
30
|
+
before = @birth.native_birth(pid)
|
|
31
|
+
return nil unless before
|
|
32
|
+
stat = File.read("/proc/#{pid}/stat")
|
|
33
|
+
match = stat.match(/\A#{pid} \(.*\) ([A-Za-z]) /m)
|
|
34
|
+
return nil unless match && !%w[Z X].include?(match[1])
|
|
35
|
+
uid = File.stat("/proc/#{pid}").uid
|
|
36
|
+
return nil unless @birth.native_birth(pid) == before
|
|
37
|
+
{"pid" => pid, "uid" => uid, "started_at" => before}
|
|
38
|
+
rescue SystemCallError, IOError
|
|
39
|
+
nil
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def initialize(executor: nil, identity: NativeIdentity.new,
|
|
44
|
+
clock: -> { (Time.now.to_f * 1000).to_i },
|
|
45
|
+
file_opener: ->(path) { File.open(path, File::RDONLY | File::NOFOLLOW | File::NONBLOCK) })
|
|
46
|
+
@executor, @identity, @clock, @file_opener = executor, identity, clock, file_opener
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def ready?(pane:, root:, prompt:, required_sources:, expected_terminal: nil)
|
|
50
|
+
executor = @executor || Ace::Herdr::Molecules::BoundedObservation.new(
|
|
51
|
+
deadline: Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5)
|
|
52
|
+
before = pane_binding(executor, pane)
|
|
53
|
+
return false unless before && (expected_terminal.nil? || before["terminal_id"] == expected_terminal)
|
|
54
|
+
path = before.fetch("path")
|
|
55
|
+
file = @file_opener.call(path)
|
|
56
|
+
stat = file.stat
|
|
57
|
+
return false unless stat.file? && stat.size.between?(1, MAX_BYTES)
|
|
58
|
+
content = file.read(MAX_BYTES + 1)
|
|
59
|
+
return false unless content && content.bytesize == stat.size && content.end_with?("\n")
|
|
60
|
+
again = file.stat
|
|
61
|
+
return false unless [stat.dev, stat.ino, stat.size, stat.mtime] == [again.dev, again.ino, again.size, again.mtime]
|
|
62
|
+
header, branch = branch_for(content)
|
|
63
|
+
return false unless header["cwd"] == root && header["id"].to_s.downcase == before.fetch("session_id").downcase
|
|
64
|
+
|
|
65
|
+
eye = branch.reverse.find { |entry| entry["type"] == "custom" && entry["customType"] == EyesEvidence::ENTRY }
|
|
66
|
+
pid = eye&.dig("data", "observerEvidence", "process", "pid")
|
|
67
|
+
return false unless pid.is_a?(Integer) && pid.positive? && foreground?(executor, pane, pid)
|
|
68
|
+
process = @identity.capture(pid)
|
|
69
|
+
return false unless process && process["uid"] == stat.uid
|
|
70
|
+
return false unless EyesEvidence.ready?(branch: branch, root: root, process: process,
|
|
71
|
+
prompt: prompt, required_sources: required_sources, now_ms: @clock.call)
|
|
72
|
+
# The owner or its session may change while the file is read.
|
|
73
|
+
return false unless @identity.capture(pid) == process && foreground?(executor, pane, pid) && pane_binding(executor, pane) == before
|
|
74
|
+
final = file.stat
|
|
75
|
+
named = File.lstat(path)
|
|
76
|
+
named.file? && !named.symlink? &&
|
|
77
|
+
[stat.dev, stat.ino, stat.size, stat.mtime] == [final.dev, final.ino, final.size, final.mtime] &&
|
|
78
|
+
[stat.dev, stat.ino] == [named.dev, named.ino]
|
|
79
|
+
rescue Ace::Herdr::ExecutorError, SystemCallError, IOError, JSON::ParserError, ArgumentError, TypeError, KeyError
|
|
80
|
+
false
|
|
81
|
+
ensure
|
|
82
|
+
file&.close
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def pane_binding(executor, pane)
|
|
88
|
+
json = executor.pane_get(pane).parsed_json
|
|
89
|
+
row = json.is_a?(Hash) && json["result"].is_a?(Hash) ? json["result"]["pane"] : nil
|
|
90
|
+
return nil unless row.is_a?(Hash) && row["pane_id"] == pane && row["agent"] == "pi" &&
|
|
91
|
+
(row["terminal_id"].is_a?(String) || row["terminal_id"].is_a?(Integer)) && !row["terminal_id"].to_s.empty?
|
|
92
|
+
session = row["agent_session"]
|
|
93
|
+
return nil unless session.is_a?(Hash) && session["agent"] == "pi" && session["kind"] == "path"
|
|
94
|
+
path = session["value"]
|
|
95
|
+
match = path.is_a?(String) && SESSION_PATH.match(path)
|
|
96
|
+
return nil unless match && path.bytesize <= 4096 && File.expand_path(path) == path
|
|
97
|
+
{"path" => path, "session_id" => match[1], "terminal_id" => row["terminal_id"].to_s}
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def foreground?(executor, pane, pid)
|
|
101
|
+
json = executor.pane_process_info(pane).parsed_json
|
|
102
|
+
info = json.is_a?(Hash) && json["result"].is_a?(Hash) ? json["result"]["process_info"] : nil
|
|
103
|
+
list = info.is_a?(Hash) ? info["foreground_processes"] : nil
|
|
104
|
+
list.is_a?(Array) && list.any? { |row| row.is_a?(Hash) && row["pid"] == pid }
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def branch_for(content)
|
|
108
|
+
lines = content.lines
|
|
109
|
+
raise ArgumentError, "session entry limit" unless lines.size.between?(2, MAX_ENTRIES + 1)
|
|
110
|
+
header = JSON.parse(lines.shift, create_additions: false, max_nesting: 64, allow_duplicate_key: false)
|
|
111
|
+
raise ArgumentError, "invalid Pi session" unless header.is_a?(Hash) && header["type"] == "session" && header["version"] == 3
|
|
112
|
+
entries = {}
|
|
113
|
+
last = nil
|
|
114
|
+
lines.each do |line|
|
|
115
|
+
entry = JSON.parse(line, create_additions: false, max_nesting: 64, allow_duplicate_key: false)
|
|
116
|
+
raise ArgumentError, "invalid session entry" unless entry.is_a?(Hash) && entry["id"].is_a?(String) &&
|
|
117
|
+
!entry["id"].empty? && !entries.key?(entry["id"]) &&
|
|
118
|
+
(entry["parentId"].nil? || entries.key?(entry["parentId"]))
|
|
119
|
+
entries[entry["id"]] = entry
|
|
120
|
+
last = entry
|
|
121
|
+
end
|
|
122
|
+
branch = []
|
|
123
|
+
while last
|
|
124
|
+
branch << last
|
|
125
|
+
last = last["parentId"] && entries.fetch(last["parentId"])
|
|
126
|
+
end
|
|
127
|
+
[header, branch.reverse]
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "ace/hitl/molecules/event_observation"
|
|
4
|
+
|
|
3
5
|
module Ace
|
|
4
6
|
module Overseer
|
|
5
7
|
module Molecules
|
|
@@ -10,8 +12,14 @@ module Ace
|
|
|
10
12
|
# until answered — both are the overseer's inbox.
|
|
11
13
|
class HitlInbox
|
|
12
14
|
# @param manager [Ace::Hitl::Organisms::HitlManager, nil]
|
|
13
|
-
def initialize(manager: nil)
|
|
15
|
+
def initialize(manager: nil, owner: "overseer", scope: nil)
|
|
14
16
|
@manager = manager || Ace::Hitl::Organisms::HitlManager.new
|
|
17
|
+
raise ArgumentError, "configured HITL owner is required" if owner.to_s.strip.empty?
|
|
18
|
+
unless scope.nil? || %w[current all].include?(scope)
|
|
19
|
+
raise ArgumentError, "HITL scope must be current or all"
|
|
20
|
+
end
|
|
21
|
+
@owner, @scope = owner, scope
|
|
22
|
+
@observation = Ace::Hitl::Molecules::EventObservation.new(manager: @manager)
|
|
15
23
|
end
|
|
16
24
|
|
|
17
25
|
# @return [Array<Hash>] {"id","type","result","task","assignment",
|
|
@@ -19,10 +27,10 @@ module Ace
|
|
|
19
27
|
# atom matches on are carried: assignment, then the managing
|
|
20
28
|
# task's project_task.
|
|
21
29
|
def incoming_events
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
30
|
+
events = @observation.incoming_events(owner: @owner, scope: @scope).select do |event|
|
|
31
|
+
event.type == "report" || (event.type == "question" && event.status == "pending")
|
|
32
|
+
end
|
|
33
|
+
events.sort_by { |event| event.created_at || Time.now }.map { |event| normalize(event) }
|
|
26
34
|
end
|
|
27
35
|
|
|
28
36
|
private
|
|
@@ -34,6 +42,7 @@ module Ace
|
|
|
34
42
|
"result" => event.result,
|
|
35
43
|
"task" => event.metadata["task"],
|
|
36
44
|
"assignment" => event.metadata["assignment"],
|
|
45
|
+
"role" => event.metadata["role"],
|
|
37
46
|
"title" => event.title
|
|
38
47
|
}
|
|
39
48
|
end
|
|
@@ -2,14 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
require "digest"
|
|
4
4
|
require "json"
|
|
5
|
+
require_relative "../atoms/observation_sources"
|
|
6
|
+
require "ace/herdr/molecules/herdr_executor"
|
|
5
7
|
|
|
6
8
|
module Ace
|
|
7
9
|
module Overseer
|
|
8
10
|
module Molecules
|
|
9
11
|
# Standing local-state observation (task 8xs.t.pu0.3): projects the
|
|
10
|
-
#
|
|
12
|
+
# local source classes the started overseer watches — project
|
|
11
13
|
# tasks, private _overseer tasks, worktrees, coworker availability,
|
|
12
|
-
# assignment progress and HITL events — each read from its STATE
|
|
14
|
+
# assignment progress, project sessions and HITL events — each read from its STATE
|
|
13
15
|
# OWNER (ace-task, ace-git-worktree, ace-coworker probe, ace-assign
|
|
14
16
|
# inventory, ace-hitl). This molecule owns no second state: every
|
|
15
17
|
# row is a {key, revision, type} projection whose revision changes
|
|
@@ -21,12 +23,18 @@ module Ace
|
|
|
21
23
|
# (an identical repeated failure never re-wakes; a new failure mode
|
|
22
24
|
# wakes once), while the healthy sources keep projecting.
|
|
23
25
|
class LocalSources
|
|
24
|
-
|
|
26
|
+
EXECUTOR_ROLES = %w[worker verifier integrator].freeze
|
|
27
|
+
|
|
28
|
+
def initialize(task_manager: nil, worktree_manager: nil, probe_factory: nil, project_sessions: nil,
|
|
29
|
+
herdr: Ace::Herdr::Molecules::HerdrExecutor.new, env: ENV)
|
|
25
30
|
@tasks = task_manager || Ace::Task::Organisms::TaskManager.new
|
|
26
31
|
@worktrees = worktree_manager || Ace::Git::Worktree::Organisms::WorktreeManager.new
|
|
27
32
|
@probe_factory = probe_factory || ->(path) {
|
|
28
33
|
Ace::Coworker::Molecules::WorktreeExecutorProbe.new(project_root: path)
|
|
29
34
|
}
|
|
35
|
+
@project_sessions = project_sessions || ProjectSessions.new
|
|
36
|
+
@herdr = herdr
|
|
37
|
+
@env = env
|
|
30
38
|
end
|
|
31
39
|
|
|
32
40
|
# @return [Hash] {"events" => Array<{key, revision, type, ...}>,
|
|
@@ -35,10 +43,25 @@ module Ace
|
|
|
35
43
|
sources = {}
|
|
36
44
|
events = []
|
|
37
45
|
|
|
38
|
-
collect(sources, events,
|
|
39
|
-
|
|
40
|
-
collect(sources, events,
|
|
41
|
-
|
|
46
|
+
collect(sources, events, Atoms::ObservationSources::PROJECT_TASKS) { project_task_events }
|
|
47
|
+
overseer_tasks = []
|
|
48
|
+
collect(sources, events, Atoms::ObservationSources::OVERSEER_TASKS) do
|
|
49
|
+
overseer_tasks = @tasks.list(in_folder: "_overseer")
|
|
50
|
+
task_events(overseer_tasks, "overseer-task")
|
|
51
|
+
end
|
|
52
|
+
collect(sources, events, Atoms::ObservationSources::WORKTREES) { worktree_events }
|
|
53
|
+
collect(sources, events, Atoms::ObservationSources::COWORKERS) { coworker_events(overseer_tasks) }
|
|
54
|
+
if @project_sessions.configured?
|
|
55
|
+
collect(sources, events, Atoms::ObservationSources::PROJECT_SESSIONS) { @project_sessions.call }
|
|
56
|
+
failed_sessions = events.select { |event| event["type"] == "project-session" && event["state"] == "unavailable" }
|
|
57
|
+
unless failed_sessions.empty?
|
|
58
|
+
sources[Atoms::ObservationSources::PROJECT_SESSIONS] = {"status" => "degraded", "error" => "one or more configured project sessions are unavailable"}
|
|
59
|
+
failures = failed_sessions.map { |event| [event.fetch("key"), event.fetch("revision")] }.sort
|
|
60
|
+
events << {"key" => "source:project_sessions", "revision" => revision(failures),
|
|
61
|
+
"type" => "source-health", "status" => "degraded",
|
|
62
|
+
"error" => "one or more configured project sessions are unavailable"}
|
|
63
|
+
end
|
|
64
|
+
end
|
|
42
65
|
|
|
43
66
|
{"events" => events, "sources" => sources}
|
|
44
67
|
end
|
|
@@ -73,10 +96,6 @@ module Ace
|
|
|
73
96
|
task_events(@tasks.list(in_folder: "next"), "task")
|
|
74
97
|
end
|
|
75
98
|
|
|
76
|
-
def overseer_task_events
|
|
77
|
-
task_events(@tasks.list(in_folder: "_overseer"), "overseer-task")
|
|
78
|
-
end
|
|
79
|
-
|
|
80
99
|
# Task rows revision on real lifecycle facts (status, dependency
|
|
81
100
|
# set, subtask statuses) — not on cosmetic frontmatter churn.
|
|
82
101
|
def task_events(tasks, type)
|
|
@@ -108,23 +127,30 @@ module Ace
|
|
|
108
127
|
# ids, never pane names), the ActiveExecutorRule decides live vs
|
|
109
128
|
# stale, and each in-flight assignment projects its own progress
|
|
110
129
|
# row so a step report revises that assignment only.
|
|
111
|
-
def coworker_events
|
|
130
|
+
def coworker_events(overseer_tasks)
|
|
112
131
|
result = @worktrees.list_all
|
|
113
132
|
raise Error, result[:error] || "failed to list worktrees" unless result[:success]
|
|
114
133
|
|
|
115
|
-
Array(result[:worktrees]).reject(&:bare)
|
|
134
|
+
worktrees = Array(result[:worktrees]).reject(&:bare)
|
|
135
|
+
native = !worktrees.empty? && (@env["ACE_RUNTIME"] == "herdr" || !@env["HERDR_PANE_ID"].to_s.empty? || !@env["HERDR_PANE"].to_s.empty?)
|
|
136
|
+
sessions = native ? executor_sessions : {}
|
|
137
|
+
bindings = native ? executor_bindings(overseer_tasks) : {}
|
|
138
|
+
worktrees.flat_map do |worktree|
|
|
116
139
|
probe = @probe_factory.call(worktree.path)
|
|
117
140
|
report = probe.probe(worktree.path)
|
|
118
141
|
verdict = Ace::Coworker::Atoms::ActiveExecutorRule.evaluate(
|
|
119
142
|
config: report[:config], assignments: report[:assignments]
|
|
120
143
|
)
|
|
121
144
|
identity = Digest::SHA256.hexdigest(worktree.path)
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
145
|
+
visible = sessions.fetch(worktree.path, [])
|
|
146
|
+
completed = report[:assignments].filter_map { |row| row["id"] if row["state"] == "completed" }
|
|
147
|
+
missing = bindings.fetch(worktree.path, []).reject do |binding|
|
|
148
|
+
completed.include?(binding["assignment"]) || visible.any? { |session| session["terminal"] == binding["terminal"] }
|
|
149
|
+
end
|
|
150
|
+
payload = {"path" => worktree.path, "active" => verdict[:active],
|
|
151
|
+
"evidence" => verdict[:evidence], "sessions" => visible, "missing_sessions" => missing}
|
|
152
|
+
executor = {"key" => "executor:#{identity}", "revision" => revision(payload),
|
|
153
|
+
"type" => "executor"}.merge(payload)
|
|
128
154
|
assignments = report[:assignments].map do |assignment|
|
|
129
155
|
{"key" => "assignment:#{assignment["id"]}",
|
|
130
156
|
"revision" => revision({"id" => assignment["id"], "state" => assignment["state"],
|
|
@@ -135,6 +161,44 @@ module Ace
|
|
|
135
161
|
[executor] + assignments
|
|
136
162
|
end
|
|
137
163
|
end
|
|
164
|
+
|
|
165
|
+
# A coordinator's durable binding also detects a session that
|
|
166
|
+
# started and exited between two native observations.
|
|
167
|
+
def executor_bindings(tasks)
|
|
168
|
+
tasks.flat_map { |task| [task, *Array(task.subtasks)] }.filter_map do |step|
|
|
169
|
+
next if %w[done canceled cancelled].include?(step.status)
|
|
170
|
+
|
|
171
|
+
metadata = step.metadata || {}
|
|
172
|
+
worktree, assignment, terminal = metadata.values_at("worktree", "assignment", "terminal")
|
|
173
|
+
next unless worktree.is_a?(String) && worktree.start_with?("/") &&
|
|
174
|
+
assignment.is_a?(String) && assignment.match?(/\A[a-zA-Z0-9_-]+\z/) &&
|
|
175
|
+
terminal.is_a?(String) && terminal.match?(/\Aterm_[0-9a-f]+\z/)
|
|
176
|
+
|
|
177
|
+
[worktree, {"task" => step.id, "assignment" => assignment, "terminal" => terminal}]
|
|
178
|
+
end.group_by(&:first).transform_values { |rows| rows.map(&:last).sort_by { |row| row.fetch("task") } }
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
# Native sessions can disappear without writing an assignment report.
|
|
182
|
+
# Project their presence on the existing stable worktree row so that
|
|
183
|
+
# even an interruption before the first assignment step is visible.
|
|
184
|
+
def executor_sessions
|
|
185
|
+
response = @herdr.agent_list.parsed_json
|
|
186
|
+
agents = response.is_a?(Hash) && !response.key?("error") && response.dig("result", "agents")
|
|
187
|
+
raise Error, "native executor inventory is unavailable" unless agents.is_a?(Array)
|
|
188
|
+
|
|
189
|
+
sessions = Hash.new { |index, cwd| index[cwd] = [] }
|
|
190
|
+
agents.each do |agent|
|
|
191
|
+
next unless agent.is_a?(Hash) && agent["name"].to_s.start_with?("coworker-")
|
|
192
|
+
|
|
193
|
+
terminal, cwd = agent.values_at("terminal_id", "cwd")
|
|
194
|
+
unless terminal.is_a?(String) && terminal.match?(/\Aterm_[0-9a-f]+\z/) &&
|
|
195
|
+
cwd.is_a?(String) && cwd.start_with?("/")
|
|
196
|
+
raise Error, "executor session identity is invalid"
|
|
197
|
+
end
|
|
198
|
+
sessions[cwd] << {"terminal" => terminal, "agent" => agent["name"]}
|
|
199
|
+
end
|
|
200
|
+
sessions.transform_values { |rows| rows.sort_by { |row| row.fetch("terminal") } }
|
|
201
|
+
end
|
|
138
202
|
end
|
|
139
203
|
end
|
|
140
204
|
end
|
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
require "json"
|
|
4
4
|
require "ace/herdr/molecules/bounded_process"
|
|
5
5
|
require "shellwords"
|
|
6
|
+
require "tempfile"
|
|
7
|
+
require "securerandom"
|
|
8
|
+
require "fileutils"
|
|
6
9
|
require "ace/support/cli/toolchain"
|
|
7
10
|
|
|
8
11
|
module Ace
|
|
@@ -59,19 +62,53 @@ module Ace
|
|
|
59
62
|
end
|
|
60
63
|
|
|
61
64
|
def self.shell_setup(root, env: ENV)
|
|
65
|
+
values = environment(root, env: env).reject { |name, _| %w[HERDR_PANE HERDR_TAB HERDR_PANE_ID HERDR_TAB_ID].include?(name) }
|
|
62
66
|
if Ace::Support::Cli::Toolchain.installed?
|
|
63
67
|
# A server-created shell may contain selectors absent from this
|
|
64
68
|
# caller. Clear the same known selectors before applying the
|
|
65
69
|
# installed closure; never inherit server-side Bundler injection.
|
|
66
70
|
names = BUNDLE_SELECTORS + BUNDLE_SELECTORS.map { |key| "BUNDLER_ORIG_#{key}" }
|
|
67
|
-
|
|
68
|
-
Ace::Support::Cli::Toolchain.shell_setup
|
|
71
|
+
values = names.to_h { |name| [name, nil] }.merge(values)
|
|
69
72
|
end
|
|
70
73
|
|
|
71
|
-
values = environment(root, env: env).reject { |name, _| %w[HERDR_PANE HERDR_TAB HERDR_PANE_ID HERDR_TAB_ID].include?(name) }
|
|
72
74
|
cleared, assigned = values.partition { |_, value| value.nil? }
|
|
73
|
-
|
|
74
|
-
|
|
75
|
+
# Guard every operation: e.g. a readonly inherited selector must
|
|
76
|
+
# refuse setup rather than print readiness after a failed unset.
|
|
77
|
+
commands = cleared.map { |name, _| "unset #{Shellwords.escape(name)}" } +
|
|
78
|
+
assigned.map { |name, value| "export #{name}=#{Shellwords.escape(value)}" }
|
|
79
|
+
commands.join(" &&\n")
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# pane_run types through a terminal: Linux canonical input truncates
|
|
83
|
+
# a line after 4095 bytes. Keep the environment in a private shared
|
|
84
|
+
# project file, not /tmp (Herdr can use a different PrivateTmp).
|
|
85
|
+
# The caller must observe the marker before launching the harness.
|
|
86
|
+
def self.with_shell_setup(root, env: ENV)
|
|
87
|
+
directory = root
|
|
88
|
+
%w[.ace-local overseer runtime-start].each do |part|
|
|
89
|
+
directory = File.join(directory, part)
|
|
90
|
+
raise Ace::Overseer::Error, "runtime setup directory is a symlink: #{directory}" if File.symlink?(directory)
|
|
91
|
+
|
|
92
|
+
Dir.mkdir(directory, 0o700) unless File.exist?(directory)
|
|
93
|
+
raise Ace::Overseer::Error, "runtime setup path is not a directory: #{directory}" unless File.directory?(directory)
|
|
94
|
+
end
|
|
95
|
+
info = File.stat(directory)
|
|
96
|
+
unless info.uid == Process.uid && (info.mode & 0o077).zero?
|
|
97
|
+
raise Ace::Overseer::Error, "runtime setup directory must be private and owned by the current user: #{directory}"
|
|
98
|
+
end
|
|
99
|
+
marker = "ACE_OVERSEER_ENV_READY_#{SecureRandom.hex(16)}"
|
|
100
|
+
Tempfile.create(["environment-", ".sh"], directory) do |file|
|
|
101
|
+
setup = shell_setup(root, env: env).strip.sub(/;\z/, "")
|
|
102
|
+
file.write("#{setup} && printf '%s\\n' #{Shellwords.escape(marker)}\n")
|
|
103
|
+
file.flush
|
|
104
|
+
file.fsync
|
|
105
|
+
# The tab is created in root. An explicit ./ prevents dot from
|
|
106
|
+
# searching PATH; a relative path also keeps terminal input short.
|
|
107
|
+
relative = "./.ace-local/overseer/runtime-start/#{File.basename(file.path)}"
|
|
108
|
+
yield ". #{Shellwords.escape(relative)}", marker
|
|
109
|
+
end
|
|
110
|
+
rescue SystemCallError => e
|
|
111
|
+
raise Ace::Overseer::Error, "runtime setup file could not be prepared: #{e.message}"
|
|
75
112
|
end
|
|
76
113
|
|
|
77
114
|
def initialize(timeout_s: 240)
|
|
@@ -97,10 +134,17 @@ module Ace
|
|
|
97
134
|
raise Ace::Overseer::Error, "project start output exceeded its limit" if result.oversized
|
|
98
135
|
out, err, status = result.stdout, result.stderr, result.status
|
|
99
136
|
report = JSON.parse(out)
|
|
137
|
+
# Startup can fail before it establishes project scope. Keep that
|
|
138
|
+
# concrete failure without admitting an unscoped report as ready.
|
|
139
|
+
if report.is_a?(Hash) && report["status"] == "blocked" &&
|
|
140
|
+
report.key?("scope") && report["scope"].nil? &&
|
|
141
|
+
report["message"].is_a?(String) && !report["message"].strip.empty?
|
|
142
|
+
raise Ace::Overseer::Error, report["message"]
|
|
143
|
+
end
|
|
100
144
|
unless report.is_a?(Hash) && report["scope"] == "project" && report["status"].is_a?(String)
|
|
101
145
|
raise Ace::Overseer::Error, "project start returned no project-scope report"
|
|
102
146
|
end
|
|
103
|
-
if !status.success? && %w[started reused ready].include?(report["status"])
|
|
147
|
+
if !status.success? && %w[started reused ready].include?(report["status"]) && report["prompt_failed"] != true
|
|
104
148
|
raise Ace::Overseer::Error, "project start exited #{status.exitstatus} despite a ready report"
|
|
105
149
|
end
|
|
106
150
|
report
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "digest"
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "../atoms/observation_sources"
|
|
5
|
+
require "ace/lab/molecules/install_manifest_loader"
|
|
6
|
+
require "ace/herdr/molecules/bounded_process"
|
|
7
|
+
require "ace/support/cli/toolchain"
|
|
8
|
+
require_relative "session_environment"
|
|
9
|
+
|
|
10
|
+
module Ace::Overseer::Molecules
|
|
11
|
+
# The lab reads the same public projection as each configured project.
|
|
12
|
+
# Tasks, reports and assignments remain in that project's existing owners.
|
|
13
|
+
class ProjectObservations
|
|
14
|
+
BUDGET_SECONDS = 20
|
|
15
|
+
def initialize(root: Dir.pwd, env: ENV, loader: -> { Ace::Lab::Molecules::InstallManifestLoader.load },
|
|
16
|
+
runner: Ace::Herdr::Molecules::BoundedProcess.method(:call),
|
|
17
|
+
clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }, environment: nil)
|
|
18
|
+
@root, @env, @loader, @runner, @clock = root, env, loader, runner, clock
|
|
19
|
+
@environment = environment || SessionEnvironment.new(env: env, manifest_loader: loader)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def call(owner:, scope:, limit:)
|
|
23
|
+
empty = {"events" => [], "sources" => {}}
|
|
24
|
+
return empty unless @env["ACE_TOOLCHAIN"] == "installed" &&
|
|
25
|
+
File.exist?(Ace::Lab.install_manifest_path)
|
|
26
|
+
document = @loader.call
|
|
27
|
+
return empty unless document.dig("coordinator", "path") == File.realpath(@root)
|
|
28
|
+
events, sources = [], {}
|
|
29
|
+
deadline = @clock.call + BUDGET_SECONDS
|
|
30
|
+
document.fetch("projects").each do |project|
|
|
31
|
+
id, root = project.values_at("id", "path")
|
|
32
|
+
source = "project:#{id}"
|
|
33
|
+
begin
|
|
34
|
+
raise Ace::Overseer::Error, "project root is not canonical" unless root != @root && File.realpath(root) == root
|
|
35
|
+
remaining = deadline - @clock.call
|
|
36
|
+
raise Timeout::Error, "project observation budget exhausted" unless remaining.positive?
|
|
37
|
+
result = @runner.call([Ace::Support::Cli::Toolchain.executable("ace-overseer"), "observe",
|
|
38
|
+
"--owner", owner, "--scope", scope, "--limit", limit.to_s],
|
|
39
|
+
environment: @environment.observation_environment(root: root),
|
|
40
|
+
chdir: root, timeout_s: remaining, output_limit: 2_097_152, cleanup_group: true)
|
|
41
|
+
raise Ace::Overseer::Error, "project observation failed" unless result.status.success? && !result.oversized
|
|
42
|
+
value = JSON.parse(result.stdout)
|
|
43
|
+
unless value["schema"] == "ace.hitl.observation/v1" && value["events"].is_a?(Array) &&
|
|
44
|
+
value["settled"].is_a?(Array) && value["sources"].is_a?(Hash) &&
|
|
45
|
+
(Ace::Overseer::Atoms::ObservationSources::REQUIRED - value["sources"].keys).empty?
|
|
46
|
+
raise Ace::Overseer::Error, "project observation is incomplete"
|
|
47
|
+
end
|
|
48
|
+
rows = (value.fetch("events") + value.fetch("settled")).map do |row|
|
|
49
|
+
unless row.is_a?(Hash) && %w[key revision id type].all? { |key| row[key].is_a?(String) }
|
|
50
|
+
raise Ace::Overseer::Error, "project event is incomplete"
|
|
51
|
+
end
|
|
52
|
+
row.merge("key" => Digest::SHA256.hexdigest("#{root}:#{row.fetch('key')}"),
|
|
53
|
+
"project" => id, "project_root" => root, "owner_key" => row.fetch("key"))
|
|
54
|
+
end
|
|
55
|
+
health = value.fetch("sources").to_h do |name, state|
|
|
56
|
+
unless name.is_a?(String) && name.match?(/\A[a-zA-Z0-9_.:-]{1,64}\z/) &&
|
|
57
|
+
state.is_a?(Hash) && %w[ok degraded].include?(state["status"])
|
|
58
|
+
raise Ace::Overseer::Error, "project source health is incomplete"
|
|
59
|
+
end
|
|
60
|
+
["#{source}:#{name}", state]
|
|
61
|
+
end
|
|
62
|
+
events.concat(rows)
|
|
63
|
+
sources.merge!(health)
|
|
64
|
+
sources[source] = {"status" => "ok"}
|
|
65
|
+
events << project_health(root, id, "ok")
|
|
66
|
+
rescue StandardError => error
|
|
67
|
+
sources[source] = {"status" => "degraded", "error" => error.class.name}
|
|
68
|
+
events << project_health(root, id, "degraded", error.class.name)
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
{"events" => events, "sources" => sources}
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
private
|
|
75
|
+
|
|
76
|
+
def project_health(root, id, status, revision = status)
|
|
77
|
+
key = Digest::SHA256.hexdigest("#{root}:project-observation")
|
|
78
|
+
{"key" => key, "id" => key, "revision" => Digest::SHA256.hexdigest(revision),
|
|
79
|
+
"type" => "source-health", "status" => status, "project" => id, "project_root" => root}
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ace/lab"
|
|
4
|
+
require "ace/herdr"
|
|
5
|
+
require "digest"
|
|
6
|
+
require "json"
|
|
7
|
+
require_relative "project_root_resolver"
|
|
8
|
+
|
|
9
|
+
module Ace
|
|
10
|
+
module Overseer
|
|
11
|
+
module Molecules
|
|
12
|
+
# Project locations and terminal sessions remain owned by ace-lab and
|
|
13
|
+
# herdr. This projection has no persisted copy of either.
|
|
14
|
+
class ProjectSessions
|
|
15
|
+
def initialize(root: Dir.pwd, control: nil, locations: Ace::Lab::Molecules::ProjectLocations,
|
|
16
|
+
root_resolver: ProjectRootResolver.new, executor: Ace::Herdr::Molecules::HerdrExecutor.new, env: ENV)
|
|
17
|
+
@root = File.realpath(root)
|
|
18
|
+
@control = control || Ace::Herdr::Organisms::ControlSurface.new
|
|
19
|
+
@locations = locations
|
|
20
|
+
@root_resolver = root_resolver
|
|
21
|
+
@executor, @env = executor, env
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def configured?
|
|
25
|
+
File.file?(File.join(@root, ".ace/lab/config.yml"))
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def call
|
|
29
|
+
return [] unless configured?
|
|
30
|
+
|
|
31
|
+
projects = @locations.read(@root)
|
|
32
|
+
return [] if projects.empty?
|
|
33
|
+
|
|
34
|
+
seen = {}
|
|
35
|
+
failures = []
|
|
36
|
+
resolved = projects.filter_map do |project|
|
|
37
|
+
begin
|
|
38
|
+
path = project.fetch("path")
|
|
39
|
+
raise Error, "project has no configured path" if path.to_s.empty?
|
|
40
|
+
canonical = File.realpath(path, @root)
|
|
41
|
+
next if canonical == @root
|
|
42
|
+
canonical = @root_resolver.resolve(cwd: canonical)
|
|
43
|
+
next if canonical == @root
|
|
44
|
+
raise Error, "multiple projects use the same root" if seen[canonical]
|
|
45
|
+
|
|
46
|
+
seen[canonical] = true
|
|
47
|
+
[project.fetch("id"), canonical, project]
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
failures << unavailable_row(project, e)
|
|
50
|
+
nil
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
return failures if resolved.empty?
|
|
54
|
+
|
|
55
|
+
tabs = @control.list_tabs
|
|
56
|
+
panes = @control.list_panes
|
|
57
|
+
failures + resolved.map do |id, root, project|
|
|
58
|
+
begin
|
|
59
|
+
project_row(id, root, tabs, panes)
|
|
60
|
+
rescue StandardError => e
|
|
61
|
+
unavailable_row(project, e)
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def unavailable_row(project, error)
|
|
69
|
+
identity = Digest::SHA256.hexdigest(JSON.generate([project["id"], project["path"]]))
|
|
70
|
+
payload = {"project" => project["id"], "state" => "unavailable", "error" => "#{error.class}: #{error.message}"}
|
|
71
|
+
{"key" => "project-session-error:#{identity}",
|
|
72
|
+
"revision" => Digest::SHA256.hexdigest(JSON.generate(payload)),
|
|
73
|
+
"type" => "project-session"}.merge(payload)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def project_row(id, root, tabs, panes)
|
|
78
|
+
label = Atoms::StartContract.session_label(root)
|
|
79
|
+
owners = tabs.select { |tab| tab[:title] == label }.flat_map do |tab|
|
|
80
|
+
matches = panes.select { |pane| pane[:tab] == tab[:id] }
|
|
81
|
+
raise Error, "project overseer tab has no unique pane: #{label}" unless matches.length == 1
|
|
82
|
+
matches
|
|
83
|
+
end
|
|
84
|
+
raise Error, "project overseer has ambiguous sessions: #{label}" if owners.length > 1
|
|
85
|
+
pane = owners.first
|
|
86
|
+
raise Error, "project overseer session did not report a working directory: #{label}" if pane && pane[:cwd].to_s.empty?
|
|
87
|
+
if pane && @root_resolver.resolve(cwd: pane[:cwd]) != root
|
|
88
|
+
raise Error, "project overseer session has a different root: #{label}"
|
|
89
|
+
end
|
|
90
|
+
state = pane ? (pane[:agent_status] || "unknown").to_s : "absent"
|
|
91
|
+
payload = {"project" => id, "root" => root, "pane" => pane && pane[:id], "state" => state}
|
|
92
|
+
{"key" => "project-session:#{Digest::SHA256.hexdigest(root)}",
|
|
93
|
+
"revision" => Digest::SHA256.hexdigest(JSON.generate(payload)),
|
|
94
|
+
"type" => "project-session"}.merge(payload)
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|