ace-overseer 0.20.0 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ace-defaults/nav/protocols/wfi-sources/ace-overseer.yml +1 -1
- data/CHANGELOG.md +27 -0
- data/README.md +14 -14
- data/docs/usage.md +66 -41
- data/handbook/workflow-instructions/overseer.wf.md +13 -9
- data/handbook/workflow-instructions/roles/lab-coordinator.wf.md +99 -0
- data/handbook/workflow-instructions/roles/project-overseer.wf.md +103 -0
- data/handbook/workflow-instructions/roles/second-commander.wf.md +93 -0
- data/lib/ace/overseer/cli/commands/agents.rb +12 -7
- data/lib/ace/overseer/cli/commands/projects.rb +11 -7
- data/lib/ace/overseer/cli/commands/prompt.rb +44 -16
- data/lib/ace/overseer/cli/commands/prune.rb +51 -123
- data/lib/ace/overseer/cli/commands/review.rb +28 -8
- data/lib/ace/overseer/cli/commands/status.rb +33 -19
- data/lib/ace/overseer/cli/commands/stop.rb +18 -8
- data/lib/ace/overseer/cli/commands/work_on.rb +52 -16
- data/lib/ace/overseer/cli.rb +6 -9
- data/lib/ace/overseer/molecules/launch_request.rb +216 -0
- data/lib/ace/overseer/molecules/original_launch_child.rb +165 -0
- data/lib/ace/overseer/molecules/proposal_tick.rb +37 -9
- data/lib/ace/overseer/molecules/protected_selection.rb +46 -0
- data/lib/ace/overseer/molecules/window_opener.rb +9 -2
- data/lib/ace/overseer/organisms/launch_recovery.rb +65 -0
- data/lib/ace/overseer/organisms/protected_prune.rb +116 -0
- data/lib/ace/overseer/organisms/protected_review.rb +167 -0
- data/lib/ace/overseer/organisms/protected_status.rb +157 -0
- data/lib/ace/overseer/organisms/protected_steering.rb +78 -0
- data/lib/ace/overseer/organisms/protected_work_on.rb +102 -0
- data/lib/ace/overseer/organisms/work_on_orchestrator.rb +4 -5
- data/lib/ace/overseer/version.rb +1 -1
- data/lib/ace/overseer.rb +1 -2
- metadata +50 -13
- data/lib/ace/overseer/cli/commands/prepare.rb +0 -38
- data/lib/ace/overseer/molecules/lab_client.rb +0 -56
- data/lib/ace/overseer/molecules/lab_prune_safety_checker.rb +0 -69
data/lib/ace/overseer/cli.rb
CHANGED
|
@@ -7,7 +7,6 @@ require_relative "cli/commands/status"
|
|
|
7
7
|
require_relative "cli/commands/prune"
|
|
8
8
|
require_relative "cli/commands/projects"
|
|
9
9
|
require_relative "cli/commands/agents"
|
|
10
|
-
require_relative "cli/commands/prepare"
|
|
11
10
|
require_relative "cli/commands/prompt"
|
|
12
11
|
require_relative "cli/commands/review"
|
|
13
12
|
require_relative "cli/commands/stop"
|
|
@@ -20,15 +19,14 @@ module Ace
|
|
|
20
19
|
PROGRAM_NAME = "ace-overseer"
|
|
21
20
|
|
|
22
21
|
REGISTERED_COMMANDS = [
|
|
23
|
-
["work-on", "
|
|
22
|
+
["work-on", "Start local task work or one retained protected launch"],
|
|
24
23
|
["status", "Show status of task worktrees"],
|
|
25
24
|
["prune", "Remove stale task worktrees"],
|
|
26
|
-
["projects", "List
|
|
27
|
-
["agents", "List
|
|
28
|
-
["
|
|
29
|
-
["
|
|
30
|
-
["
|
|
31
|
-
["stop", "Stop a Lab Work"]
|
|
25
|
+
["projects", "List visible public projects"],
|
|
26
|
+
["agents", "List visible public project agents"],
|
|
27
|
+
["prompt", "Prompt an exact protected attempt or inspect original prompt status"],
|
|
28
|
+
["review", "Review an exact protected candidate under its original capability"],
|
|
29
|
+
["stop", "Request proof-authenticated stop of an exact protected attempt"]
|
|
32
30
|
].freeze
|
|
33
31
|
|
|
34
32
|
HELP_EXAMPLES = [
|
|
@@ -42,7 +40,6 @@ module Ace
|
|
|
42
40
|
register "prune", Commands::Prune
|
|
43
41
|
register "projects", Commands::Projects
|
|
44
42
|
register "agents", Commands::Agents
|
|
45
|
-
register "prepare", Commands::Prepare
|
|
46
43
|
register "prompt", Commands::Prompt
|
|
47
44
|
register "review", Commands::Review
|
|
48
45
|
register "stop", Commands::Stop
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "digest"
|
|
5
|
+
require "ace/assign/authority/prepared_work"
|
|
6
|
+
require "ace/assign/authority/private_directory"
|
|
7
|
+
|
|
8
|
+
module Ace
|
|
9
|
+
module Overseer
|
|
10
|
+
module Molecules
|
|
11
|
+
# Retained inputs, never an authority grant or an attempt/progress ledger.
|
|
12
|
+
class LaunchRequest
|
|
13
|
+
KEYS = %w[version project_id mapping_id assignment_id task_id scope base_head mutation_id definition_bytes definition_sha256 prepared_bundle].sort.freeze
|
|
14
|
+
BUNDLE_KEYS = %w[filename bytes sha256].sort.freeze
|
|
15
|
+
MAX_REQUEST = 131_072
|
|
16
|
+
MAX_DEFINITION = 32_768
|
|
17
|
+
MAX_BUNDLE = 64 * 1024 * 1024
|
|
18
|
+
SHA = /\A[0-9a-f]{64}\z/
|
|
19
|
+
GIT = /\A(?:[0-9a-f]{40}|[0-9a-f]{64})\z/
|
|
20
|
+
MUTATION = /\A[a-zA-Z0-9][a-zA-Z0-9._-]{0,115}\z/
|
|
21
|
+
|
|
22
|
+
def initialize(root:)
|
|
23
|
+
@root = Ace::Assign::Authority::PrivateDirectory.verify!(root)
|
|
24
|
+
rescue Ace::Assign::AttemptErrors::ReceiptRejected => error
|
|
25
|
+
raise Error, error.message
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def publish(project_id:, mapping_id:, task_id:, base_head:, mutation_id:, prepared:)
|
|
29
|
+
bundle = prepared.fetch("bundle")
|
|
30
|
+
document = {"version" => 1, "project_id" => project_id, "mapping_id" => mapping_id,
|
|
31
|
+
"assignment_id" => prepared.fetch("assignment_id"), "task_id" => task_id,
|
|
32
|
+
"scope" => prepared.fetch("scope"), "base_head" => base_head, "mutation_id" => mutation_id,
|
|
33
|
+
"definition_bytes" => prepared.fetch("definition_bytes"),
|
|
34
|
+
"definition_sha256" => Digest::SHA256.hexdigest(prepared.fetch("definition_bytes")),
|
|
35
|
+
"prepared_bundle" => {"filename" => "#{mutation_id}.prepared.bundle", "bytes" => prepared.fetch("bytes"), "sha256" => prepared.fetch("sha256")}}
|
|
36
|
+
validate!(document)
|
|
37
|
+
verify_bundle!(document, bundle)
|
|
38
|
+
bytes = JSON.generate(document)
|
|
39
|
+
raise Error, "Retained launch request exceeds bound" if bytes.bytesize > MAX_REQUEST
|
|
40
|
+
request_path = File.join(@root, "#{mutation_id}.json")
|
|
41
|
+
bundle_path = File.join(@root, document.fetch("prepared_bundle").fetch("filename"))
|
|
42
|
+
sidecar_path = definition_path(document)
|
|
43
|
+
[request_path, bundle_path, sidecar_path].each do |path|
|
|
44
|
+
begin
|
|
45
|
+
File.lstat(path)
|
|
46
|
+
raise Error, "Retained launch invocation already exists"
|
|
47
|
+
rescue Errno::ENOENT
|
|
48
|
+
# Both names must be absent; exclusive creation closes the race.
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
# Never overwrite either input. Failure can leave an unselected orphan;
|
|
52
|
+
# it cannot create a different accepted invocation under the same ID.
|
|
53
|
+
write_exclusive(bundle_path, bundle)
|
|
54
|
+
write_exclusive(sidecar_path, document.fetch("definition_bytes"))
|
|
55
|
+
write_exclusive(request_path, bytes)
|
|
56
|
+
File.open(@root, File::RDONLY) { |directory| directory.fsync }
|
|
57
|
+
load(request_path)
|
|
58
|
+
rescue KeyError, JSON::GeneratorError, SystemCallError => error
|
|
59
|
+
raise Error, "Retained launch publication failed: #{error.message}"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def load(path)
|
|
63
|
+
path = File.expand_path(path)
|
|
64
|
+
raise Error, "Retained launch request is outside its private owner" unless File.dirname(path) == @root
|
|
65
|
+
bytes = held_read(path, MAX_REQUEST)
|
|
66
|
+
value = JSON.parse(bytes, create_additions: false, max_nesting: 32, allow_duplicate_key: false, allow_comments: false)
|
|
67
|
+
validate!(value)
|
|
68
|
+
raise Error, "Retained launch filename differs" unless File.basename(path) == "#{value.fetch('mutation_id')}.json"
|
|
69
|
+
freeze_value(value)
|
|
70
|
+
rescue JSON::ParserError, KeyError, TypeError, SystemCallError => error
|
|
71
|
+
raise Error, "Retained launch request is invalid: #{error.message}"
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Fresh launch requires the original complete bundle. Recovery only
|
|
75
|
+
# loads the document and observes availability; it never reconstructs.
|
|
76
|
+
def verify_fresh!(document)
|
|
77
|
+
validate!(document)
|
|
78
|
+
definition = held_read(definition_path(document), MAX_DEFINITION)
|
|
79
|
+
raise Error, "Retained definition sidecar differs" unless definition == document.fetch("definition_bytes")
|
|
80
|
+
bytes = held_read(File.join(@root, document.fetch("prepared_bundle").fetch("filename")), MAX_BUNDLE)
|
|
81
|
+
verify_bundle!(document, bytes)
|
|
82
|
+
bytes.freeze
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def identity(document)
|
|
86
|
+
validate!(document)
|
|
87
|
+
path = File.join(@root, document.fetch("mutation_id") + ".json")
|
|
88
|
+
bytes = held_read(path, MAX_REQUEST)
|
|
89
|
+
raise Error, "Retained request changed before identity output" unless bytes == JSON.generate(document)
|
|
90
|
+
reference = JSON.parse(document.fetch("definition_bytes")).fetch("prepared_work")
|
|
91
|
+
document.slice("project_id", "mapping_id", "assignment_id", "task_id", "scope", "base_head", "mutation_id", "definition_sha256").merge(
|
|
92
|
+
"type" => "launch_inputs_retained", "request_path" => path, "request_sha256" => Digest::SHA256.hexdigest(bytes),
|
|
93
|
+
"definition_path" => definition_path(document), "prepared_bundle" => document.fetch("prepared_bundle"),
|
|
94
|
+
"prepared_bundle_path" => File.join(@root, document.fetch("prepared_bundle").fetch("filename")),
|
|
95
|
+
"selection_sha256" => reference.fetch("selection_sha256"))
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def definition_path(document)
|
|
99
|
+
validate!(document)
|
|
100
|
+
File.join(@root, document.fetch("mutation_id") + ".definition.json")
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def definition_availability(document)
|
|
104
|
+
bytes = held_read(definition_path(document), MAX_DEFINITION)
|
|
105
|
+
bytes == document.fetch("definition_bytes") ? "available" : "mismatch"
|
|
106
|
+
rescue Errno::ENOENT
|
|
107
|
+
"unavailable"
|
|
108
|
+
rescue Error, SystemCallError
|
|
109
|
+
"mismatch"
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def bundle_availability(document)
|
|
113
|
+
validate!(document)
|
|
114
|
+
bytes = held_read(File.join(@root, document.fetch("prepared_bundle").fetch("filename")), MAX_BUNDLE)
|
|
115
|
+
ref = document.fetch("prepared_bundle")
|
|
116
|
+
bytes.bytesize == ref.fetch("bytes") && Digest::SHA256.hexdigest(bytes) == ref.fetch("sha256") ? "available" : "mismatch"
|
|
117
|
+
rescue Errno::ENOENT
|
|
118
|
+
"unavailable"
|
|
119
|
+
rescue Error, SystemCallError
|
|
120
|
+
"mismatch"
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
private
|
|
124
|
+
|
|
125
|
+
def validate!(value)
|
|
126
|
+
closed!(value, KEYS)
|
|
127
|
+
raise Error, "Retained launch version differs" unless value["version"].is_a?(Integer) && value["version"] == 1
|
|
128
|
+
%w[project_id mapping_id assignment_id task_id].each do |key|
|
|
129
|
+
raise Error, "Retained launch identity is invalid" unless value[key].is_a?(String) && value[key].match?(Ace::Assign::Authority::PreparedWork::TOKEN)
|
|
130
|
+
end
|
|
131
|
+
raise Error, "Retained launch mutation is invalid" unless value["mutation_id"].is_a?(String) && value["mutation_id"].match?(MUTATION)
|
|
132
|
+
raise Error, "Retained launch scope is invalid" unless value["scope"].is_a?(String) && value["scope"].match?(Ace::Assign::Authority::PreparedWork::SCOPE)
|
|
133
|
+
raise Error, "Retained launch base is invalid" unless value["base_head"].is_a?(String) && value["base_head"].match?(GIT)
|
|
134
|
+
definition = value["definition_bytes"]
|
|
135
|
+
raise Error, "Retained launch definition exceeds bound" unless definition.is_a?(String) && definition.bytesize.between?(1, MAX_DEFINITION) && definition.valid_encoding?
|
|
136
|
+
raise Error, "Retained launch definition digest differs" unless value["definition_sha256"].is_a?(String) && value["definition_sha256"].match?(SHA) && Digest::SHA256.hexdigest(definition) == value["definition_sha256"]
|
|
137
|
+
parsed = JSON.parse(definition, create_additions: false, max_nesting: 32, allow_duplicate_key: false, allow_comments: false)
|
|
138
|
+
unless parsed.is_a?(Hash) && parsed.values_at("session_id", "project_id", "task_id") == value.values_at("assignment_id", "project_id", "task_id")
|
|
139
|
+
raise Error, "Retained definition association differs"
|
|
140
|
+
end
|
|
141
|
+
reference = parsed.fetch("prepared_work")
|
|
142
|
+
closed!(reference, %w[version task_id scope prepared_head prepared_tree manifest_bytes manifest_sha256 selection_sha256].sort)
|
|
143
|
+
unless reference["version"].is_a?(Integer) && reference["version"] == 1 && reference.values_at("task_id", "scope") == value.values_at("task_id", "scope") &&
|
|
144
|
+
%w[prepared_head prepared_tree].all? { |key| reference[key].is_a?(String) && reference[key].match?(/\A[0-9a-f]{40}\z/) } &&
|
|
145
|
+
reference["manifest_bytes"].is_a?(Integer) && reference["manifest_bytes"].between?(1, MAX_DEFINITION) &&
|
|
146
|
+
%w[manifest_sha256 selection_sha256].all? { |key| reference[key].is_a?(String) && reference[key].match?(SHA) }
|
|
147
|
+
raise Error, "Retained prepared selection differs"
|
|
148
|
+
end
|
|
149
|
+
bundle = value["prepared_bundle"]
|
|
150
|
+
closed!(bundle, BUNDLE_KEYS)
|
|
151
|
+
unless bundle["filename"] == "#{value['mutation_id']}.prepared.bundle" && bundle["bytes"].is_a?(Integer) && bundle["bytes"].between?(1, MAX_BUNDLE) && bundle["sha256"].is_a?(String) && bundle["sha256"].match?(SHA)
|
|
152
|
+
raise Error, "Retained bundle reference differs"
|
|
153
|
+
end
|
|
154
|
+
value
|
|
155
|
+
rescue JSON::ParserError, KeyError, TypeError
|
|
156
|
+
raise Error, "Retained launch definition is invalid"
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def verify_bundle!(document, bytes)
|
|
160
|
+
ref = document.fetch("prepared_bundle")
|
|
161
|
+
raise Error, "Retained bundle bytes differ" unless bytes.is_a?(String) && bytes.bytesize == ref.fetch("bytes") && Digest::SHA256.hexdigest(bytes) == ref.fetch("sha256")
|
|
162
|
+
# One instance-owned proof for the last exact admitted input. Every
|
|
163
|
+
# caller still validates the closed document and freshly held bytes;
|
|
164
|
+
# a new owner or any tuple/content change requires actual admission.
|
|
165
|
+
key = JSON.generate(document)
|
|
166
|
+
raise Error, "Retained launch request exceeds bound" if key.bytesize > MAX_REQUEST
|
|
167
|
+
return if @admitted_key == key
|
|
168
|
+
reference = JSON.parse(document.fetch("definition_bytes")).fetch("prepared_work")
|
|
169
|
+
work = Ace::Assign::Authority::PreparedWork.admit(bytes: bytes, head: reference.fetch("prepared_head"), tree: reference.fetch("prepared_tree"), sha256: ref.fetch("sha256"), size: ref.fetch("bytes"), root: @root)
|
|
170
|
+
unless work.definition_bytes(head: reference.fetch("prepared_head"), tree: reference.fetch("prepared_tree")) == document.fetch("definition_bytes")
|
|
171
|
+
raise Error, "Retained prepared definition differs"
|
|
172
|
+
end
|
|
173
|
+
@admitted_key = key.freeze
|
|
174
|
+
rescue ArgumentError, Ace::Assign::AttemptErrors::ReceiptRejected => error
|
|
175
|
+
raise Error, "Retained prepared work is invalid: #{error.message}"
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def held_read(path, limit)
|
|
179
|
+
File.open(path, File::RDONLY | File::NOFOLLOW | File::NONBLOCK) do |file|
|
|
180
|
+
before = file.stat
|
|
181
|
+
unless before.file? && before.uid == Process.uid && (before.mode & 0077).zero? && before.size.between?(1, limit)
|
|
182
|
+
raise Error, "Retained input is not a bounded private regular file"
|
|
183
|
+
end
|
|
184
|
+
bytes = file.read(limit + 1)
|
|
185
|
+
after = file.stat
|
|
186
|
+
fields = %i[dev ino mode uid gid size mtime ctime]
|
|
187
|
+
unless fields.all? { |field| before.public_send(field) == after.public_send(field) } && bytes.bytesize == before.size
|
|
188
|
+
raise Error, "Retained input changed during held read"
|
|
189
|
+
end
|
|
190
|
+
bytes
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def write_exclusive(path, bytes)
|
|
195
|
+
File.open(path, File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW, 0600) do |file|
|
|
196
|
+
file.write(bytes)
|
|
197
|
+
file.flush
|
|
198
|
+
file.fsync
|
|
199
|
+
end
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def closed!(value, keys)
|
|
203
|
+
raise Error, "Retained launch fields differ" unless value.is_a?(Hash) && value.keys.all? { |key| key.is_a?(String) } && value.keys.sort == keys
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
def freeze_value(value)
|
|
207
|
+
case value
|
|
208
|
+
when Hash then value.each { |key, item| freeze_value(key); freeze_value(item) }
|
|
209
|
+
when Array then value.each { |item| freeze_value(item) }
|
|
210
|
+
end
|
|
211
|
+
value.freeze
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "ace/assign/cli"
|
|
5
|
+
require "ace/runtime/molecules/process_identity"
|
|
6
|
+
require_relative "proposal_tick"
|
|
7
|
+
require_relative "../organisms/launch_recovery"
|
|
8
|
+
|
|
9
|
+
module Ace
|
|
10
|
+
module Overseer
|
|
11
|
+
module Molecules
|
|
12
|
+
# Own one foreground child of the already loaded fixed Assign CLI. The
|
|
13
|
+
# process boundary is injected in controlled tests, never selected by input.
|
|
14
|
+
class OriginalLaunchChild
|
|
15
|
+
LIMIT = 16_384
|
|
16
|
+
DEADLINE = 30
|
|
17
|
+
DRAIN_LIMIT = 65_536
|
|
18
|
+
READY_FIELDS = %w[assignment_id attempt_id generation journal_commit mapping_id original_binding_digest type version].sort.freeze
|
|
19
|
+
class ProcessBoundary
|
|
20
|
+
def supported? = Process.respond_to?(:fork)
|
|
21
|
+
def pipe = IO.pipe
|
|
22
|
+
def fork_loaded(argv:, reader:, writer:)
|
|
23
|
+
Process.fork do
|
|
24
|
+
reader.close
|
|
25
|
+
STDOUT.reopen(writer)
|
|
26
|
+
writer.close
|
|
27
|
+
$stdout = STDOUT
|
|
28
|
+
begin
|
|
29
|
+
code = Ace::Assign::CLI.start(argv)
|
|
30
|
+
Process.exit!(code.is_a?(Integer) ? code : 0)
|
|
31
|
+
rescue SystemExit => error
|
|
32
|
+
Process.exit!(error.status)
|
|
33
|
+
rescue Exception => error
|
|
34
|
+
warn "Original Assign child failed (#{error.class}): #{error.message}"
|
|
35
|
+
Process.exit!(1)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
def wait(pid) = Process.waitpid2(pid, Process::WNOHANG)
|
|
40
|
+
def readable?(reader, timeout) = !!IO.select([reader], nil, nil, timeout)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
attr_reader :pid, :identity, :state, :error, :ready, :exit_status
|
|
44
|
+
|
|
45
|
+
def initialize(process: ProcessBoundary.new, kernel: Ace::Runtime::Molecules::ProcessIdentity.new,
|
|
46
|
+
threads: -> { Thread.list }, quiescent: -> { ProposalTick.quiescent? },
|
|
47
|
+
clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
48
|
+
@process, @kernel, @threads, @quiescent, @clock = process, kernel, threads, quiescent, clock
|
|
49
|
+
@state = "not_started"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def start(request:, definition_path:, bundle_path:)
|
|
53
|
+
raise Error, "Original child already selected" unless state == "not_started"
|
|
54
|
+
raise Error, "Loaded CLI fork is unsupported" unless @process.supported?
|
|
55
|
+
safe_boundary!
|
|
56
|
+
argv = ["authority", "launch", "--mapping", request.fetch("mapping_id"), "--assignment", request.fetch("assignment_id"),
|
|
57
|
+
"--definition", definition_path, "--prepared-bundle", bundle_path, "--step", request.fetch("scope"),
|
|
58
|
+
"--base-head", request.fetch("base_head"), "--mutation", request.fetch("mutation_id")].freeze
|
|
59
|
+
@request = request
|
|
60
|
+
@reader, writer = @process.pipe
|
|
61
|
+
opened_here = true
|
|
62
|
+
safe_boundary!
|
|
63
|
+
@readiness_deadline = @clock.call + DEADLINE
|
|
64
|
+
@pid = @process.fork_loaded(argv: argv, reader: @reader, writer: writer)
|
|
65
|
+
started_here = true
|
|
66
|
+
@state = "awaiting_ready"
|
|
67
|
+
writer.close
|
|
68
|
+
@identity = @kernel.capture(pid)
|
|
69
|
+
if identity
|
|
70
|
+
identity.each_value(&:freeze)
|
|
71
|
+
identity.freeze
|
|
72
|
+
end
|
|
73
|
+
uncertain!("Original child birth is unavailable") unless identity
|
|
74
|
+
self
|
|
75
|
+
rescue StandardError, NotImplementedError
|
|
76
|
+
if started_here
|
|
77
|
+
uncertain!("Original child startup observation failed")
|
|
78
|
+
self
|
|
79
|
+
else
|
|
80
|
+
@reader&.close if opened_here
|
|
81
|
+
raise
|
|
82
|
+
end
|
|
83
|
+
ensure
|
|
84
|
+
writer&.close unless writer&.closed?
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def await_ready(status:)
|
|
88
|
+
return self unless state == "awaiting_ready"
|
|
89
|
+
bytes = +"".b
|
|
90
|
+
loop do
|
|
91
|
+
remaining = @readiness_deadline - @clock.call
|
|
92
|
+
return uncertain!("Original child readiness timed out") unless remaining.positive? && @process.readable?(@reader, remaining)
|
|
93
|
+
chunk = @reader.read_nonblock(LIMIT + 1 - bytes.bytesize, exception: false)
|
|
94
|
+
next if chunk == :wait_readable
|
|
95
|
+
return uncertain!("Original child ended before readiness") if chunk.nil?
|
|
96
|
+
return uncertain!("Original child readiness timed out after receipt") unless @clock.call < @readiness_deadline
|
|
97
|
+
bytes << chunk
|
|
98
|
+
return uncertain!("Original child readiness exceeds bound") if bytes.bytesize > LIMIT
|
|
99
|
+
next unless bytes.include?("\n")
|
|
100
|
+
return uncertain!("Original child readiness has extra output") unless bytes.end_with?("\n") && bytes.count("\n") == 1
|
|
101
|
+
text = bytes.dup.force_encoding(Encoding::UTF_8)
|
|
102
|
+
raise Error, "Original child readiness is not UTF-8" unless text.valid_encoding?
|
|
103
|
+
ready = JSON.parse(text, create_additions: false, max_nesting: 32, allow_duplicate_key: false, allow_comments: false)
|
|
104
|
+
unless ready.is_a?(Hash) && ready.keys.sort == READY_FIELDS && ready["version"].is_a?(Integer) && ready["version"] == 1 &&
|
|
105
|
+
ready["type"] == "launch_ready" && ready.values_at("mapping_id", "assignment_id") == @request.values_at("mapping_id", "assignment_id") &&
|
|
106
|
+
@kernel.capture(pid) == identity
|
|
107
|
+
raise Error, "Original child readiness differs from original selection"
|
|
108
|
+
end
|
|
109
|
+
return uncertain!("Original child readiness timed out before canonical join") unless @clock.call < @readiness_deadline
|
|
110
|
+
canonical = status.join_ready!(project: @request.fetch("project_id"), agent: @request.fetch("mapping_id"), ready: ready)
|
|
111
|
+
Organisms::LaunchRecovery.verify_original!(request: @request, row: canonical.fetch("item"))
|
|
112
|
+
return uncertain!("Original child readiness timed out during canonical join") unless @clock.call < @readiness_deadline
|
|
113
|
+
raise Error, "Original child birth changed during readiness join" unless @kernel.capture(pid) == identity
|
|
114
|
+
ready.each_value(&:freeze)
|
|
115
|
+
@ready = ready.freeze
|
|
116
|
+
@state = "ready"
|
|
117
|
+
return self
|
|
118
|
+
end
|
|
119
|
+
rescue StandardError => error
|
|
120
|
+
uncertain!("Original child readiness unavailable: #{error.message}")
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# No lifetime deadline, replacement, signal or detached waiter. Reaping
|
|
124
|
+
# this exact owned child is observation, never canonical release proof.
|
|
125
|
+
def observe
|
|
126
|
+
return self unless pid && state != "exited"
|
|
127
|
+
if %w[ready uncertain].include?(state) && @process.readable?(@reader, 0)
|
|
128
|
+
# Discard at most one bounded chunk per observation, including
|
|
129
|
+
# uncertain children. Never retain arbitrary output or let an
|
|
130
|
+
# ignored stdout pipe deadlock the same original foreground owner.
|
|
131
|
+
extra = @reader.read_nonblock(DRAIN_LIMIT, exception: false)
|
|
132
|
+
uncertain!("Original child emitted output outside readiness") if extra.is_a?(String) && !extra.empty?
|
|
133
|
+
end
|
|
134
|
+
result = @process.wait(pid)
|
|
135
|
+
if result
|
|
136
|
+
raise Error, "Wait selected a different child" unless result.first == pid
|
|
137
|
+
@exit_status = result.last
|
|
138
|
+
@reader.close unless @reader.closed?
|
|
139
|
+
@state = "exited"
|
|
140
|
+
elsif identity && @kernel.capture(pid) != identity
|
|
141
|
+
uncertain!("Original child birth is no longer observable")
|
|
142
|
+
end
|
|
143
|
+
self
|
|
144
|
+
rescue StandardError => error
|
|
145
|
+
uncertain!("Original child observation unavailable: #{error.message}")
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
private
|
|
149
|
+
|
|
150
|
+
def safe_boundary!
|
|
151
|
+
live = @threads.call.select(&:alive?)
|
|
152
|
+
unless live == [Thread.current] && @quiescent.call
|
|
153
|
+
raise Error, "Loaded CLI fork requires one quiescent coordinator thread with no held coordination lock"
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
def uncertain!(message)
|
|
158
|
+
@error = message
|
|
159
|
+
@state = "uncertain"
|
|
160
|
+
self
|
|
161
|
+
end
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
end
|
|
@@ -1,26 +1,54 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "open3"
|
|
4
3
|
require "json"
|
|
4
|
+
require "ace/herdr/molecules/bounded_process"
|
|
5
5
|
|
|
6
6
|
module Ace
|
|
7
7
|
module Overseer
|
|
8
8
|
module Molecules
|
|
9
|
-
#
|
|
10
|
-
#
|
|
9
|
+
# One bounded policy wake. These process-local active markers coalesce
|
|
10
|
+
# callers; they never retain proposal outcomes or grant permission.
|
|
11
11
|
class ProposalTick
|
|
12
|
-
|
|
12
|
+
DEADLINE = 5
|
|
13
|
+
LIMIT = 65_536
|
|
14
|
+
@mutex = Mutex.new
|
|
15
|
+
@active = {}
|
|
16
|
+
|
|
17
|
+
def self.quiescent?
|
|
18
|
+
return false if @mutex.owned?
|
|
19
|
+
@mutex.synchronize { @active.empty? }
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.acquire(key)
|
|
23
|
+
@mutex.synchronize do
|
|
24
|
+
return false if @active.key?(key)
|
|
25
|
+
@active[key] = true
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.release(key)
|
|
30
|
+
@mutex.synchronize { @active.delete(key) }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def initialize(runner: Ace::Herdr::Molecules::BoundedProcess, binary: "/usr/local/bin/ace-hitl", socket: ENV["ACE_HITL_SOCKET"], project: ENV["ACE_HITL_PROJECT"])
|
|
13
34
|
@runner, @binary, @socket, @project = runner, binary, socket, project
|
|
14
35
|
end
|
|
15
36
|
|
|
16
37
|
def call
|
|
17
38
|
return [] if @socket.to_s.empty? || @project.to_s.empty?
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
39
|
+
key = [@binary, @socket, @project].map { |value| value.to_s.dup.freeze }.freeze
|
|
40
|
+
return [] unless self.class.acquire(key)
|
|
41
|
+
acquired = true
|
|
42
|
+
result = @runner.call([{"ACE_HITL_SOCKET" => @socket}, @binary, "proposal", "resolve-due", "--project", @project],
|
|
43
|
+
timeout_s: DEADLINE, output_limit: LIMIT, stderr_limit: LIMIT, cleanup_group: true)
|
|
44
|
+
raise Error, "Proposal resolution unavailable; deadlines deferred" unless result.status&.success? && !result.oversized
|
|
45
|
+
value = JSON.parse(result.stdout)
|
|
46
|
+
raise Error, "Proposal resolution malformed; deadlines deferred" unless value.is_a?(Array) && value.all? { |item| item.is_a?(Hash) }
|
|
47
|
+
value
|
|
48
|
+
rescue JSON::ParserError, SystemCallError, Timeout::Error, Ace::Herdr::Molecules::BoundedProcess::PostLaunchError
|
|
23
49
|
raise Error, "Proposal resolution unavailable; deadlines deferred"
|
|
50
|
+
ensure
|
|
51
|
+
self.class.release(key) if acquired
|
|
24
52
|
end
|
|
25
53
|
end
|
|
26
54
|
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ace/lab"
|
|
4
|
+
require "ace/assign/authority/deployment"
|
|
5
|
+
|
|
6
|
+
module Ace
|
|
7
|
+
module Overseer
|
|
8
|
+
module Molecules
|
|
9
|
+
# Visibility narrows the installed project pool; it never grants authority.
|
|
10
|
+
class ProtectedSelection
|
|
11
|
+
def initialize(topology: nil, deployment_loader: nil)
|
|
12
|
+
@topology = topology || Ace::Lab::Organisms::TopologyService.from_config
|
|
13
|
+
@deployment_loader = deployment_loader || -> { Ace::Assign::Authority::Deployment.load }
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def call(project:, agent: nil)
|
|
17
|
+
token!(project)
|
|
18
|
+
token!(agent) if agent
|
|
19
|
+
topology = @topology.agents(project: project)
|
|
20
|
+
raise Error, "Protected topology unavailable (#{topology.error_code})" unless topology.ok?
|
|
21
|
+
deployment = @deployment_loader.call
|
|
22
|
+
pool = deployment.data.fetch("launch_mappings").select { |_id, map| map.fetch("project_id") == project }.keys.sort
|
|
23
|
+
raise Error, "Protected project has no provisioned mappings" if pool.empty?
|
|
24
|
+
visible = topology.data.fetch("agents")
|
|
25
|
+
unless visible.is_a?(Array) && visible.all? { |entry| entry.is_a?(Hash) && entry["project"] == project && pool.include?(entry["id"]) } &&
|
|
26
|
+
visible.map { |entry| entry.fetch("id") }.uniq.size == visible.size
|
|
27
|
+
raise Error, "Public agent visibility differs from protected project mappings"
|
|
28
|
+
end
|
|
29
|
+
ids = visible.map { |entry| entry.fetch("id") }.sort
|
|
30
|
+
if agent && !ids.include?(agent)
|
|
31
|
+
raise Error, "Requested protected agent is not visible in this project"
|
|
32
|
+
end
|
|
33
|
+
[deployment, pool.freeze, ids.freeze].freeze
|
|
34
|
+
rescue KeyError, TypeError, NoMethodError
|
|
35
|
+
raise Error, "Protected selection metadata is malformed"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
def token!(value)
|
|
41
|
+
raise Error, "Protected identity is invalid" unless value.is_a?(String) && value.match?(Ace::Assign::Authority::Deployment::TOKEN)
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -15,9 +15,9 @@ module Ace
|
|
|
15
15
|
@env = env
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
def open(worktree_path:, preset: nil)
|
|
18
|
+
def open(worktree_path:, preset: nil, runtime: nil)
|
|
19
19
|
name = Ace::Runtime.sanitize_name(File.basename(worktree_path.to_s))
|
|
20
|
-
|
|
20
|
+
selected_adapter(runtime).ensure_window(
|
|
21
21
|
name: name,
|
|
22
22
|
root: File.expand_path(worktree_path.to_s),
|
|
23
23
|
preset: preset
|
|
@@ -31,6 +31,13 @@ module Ace
|
|
|
31
31
|
|
|
32
32
|
attr_reader :config, :env
|
|
33
33
|
|
|
34
|
+
def selected_adapter(runtime)
|
|
35
|
+
return adapter if runtime.nil? || @runtime
|
|
36
|
+
Ace::Runtime::Molecules::RuntimeSelector.new(
|
|
37
|
+
config: {runtime: Atoms::RuntimeSetting.normalize(runtime)}, env: env
|
|
38
|
+
).resolve
|
|
39
|
+
end
|
|
40
|
+
|
|
34
41
|
def adapter
|
|
35
42
|
@adapter ||= @runtime || Ace::Runtime::Molecules::RuntimeSelector.new(
|
|
36
43
|
config: {runtime: Atoms::RuntimeSetting.normalize(config["runtime"])},
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../molecules/launch_request"
|
|
4
|
+
require_relative "protected_status"
|
|
5
|
+
|
|
6
|
+
module Ace
|
|
7
|
+
module Overseer
|
|
8
|
+
module Organisms
|
|
9
|
+
# Canonical discovery only. Missing input or missing attribution never
|
|
10
|
+
# authorizes replacement, registration, upload, cleanup or a new launch.
|
|
11
|
+
class LaunchRecovery
|
|
12
|
+
def initialize(status: nil, request_factory: nil)
|
|
13
|
+
@status = status || ProtectedStatus.new
|
|
14
|
+
@request_factory = request_factory || ->(root) { Molecules::LaunchRequest.new(root: root) }
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# Shared readiness/recovery association. A valid canonical row is not
|
|
18
|
+
# enough: it must belong to this retained invocation and exact inputs.
|
|
19
|
+
def self.verify_original!(request:, row:)
|
|
20
|
+
reference = JSON.parse(request.fetch("definition_bytes")).fetch("prepared_work")
|
|
21
|
+
bundle = request.fetch("prepared_bundle")
|
|
22
|
+
expected = {"assignment_id" => request.fetch("assignment_id"), "task_id" => request.fetch("task_id"),
|
|
23
|
+
"scope" => request.fetch("scope"), "base_head" => request.fetch("base_head"),
|
|
24
|
+
"reservation_mutation_id" => request.fetch("mutation_id") + "-reserve",
|
|
25
|
+
"definition_digest" => request.fetch("definition_sha256"), "selection_sha256" => reference.fetch("selection_sha256"),
|
|
26
|
+
"prepared_bundle_bytes" => bundle.fetch("bytes"), "prepared_bundle_sha256" => bundle.fetch("sha256"),
|
|
27
|
+
"prepared_bundle_ref" => "execution/prepared/#{request.fetch('assignment_id')}-#{bundle.fetch('sha256')}.bundle"}
|
|
28
|
+
unless expected.all? { |key, value| row.fetch(key) == value && row.fetch(key).class == value.class }
|
|
29
|
+
raise Error, "Retained invocation differs from original reservation"
|
|
30
|
+
end
|
|
31
|
+
row
|
|
32
|
+
rescue KeyError, TypeError, JSON::ParserError
|
|
33
|
+
raise Error, "Retained invocation association is malformed"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def call(path:)
|
|
37
|
+
path = File.expand_path(path)
|
|
38
|
+
owner = @request_factory.call(File.dirname(path))
|
|
39
|
+
request = owner.load(path)
|
|
40
|
+
availability = owner.bundle_availability(request)
|
|
41
|
+
definition_availability = owner.definition_availability(request)
|
|
42
|
+
snapshot = @status.collect(project: request.fetch("project_id"), agent: request.fetch("mapping_id"))
|
|
43
|
+
mapping = snapshot.fetch("agents").find { |entry| entry.fetch("agent_id") == request.fetch("mapping_id") }
|
|
44
|
+
raise Error, "Retained invocation canonical inventory is unavailable" unless mapping && mapping.fetch("status") == "ok"
|
|
45
|
+
inventory = mapping.fetch("inventory")
|
|
46
|
+
reference = JSON.parse(request.fetch("definition_bytes")).fetch("prepared_work")
|
|
47
|
+
bundle = request.fetch("prepared_bundle")
|
|
48
|
+
matches = inventory.fetch("items").select do |row|
|
|
49
|
+
row.fetch("reservation_mutation_id") == request.fetch("mutation_id") + "-reserve"
|
|
50
|
+
end
|
|
51
|
+
raise Error, "Retained invocation reservation is ambiguous" if matches.size > 1
|
|
52
|
+
row = matches.first
|
|
53
|
+
self.class.verify_original!(request: request, row: row) if row
|
|
54
|
+
{"request_path" => path, "project_id" => request.fetch("project_id"), "mapping_id" => request.fetch("mapping_id"),
|
|
55
|
+
"assignment_id" => request.fetch("assignment_id"), "mutation_id" => request.fetch("mutation_id"),
|
|
56
|
+
"definition_sha256" => request.fetch("definition_sha256"), "selection_sha256" => reference.fetch("selection_sha256"),
|
|
57
|
+
"prepared_bundle" => bundle, "local_bundle" => availability, "local_definition" => definition_availability,
|
|
58
|
+
"attribution" => row ? "original_reservation" : "unattributed", "journal_commit" => inventory.fetch("journal_commit"), "item" => row}
|
|
59
|
+
rescue KeyError, TypeError, JSON::ParserError
|
|
60
|
+
raise Error, "Retained invocation recovery metadata is malformed"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|