ace-assign 0.58.0 → 0.61.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +50 -12
- data/docs/usage.md +32 -0
- data/lib/ace/assign/cli/commands/attempt/evidence.rb +28 -0
- data/lib/ace/assign/cli.rb +4 -0
- data/lib/ace/assign/models/evidence_event.rb +1 -0
- data/lib/ace/assign/models/execution_receipt.rb +7 -3
- data/lib/ace/assign/molecules/evidence_calculator.rb +51 -1
- data/lib/ace/assign/molecules/evidence_journal.rb +291 -12
- data/lib/ace/assign/molecules/fork_session_launcher.rb +38 -19
- data/lib/ace/assign/molecules/lifecycle_exclusion.rb +209 -0
- data/lib/ace/assign/molecules/receipt_verifier.rb +45 -0
- data/lib/ace/assign/organisms/assignment_executor.rb +3 -1
- data/lib/ace/assign/organisms/attempt_coordinator.rb +378 -42
- data/lib/ace/assign/version.rb +1 -1
- metadata +20 -4
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "digest"
|
|
4
|
+
require "digest"
|
|
3
5
|
require "fileutils"
|
|
4
6
|
require "json"
|
|
5
7
|
require "open3"
|
|
@@ -97,11 +99,15 @@ module Ace
|
|
|
97
99
|
value = ref_value
|
|
98
100
|
return [] if value.nil?
|
|
99
101
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
102
|
+
paths, stderr, status = git("ls-tree", "-r", "--name-only", value, "--",
|
|
103
|
+
"execution/#{assignment_id}/events/")
|
|
104
|
+
raise AttemptErrors::EvidenceUnavailable, "Cannot read journal events: #{stderr}" unless status.success?
|
|
105
|
+
events = paths.lines.map(&:strip).select { |path| path.end_with?(".json") }.map do |path|
|
|
106
|
+
content, error, read_status = git("show", "#{value}:#{path}")
|
|
107
|
+
raise AttemptErrors::EvidenceUnavailable, "Cannot read journal event: #{error}" unless read_status.success?
|
|
108
|
+
JSON.parse(content)
|
|
104
109
|
end
|
|
110
|
+
order_by_chain(events)
|
|
105
111
|
rescue JSON::ParserError => e
|
|
106
112
|
raise AttemptErrors::EvidenceUnavailable, "Corrupt journal event for #{assignment_id}: #{e.message}"
|
|
107
113
|
end
|
|
@@ -117,6 +123,215 @@ module Ace
|
|
|
117
123
|
.compact
|
|
118
124
|
end
|
|
119
125
|
|
|
126
|
+
# Reserve a globally unique service request in the same evidence ref
|
|
127
|
+
# as its owning attempt. The request index and attempt event are one
|
|
128
|
+
# Git commit, so a ref race cannot admit two effects for one ID. An
|
|
129
|
+
# exact authorization reference is consumed by its first live claim:
|
|
130
|
+
# a second request presenting the same operation/project/target
|
|
131
|
+
# authorization is rejected instead of dispatching a duplicate effect.
|
|
132
|
+
# Dispatching callers claim directly as +uncertain+ so no commit
|
|
133
|
+
# window exists where a stranded claim reads as accepted.
|
|
134
|
+
def claim_service_request(binding, state: "accepted", guard: nil)
|
|
135
|
+
guard ||= -> { authorization_conflict(binding) }
|
|
136
|
+
update_service_request(binding.fetch("request_id"), expected: nil,
|
|
137
|
+
replacement: binding.merge("state" => state, "claimed_at" => Time.now.utc.iso8601(9)),
|
|
138
|
+
event_type: "service_claim", guard: guard)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Reject a service request. A request not yet on file gets its
|
|
142
|
+
# auditable rejection claim (which never consumed the authorization);
|
|
143
|
+
# an existing live claim transitions to rejected but stays consuming:
|
|
144
|
+
# only attributable evidence that the effect did not occur may free a
|
|
145
|
+
# dispatched authorization, never a bare rejection.
|
|
146
|
+
def reject_service_request(binding, reason:)
|
|
147
|
+
request_id = binding.fetch("request_id")
|
|
148
|
+
existing = service_request(request_id)
|
|
149
|
+
if existing
|
|
150
|
+
unless existing.except("state", "receipt", "reason", "claimed_at", "failed_at") == binding.except("state", "receipt", "reason", "claimed_at", "failed_at")
|
|
151
|
+
raise AttemptErrors::Conflict, "Service request #{request_id} has different input"
|
|
152
|
+
end
|
|
153
|
+
return existing if existing["state"] == "rejected"
|
|
154
|
+
return update_service_request(request_id, expected: existing,
|
|
155
|
+
replacement: existing.merge("state" => "rejected", "reason" => reason),
|
|
156
|
+
event_type: "service_transition")
|
|
157
|
+
end
|
|
158
|
+
update_service_request(request_id, expected: nil,
|
|
159
|
+
replacement: binding.merge("state" => "rejected", "reason" => reason, "consumed" => false),
|
|
160
|
+
event_type: "service_claim")
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Terminal writes are validated HERE at the journal boundary —
|
|
164
|
+
# receipt schema, binding against the current record, and attested
|
|
165
|
+
# outcome — so no caller-supplied flag decides trust. The
|
|
166
|
+
# coordinator additionally performs content-level attestation and
|
|
167
|
+
# file verification.
|
|
168
|
+
def transition_service_request(request_id, state:, receipt: nil, validated: false)
|
|
169
|
+
unless %w[succeeded failed uncertain rejected failed-settled].include?(state)
|
|
170
|
+
raise ArgumentError, "invalid service request state"
|
|
171
|
+
end
|
|
172
|
+
current = service_request(request_id)
|
|
173
|
+
raise AttemptErrors::NotFound, "Service request #{request_id} not found" unless current
|
|
174
|
+
# Terminal states are immutable; failed-settled is reached only
|
|
175
|
+
# from a dispatched (failed, uncertain, or consumed-rejected)
|
|
176
|
+
# effect with a validated no-effect receipt.
|
|
177
|
+
consumed_rejected = current["state"] == "rejected" && current["consumed"] != false
|
|
178
|
+
if %w[succeeded failed-settled].include?(current["state"]) ||
|
|
179
|
+
(current["state"] == "rejected" && !consumed_rejected) ||
|
|
180
|
+
(consumed_rejected && state != "failed-settled")
|
|
181
|
+
raise AttemptErrors::InvalidState, "Service request #{request_id} is terminal"
|
|
182
|
+
end
|
|
183
|
+
validate_terminal_receipt!(current, state, receipt) if terminal_state?(state)
|
|
184
|
+
stamped = current.merge("state" => state, "receipt" => receipt)
|
|
185
|
+
stamped["failed_at"] = Time.now.utc.iso8601(9) if state == "failed"
|
|
186
|
+
update_service_request(request_id, expected: current,
|
|
187
|
+
replacement: stamped,
|
|
188
|
+
event_type: "service_transition")
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
TERMINAL_RECEIPT_FIELDS = %w[assignment_id attempt_id candidate_head evidence executor_uid
|
|
192
|
+
input_digest operation outcome project_id request_id target transport].freeze
|
|
193
|
+
TERMINAL_BINDING_FIELDS = %w[request_id assignment_id attempt_id project_id operation
|
|
194
|
+
input_digest target candidate_head executor_uid transport].freeze
|
|
195
|
+
|
|
196
|
+
def terminal_state?(state)
|
|
197
|
+
%w[succeeded failed failed-settled].include?(state)
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
def validate_terminal_receipt!(current, state, receipt)
|
|
201
|
+
unless receipt.is_a?(Hash) && receipt.keys.sort == TERMINAL_RECEIPT_FIELDS.sort
|
|
202
|
+
raise AttemptErrors::ReceiptRejected,
|
|
203
|
+
"Service terminal receipt has invalid fields"
|
|
204
|
+
end
|
|
205
|
+
TERMINAL_BINDING_FIELDS.each do |key|
|
|
206
|
+
unless receipt[key] == current[key]
|
|
207
|
+
raise AttemptErrors::ReceiptRejected, "Service terminal receipt does not match #{key}"
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
expected_outcome = (state == "failed-settled") ? "failed" : state
|
|
211
|
+
evidence_items = receipt["evidence"]
|
|
212
|
+
valid_evidence = evidence_items.is_a?(Array) && !evidence_items.empty? &&
|
|
213
|
+
evidence_items.all? do |item|
|
|
214
|
+
item.is_a?(Hash) && item.keys.sort == %w[ref sha256] &&
|
|
215
|
+
item["ref"].is_a?(String) && item["ref"].match?(/\A[a-zA-Z0-9_.:\/-]{1,256}\z/) &&
|
|
216
|
+
item["sha256"].is_a?(String) && item["sha256"].match?(/\A[0-9a-f]{64}\z/)
|
|
217
|
+
end
|
|
218
|
+
executor_valid = receipt["executor_uid"].is_a?(Integer) && receipt["executor_uid"] >= 0
|
|
219
|
+
unless receipt["outcome"] == expected_outcome && executor_valid && valid_evidence
|
|
220
|
+
raise AttemptErrors::ReceiptRejected, "Service terminal receipt has invalid executor or evidence"
|
|
221
|
+
end
|
|
222
|
+
if state == "failed-settled" &&
|
|
223
|
+
!receipt["evidence"].any? { |item| item["ref"].end_with?("no-effect") }
|
|
224
|
+
raise AttemptErrors::ReceiptRejected,
|
|
225
|
+
"Service settlement requires a no-effect attestation artifact"
|
|
226
|
+
end
|
|
227
|
+
repo_root = File.realpath(@repo_root)
|
|
228
|
+
receipt["evidence"].each do |item|
|
|
229
|
+
path = File.expand_path(item["ref"], repo_root)
|
|
230
|
+
real = begin
|
|
231
|
+
File.realpath(path)
|
|
232
|
+
rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
|
|
233
|
+
raise AttemptErrors::ReceiptRejected,
|
|
234
|
+
"Service terminal receipt evidence is unverifiable: #{item["ref"]}"
|
|
235
|
+
end
|
|
236
|
+
intact = begin
|
|
237
|
+
real.start_with?(repo_root + File::SEPARATOR) &&
|
|
238
|
+
Digest::SHA256.file(real).hexdigest == item["sha256"]
|
|
239
|
+
rescue Errno::EACCES, Errno::ELOOP
|
|
240
|
+
false
|
|
241
|
+
end
|
|
242
|
+
raise AttemptErrors::ReceiptRejected,
|
|
243
|
+
"Service terminal receipt evidence is unverifiable: #{item["ref"]}" unless intact
|
|
244
|
+
attestation = /^ace-service-attestation request:#{Regexp.escape(current["request_id"])} \
|
|
245
|
+
input:#{Regexp.escape(current["input_digest"])} outcome:(\S+)( no-effect:(\S+))?$/
|
|
246
|
+
attested = File.read(real).scan(attestation).first
|
|
247
|
+
attested_outcome = state == "failed-settled" ? "failed" : state
|
|
248
|
+
unless attested && attested.first == attested_outcome &&
|
|
249
|
+
(state == "failed-settled" ? attested[2] == "true" : true)
|
|
250
|
+
raise AttemptErrors::ReceiptRejected,
|
|
251
|
+
"Service terminal receipt evidence does not attest #{attested_outcome}: #{item["ref"]}"
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def service_request(request_id)
|
|
257
|
+
validate_request_id!(request_id)
|
|
258
|
+
value = ref_value
|
|
259
|
+
return nil unless value
|
|
260
|
+
out, stderr, status = git("show", "#{value}:#{service_request_path(request_id)}")
|
|
261
|
+
return JSON.parse(out) if status.success?
|
|
262
|
+
return nil if stderr.include?("does not exist") || stderr.include?("exists on disk")
|
|
263
|
+
raise AttemptErrors::EvidenceUnavailable, "Cannot read service request: #{stderr}"
|
|
264
|
+
rescue JSON::ParserError
|
|
265
|
+
raise AttemptErrors::EvidenceUnavailable, "Corrupt service request #{request_id}"
|
|
266
|
+
end
|
|
267
|
+
|
|
268
|
+
# Every service request recorded in this evidence ref, whatever the
|
|
269
|
+
# owning assignment. Backs the journal-wide request-ID and
|
|
270
|
+
# authorization-consumption checks.
|
|
271
|
+
# Another live request holding the same exact authorization for the
|
|
272
|
+
# same operation, project and target. Only requests that were born
|
|
273
|
+
# rejected (never claimed, never dispatched) leave the decision
|
|
274
|
+
# unconsumed: a withdrawn or failed claim keeps it consumed until
|
|
275
|
+
# attributable evidence proves the effect did not occur.
|
|
276
|
+
def authorization_conflict(binding)
|
|
277
|
+
authorization = binding["authorization"]
|
|
278
|
+
return nil unless authorization.is_a?(String) && !authorization.empty?
|
|
279
|
+
service_request_records.find do |other|
|
|
280
|
+
next false if other["request_id"] == binding.fetch("request_id")
|
|
281
|
+
next false if other["state"] == "rejected" && other["consumed"] == false
|
|
282
|
+
# A settled failure frees the exact authorization only while its
|
|
283
|
+
# no-effect evidence remains verifiable in the repository.
|
|
284
|
+
next false if other["state"] == "failed-settled" && settlement_evidence_intact?(other)
|
|
285
|
+
other["authorization"] == authorization &&
|
|
286
|
+
other["operation"] == binding.fetch("operation") &&
|
|
287
|
+
other["project_id"] == binding.fetch("project_id") &&
|
|
288
|
+
other["target"] == binding.fetch("target")
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def settlement_evidence_intact?(record)
|
|
293
|
+
receipt = record["receipt"]
|
|
294
|
+
return false unless receipt.is_a?(Hash)
|
|
295
|
+
repo_root = File.realpath(@repo_root)
|
|
296
|
+
Array(receipt["evidence"]).all? do |item|
|
|
297
|
+
path = File.expand_path(item["ref"].to_s, repo_root)
|
|
298
|
+
real = begin
|
|
299
|
+
File.realpath(path)
|
|
300
|
+
rescue Errno::ENOENT, Errno::EACCES, Errno::ELOOP
|
|
301
|
+
next false
|
|
302
|
+
end
|
|
303
|
+
begin
|
|
304
|
+
real.start_with?(repo_root + File::SEPARATOR) &&
|
|
305
|
+
Digest::SHA256.file(real).hexdigest == item["sha256"]
|
|
306
|
+
rescue Errno::EACCES, Errno::ELOOP
|
|
307
|
+
false
|
|
308
|
+
end
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
def service_request_records
|
|
313
|
+
value = ref_value
|
|
314
|
+
return [] unless value
|
|
315
|
+
paths, stderr, status = git("ls-tree", "-r", "--name-only", value, "--", "execution/requests/")
|
|
316
|
+
raise AttemptErrors::EvidenceUnavailable, "Cannot read service request index: #{stderr}" unless status.success?
|
|
317
|
+
paths.lines.map(&:strip).select { |path| path.end_with?(".json") }.filter_map do |path|
|
|
318
|
+
content, error, read_status = git("show", "#{value}:#{path}")
|
|
319
|
+
unless read_status.success?
|
|
320
|
+
raise AttemptErrors::EvidenceUnavailable, "Cannot read service request #{path}: #{error}"
|
|
321
|
+
end
|
|
322
|
+
JSON.parse(content)
|
|
323
|
+
end
|
|
324
|
+
rescue JSON::ParserError
|
|
325
|
+
raise AttemptErrors::EvidenceUnavailable, "Corrupt service request index"
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
def service_requests(assignment_id)
|
|
329
|
+
request_ids = read_events(assignment_id)
|
|
330
|
+
.select { |event| event["type"] == "service_claim" }
|
|
331
|
+
.map { |event| event.dig("payload", "request_id") }.compact.uniq
|
|
332
|
+
request_ids.filter_map { |request_id| service_request(request_id) }
|
|
333
|
+
end
|
|
334
|
+
|
|
120
335
|
# All attempts derivable from the journal, reconstructed from intent
|
|
121
336
|
# and process_start facts with journal-authoritative state. The
|
|
122
337
|
# journal is the owner of attempt-state interpretation.
|
|
@@ -155,17 +370,81 @@ module Ace
|
|
|
155
370
|
value = ref_value
|
|
156
371
|
return [] if value.nil?
|
|
157
372
|
|
|
373
|
+
paths, stderr, status = git("ls-tree", "-d", "--name-only", value, "--", "execution/")
|
|
374
|
+
raise AttemptErrors::EvidenceUnavailable, "Cannot discover journal assignments: #{stderr}" unless status.success?
|
|
375
|
+
paths.lines.map { |path| path.strip.delete_prefix("execution/") }
|
|
376
|
+
.reject { |id| id == "requests" }.sort
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
private
|
|
380
|
+
|
|
381
|
+
def update_service_request(request_id, expected:, replacement:, event_type:, guard: nil)
|
|
382
|
+
validate_request_id!(request_id)
|
|
158
383
|
with_lock do
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
384
|
+
CAS_ATTEMPTS.times do
|
|
385
|
+
old = ref_value
|
|
386
|
+
ensure_checkout!
|
|
387
|
+
old = ref_value if old.nil?
|
|
388
|
+
sync_checkout(old)
|
|
389
|
+
# Guards (for example authorization consumption) re-run inside
|
|
390
|
+
# the lock against the ref being committed, so a racing claim
|
|
391
|
+
# cannot slip through between the check and the CAS.
|
|
392
|
+
if guard
|
|
393
|
+
conflict = guard.call
|
|
394
|
+
if conflict
|
|
395
|
+
raise AttemptErrors::Conflict,
|
|
396
|
+
"Authorization reference already consumed by request #{conflict.fetch("request_id")}"
|
|
397
|
+
end
|
|
398
|
+
end
|
|
399
|
+
path = File.join(checkout_dir, service_request_path(request_id))
|
|
400
|
+
existing = File.exist?(path) ? JSON.parse(File.read(path)) : nil
|
|
401
|
+
if expected.nil? && existing
|
|
402
|
+
return existing if existing.except("state", "receipt", "reason", "claimed_at") ==
|
|
403
|
+
replacement.except("state", "receipt", "reason", "claimed_at")
|
|
404
|
+
raise AttemptErrors::Conflict, "Service request #{request_id} has different input"
|
|
405
|
+
end
|
|
406
|
+
if expected && existing != expected
|
|
407
|
+
raise AttemptErrors::Conflict, "Service request #{request_id} changed during transition"
|
|
408
|
+
end
|
|
409
|
+
terminal = existing && %w[succeeded failed rejected].include?(existing["state"])
|
|
410
|
+
settlement = existing && existing["state"] == "failed" && replacement["state"] == "failed-settled"
|
|
411
|
+
if expected && existing && terminal && !settlement
|
|
412
|
+
raise AttemptErrors::InvalidState, "Service request #{request_id} is terminal"
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
416
|
+
File.write(path, JSON.pretty_generate(replacement))
|
|
417
|
+
assignment_id = replacement.fetch("assignment_id")
|
|
418
|
+
attempt_id = replacement.fetch("attempt_id")
|
|
419
|
+
prior = read_events(assignment_id).reverse
|
|
420
|
+
.find { |entry| entry["attempt_id"] == attempt_id }&.fetch("digest")
|
|
421
|
+
payload = {"request_id" => request_id, "state" => replacement.fetch("state"),
|
|
422
|
+
"input_digest" => replacement.fetch("input_digest"),
|
|
423
|
+
"receipt_digest" => replacement["receipt"] &&
|
|
424
|
+
Atoms::EvidenceDigest.digest(replacement["receipt"])}
|
|
425
|
+
event = Models::EvidenceEvent.build(type: event_type, attempt_id: attempt_id,
|
|
426
|
+
payload: payload, previous_digest: prior)
|
|
427
|
+
write_event_files(assignment_id, [event])
|
|
428
|
+
git!("-C", checkout_dir, "add", "--", service_request_path(request_id),
|
|
429
|
+
"execution/#{assignment_id}/events")
|
|
430
|
+
state = replacement.fetch("state")
|
|
431
|
+
git!("-C", checkout_dir, "-c", "user.name=ace-assign", "-c", "user.email=ace-assign@localhost",
|
|
432
|
+
"commit", "-m", "evidence: service request #{request_id} #{state}")
|
|
433
|
+
commit = git!("-C", checkout_dir, "rev-parse", "HEAD").first
|
|
434
|
+
return replacement.merge("journal_commit" => commit) if update_ref_cas(commit, old)
|
|
435
|
+
end
|
|
436
|
+
raise AttemptErrors::EvidenceUnavailable, "Service request ref stayed conflicting"
|
|
165
437
|
end
|
|
166
438
|
end
|
|
167
439
|
|
|
168
|
-
|
|
440
|
+
def service_request_path(request_id)
|
|
441
|
+
"execution/requests/#{request_id}.json"
|
|
442
|
+
end
|
|
443
|
+
|
|
444
|
+
def validate_request_id!(request_id)
|
|
445
|
+
return if request_id.is_a?(String) && request_id.match?(/\A[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}\z/)
|
|
446
|
+
raise ArgumentError, "invalid service request ID"
|
|
447
|
+
end
|
|
169
448
|
|
|
170
449
|
# Latest lifecycle state implied by the events, or nil when the
|
|
171
450
|
# events do not describe a full attempt (intent missing).
|
|
@@ -388,7 +667,7 @@ module Ace
|
|
|
388
667
|
def git!(*argv)
|
|
389
668
|
out, stderr, status = git(*argv)
|
|
390
669
|
unless status.success?
|
|
391
|
-
raise AttemptErrors::EvidenceUnavailable, "git #{argv.join(
|
|
670
|
+
raise AttemptErrors::EvidenceUnavailable, "git #{argv.join(" ")} failed: #{stderr}"
|
|
392
671
|
end
|
|
393
672
|
|
|
394
673
|
[out, stderr, status]
|
|
@@ -42,31 +42,46 @@ module Ace
|
|
|
42
42
|
@interactive_builder = interactive_builder || Ace::LLM::Molecules::InteractiveCommandBuilder.new
|
|
43
43
|
end
|
|
44
44
|
|
|
45
|
+
def initialize(config: nil, query_interface: Ace::LLM::QueryInterface, tmux_runner: nil, interactive_builder: nil,
|
|
46
|
+
lifecycle_exclusion: nil)
|
|
47
|
+
@config = config || Ace::Assign.config
|
|
48
|
+
@query_interface = query_interface
|
|
49
|
+
@tmux_runner = tmux_runner || TmuxControlSurfaceRunner.new
|
|
50
|
+
@interactive_builder = interactive_builder || Ace::LLM::Molecules::InteractiveCommandBuilder.new
|
|
51
|
+
@lifecycle_exclusion = lifecycle_exclusion
|
|
52
|
+
end
|
|
53
|
+
|
|
45
54
|
def launch(assignment_id:, fork_root:, provider: nil, cli_args: nil, timeout: nil, cache_dir: nil, launch_mode: nil, callback_pane: nil)
|
|
46
55
|
ensure_not_same_scoped_refork!(assignment_id: assignment_id, fork_root: fork_root)
|
|
47
56
|
resolved_provider = provider || config.dig("execution", "provider") || DEFAULT_PROVIDER
|
|
48
57
|
resolved_timeout = timeout || config.dig("execution", "timeout") || DEFAULT_TIMEOUT
|
|
49
58
|
resolved_mode = resolve_launch_mode(launch_mode)
|
|
50
59
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
60
|
+
# Driver startup participates in prune exclusion: prune holds the
|
|
61
|
+
# exclusive side from final evidence reads through removal; a removed
|
|
62
|
+
# marker fails the launch closed instead of writing into a deleted
|
|
63
|
+
# assignment.
|
|
64
|
+
lifecycle_exclusion.with_shared(lifecycle_exclusion.assignment_key(assignment_id)) do
|
|
65
|
+
if resolved_mode == "tmux"
|
|
66
|
+
launch_tmux(
|
|
67
|
+
assignment_id: assignment_id,
|
|
68
|
+
fork_root: fork_root,
|
|
69
|
+
provider: resolved_provider,
|
|
70
|
+
cli_args: cli_args,
|
|
71
|
+
timeout: resolved_timeout,
|
|
72
|
+
cache_dir: cache_dir,
|
|
73
|
+
callback_pane: callback_pane
|
|
74
|
+
)
|
|
75
|
+
else
|
|
76
|
+
launch_provider_session(
|
|
77
|
+
assignment_id: assignment_id,
|
|
78
|
+
fork_root: fork_root,
|
|
79
|
+
provider: resolved_provider,
|
|
80
|
+
cli_args: cli_args,
|
|
81
|
+
timeout: resolved_timeout,
|
|
82
|
+
cache_dir: cache_dir
|
|
83
|
+
)
|
|
84
|
+
end
|
|
70
85
|
end
|
|
71
86
|
end
|
|
72
87
|
|
|
@@ -111,6 +126,10 @@ module Ace
|
|
|
111
126
|
|
|
112
127
|
attr_reader :config, :query_interface, :tmux_runner, :interactive_builder
|
|
113
128
|
|
|
129
|
+
def lifecycle_exclusion
|
|
130
|
+
@lifecycle_exclusion ||= Molecules::LifecycleExclusion.new
|
|
131
|
+
end
|
|
132
|
+
|
|
114
133
|
def resolve_launch_mode(explicit_mode)
|
|
115
134
|
mode = explicit_mode.to_s.strip
|
|
116
135
|
mode = config.dig("execution", "launch_mode").to_s.strip if mode.empty?
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "digest"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "open3"
|
|
7
|
+
require "time"
|
|
8
|
+
|
|
9
|
+
module Ace
|
|
10
|
+
module Assign
|
|
11
|
+
module Molecules
|
|
12
|
+
# Durable start/prune exclusion for assignment and worktree identities.
|
|
13
|
+
#
|
|
14
|
+
# Lock files live OUTSIDE every deletion target — under the shared Git
|
|
15
|
+
# common dir — so they survive removal of the assignment cache, the
|
|
16
|
+
# worktree, or a recreated lock pathname. Prune holds the exclusive
|
|
17
|
+
# side from its final evidence reads through completion of removal;
|
|
18
|
+
# every supported start path (attempt registration, fork/driver
|
|
19
|
+
# session launch, worktree provisioning) holds the shared side while
|
|
20
|
+
# registering a writer.
|
|
21
|
+
#
|
|
22
|
+
# A `removed` marker recorded by prune makes post-deletion starts fail
|
|
23
|
+
# closed instead of recreating the deleted identity: acquiring the
|
|
24
|
+
# shared side raises {AttemptErrors::Conflict} when the marker is
|
|
25
|
+
# present. Legitimate fresh provisioning (a new worktree for a task)
|
|
26
|
+
# clears the marker explicitly via `reset_removed`.
|
|
27
|
+
class LifecycleExclusion
|
|
28
|
+
# @param repo_root [String, nil] Repository used to resolve the
|
|
29
|
+
# shared Git common dir (default: project root)
|
|
30
|
+
# @param root [String, nil] Explicit exclusion root (tests)
|
|
31
|
+
def initialize(repo_root: nil, root: nil)
|
|
32
|
+
@root = root || self.class.default_root(repo_root)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# The exclusion root shared by every writer and prune for this repo.
|
|
36
|
+
# Sandboxed environments (CACHE_BASE) scope it under the cache base,
|
|
37
|
+
# which itself is never a deletion target; otherwise it lives under
|
|
38
|
+
# the shared Git common dir, which survives worktree removal.
|
|
39
|
+
def self.default_root(_repo_root = nil)
|
|
40
|
+
cache_base = ENV["CACHE_BASE"]
|
|
41
|
+
if cache_base && !cache_base.empty?
|
|
42
|
+
return File.join(File.expand_path(cache_base), ".exclusion")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
repo_root = Ace::Support::Fs::Molecules::ProjectRootFinder.find_or_current
|
|
46
|
+
common_dir, _stderr, status = Open3.capture3(
|
|
47
|
+
"git", "-C", repo_root, "rev-parse", "--path-format=absolute", "--git-common-dir"
|
|
48
|
+
)
|
|
49
|
+
base = status.success? && !common_dir.to_s.strip.empty? ? common_dir.strip : File.join(repo_root, ".git")
|
|
50
|
+
File.join(base, "ace", "lifecycle-exclusion")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def assignment_key(assignment_id)
|
|
54
|
+
"assignment:#{assignment_id}"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Task identity is the canonical worktree identity: work-on derives
|
|
58
|
+
# the worktree path from the task ref, and prune candidates carry
|
|
59
|
+
# the same task id.
|
|
60
|
+
def task_key(task_id)
|
|
61
|
+
"task:#{task_id}"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def worktree_key(worktree_path)
|
|
65
|
+
canonical = begin
|
|
66
|
+
File.realpath(worktree_path.to_s)
|
|
67
|
+
rescue Errno::ENOENT, Errno::EACCES
|
|
68
|
+
File.expand_path(worktree_path.to_s)
|
|
69
|
+
end
|
|
70
|
+
"worktree:#{canonical}"
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
# Hold the exclusive side while recomputing proofs and removing the
|
|
74
|
+
# target. Start paths block for the whole block.
|
|
75
|
+
#
|
|
76
|
+
# @param key [String] Identity key
|
|
77
|
+
# @yield Required block executed under exclusive exclusion
|
|
78
|
+
def with_exclusive(key)
|
|
79
|
+
raise ArgumentError, "exclusion key is required" if key.to_s.empty?
|
|
80
|
+
|
|
81
|
+
lock = open_lock(key)
|
|
82
|
+
lock.flock(File::LOCK_EX)
|
|
83
|
+
begin
|
|
84
|
+
yield
|
|
85
|
+
ensure
|
|
86
|
+
lock.flock(File::LOCK_UN)
|
|
87
|
+
lock.close
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Hold the shared side while registering a writer or creating a
|
|
92
|
+
# resource. Fails closed when the identity was pruned.
|
|
93
|
+
#
|
|
94
|
+
# @param key [String] Identity key
|
|
95
|
+
# @param reset_removed [Boolean] Clear a stale removed marker instead
|
|
96
|
+
# of raising (legitimate fresh provisioning of the same pathname)
|
|
97
|
+
def with_shared(key, reset_removed: false)
|
|
98
|
+
raise ArgumentError, "exclusion key is required" if key.to_s.empty?
|
|
99
|
+
|
|
100
|
+
lock = open_lock(key)
|
|
101
|
+
lock.flock(File::LOCK_SH)
|
|
102
|
+
begin
|
|
103
|
+
if removed?(key)
|
|
104
|
+
unless reset_removed
|
|
105
|
+
raise AttemptErrors::Conflict, removed_message(key)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
clear_removed!(key)
|
|
109
|
+
end
|
|
110
|
+
yield
|
|
111
|
+
ensure
|
|
112
|
+
lock.flock(File::LOCK_UN)
|
|
113
|
+
lock.close
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Hold the shared side on several identities at once (acquired in the
|
|
118
|
+
# given order, released in reverse — callers must use one consistent
|
|
119
|
+
# global order, task before assignment, to avoid deadlock). Fails
|
|
120
|
+
# closed when any identity was pruned.
|
|
121
|
+
def with_shared_multi(keys, reset_removed: false)
|
|
122
|
+
ordered = Array(keys).compact.reject { |key| key.to_s.empty? }.uniq
|
|
123
|
+
raise ArgumentError, "exclusion keys are required" if ordered.empty?
|
|
124
|
+
|
|
125
|
+
locks = ordered.map { |key| open_lock(key) }
|
|
126
|
+
begin
|
|
127
|
+
locks.each { |lock| lock.flock(File::LOCK_SH) }
|
|
128
|
+
ordered.each do |key|
|
|
129
|
+
if removed?(key)
|
|
130
|
+
unless reset_removed
|
|
131
|
+
raise AttemptErrors::Conflict, removed_message(key)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
clear_removed!(key)
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
yield
|
|
138
|
+
ensure
|
|
139
|
+
locks.reverse_each do |lock|
|
|
140
|
+
lock.flock(File::LOCK_UN)
|
|
141
|
+
lock.close
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Record that the identity was removed. Call while holding the
|
|
147
|
+
# exclusive side, BEFORE deleting the target.
|
|
148
|
+
def record_removed!(key)
|
|
149
|
+
atomic_write(
|
|
150
|
+
state_path(key),
|
|
151
|
+
JSON.generate({"removed" => true, "removed_at" => Time.now.utc.iso8601, "key" => key})
|
|
152
|
+
)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def removed?(key)
|
|
156
|
+
state = JSON.parse(File.read(state_path(key)))
|
|
157
|
+
state["removed"] == true
|
|
158
|
+
rescue Errno::ENOENT, JSON::ParserError, TypeError
|
|
159
|
+
false
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def clear_removed!(key)
|
|
163
|
+
File.delete(state_path(key))
|
|
164
|
+
rescue Errno::ENOENT
|
|
165
|
+
nil
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
private
|
|
169
|
+
|
|
170
|
+
attr_reader :root
|
|
171
|
+
|
|
172
|
+
def open_lock(key)
|
|
173
|
+
FileUtils.mkdir_p(root)
|
|
174
|
+
File.open(lock_path(key), File::RDWR | File::CREAT)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def lock_path(key)
|
|
178
|
+
File.join(root, "#{Digest::SHA256.hexdigest(key)}.lock")
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def state_path(key)
|
|
182
|
+
File.join(root, "#{Digest::SHA256.hexdigest(key)}.state.json")
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def removed_message(key)
|
|
186
|
+
identity = key.to_s.sub(/\A[^:]+:/, "")
|
|
187
|
+
case key
|
|
188
|
+
when /\Aassignment:/
|
|
189
|
+
"Assignment #{identity} was pruned; refusing to start a writer on a removed identity"
|
|
190
|
+
when /\Atask:/
|
|
191
|
+
"Task #{identity} worktree was pruned; refusing to start a writer on a removed identity"
|
|
192
|
+
else
|
|
193
|
+
"Worktree #{identity} was pruned; refusing to start a writer on a removed identity"
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def atomic_write(path, content)
|
|
198
|
+
dir = File.dirname(path)
|
|
199
|
+
FileUtils.mkdir_p(dir)
|
|
200
|
+
temp = File.join(dir, ".tmp-#{Process.pid}-#{rand(1_000_000)}")
|
|
201
|
+
File.write(temp, content)
|
|
202
|
+
File.rename(temp, path)
|
|
203
|
+
ensure
|
|
204
|
+
File.delete(temp) if temp && File.exist?(temp)
|
|
205
|
+
end
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
end
|